SERVER-70712 improve scons splunk upload performance

etc/evergreen_yml_components/definitions.yml

@@ -1240,6 +1240,13 @@ functions:
   - *get_version_expansions
   - *apply_version_expansions
   - *f_expansions_write
+  - command: subprocess.exec
+    params:
+      background: true
+      continue_on_err: true
+      binary: bash
+      args:
+      - "./src/evergreen/scons_splunk.sh"
   - command: subprocess.exec
     type: test
     params:
@@ -2094,14 +2101,6 @@ functions:
       permissions: public-read
       display_name: SCons stdout log
 
-  - *f_expansions_write
-  - command: subprocess.exec
-    params:
-      continue_on_err: true
-      binary: bash
-      args:
-      - "./src/evergreen/scons_splunk.sh"
-
   "send scons cedar report":
     - command: subprocess.exec
       params:

evergreen/scons_splunk.sh

@@ -1,4 +1,6 @@
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$DIR/prelude.sh"
 
-curator --level warning splunk --json --url=${scons_splunk_server} --token=${scons_splunk_token} --annotation=project:${project} --annotation=task_id:${task_id} --annotation=build_variant:${build_variant} --annotation=git_revision:${revision} pipe < src/scons_cache.log.json
+touch src/scons_cache.log.json
+# TODO EVG-18207 remove GODEBUG=asyncpreemptoff=1
+GODEBUG=asyncpreemptoff=1 curator --level warning splunk --json --url=${scons_splunk_server} --token=${scons_splunk_token} --annotation=project:${project} --annotation=task_id:${task_id} --annotation=build_variant:${build_variant} --annotation=git_revision:${revision} follow --file src/scons_cache.log.json

site_scons/site_tools/validate_cache_dir.py

@@ -192,7 +192,7 @@ class CacheDirValidate(SCons.CacheDir.CacheDir):
     def CacheDebugJson(self, json_data, target, cachefile):
 
         if SCons.CacheDir.cache_debug and SCons.CacheDir.cache_debug != '-' and self.json_log is None:
-            self.json_log = open(SCons.CacheDir.cache_debug + '.json', 'w')
+            self.json_log = open(SCons.CacheDir.cache_debug + '.json', 'a')
 
         if self.json_log is not None:
             cksum_cachefile = str(pathlib.Path(cachefile).parent)