SERVER-69353 Generate burn-in tasks separately

etc/evergreen.yml

@@ -1227,6 +1227,8 @@ buildvariants:
     exec_timeout_factor: 1.5
     large_distro_name: rhel80-medium
     burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
   tasks:
   - name: compile_test_and_package_parallel_core_stream_TG
@@ -1519,6 +1521,8 @@ buildvariants:
     max_sub_suites: 5
     large_distro_name: rhel80-medium
     burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
     test_flags: >-
       --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"

etc/evergreen_yml_components/definitions.yml

@@ -102,6 +102,17 @@ variables:
     vars:
       resmoke_args: --help
 
+- &gen_burn_in_task_template
+  name: gen_burn_in_task_template
+  depends_on:
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+  - name: archive_dist_test_debug
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      resmoke_args: --help
+
 - &benchmark_template
   name: benchmark_template
   depends_on:
@@ -1037,6 +1048,38 @@ functions:
       files:
       - src/generated_resmoke_config/*.json
 
+  "generate version burn in":
+  - *f_expansions_write
+  - *configure_evergreen_api_credentials
+  - command: subprocess.exec
+    type: test
+    params:
+      binary: bash
+      args:
+      - "./src/evergreen/generate_version_burn_in.sh"
+  - command: archive.targz_pack
+    params:
+      target: generate_tasks_config.tgz
+      source_dir: src/generated_resmoke_config
+      include:
+      - "*"
+  - command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: generate_tasks_config.tgz
+      remote_file: ${project}/${revision}/generate_tasks/generated-burn-in-config-${version_id}.tgz
+      bucket: mciuploads
+      permissions: public-read
+      content_type: ${content_type|application/gzip}
+      display_name: Generated Burn In Task Config - Execution ${execution}
+      optional: true
+  - command: generate.tasks
+    params:
+      optional: true
+      files:
+      - src/generated_resmoke_config/*.json
+
   "generate resmoke tasks":
   - *fetch_artifacts
   - *f_expansions_write
@@ -3367,7 +3410,7 @@ tasks:
       args:
       - "src/evergreen/check_idl_compat.sh"
 
-- <<: *gen_task_template
+- <<: *gen_burn_in_task_template
   name: burn_in_tests_gen
   tags: []
   commands:
@@ -4432,7 +4475,7 @@ tasks:
   commands:
   - func: "generate resmoke tasks"
 
-- <<: *gen_task_template
+- <<: *gen_burn_in_task_template
   name: burn_in_tags_gen
   tags: []
   commands:
@@ -4450,6 +4493,18 @@ tasks:
   - func: "upload pip requirements"
   - func: "generate version"
 
+- name: version_burn_in_gen
+  commands:
+  - command: manifest.load
+  - *git_get_project
+  - *f_expansions_write
+  - *add_git_tag
+  - *kill_processes
+  - *cleanup_environment
+  - func: "set up venv"
+  - func: "upload pip requirements"
+  - func: "generate version burn in"
+
 - name: version_expansions_gen
   commands:
   - command: manifest.load

etc/evergreen_yml_components/variants/task_generation.yml

@@ -10,4 +10,5 @@ buildvariants:
   - rhel80-medium
   tasks:
   - name: version_gen
+  - name: version_burn_in_gen
   - name: version_expansions_gen

evergreen/generate_version.sh

@@ -6,25 +6,11 @@ cd src
 set -o errexit
 set -o verbose
 
-curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.5.3/mongo-task-generator --output mongo-task-generator
-chmod +x mongo-task-generator
-
-## Comment above and uncomment below to test unreleased mongo-task-generator changes pushed to <branch-name>
-#curl https://sh.rustup.rs -sSf | sh -s -- -y
-#source "$HOME/.cargo/env"
-#git clone git@github.com:mongodb/mongo-task-generator.git unreleased-mongo-task-generator
-#pushd unreleased-mongo-task-generator
-#git checkout <branch-name>
-#cargo build --release --locked
-#generator_path="$(pwd)/target/release/mongo-task-generator"
-#popd
-#cp "$generator_path" mongo-task-generator
-
+setup_mongo_task_generator
 activate_venv
 PATH=$PATH:$HOME:/ ./mongo-task-generator \
   --expansion-file ../expansions.yml \
   --evg-auth-file ./.evergreen.yml \
   --evg-project-file ${evergreen_config_file_path} \
   --generate-sub-tasks-config etc/generate_subtasks_config.yml \
-  --burn-in \
   $@

evergreen/generate_version_burn_in.sh

@@ -0,0 +1,17 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+setup_mongo_task_generator
+activate_venv
+PATH=$PATH:$HOME:/ ./mongo-task-generator \
+  --expansion-file ../expansions.yml \
+  --evg-auth-file ./.evergreen.yml \
+  --evg-project-file ${evergreen_config_file_path} \
+  --generate-sub-tasks-config etc/generate_subtasks_config.yml \
+  --burn-in \
+  $@

evergreen/prelude.sh

@@ -11,6 +11,7 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$evergreen_dir/prelude_python.sh"
 . "$evergreen_dir/prelude_venv.sh"
 . "$evergreen_dir/prelude_db_contrib_tool.sh"
+. "$evergreen_dir/prelude_mongo_task_generator.sh"
 
 expansions_yaml="$evergreen_dir/../../expansions.yml"
 expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"

evergreen/prelude_mongo_task_generator.sh

@@ -0,0 +1,24 @@
+function setup_mongo_task_generator {
+  if [ ! -f mongo-task-generator ]; then
+    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.6.2/mongo-task-generator --output mongo-task-generator
+    chmod +x mongo-task-generator
+  fi
+}
+
+## Comment above and uncomment below to test unreleased mongo-task-generator changes that are
+## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git`
+## repo
+#function setup_mongo_task_generator {
+#  if [ ! -f mongo-task-generator ]; then
+#
+#    curl https://sh.rustup.rs -sSf | sh -s -- -y
+#    source "$HOME/.cargo/env"
+#    git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator
+#    pushd unreleased-mongo-task-generator
+#    git checkout <branch-name>
+#    cargo build --release --locked
+#    generator_path="$(pwd)/target/release/mongo-task-generator"
+#    popd
+#    cp "$generator_path" mongo-task-generator
+#  fi
+#}