diff --git a/etc/evergreen.yml b/etc/evergreen.yml index c44e6a62dbd..d3b973ab952 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -60,6 +60,7 @@ include: - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml - filename: etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml - filename: etc/evergreen_yml_components/tasks/compile_tasks.yml + - filename: etc/evergreen_yml_components/tasks/compile_tasks_shared.yml - filename: etc/evergreen_yml_components/tasks/misc_tasks.yml - filename: etc/evergreen_yml_components/variants/misc/task_generation.yml diff --git a/etc/evergreen_nightly.yml b/etc/evergreen_nightly.yml index bd7ae3fe731..7928e938297 100644 --- a/etc/evergreen_nightly.yml +++ b/etc/evergreen_nightly.yml @@ -9,6 +9,7 @@ include: - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml - filename: etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml - filename: etc/evergreen_yml_components/tasks/compile_tasks.yml + - filename: etc/evergreen_yml_components/tasks/compile_tasks_shared.yml - filename: etc/evergreen_yml_components/tasks/misc_tasks.yml - filename: etc/evergreen_yml_components/variants/misc/task_generation.yml diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index a6f55774959..1b3e6e4db62 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -26,6 +26,7 @@ variables: # THIS HAS COPIES IN # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml # - etc/evergreen_yml_components/tasks/compile_tasks.yml + # - etc/evergreen_yml_components/tasks/compile_tasks_shared.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &compile_task_group_template name: compile_task_group_template @@ -170,27 +171,6 @@ variables: # Tasks # ################################################ tasks: - ## compile - build all scons targets except unittests ## - - &compile_dist_test - name: compile_dist_test - tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - targets: >- - install-dist-test - ${additional_compile_targets|} - task_compile_flags: >- - PREFIX=dist-test - - - <<: *compile_dist_test - name: compile_dist_test_future_git_tag_multiversion - depends_on: - - name: version_expansions_future_git_tag_multiversion_gen - ## compile - build bazel targets ## ## The bazel migration is in progress & this ## task ensures we can build with bazel. @@ -479,158 +459,6 @@ tasks: content_type: application/tar display_name: Benchmarks Debug - - &archive_dist_test - name: archive_dist_test - tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] - depends_on: - - name: compile_dist_test - commands: - - func: "scons compile" - vars: - targets: >- - archive-dist-test - task_compile_flags: >- - PREFIX=dist-test - - func: "f_expansions_write" - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - args: - - "src/evergreen/run_python_script.sh" - - "evergreen/macos_notary.py" - - "mongodb-binaries.${ext|tgz}" - - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb-binaries.${ext|tgz} - remote_file: ${mongo_binaries} - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Binaries - - - func: "f_expansions_write" - - func: "gen feature flags" - - func: "fetch resmoke constants" - - - command: archive.targz_pack - params: - target: "artifacts.tgz" - source_dir: "src" - include: - - ".resmoke_mongo_version.yml" - - ".resmoke_mongo_release_values.yml" - - "patch_test_tags.tgz" - - "./build/**.gcno" - - "./etc/*san.suppressions" - - "./etc/backports_required_for_multiversion_tests.yml" - - "./etc/evergreen_timeouts.yml" - - "./etc/expansions.default.yml" - - "./etc/evergreen_nightly.yml" - - "./etc/evergreen.yml" - - "./etc/evergreen_yml_components/**" - - "./etc/repo_config.yaml" - - "./etc/scons/**" - - "./etc/macos_dev_entitlements.xml" - - "docker_compose/**" - - "buildscripts/**" - - "jstests/**" - - "patch_files.txt" - - "evergreen/**" - - "src/**.idl" - - "src/mongo/client/sdam/json_tests/sdam_tests/**" - - "src/mongo/client/sdam/json_tests/server_selection_tests/**" - - "src/mongo/db/modules/enterprise/docs/**" - - "src/mongo/db/modules/enterprise/jstests/**" - - "src/mongo/db/modules/subscription/jstests/**" - - "src/mongo/util/options_parser/test_config_files/**" - - "src/third_party/JSON-Schema-Test-Suite/tests/draft4/**" - - "src/third_party/mock_ocsp_responder/**" - - "src/third_party/schemastore.org/**" - - "poetry.lock" - - "pyproject.toml" - exclude_files: - - "*_test.pdb" - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: artifacts.tgz - remote_file: ${mongo_artifacts} - bucket: mciuploads - permissions: private - visibility: signed - content_type: application/tar - display_name: Artifacts - - - command: archive.targz_pack - params: - target: "venv.tgz" - source_dir: "./" - include: - - "./venv/**" - - "./venv_readme.txt" - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: venv.tgz - remote_file: ${mongo_venv} - bucket: mciuploads - permissions: private - visibility: signed - content_type: application/tar - display_name: Python venv (see included venv_readme.txt) - - - <<: *archive_dist_test - name: archive_dist_test_future_git_tag_multiversion - depends_on: - - name: compile_dist_test_future_git_tag_multiversion - - - &archive_dist_test_debug - name: archive_dist_test_debug - tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] - depends_on: - - name: archive_dist_test - commands: - - func: "scons compile" - vars: - targets: >- - archive-dist-test-debug - task_compile_flags: >- - PREFIX=dist-test - - func: "upload debugsymbols" - - func: "f_expansions_write" - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/record_mongodb_server_version.sh" - - "./build/install/dist-test/bin/mongod" - - "./version_info_mongod.txt" - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/version_info_mongod.txt - remote_file: ${project}/${build_variant}/${version_id}/version-info-mongod-${task_id}.txt - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: mongod --version output - - - <<: *archive_dist_test_debug - name: archive_dist_test_debug_future_git_tag_multiversion - depends_on: - - name: archive_dist_test_future_git_tag_multiversion - - name: compile_ninja tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] depends_on: @@ -1381,53 +1209,6 @@ tasks: args: - "src/evergreen/stitch_support_run_tests.sh" - - name: crypt_create_lib - tags: - [ - "assigned_to_jira_team_server_security", - "release_critical", - "incompatible_community", - "incompatible_tsan", - "incompatible_aubsan", - "requires_compile_variant", - "requires_large_host_debug_mode", - "crypt", - "bazel_check", - ] - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "do scons setup" - - func: "f_expansions_write" - - func: "scons compile" - vars: - targets: archive-mongo-crypt-dev - task_compile_flags: >- - --allocator=system - --enterprise-features=fle - --js-engine=none - --link-model=dynamic-sdk - --enable-http-client=off - --ssl=off - ${crypt_task_compile_flags} - DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' - - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/crypt_run_tests.sh" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: "src/mongo-crypt-dev.${ext|tgz}" - remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1-${version}.${ext|tgz}" - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/tar} - display_name: "Mongo crypt Library" - - name: crypt_create_debug_lib tags: ["assigned_to_jira_team_server_security", "auxiliary", "bazel_check"] depends_on: @@ -1509,133 +1290,6 @@ tasks: args: - "src/evergreen/crypt_run_tests.sh" - - name: package - tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] - depends_on: - - name: compile_dist_test - commands: - - func: "f_expansions_write" - - func: "scons compile" - vars: - targets: >- - archive-dist - archive-dist-debug - archive-jstestshell - archive-jstestshell-debug - ${additional_package_targets|} - task_compile_flags: >- - --legacy-tarball - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - args: - - "src/evergreen/run_python_script.sh" - - "evergreen/macos_notary.py" - - "mongodb-dist.${ext|tgz}" - - func: "f_expansions_write" - - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/package.sh" - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb-dist.${ext|tgz} - remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}.${ext|tgz} - bucket: mciuploads - permissions: public-read - content_type: application/tar - display_name: Dist Tarball - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb-dist-debugsymbols.${ext|tgz} - remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}-debugsymbols.${ext|tgz} - bucket: mciuploads - permissions: public-read - content_type: application/tar - display_name: Dist Debugsymbols - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb-jstestshell.${ext|tgz} - remote_file: ${mongo_jstestshell} - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Jstestshell - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb-jstestshell-debugsymbols.${ext|tgz} - remote_file: ${mongo_jstestshell_debugsymbols} - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Jstestshell Debugsymbols - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb-cryptd.${ext|tgz} - remote_file: ${mongo_cryptd} - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: CryptD Binaries - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb-cryptd-debugsymbols.${ext|tgz} - remote_file: ${mongo_cryptd_debugsymbols} - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: CryptD Debugsymbols - - func: "run packager.py" - - command: archive.targz_pack - params: - target: "packages.tgz" - source_dir: "src" - include: - - "repo/**" - - "./**.msi" - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: packages.tgz - remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz - bucket: mciuploads - permissions: public-read - content_type: application/tar - display_name: Packages - # We perform package task generally in the middle of larger task groups running - # on the same host serially. In order to reduce peak required disk space, we can delete - # these generated files since we have already uploaded them to aws at this point. - - func: "remove files" - vars: - files: >- - packages.tgz - src/mongodb-cryptd-debugsymbols.${ext|tgz} - src/mongodb-cryptd.${ext|tgz} - src/mongodb-jstestshell-debugsymbols.${ext|tgz} - src/mongodb-jstestshell.${ext|tgz} - src/mongodb-dist-debugsymbols.${ext|tgz} - src/mongodb-dist.${ext|tgz} - - name: publish_packages run_on: rhel8.7-small tags: @@ -2548,7 +2202,6 @@ task_groups: name: compile_dist_test_TG tasks: - compile_dist_test - - <<: *compile_bazel_task_group_template name: compile_bazel_TG tasks: @@ -2764,13 +2417,6 @@ task_groups: # - archive_dbtest # - compile_all_but_not_unittests - - <<: *compile_task_group_template - name: compile_and_package_serial_no_unittests_TG - tasks: - - compile_dist_test - - archive_dist_test - - archive_dist_test_debug - # The *no_unittests_TG taskgroups are useful for static builds, # were the static binaries of the unittests can take up to # a terabyte of data. diff --git a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml new file mode 100644 index 00000000000..40843e3e139 --- /dev/null +++ b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml @@ -0,0 +1,463 @@ +variables: + # THIS HAS COPIES IN + # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml + # - etc/evergreen_yml_components/tasks/compile_tasks.yml + # - etc/evergreen_yml_components/tasks/compile_tasks_shared.yml + # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES + - &compile_task_group_template + name: compile_task_group_template + max_hosts: 1 + tasks: [] + setup_task: + - func: "f_expansions_write" + - func: "apply compile expansions" + - func: "f_expansions_write" + - func: "set task expansion macros" + - func: "f_expansions_write" + - func: "get engflow creds" + - func: "fetch pgo profile" + teardown_task: + - func: "f_expansions_write" + - func: "attach scons logs" + - func: "attach report" + - func: "attach artifacts" + - func: "attach local resmoke invocation" + - func: "attach multiversion download links" + - func: "kill processes" + - func: "save code coverage data" + - func: "save mongo coredumps" + - func: "generate hang analyzer tasks" + - func: "save failed unittests" + - func: "save UndoDB recordings" + - func: "save unstripped dbtest" + - func: "save bazel headers" + - func: "save hang analyzer debugger files" + - func: "save disk statistics" + - func: "save system resource information" + - func: "save libfuzzertest corpora" + - func: "remove files" + vars: + files: >- + src/resmoke_error_code + src/build/scons/config.log + src/*.gcda.gcov + src/gcov-intermediate-files.tgz + src/*.core src/*.mdmp src/*.core.gz src/*.mdmp.gz + mongo-coredumps.json + src/dist-unittests/bin/* + src/dist-unittests/lib/* + mongo-unittests.tgz + src/debugger*.* + src/mongo-hanganalyzer.tgz + diskstats.tgz + system-resource-info.tgz + ${report_file|src/report.json} + ${archive_file|src/archive.json} + setup_group_can_fail_task: true + setup_group: + - command: manifest.load + - func: "git get project and add git tag" + - func: "set task expansion macros" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + # The python virtual environment is installed in ${workdir}, which is created in + # "set up venv". + - func: "set up venv" + - func: "upload pip requirements" + - func: "f_expansions_write" + - func: "configure evergreen api credentials" + - func: "get buildnumber" + - func: "f_expansions_write" + - func: "set up credentials" + - func: "f_expansions_write" + - func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true" + - func: "f_expansions_write" + - func: "set up win mount script" + - func: "f_expansions_write" + - func: "generate compile expansions" + - func: "f_expansions_write" + teardown_group: + - func: "f_expansions_write" + - func: "umount shared scons directory" + - func: "cleanup environment" + timeout: + - func: "f_expansions_write" + - func: "run hang analyzer" + - func: "wait for resmoke to shutdown" + +task_groups: + - <<: *compile_task_group_template + name: compile_and_package_serial_no_unittests_TG + tasks: + - compile_dist_test + - archive_dist_test + - archive_dist_test_debug + +tasks: + - &version_expansions_gen + name: version_expansions_gen + tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] + priority: 10 + commands: + - command: manifest.load + - func: "git get shallow project" + - func: "f_expansions_write" + - func: "restore git history and tags" + - func: "add git tag" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "generate and upload version expansions" + - func: "generate and upload resmoke constants" + + - <<: *version_expansions_gen + name: version_expansions_future_git_tag_multiversion_gen + + ## compile - build all scons targets except unittests ## + - &compile_dist_test + name: compile_dist_test + tags: ["assigned_to_jira_team_devprod_build", "auxiliary", "bazel_check"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + targets: >- + install-dist-test + ${additional_compile_targets|} + task_compile_flags: >- + PREFIX=dist-test + + - <<: *compile_dist_test + name: compile_dist_test_future_git_tag_multiversion + depends_on: + - name: version_expansions_future_git_tag_multiversion_gen + + - &archive_dist_test + name: archive_dist_test + tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] + depends_on: + - name: compile_dist_test + commands: + - func: "scons compile" + vars: + targets: >- + archive-dist-test + task_compile_flags: >- + PREFIX=dist-test + - func: "f_expansions_write" + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + args: + - "src/evergreen/run_python_script.sh" + - "evergreen/macos_notary.py" + - "mongodb-binaries.${ext|tgz}" + + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-binaries.${ext|tgz} + remote_file: ${mongo_binaries} + bucket: mciuploads + permissions: public-read + content_type: application/gzip + display_name: Binaries + + - func: "f_expansions_write" + - func: "gen feature flags" + - func: "fetch resmoke constants" + + - command: archive.targz_pack + params: + target: "artifacts.tgz" + source_dir: "src" + include: + - ".resmoke_mongo_version.yml" + - ".resmoke_mongo_release_values.yml" + - "patch_test_tags.tgz" + - "./build/**.gcno" + - "./etc/*san.suppressions" + - "./etc/backports_required_for_multiversion_tests.yml" + - "./etc/evergreen_timeouts.yml" + - "./etc/expansions.default.yml" + - "./etc/evergreen_nightly.yml" + - "./etc/evergreen.yml" + - "./etc/evergreen_yml_components/**" + - "./etc/repo_config.yaml" + - "./etc/scons/**" + - "./etc/macos_dev_entitlements.xml" + - "docker_compose/**" + - "buildscripts/**" + - "jstests/**" + - "patch_files.txt" + - "evergreen/**" + - "src/**.idl" + - "src/mongo/client/sdam/json_tests/sdam_tests/**" + - "src/mongo/client/sdam/json_tests/server_selection_tests/**" + - "src/mongo/db/modules/enterprise/docs/**" + - "src/mongo/db/modules/enterprise/jstests/**" + - "src/mongo/db/modules/subscription/jstests/**" + - "src/mongo/util/options_parser/test_config_files/**" + - "src/third_party/JSON-Schema-Test-Suite/tests/draft4/**" + - "src/third_party/mock_ocsp_responder/**" + - "src/third_party/schemastore.org/**" + - "poetry.lock" + - "pyproject.toml" + exclude_files: + - "*_test.pdb" + + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: artifacts.tgz + remote_file: ${mongo_artifacts} + bucket: mciuploads + permissions: private + visibility: signed + content_type: application/tar + display_name: Artifacts + + - command: archive.targz_pack + params: + target: "venv.tgz" + source_dir: "./" + include: + - "./venv/**" + - "./venv_readme.txt" + + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: venv.tgz + remote_file: ${mongo_venv} + bucket: mciuploads + permissions: private + visibility: signed + content_type: application/tar + display_name: Python venv (see included venv_readme.txt) + + - <<: *archive_dist_test + name: archive_dist_test_future_git_tag_multiversion + depends_on: + - name: compile_dist_test_future_git_tag_multiversion + + - &archive_dist_test_debug + name: archive_dist_test_debug + tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] + depends_on: + - name: archive_dist_test + commands: + - func: "scons compile" + vars: + targets: >- + archive-dist-test-debug + task_compile_flags: >- + PREFIX=dist-test + - func: "upload debugsymbols" + - func: "f_expansions_write" + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/record_mongodb_server_version.sh" + - "./build/install/dist-test/bin/mongod" + - "./version_info_mongod.txt" + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/version_info_mongod.txt + remote_file: ${project}/${build_variant}/${version_id}/version-info-mongod-${task_id}.txt + bucket: mciuploads + permissions: public-read + content_type: text/plain + display_name: mongod --version output + + - <<: *archive_dist_test_debug + name: archive_dist_test_debug_future_git_tag_multiversion + depends_on: + - name: archive_dist_test_future_git_tag_multiversion + + - name: crypt_create_lib + tags: + [ + "assigned_to_jira_team_server_security", + "release_critical", + "incompatible_community", + "incompatible_tsan", + "incompatible_aubsan", + "requires_compile_variant", + "requires_large_host_debug_mode", + "crypt", + "bazel_check", + ] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "do scons setup" + - func: "f_expansions_write" + - func: "scons compile" + vars: + targets: archive-mongo-crypt-dev + task_compile_flags: >- + --allocator=system + --enterprise-features=fle + --js-engine=none + --link-model=dynamic-sdk + --enable-http-client=off + --ssl=off + ${crypt_task_compile_flags} + DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/crypt_run_tests.sh" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: "src/mongo-crypt-dev.${ext|tgz}" + remote_file: "${project}/mongo_crypt/${build_variant}/${revision}/mongo_crypt_shared_v1-${version}.${ext|tgz}" + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/tar} + display_name: "Mongo crypt Library" + + - name: package + tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] + depends_on: + - name: compile_dist_test + commands: + - func: "f_expansions_write" + - func: "scons compile" + vars: + targets: >- + archive-dist + archive-dist-debug + archive-jstestshell + archive-jstestshell-debug + ${additional_package_targets|} + task_compile_flags: >- + --legacy-tarball + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + args: + - "src/evergreen/run_python_script.sh" + - "evergreen/macos_notary.py" + - "mongodb-dist.${ext|tgz}" + - func: "f_expansions_write" + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/package.sh" + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-dist.${ext|tgz} + remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}.${ext|tgz} + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: Dist Tarball + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-dist-debugsymbols.${ext|tgz} + remote_file: ${project}/${build_variant}/${revision}/dist/mongo-${build_id}-debugsymbols.${ext|tgz} + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: Dist Debugsymbols + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-jstestshell.${ext|tgz} + remote_file: ${mongo_jstestshell} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Jstestshell + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-jstestshell-debugsymbols.${ext|tgz} + remote_file: ${mongo_jstestshell_debugsymbols} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Jstestshell Debugsymbols + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-cryptd.${ext|tgz} + remote_file: ${mongo_cryptd} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: CryptD Binaries + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb-cryptd-debugsymbols.${ext|tgz} + remote_file: ${mongo_cryptd_debugsymbols} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: CryptD Debugsymbols + - func: "run packager.py" + - command: archive.targz_pack + params: + target: "packages.tgz" + source_dir: "src" + include: + - "repo/**" + - "./**.msi" + - command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: packages.tgz + remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: Packages + # We perform package task generally in the middle of larger task groups running + # on the same host serially. In order to reduce peak required disk space, we can delete + # these generated files since we have already uploaded them to aws at this point. + - func: "remove files" + vars: + files: >- + packages.tgz + src/mongodb-cryptd-debugsymbols.${ext|tgz} + src/mongodb-cryptd.${ext|tgz} + src/mongodb-jstestshell-debugsymbols.${ext|tgz} + src/mongodb-jstestshell.${ext|tgz} + src/mongodb-dist-debugsymbols.${ext|tgz} + src/mongodb-dist.${ext|tgz} diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index 50eaabca68c..2f1f2e09f7e 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -1602,26 +1602,6 @@ tasks: - func: "upload pip requirements" - func: "generate version burn in" - - &version_expansions_gen - name: version_expansions_gen - tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] - priority: 10 - commands: - - command: manifest.load - - func: "git get shallow project" - - func: "f_expansions_write" - - func: "restore git history and tags" - - func: "add git tag" - - func: "kill processes" - - func: "cleanup environment" - - func: "set up venv" - - func: "upload pip requirements" - - func: "generate and upload version expansions" - - func: "generate and upload resmoke constants" - - - <<: *version_expansions_gen - name: version_expansions_future_git_tag_multiversion_gen - - name: version_gen run_on: ubuntu2004-medium tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml index ba40851af74..eaee607424e 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml @@ -106,6 +106,7 @@ variables: # THIS HAS COPIES IN # - etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml # - etc/evergreen_yml_components/tasks/compile_tasks.yml + # - etc/evergreen_yml_components/tasks/compile_tasks_shared.yml # ANY MODIFICATIONS HERE SHOULD ALSO BE MADE IN THOSE FILES - &compile_task_group_template name: compile_task_group_template diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 5712da87188..73d9b830733 100755 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -5,13 +5,7 @@ timeout_secs: &timeout_secs 7200 include: - filename: etc/evergreen_yml_components/definitions.yml - - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml - - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml - - filename: etc/evergreen_yml_components/tasks/compile_tasks.yml - - filename: etc/evergreen_yml_components/tasks/misc_tasks.yml - # This is a substitute task_generation.yml for sys-perf that doesn't - # generate resmoke tasks. - - filename: etc/system_perf_yml_components/variants/task_generation.yml + - filename: etc/evergreen_yml_components/tasks/compile_tasks_shared.yml ### lint_yaml trim start ### # # The lines below are trimmed by yamllinters.sh so that `evergreen evaluate` works on this file. @@ -391,6 +385,16 @@ tasks: - func: "upload supplementary data" buildvariants: + # Replacement for generate-tasks-for-version from + # evergreen_yml_components/variants/misc/task_generation.yml. + - name: generate-tasks-for-version + display_name: "~ Generate tasks for evergreen version" + activate: true + run_on: + - rhel8.8-medium + tasks: + - name: version_expansions_gen + - name: amazon2-x86-compile display_name: "* Compile for Amazon Linux 2 x86" expansions: diff --git a/etc/system_perf_yml_components/variants/task_generation.yml b/etc/system_perf_yml_components/variants/task_generation.yml deleted file mode 100644 index 6868f2bf4d4..00000000000 --- a/etc/system_perf_yml_components/variants/task_generation.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Replacement for generate-tasks-for-version from evergreen_yml_components. -# -# This is similar to generate-tasks-for-version in evergreen_yml_components, -# but doesn't include actual task generation, which does nothing -# for sys-perf, and breaks when it tries to process the sys-perf project. -# - -buildvariants: - - name: generate-tasks-for-version - display_name: "! Generate tasks for evergreen version" - activate: true - run_on: - - rhel8.8-medium - tasks: - - name: version_expansions_gen