From a6349b5605e1689cda61de4992e8afc7896744cf Mon Sep 17 00:00:00 2001 From: Mikhail Shchatko Date: Thu, 16 May 2024 20:59:58 +0300 Subject: [PATCH] SERVER-90565 Re-enable evergreen yml lint rules (#22248) GitOrigin-RevId: 665a9e3df84103ad40bce85cb5daecfb247b85fb --- etc/evergreen_lint.yml | 8 +-- etc/evergreen_yml_components/definitions.yml | 61 +++++++++++++------ .../tasks/compile_tasks.yml | 29 +++++---- .../tasks/misc_tasks.yml | 10 ++- etc/yamllint_config.yml | 7 ++- evergreen/container_registry_login.sh | 7 +++ evergreen/verify_build_output_present.sh | 7 +++ ...ngo_binary_url_to_downstream_expansions.sh | 5 ++ poetry.lock | 8 +-- pyproject.toml | 2 +- 10 files changed, 99 insertions(+), 45 deletions(-) create mode 100644 evergreen/container_registry_login.sh create mode 100644 evergreen/verify_build_output_present.sh create mode 100644 evergreen/write_mongo_binary_url_to_downstream_expansions.sh diff --git a/etc/evergreen_lint.yml b/etc/evergreen_lint.yml index 84bc11b37ad..b14deace4d1 100644 --- a/etc/evergreen_lint.yml +++ b/etc/evergreen_lint.yml @@ -15,7 +15,7 @@ rules: # # users to source a script called `prelude.sh`. This script requires that # # the user be in ${workdir}, so we forbid users from `cd`ing with # # `working_dir` parameter. - # - rule: "no-working-dir-on-shell" + - rule: "no-working-dir-on-shell" # # Embedding shell.exec scripts in the Evergreen yaml has been a historical # # source of pain: @@ -25,7 +25,7 @@ rules: # # We completed a project to completely eject all shell scripts from the # # YAML, and now require this of all shell scripts. Add your scripts to the # # evergreen/ directory, and make use of subprocess.exec. - # - rule: "no-shell-exec" + - rule: "no-shell-exec" # Historically, expansions.update has been used to inject shell scripts # into expansions, which were then used to inject commonly used snippets @@ -41,8 +41,8 @@ rules: # # To ensure that the expansions file loaded by prelude.sh is not stale, we # # require the user to add sometimes superfluous expansions.write calls to # # ere on side of correctness. - # - rule: "required-expansions-write" - # regex: .*\/evergreen\/.*\.sh + - rule: "required-expansions-write" + regex: .*\/evergreen\/.*\.sh # Generated tests must depend on build_variant_gen (which generates the # tasks that contain them) and archive_dist_test (which contains diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 56d1bf6853d..7c9fee68e7b 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -188,6 +188,7 @@ functions: "get engflow creds": - *get_engflow_key - *get_engflow_cert + - *f_expansions_write - *generate_evergreen_bazelrc "adjust venv": &adjust_venv @@ -241,12 +242,11 @@ functions: local_file: src/mongo-binaries.tgz "write mongo binary URL to downstream_expansions.yml": &echo_mongo_binary_url - command: shell.exec + command: subprocess.exec params: - shell: bash - script: | - touch downstream_expansions.yaml - echo "${build_variant}-mongo-binaries: https://mciuploads.s3.amazonaws.com/${mongo_binaries}" 2>&1 | tee downstream_expansions.yaml + binary: bash + args: + - "./src/evergreen/write_mongo_binary_url_to_downstream_expansions.sh" "set downstreams expansions": &downstream_expansions command: downstream_expansions.set @@ -299,6 +299,7 @@ functions: "get and apply version expansions": - *get_version_expansions - *apply_version_expansions + - *f_expansions_write "fetch benchmarks": &fetch_benchmarks command: s3.get @@ -601,6 +602,7 @@ functions: display_name: Pip Requirements "run clang tidy": &run_clang_tidy + - *f_expansions_write - command: subprocess.exec params: binary: bash @@ -687,6 +689,12 @@ functions: - *collect_ulimit_info "write and set downstream expansions": + - *f_expansions_write + - *fetch_artifacts + - *kill_processes + - *cleanup_environment + - *fetch_venv + - *adjust_venv - *echo_mongo_binary_url - *downstream_expansions @@ -698,6 +706,7 @@ functions: remote_file: ${mongo_debugsymbols} bucket: mciuploads local_file: src/mongo-debugsymbols.${ext|tgz} + - *f_expansions_write - command: subprocess.exec params: binary: bash @@ -969,9 +978,12 @@ functions: - *generate_resmoke_tasks_config "initialize multiversion tasks": &initialize_multiversion_tasks - - command: shell.exec + - *f_expansions_write + - command: subprocess.exec params: - script: "echo 'noop'" + binary: echo + args: + - "noop" "generate resmoke tasks": - *fetch_artifacts @@ -980,6 +992,7 @@ functions: - *cleanup_environment - *get_version_expansions - *apply_version_expansions + - *f_expansions_write - *fetch_venv - *adjust_venv - *f_expansions_write @@ -1055,6 +1068,7 @@ functions: - *configure_evergreen_api_credentials - *determine_task_timeout - *update_task_timeout_expansions + - *f_expansions_write - *update_task_timeout - *f_expansions_write - *sign_macos_dev_binaries @@ -1177,6 +1191,7 @@ functions: updates: - key: install_dir value: ${install_dir} + - *f_expansions_write - command: s3.put display_name: "attach bazel scons diff file" params: @@ -1231,6 +1246,7 @@ functions: display_name: Bazel-SCons Diff File "fetch bazel": + - *f_expansions_write - command: subprocess.exec display_name: "fetch bazel" type: test @@ -1242,12 +1258,13 @@ functions: "verify build output present": # Args: # $output (string) - relpath to the file-to-be-checked - - command: shell.exec + - *f_expansions_write + - command: subprocess.exec display_name: "verify build output present" params: - working_dir: src - script: | - test -f ${output} + binary: bash + args: + - "src/evergreen/verify_build_output_present.sh" "bazel compile sh": &bazel_compile_sh command: subprocess.exec @@ -1326,6 +1343,7 @@ functions: display_name: version expansions "generate and upload version expansions": + - *f_expansions_write - *generate_version_expansions - *upload_version_expansions @@ -1367,6 +1385,7 @@ functions: display_name: resmoke constants "generate and upload resmoke constants": + - *f_expansions_write - *generate_resmoke_constants - *archive_resmoke_constants - *upload_remoske_constants @@ -1712,11 +1731,12 @@ functions: - "./src/evergreen/packager.py_run.sh" "do watchdog setup": - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/do_watchdog_setup.sh" + - *f_expansions_write + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/do_watchdog_setup.sh" "run package test": - *get_version_expansions @@ -2343,6 +2363,7 @@ functions: file_location: ${report_file|src/report.json} "combine build metrics": + - *f_expansions_write - command: subprocess.exec params: binary: bash @@ -2350,6 +2371,7 @@ functions: args: - "src/evergreen/run_python_script.sh" - "site_scons/site_tools/build_metrics/combine_metrics_unittest.py" + - *f_expansions_write - command: subprocess.exec params: binary: bash @@ -2362,6 +2384,7 @@ functions: - "--prefix-name=pull_cache" "print top N metrics": + - *f_expansions_write - command: subprocess.exec params: binary: bash @@ -2405,6 +2428,7 @@ functions: content_type: application/gzip display_name: Metrics JSON + - *f_expansions_write - command: subprocess.exec params: binary: bash @@ -2561,22 +2585,25 @@ functions: - "./src/evergreen/antithesis_image_build_and_push.sh" "cleanup external auth OIDC resources": + - *f_expansions_write - command: subprocess.exec params: binary: bash args: - "./src/evergreen/external_auth_gcp_teardown.sh" + - *f_expansions_write - command: subprocess.exec params: binary: bash args: - "./src/evergreen/external_auth_azure_teardown.sh" + - *f_expansions_write - command: subprocess.exec params: binary: bash args: - "./src/evergreen/external_auth_oidc_teardown.sh" - + - *f_expansions_write - command: subprocess.exec params: binary: bash diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index d6668359b78..471d4c45b72 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -183,6 +183,7 @@ tasks: ${additional_compile_targets|} task_compile_flags: >- PREFIX=dist-test + - func: "f_expansions_write" - command: subprocess.exec params: binary: bash @@ -485,6 +486,7 @@ tasks: archive-dist-test task_compile_flags: >- PREFIX=dist-test + - func: "f_expansions_write" - command: subprocess.exec params: binary: bash @@ -1573,13 +1575,12 @@ tasks: aws_secret_remote: ${repo_aws_secret} - func: "set up notary client credentials" - func: "f_expansions_write" - - command: shell.exec + - command: subprocess.exec params: - shell: bash + binary: bash silent: true - script: | - set -oe - echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry} + args: + - "./src/evergreen/container_registry_login.sh" - command: subprocess.exec params: binary: bash @@ -1641,13 +1642,12 @@ tasks: - func: "f_expansions_write" # login to container registry - - command: shell.exec + - command: subprocess.exec params: - shell: bash + binary: bash silent: true - script: | - set -oe - echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry} + args: + - "./src/evergreen/container_registry_login.sh" # signing windows artifacts - command: subprocess.exec @@ -2271,13 +2271,12 @@ tasks: aws_secret_remote: ${repo_aws_secret} - func: "f_expansions_write" # login to container registry - - command: shell.exec + - command: subprocess.exec params: - shell: bash + binary: bash silent: true - script: | - set -oe - echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry} + args: + - "./src/evergreen/container_registry_login.sh" - command: subprocess.exec params: binary: bash diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index 6411c4a2cde..bf0178bb625 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -56,9 +56,12 @@ variables: tags: - assigned_to_jira_team_devprod_correctness commands: - - command: shell.exec + - func: "f_expansions_write" + - command: subprocess.exec params: - script: "echo 'noop'" + binary: echo + args: + - "noop" ################################################ # Tasks # @@ -383,6 +386,7 @@ tasks: variant: generate-tasks-for-version commands: - func: "do scons setup" + - func: "f_expansions_write" - command: subprocess.exec params: binary: bash @@ -531,6 +535,7 @@ tasks: ] commands: - func: "do scons setup" + - func: "f_expansions_write" - command: subprocess.exec params: binary: bash @@ -1508,6 +1513,7 @@ tasks: - func: "set up venv" - func: "upload pip requirements" - func: "configure evergreen api credentials" + - func: "f_expansions_write" - command: subprocess.exec type: test params: diff --git a/etc/yamllint_config.yml b/etc/yamllint_config.yml index ae095fd0de5..9ff20db64ee 100644 --- a/etc/yamllint_config.yml +++ b/etc/yamllint_config.yml @@ -8,5 +8,8 @@ rules: comments-indentation: disable truthy: disable -ignore: | - buildscripts/packaging/msi/mongod.yaml +ignore: + - buildscripts/packaging/msi/mongod.yaml + - etc/evaluated_evergreen.yml + - etc/evaluated_evergreen_nightly.yml + - etc/evaluated_system_perf.yml diff --git a/evergreen/container_registry_login.sh b/evergreen/container_registry_login.sh new file mode 100644 index 00000000000..50988d0f540 --- /dev/null +++ b/evergreen/container_registry_login.sh @@ -0,0 +1,7 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -oe +echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry} diff --git a/evergreen/verify_build_output_present.sh b/evergreen/verify_build_output_present.sh new file mode 100644 index 00000000000..21b553e5237 --- /dev/null +++ b/evergreen/verify_build_output_present.sh @@ -0,0 +1,7 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +test -f ${output} diff --git a/evergreen/write_mongo_binary_url_to_downstream_expansions.sh b/evergreen/write_mongo_binary_url_to_downstream_expansions.sh new file mode 100644 index 00000000000..4ef24c2c507 --- /dev/null +++ b/evergreen/write_mongo_binary_url_to_downstream_expansions.sh @@ -0,0 +1,5 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +touch downstream_expansions.yaml +echo "${build_variant}-mongo-binaries: https://mciuploads.s3.amazonaws.com/${mongo_binaries}" 2>&1 | tee downstream_expansions.yaml diff --git a/poetry.lock b/poetry.lock index 987b150ec07..1bb1d50fdf3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -853,13 +853,13 @@ dev = ["black", "build", "commitizen", "isort", "pip-tools", "pre-commit", "twin [[package]] name = "evergreen-lint" -version = "0.1.6" +version = "0.1.7" description = "" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "evergreen_lint-0.1.6-py3-none-any.whl", hash = "sha256:75b0605682d0382ad7cb3ed620869abf719b52fc8fb29ef77e6c4b8f1e52d343"}, - {file = "evergreen_lint-0.1.6.tar.gz", hash = "sha256:fe7a4bfdcd1c3fc02ab7bc4b566b97bdb581fb274515ae3a44e9068baebd57b3"}, + {file = "evergreen_lint-0.1.7-py3-none-any.whl", hash = "sha256:b88523bd495e55c428a231e08cf7090a334d48758d5d2a8185c36a86c1f29395"}, + {file = "evergreen_lint-0.1.7.tar.gz", hash = "sha256:f9bbc9a38f7129083415c19932f8e0ed6ae26db363f8cf025cba5c90d6f33939"}, ] [package.dependencies] @@ -4281,4 +4281,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "40ee186af26d02e11e401a1b331c47db9c9228a078ae2020fcc12e1468b03f7f" +content-hash = "5e4c3efcad5d92501d4c494eeccc4bdf140389c4c237cfe54000d86e49a3cc41" diff --git a/pyproject.toml b/pyproject.toml index 61431e160e7..1a8c488dee3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,7 @@ types-setuptools = "57.4.12" # latest is 68.0.0.0, but that causes pip install r types-requests = "2.31.0.1" tqdm = "*" colorama = "^0.4.6" -evergreen-lint = "^0.1.6" +evergreen-lint = "^0.1.7" ruff = "^0.4.4" [tool.poetry.group.platform.dependencies]