mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-11-21 12:39:08 +01:00
Code Issues Releases Activity

SERVER-90565 Re-enable evergreen yml lint rules (#22248)

Browse Source 
GitOrigin-RevId: 665a9e3df84103ad40bce85cb5daecfb247b85fb
This commit is contained in:
Mikhail Shchatko 2024-05-16 20:59:58 +03:00 committed by MongoDB Bot
parent c53dd2959a
commit a6349b5605
10 changed files with 99 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
etc/evergreen_lint.yml
View File

@ -15,7 +15,7 @@ rules:
# # users to source a script called `prelude.sh`. This script requires that
# # the user be in ${workdir}, so we forbid users from `cd`ing with
# # `working_dir` parameter.
# - rule: "no-working-dir-on-shell"
- rule: "no-working-dir-on-shell"
# # Embedding shell.exec scripts in the Evergreen yaml has been a historical
# # source of pain:
@ -25,7 +25,7 @@ rules:
# # We completed a project to completely eject all shell scripts from the
# # YAML, and now require this of all shell scripts. Add your scripts to the
# # evergreen/ directory, and make use of subprocess.exec.
# - rule: "no-shell-exec"
- rule: "no-shell-exec"
# Historically, expansions.update has been used to inject shell scripts
# into expansions, which were then used to inject commonly used snippets
@ -41,8 +41,8 @@ rules:
# # To ensure that the expansions file loaded by prelude.sh is not stale, we
# # require the user to add sometimes superfluous expansions.write calls to
# # ere on side of correctness.
# - rule: "required-expansions-write"
# regex: .*\/evergreen\/.*\.sh
- rule: "required-expansions-write"
regex: .*\/evergreen\/.*\.sh
# Generated tests must depend on build_variant_gen (which generates the
# tasks that contain them) and archive_dist_test (which contains

61
etc/evergreen_yml_components/definitions.yml
View File

@ -188,6 +188,7 @@ functions:
"get engflow creds":
- *get_engflow_key
- *get_engflow_cert
- *f_expansions_write
- *generate_evergreen_bazelrc
"adjust venv": &adjust_venv
@ -241,12 +242,11 @@ functions:
local_file: src/mongo-binaries.tgz
"write mongo binary URL to downstream_expansions.yml": &echo_mongo_binary_url
command: shell.exec
command: subprocess.exec
params:
shell: bash
script: |
touch downstream_expansions.yaml
echo "${build_variant}-mongo-binaries: https://mciuploads.s3.amazonaws.com/${mongo_binaries}" 2>&1 | tee downstream_expansions.yaml
binary: bash
args:
- "./src/evergreen/write_mongo_binary_url_to_downstream_expansions.sh"
"set downstreams expansions": &downstream_expansions
command: downstream_expansions.set
@ -299,6 +299,7 @@ functions:
"get and apply version expansions":
- *get_version_expansions
- *apply_version_expansions
- *f_expansions_write
"fetch benchmarks": &fetch_benchmarks
command: s3.get
@ -601,6 +602,7 @@ functions:
display_name: Pip Requirements
"run clang tidy": &run_clang_tidy
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
@ -687,6 +689,12 @@ functions:
- *collect_ulimit_info
"write and set downstream expansions":
- *f_expansions_write
- *fetch_artifacts
- *kill_processes
- *cleanup_environment
- *fetch_venv
- *adjust_venv
- *echo_mongo_binary_url
- *downstream_expansions
@ -698,6 +706,7 @@ functions:
remote_file: ${mongo_debugsymbols}
bucket: mciuploads
local_file: src/mongo-debugsymbols.${ext|tgz}
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
@ -969,9 +978,12 @@ functions:
- *generate_resmoke_tasks_config
"initialize multiversion tasks": &initialize_multiversion_tasks
- command: shell.exec
- *f_expansions_write
- command: subprocess.exec
params:
script: "echo 'noop'"
binary: echo
args:
- "noop"
"generate resmoke tasks":
- *fetch_artifacts
@ -980,6 +992,7 @@ functions:
- *cleanup_environment
- *get_version_expansions
- *apply_version_expansions
- *f_expansions_write
- *fetch_venv
- *adjust_venv
- *f_expansions_write
@ -1055,6 +1068,7 @@ functions:
- *configure_evergreen_api_credentials
- *determine_task_timeout
- *update_task_timeout_expansions
- *f_expansions_write
- *update_task_timeout
- *f_expansions_write
- *sign_macos_dev_binaries
@ -1177,6 +1191,7 @@ functions:
updates:
- key: install_dir
value: ${install_dir}
- *f_expansions_write
- command: s3.put
display_name: "attach bazel scons diff file"
params:
@ -1231,6 +1246,7 @@ functions:
display_name: Bazel-SCons Diff File
"fetch bazel":
- *f_expansions_write
- command: subprocess.exec
display_name: "fetch bazel"
type: test
@ -1242,12 +1258,13 @@ functions:
"verify build output present":
# Args:
# $output (string) - relpath to the file-to-be-checked
- command: shell.exec
- *f_expansions_write
- command: subprocess.exec
display_name: "verify build output present"
params:
working_dir: src
script: |
test -f ${output}
binary: bash
args:
- "src/evergreen/verify_build_output_present.sh"
"bazel compile sh": &bazel_compile_sh
command: subprocess.exec
@ -1326,6 +1343,7 @@ functions:
display_name: version expansions
"generate and upload version expansions":
- *f_expansions_write
- *generate_version_expansions
- *upload_version_expansions
@ -1367,6 +1385,7 @@ functions:
display_name: resmoke constants
"generate and upload resmoke constants":
- *f_expansions_write
- *generate_resmoke_constants
- *archive_resmoke_constants
- *upload_remoske_constants
@ -1712,11 +1731,12 @@ functions:
- "./src/evergreen/packager.py_run.sh"
"do watchdog setup":
command: subprocess.exec
params:
binary: bash
args:
- "./src/evergreen/do_watchdog_setup.sh"
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
args:
- "./src/evergreen/do_watchdog_setup.sh"
"run package test":
- *get_version_expansions
@ -2343,6 +2363,7 @@ functions:
file_location: ${report_file|src/report.json}
"combine build metrics":
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
@ -2350,6 +2371,7 @@ functions:
args:
- "src/evergreen/run_python_script.sh"
- "site_scons/site_tools/build_metrics/combine_metrics_unittest.py"
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
@ -2362,6 +2384,7 @@ functions:
- "--prefix-name=pull_cache"
"print top N metrics":
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
@ -2405,6 +2428,7 @@ functions:
content_type: application/gzip
display_name: Metrics JSON
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
@ -2561,22 +2585,25 @@ functions:
- "./src/evergreen/antithesis_image_build_and_push.sh"
"cleanup external auth OIDC resources":
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
args:
- "./src/evergreen/external_auth_gcp_teardown.sh"
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
args:
- "./src/evergreen/external_auth_azure_teardown.sh"
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash
args:
- "./src/evergreen/external_auth_oidc_teardown.sh"
- *f_expansions_write
- command: subprocess.exec
params:
binary: bash

29
etc/evergreen_yml_components/tasks/compile_tasks.yml
View File

@ -183,6 +183,7 @@ tasks:
${additional_compile_targets|}
task_compile_flags: >-
PREFIX=dist-test
- func: "f_expansions_write"
- command: subprocess.exec
params:
binary: bash
@ -485,6 +486,7 @@ tasks:
archive-dist-test
task_compile_flags: >-
PREFIX=dist-test
- func: "f_expansions_write"
- command: subprocess.exec
params:
binary: bash
@ -1573,13 +1575,12 @@ tasks:
aws_secret_remote: ${repo_aws_secret}
- func: "set up notary client credentials"
- func: "f_expansions_write"
- command: shell.exec
- command: subprocess.exec
params:
shell: bash
binary: bash
silent: true
script: |
set -oe
echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry}
args:
- "./src/evergreen/container_registry_login.sh"
- command: subprocess.exec
params:
binary: bash
@ -1641,13 +1642,12 @@ tasks:
- func: "f_expansions_write"
# login to container registry
- command: shell.exec
- command: subprocess.exec
params:
shell: bash
binary: bash
silent: true
script: |
set -oe
echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry}
args:
- "./src/evergreen/container_registry_login.sh"
# signing windows artifacts
- command: subprocess.exec
@ -2271,13 +2271,12 @@ tasks:
aws_secret_remote: ${repo_aws_secret}
- func: "f_expansions_write"
# login to container registry
- command: shell.exec
- command: subprocess.exec
params:
shell: bash
binary: bash
silent: true
script: |
set -oe
echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry}
args:
- "./src/evergreen/container_registry_login.sh"
- command: subprocess.exec
params:
binary: bash

10
etc/evergreen_yml_components/tasks/misc_tasks.yml
View File

@ -56,9 +56,12 @@ variables:
tags:
- assigned_to_jira_team_devprod_correctness
commands:
- command: shell.exec
- func: "f_expansions_write"
- command: subprocess.exec
params:
script: "echo 'noop'"
binary: echo
args:
- "noop"
################################################
# Tasks #
@ -383,6 +386,7 @@ tasks:
variant: generate-tasks-for-version
commands:
- func: "do scons setup"
- func: "f_expansions_write"
- command: subprocess.exec
params:
binary: bash
@ -531,6 +535,7 @@ tasks:
]
commands:
- func: "do scons setup"
- func: "f_expansions_write"
- command: subprocess.exec
params:
binary: bash
@ -1508,6 +1513,7 @@ tasks:
- func: "set up venv"
- func: "upload pip requirements"
- func: "configure evergreen api credentials"
- func: "f_expansions_write"
- command: subprocess.exec
type: test
params:

7
etc/yamllint_config.yml
View File

@ -8,5 +8,8 @@ rules:
comments-indentation: disable
truthy: disable
ignore: |
buildscripts/packaging/msi/mongod.yaml
ignore:
- buildscripts/packaging/msi/mongod.yaml
- etc/evaluated_evergreen.yml
- etc/evaluated_evergreen_nightly.yml
- etc/evaluated_system_perf.yml

7
evergreen/container_registry_login.sh Normal file
View File

@ -0,0 +1,7 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
set -oe
echo "${release_tools_container_registry_password}" | podman login --password-stdin --username ${release_tools_container_registry_username} ${release_tools_container_registry}

7
evergreen/verify_build_output_present.sh Normal file
View File

@ -0,0 +1,7 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
set -o errexit
test -f ${output}

5
evergreen/write_mongo_binary_url_to_downstream_expansions.sh Normal file
View File

@ -0,0 +1,5 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
touch downstream_expansions.yaml
echo "${build_variant}-mongo-binaries: https://mciuploads.s3.amazonaws.com/${mongo_binaries}" 2>&1 | tee downstream_expansions.yaml

8
poetry.lock generated
View File

@ -853,13 +853,13 @@ dev = ["black", "build", "commitizen", "isort", "pip-tools", "pre-commit", "twin
[[package]]
name = "evergreen-lint"
version = "0.1.6"
version = "0.1.7"
description = ""
optional = false
python-versions = "<4.0,>=3.9"
files = [
{file = "evergreen_lint-0.1.6-py3-none-any.whl", hash = "sha256:75b0605682d0382ad7cb3ed620869abf719b52fc8fb29ef77e6c4b8f1e52d343"},
{file = "evergreen_lint-0.1.6.tar.gz", hash = "sha256:fe7a4bfdcd1c3fc02ab7bc4b566b97bdb581fb274515ae3a44e9068baebd57b3"},
{file = "evergreen_lint-0.1.7-py3-none-any.whl", hash = "sha256:b88523bd495e55c428a231e08cf7090a334d48758d5d2a8185c36a86c1f29395"},
{file = "evergreen_lint-0.1.7.tar.gz", hash = "sha256:f9bbc9a38f7129083415c19932f8e0ed6ae26db363f8cf025cba5c90d6f33939"},
]
[package.dependencies]
@ -4281,4 +4281,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<4.0"
content-hash = "40ee186af26d02e11e401a1b331c47db9c9228a078ae2020fcc12e1468b03f7f"
content-hash = "5e4c3efcad5d92501d4c494eeccc4bdf140389c4c237cfe54000d86e49a3cc41"

2
pyproject.toml
View File

@ -103,7 +103,7 @@ types-setuptools = "57.4.12" # latest is 68.0.0.0, but that causes pip install r
types-requests = "2.31.0.1"
tqdm = "*"
colorama = "^0.4.6"
evergreen-lint = "^0.1.6"
evergreen-lint = "^0.1.7"
ruff = "^0.4.4"
[tool.poetry.group.platform.dependencies]