From a36cc452e1e49305eb4f2505eae770b75fe08d5f Mon Sep 17 00:00:00 2001 From: Robert Guo Date: Fri, 2 Apr 2021 09:25:49 -0400 Subject: [PATCH] Revert "SERVER-55300 Implement expansions handling for standalone shell" This reverts commit 24aa56a6ed096a470f1380709b87084ff74106b5. --- buildscripts/evergreen_expansions2bash.py | 46 ------- buildscripts/generate_compile_expansions.py | 5 +- ...enerate_compile_expansions_shared_cache.py | 9 +- etc/evergreen.yml | 129 ++++++++++++++---- etc/expansions.default.yml | 35 ----- evergreen/prelude.sh | 106 -------------- evergreen/scons_compile.sh | 77 ----------- evergreen/scons_lint.sh | 24 ---- 8 files changed, 112 insertions(+), 319 deletions(-) delete mode 100644 buildscripts/evergreen_expansions2bash.py delete mode 100644 etc/expansions.default.yml delete mode 100755 evergreen/prelude.sh delete mode 100755 evergreen/scons_compile.sh delete mode 100755 evergreen/scons_lint.sh diff --git a/buildscripts/evergreen_expansions2bash.py b/buildscripts/evergreen_expansions2bash.py deleted file mode 100644 index ab1a0e09449..00000000000 --- a/buildscripts/evergreen_expansions2bash.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Convert Evergreen's expansions.yml to an eval-able shell script.""" -import sys -import platform -from shlex import quote -from typing import Any - -import yaml -import click - - -def _error(msg: str) -> None: - print(f"___expansions_error={quote(msg)}") - sys.exit(1) - - -@click.command() -@click.argument("expansions_file", type=str) -@click.argument("defaults_file", type=str) -def _main(expansions_file, defaults_file): - try: - with open(defaults_file) as fh: - defaults = yaml.safe_load(fh) - with open(expansions_file) as fh: - expansions = yaml.safe_load(fh) - - # inject defaults into expansions - for key, value in defaults.items(): - if key not in expansions: - expansions[key] = str(value) - - if not isinstance(expansions, dict): - _error("ERROR: expected to read a dictionary. Has the output format " - "of expansions.write changed?") - - if not expansions: - _error("ERROR: found 0 expansions. This is almost certainly wrong.") - - for key, value in expansions.items(): - print(f"{key}={quote(value)}; ", end="") - - except Exception as ex: # pylint: disable=broad-except - _error(ex) - - -if __name__ == "__main__": - _main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py index cd3b7c63b0c..653f01502cb 100755 --- a/buildscripts/generate_compile_expansions.py +++ b/buildscripts/generate_compile_expansions.py @@ -11,7 +11,6 @@ import json import os import re import sys -import shlex import yaml VERSION_JSON = "version.json" @@ -95,8 +94,8 @@ def generate_scons_cache_expansions(): scons_cache_mode = "nolinked" if os.getenv("USE_SCONS_CACHE") not in (None, False, "false", ""): - expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format( - scons_cache_mode, shlex.quote(default_cache_path)) + expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format( + scons_cache_mode, default_cache_path) return expansions diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py index 4f31341c806..685eedb629f 100755 --- a/buildscripts/generate_compile_expansions_shared_cache.py +++ b/buildscripts/generate_compile_expansions_shared_cache.py @@ -11,7 +11,6 @@ import json import os import re import sys -import shlex import yaml VERSION_JSON = "version.json" @@ -100,8 +99,8 @@ def generate_scons_cache_expansions(): shared_mount_root = '/efs' default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache") expansions["scons_cache_path"] = default_cache_path - expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format( - scons_cache_mode, shlex.quote(default_cache_path)) + expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format( + scons_cache_mode, default_cache_path) # Local shared cache - host-based elif os.getenv("SCONS_CACHE_SCOPE") == "local": @@ -113,8 +112,8 @@ def generate_scons_cache_expansions(): default_cache_path = os.path.join(default_cache_path_base, system_uuid) expansions["scons_cache_path"] = default_cache_path - expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format( - scons_cache_mode, shlex.quote(default_cache_path)) + expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format( + scons_cache_mode, default_cache_path) # No cache else: # Anything else is 'none' diff --git a/etc/evergreen.yml b/etc/evergreen.yml index c9eb6b9b696..e0773a93087 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -382,6 +382,7 @@ variables: ####################################### functions: + "remove files": &remove_files command: shell.exec params: @@ -432,7 +433,7 @@ functions: "git get project": &git_get_project command: git.get_project params: - directory: src + directory: ${git_project_directory|src} revisions: # for each module include revision as : ${_rev} enterprise: ${enterprise_rev} wtdevelop: ${wtdevelop_rev} @@ -1096,6 +1097,8 @@ functions: # exit immediately if virtualenv is not found set -o errexit + virtualenv_loc=$(which ${virtualenv|virtualenv}) + python_loc=$(which ${python|/opt/mongodbtoolchain/v3/bin/python3}) venv_dir="${workdir}/venv" "$python_loc" -m venv --system-site-packages "$venv_dir" @@ -1709,32 +1712,112 @@ functions: - "./src/evergreen/check_run_tests_infrastructure_failure.sh" "scons lint": - - command: expansions.write - params: - file: expansions.yml - - command: subprocess.exec + - command: shell.exec type: test params: - binary: bash - args: - - "src/evergreen/scons_lint.sh" - env: - python: ${python} - workdir: ${workdir} + working_dir: src + shell: bash + script: | + set -o errexit + set -o verbose + + ${activate_virtualenv} + export MYPY="$( + if which cygpath 2>/dev/null; then + PATH+=":$(cypath "${workdir}")/venv_3/Scripts" + else + PATH+=":${workdir}/venv_3/bin" + fi + PATH+=':/opt/mongodbtoolchain/v3/bin' + which mypy + )" + echo "Found mypy executable at '$MYPY'" + export extra_flags="" + if [[ ${is_patch|false} == "true" ]]; then + extra_flags="--lint-scope=changed" + fi + + ${compile_env|} python3 ./buildscripts/scons.py ${compile_flags|} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision|}" ENTERPRISE_REV="${enterprise_rev|}" ${targets} "scons compile": - - command: expansions.write - params: - file: expansions.yml - - command: subprocess.exec - type: test - params: - binary: bash - args: - - "src/evergreen/scons_compile.sh" - env: - python: ${python} - workdir: ${workdir} + command: shell.exec + type: test + params: + working_dir: src + shell: bash + script: | + set -o errexit + set -o verbose + + rm -rf ${install_directory|/data/mongo-install-directory} + + # Use hardlinks to reduce the disk space impact of installing + # all of the binaries and associated debug info. + + # The expansion here is a workaround to let us set a different install-action + # for tasks that don't support the one we set here. A better plan would be + # to support install-action for Ninja builds directly. + # TODO: https://jira.mongodb.org/browse/SERVER-48203 + extra_args="--install-action=${task_install_action|hardlink}" + + # By default, limit link jobs to one quarter of our overall -j + # concurrency unless locally overridden. We do this because in + # static link environments, the memory consumption of each + # link job is so high that without constraining the number of + # links we are likely to OOM or thrash the machine. Dynamic + # builds, where htis is not a concern, override this value. + echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|0.25}" + extra_args="$extra_args --jlink=${num_scons_link_jobs_available|0.25}" + + if [ "${scons_cache_scope|}" = "shared" ]; then + extra_args="$extra_args --cache-debug=scons_cache.log" + fi + + # Conditionally enable scons time debugging + if [ "${show_scons_timings|true}" = "true" ]; then + extra_args="$extra_args --debug=time" + fi + + # Build packages where the upload tasks expect them + if [ -n "${git_project_directory|}" ]; then + extra_args="$extra_args PKGDIR=${git_project_directory}" + else + extra_args="$extra_args PKGDIR=${workdir}/src" + fi + + # If we are doing a patch build or we are building a non-push + # build on the waterfall, then we don't need the --release + # flag. Otherwise, this is potentially a build that "leaves + # the building", so we do want that flag. The non --release + # case should auto enale the faster decider when + # applicable. Furthermore, for the non --release cases we can + # accelerate the build slightly for situations where we invoke + # SCons multiple times on the same machine by allowing SCons + # to assume that implicit dependencies are cacheable across + # runs. + if [ "${is_patch|false}" = "true" ] || [ -z "${push_bucket|}" ] || [ "${compiling_for_test|false}" = "true" ]; then + extra_args="$extra_args --implicit-cache --build-fast-and-loose=on" + else + extra_args="$extra_args --release" + fi + + if [ "Windows_NT" = "$OS" ]; then + vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" + export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH + fi + + ${activate_virtualenv} + + ${compile_env|} $python ./buildscripts/scons.py \ + ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} \ + ${scons_cache_args|} $extra_args \ + ${targets} MONGO_VERSION=${version} ${patch_compile_flags|} || exit_status=$? + + # If compile fails we do not run any tests + if [[ $exit_status -ne 0 ]]; then + touch ${skip_tests} + fi + exit $exit_status "generate compile expansions": command: shell.exec diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml deleted file mode 100644 index 43797f8dd98..00000000000 --- a/etc/expansions.default.yml +++ /dev/null @@ -1,35 +0,0 @@ -report_file: src/report.json -archive_file: src/archive.json -decompress: tar xzvf -private_key_file: /dev/null -aws_profile_remote: default -resmoke_jobs_factor: 1 -resmoke_jobs_max: 0 -timeout_secs: 0 -exec_timeout_secs: 0 -python: /opt/mongodbtoolchain/v3/bin/python3 -multiversion_edition: base -multiversion_platform: linux_x86_64 -multiversion_architecture: x86_64 -disable_unit_tests: false -skip_tests: /dev/null -resmoke_jobs: 1 -should_shuffle: true -continue_on_failure: true -install_dir: dist-test/bin -is_patch: false -install_directory: /data/mongo-install-directory -task_install_action: hardlink -num_scons_link_jobs_available: 0.25 -show_scons_timings: true -compiling_for_test: false -scons_cache_mode: nolinked -is_commit_queue: false -use_scons_cache: false -npm_command: jstestfuzz -connection_attempts: 25 -tar: tar -cmake_path: /opt/cmake/bin/cmake -content_type: application/x-gzip -jstestfuzz_concurrent_num_files: 10 -curator_release: "latest" diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh deleted file mode 100755 index 823b29449af..00000000000 --- a/evergreen/prelude.sh +++ /dev/null @@ -1,106 +0,0 @@ -if [[ "$0" == *"/evergreen/prelude.sh" ]]; then - echo "ERROR: do not execute this script. source it instead. ie: . prelude.sh" - exit 1 -fi - -# path the directory that contains this script. -evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" - -# bootstrapping python assumes that the user has not cd'd before the prelude. -# Ensure that here. -calculated_workdir=$(cd "$evergreen_dir/../.." && echo $PWD) -if [ "$PWD" != "$calculated_workdir" ]; then - echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that" - echo "\$PWD: $PWD" - echo "\$calculated_workdir: $calculated_workdir" - exit 1 -fi - -function activate_venv { - set -e - # check if virtualenv is set up - if [ -d "${workdir}/venv" ]; then - if [ "Windows_NT" = "$OS" ]; then - # Need to quote the path on Windows to preserve the separator. - . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log - else - . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log - fi - if [ $? -ne 0 ]; then - echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)" - fi - python=python - else - python=${python:-/opt/mongodbtoolchain/v3/bin/python3} - fi - - if [ "Windows_NT" = "$OS" ]; then - export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)" - else - export PYTHONPATH="$PYTHONPATH:${workdir}/src" - fi - - echo "python set to $(which $python)" - set +e -} - -expansions_yaml="$evergreen_dir/../../expansions.yml" -expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml" -script="$evergreen_dir/../buildscripts/evergreen_expansions2bash.py" -if [ "Windows_NT" = "$OS" ]; then - expansions_yaml=$(cygpath -w "$expansions_yaml") - expansions_default_yaml=$(cygpath -w "$expansions_default_yaml") - script=$(cygpath -w "$script") -fi - -eval $(activate_venv >/dev/null && $python "$script" "$expansions_yaml" "$expansions_default_yaml") -if [ -n "$___expansions_error" ]; then - echo $___expansions_error - exit 1 -fi -unset expansions_yaml -unset expansions_default_yaml -unset script -unset evergreen_dir - -function add_nodejs_to_path { - # Add node and npm binaries to PATH - if [ "Windows_NT" = "$OS" ]; then - # An "npm" directory might not have been created in %APPDATA% by the Windows installer. - # Work around the issue by specifying a different %APPDATA% path. - # See: https://github.com/nodejs/node-v0.x-archive/issues/8141 - export APPDATA=${workdir}/npm-app-data - export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location - # TODO: this is to work around BUILD-8652 - cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')" - else - export PATH="$PATH:/opt/node/bin" - fi -} - -function posix_workdir { - if [ "Windows_NT" = "$OS" ]; then - echo $(cygpath -u "${workdir}") - else - echo ${workdir} - fi -} - -function set_sudo { - set -o >/tmp/settings.log - set +o errexit - grep errexit /tmp/settings.log | grep on - errexit_on=$? - # Set errexit "off". - set +o errexit - sudo= - # Use sudo, if it is supported. - sudo date >/dev/null 2>&1 - if [ $? -eq 0 ]; then - sudo=sudo - fi - # Set errexit "on", if previously enabled. - if [ $errexit_on -eq 0 ]; then - set -o errexit - fi -} diff --git a/evergreen/scons_compile.sh b/evergreen/scons_compile.sh deleted file mode 100755 index a69c07b29e1..00000000000 --- a/evergreen/scons_compile.sh +++ /dev/null @@ -1,77 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -set -o errexit -set -o verbose - -rm -rf ${install_directory} - -# Use hardlinks to reduce the disk space impact of installing -# all of the binaries and associated debug info. - -# The expansion here is a workaround to let us set a different install-action -# for tasks that don't support the one we set here. A better plan would be -# to support install-action for Ninja builds directly. -# TODO: https://jira.mongodb.org/browse/SERVER-48203 -extra_args="--install-action=${task_install_action}" - -# By default, limit link jobs to one quarter of our overall -j -# concurrency unless locally overridden. We do this because in -# static link environments, the memory consumption of each -# link job is so high that without constraining the number of -# links we are likely to OOM or thrash the machine. Dynamic -# builds, where htis is not a concern, override this value. -echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available}" -extra_args="$extra_args --jlink=${num_scons_link_jobs_available}" - -if [ "${scons_cache_scope}" = "shared" ]; then - extra_args="$extra_args --cache-debug=scons_cache.log" -fi - -# Conditionally enable scons time debugging -if [ "${show_scons_timings}" = "true" ]; then - extra_args="$extra_args --debug=time" -fi - -# Build packages where the upload tasks expect them -if [ -n "${git_project_directory}" ]; then - extra_args="$extra_args PKGDIR='${git_project_directory}'" -else - extra_args="$extra_args PKGDIR='${workdir}/src'" -fi - -# If we are doing a patch build or we are building a non-push -# build on the waterfall, then we don't need the --release -# flag. Otherwise, this is potentially a build that "leaves -# the building", so we do want that flag. The non --release -# case should auto enale the faster decider when -# applicable. Furthermore, for the non --release cases we can -# accelerate the build slightly for situations where we invoke -# SCons multiple times on the same machine by allowing SCons -# to assume that implicit dependencies are cacheable across -# runs. -if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then - extra_args="$extra_args --implicit-cache --build-fast-and-loose=on" -else - extra_args="$extra_args --release" -fi - -if [ "Windows_NT" = "$OS" ]; then - vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" - export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH -fi - -activate_venv - -eval ${compile_env} $python ./buildscripts/scons.py \ - ${compile_flags} ${task_compile_flags} ${task_compile_flags_extra} \ - ${scons_cache_args} $extra_args \ - ${targets} MONGO_VERSION=${version} ${patch_compile_flags} || exit_status=$? - -# If compile fails we do not run any tests -if [[ $exit_status -ne 0 ]]; then - touch ${skip_tests} -fi -exit $exit_status diff --git a/evergreen/scons_lint.sh b/evergreen/scons_lint.sh deleted file mode 100755 index 708ef4168a5..00000000000 --- a/evergreen/scons_lint.sh +++ /dev/null @@ -1,24 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -set -o errexit -set -o verbose - -activate_venv -export MYPY="$( - if which cygpath 2>/dev/null; then - PATH+=":$(cypath "${workdir}")/venv_3/Scripts" - else - PATH+=":${workdir}/venv_3/bin" - fi - PATH+=':/opt/mongodbtoolchain/v3/bin' - which mypy -)" -echo "Found mypy executable at '$MYPY'" -export extra_flags="" -if [[ ${is_patch} == "true" ]]; then - extra_flags="--lint-scope=changed" -fi -eval ${compile_env} python3 ./buildscripts/scons.py ${compile_flags} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision}" ENTERPRISE_REV="${enterprise_rev}" ${targets}