SERVER-55300 SERVER-55731 Implement expansions handling for standalone shell

buildscripts/evergreen_expansions2bash.py

@@ -0,0 +1,92 @@
+"""Convert Evergreen's expansions.yml to an eval-able shell script."""
+import sys
+import platform
+from shlex import quote
+from typing import Any
+
+import yaml
+import click
+
+
+def _error(msg: str) -> None:
+    print(f"___expansions_error={quote(msg)}")
+    sys.exit(1)
+
+
+def _load_defaults(defaults_file: str) -> dict:
+    with open(defaults_file) as fh:
+        defaults = yaml.safe_load(fh)
+        if not isinstance(defaults, dict):
+            _error("ERROR: expected to read a dictionary. expansions.defaults.yml"
+                   "must be a dictionary. Check the indentation.")
+
+        # expansions MUST be strings. Reject any that are not
+        bad_expansions = set()
+        for key, value in defaults.items():
+            if not isinstance(value, str):
+                bad_expansions.add(key)
+
+        if bad_expansions:
+            _error("ERROR: all default expansions must be strings. You can "
+                   " fix this error by quoting the values in expansions.defaults.yml. "
+                   "Integers, floating points, 'true', 'false', and 'null' "
+                   "must be quoted. The following keys were interpreted as "
+                   f"other types: {bad_expansions}")
+
+        # These values show up if 1. Python's str is used to naively convert
+        # a boolean to str, 2. A human manually entered one of those strings.
+        # Either way, our shell scripts expect 'true' or 'false' (leading
+        # lowercase), and we reject them as errors. This will probably save
+        # someone a lot of time, but if this assumption proves wrong, start
+        # a conversation in #server-testing.
+        risky_boolean_keys = set()
+        for key, value in defaults.items():
+            if value in ("True", "False"):
+                risky_boolean_keys.add(key)
+
+        if risky_boolean_keys:
+            _error("ERROR: Found keys which had 'True' or 'False' as values. "
+                   "Shell scripts assume that booleans are represented as 'true'"
+                   " or 'false' (leading lowercase). If you added a new boolean, "
+                   "ensure that it's represented in lowercase. If not, please report this in "
+                   f"#server-testing. Keys with bad values: {risky_boolean_keys}")
+
+        return defaults
+
+
+def _load_expansions(expansions_file) -> dict:
+    with open(expansions_file) as fh:
+        expansions = yaml.safe_load(fh)
+
+        if not isinstance(expansions, dict):
+            _error("ERROR: expected to read a dictionary. Has the output format "
+                   "of expansions.write changed?")
+
+        if not expansions:
+            _error("ERROR: found 0 expansions. This is almost certainly wrong.")
+
+        return expansions
+
+
+@click.command()
+@click.argument("expansions_file", type=str)
+@click.argument("defaults_file", type=str)
+def _main(expansions_file: str, defaults_file: str):
+    try:
+        defaults = _load_defaults(defaults_file)
+        expansions = _load_expansions(expansions_file)
+
+        # inject defaults into expansions
+        for key, value in defaults.items():
+            if key not in expansions:
+                expansions[key] = value
+
+        for key, value in expansions.items():
+            print(f"{key}={quote(value)}; ", end="")
+
+    except Exception as ex:  # pylint: disable=broad-except
+        _error(ex)
+
+
+if __name__ == "__main__":
+    _main()  # pylint: disable=no-value-for-parameter

buildscripts/generate_compile_expansions.py

@@ -11,6 +11,7 @@ import json
 import os
 import re
 import sys
+import shlex
 import yaml
 
 VERSION_JSON = "version.json"
@@ -94,8 +95,8 @@ def generate_scons_cache_expansions():
                 scons_cache_mode = "nolinked"
 
             if os.getenv("USE_SCONS_CACHE") not in (None, False, "false", ""):
-                expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
-                    scons_cache_mode, default_cache_path)
+                expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
+                    scons_cache_mode, shlex.quote(default_cache_path))
     return expansions
 
 

buildscripts/generate_compile_expansions_shared_cache.py

@@ -11,6 +11,7 @@ import json
 import os
 import re
 import sys
+import shlex
 import yaml
 
 VERSION_JSON = "version.json"
@@ -99,8 +100,8 @@ def generate_scons_cache_expansions():
             shared_mount_root = '/efs'
         default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache")
         expansions["scons_cache_path"] = default_cache_path
-        expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
-            scons_cache_mode, default_cache_path)
+        expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
+            scons_cache_mode, shlex.quote(default_cache_path))
 
     # Local shared cache - host-based
     elif os.getenv("SCONS_CACHE_SCOPE") == "local":
@@ -112,8 +113,8 @@ def generate_scons_cache_expansions():
 
         default_cache_path = os.path.join(default_cache_path_base, system_uuid)
         expansions["scons_cache_path"] = default_cache_path
-        expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
-            scons_cache_mode, default_cache_path)
+        expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
+            scons_cache_mode, shlex.quote(default_cache_path))
     # No cache
     else:
         # Anything else is 'none'

etc/evergreen.yml

@@ -382,7 +382,6 @@ variables:
 #######################################
 
 functions:
-
   "remove files": &remove_files
     command: shell.exec
     params:
@@ -433,7 +432,7 @@ functions:
   "git get project": &git_get_project
     command: git.get_project
     params:
-      directory: ${git_project_directory|src}
+      directory: src
       revisions: # for each module include revision as <module_name> : ${<module_name>_rev}
         enterprise: ${enterprise_rev}
         wtdevelop: ${wtdevelop_rev}
@@ -1097,8 +1096,6 @@ functions:
           # exit immediately if virtualenv is not found
           set -o errexit
 
-          virtualenv_loc=$(which ${virtualenv|virtualenv})
-
           python_loc=$(which ${python|/opt/mongodbtoolchain/v3/bin/python3})
           venv_dir="${workdir}/venv"
           "$python_loc" -m venv --system-site-packages "$venv_dir"
@@ -1712,112 +1709,32 @@ functions:
           - "./src/evergreen/check_run_tests_infrastructure_failure.sh"
 
   "scons lint":
-    - command: shell.exec
+    - command: expansions.write
+      params:
+        file: expansions.yml
+    - command: subprocess.exec
       type: test
       params:
-        working_dir: src
-        shell: bash
-        script: |
-          set -o errexit
-          set -o verbose
-
-          ${activate_virtualenv}
-          export MYPY="$(
-            if which cygpath 2>/dev/null; then
-                PATH+=":$(cypath "${workdir}")/venv_3/Scripts"
-            else
-                PATH+=":${workdir}/venv_3/bin"
-            fi
-            PATH+=':/opt/mongodbtoolchain/v3/bin'
-            which mypy
-          )"
-          echo "Found mypy executable at '$MYPY'"
-          export extra_flags=""
-          if [[ ${is_patch|false} == "true" ]]; then
-            extra_flags="--lint-scope=changed"
-          fi
-
-          ${compile_env|} python3 ./buildscripts/scons.py ${compile_flags|} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision|}" ENTERPRISE_REV="${enterprise_rev|}" ${targets}
+        binary: bash
+        args:
+          - "src/evergreen/scons_lint.sh"
+        env:
+          python: ${python}
+          workdir: ${workdir}
 
   "scons compile":
-    command: shell.exec
-    type: test
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        set -o verbose
-
-        rm -rf ${install_directory|/data/mongo-install-directory}
-
-        # Use hardlinks to reduce the disk space impact of installing
-        # all of the binaries and associated debug info.
-
-        # The expansion here is a workaround to let us set a different install-action
-        # for tasks that don't support the one we set here. A better plan would be
-        # to support install-action for Ninja builds directly.
-        # TODO: https://jira.mongodb.org/browse/SERVER-48203
-        extra_args="--install-action=${task_install_action|hardlink}"
-
-        # By default, limit link jobs to one quarter of our overall -j
-        # concurrency unless locally overridden. We do this because in
-        # static link environments, the memory consumption of each
-        # link job is so high that without constraining the number of
-        # links we are likely to OOM or thrash the machine. Dynamic
-        # builds, where htis is not a concern, override this value.
-        echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|0.25}"
-        extra_args="$extra_args --jlink=${num_scons_link_jobs_available|0.25}"
-
-        if [ "${scons_cache_scope|}" = "shared" ]; then
-          extra_args="$extra_args --cache-debug=scons_cache.log"
-        fi
-
-        # Conditionally enable scons time debugging
-        if [ "${show_scons_timings|true}" = "true" ]; then
-          extra_args="$extra_args --debug=time"
-        fi
-
-        # Build packages where the upload tasks expect them
-        if [ -n "${git_project_directory|}" ]; then
-            extra_args="$extra_args PKGDIR=${git_project_directory}"
-        else
-            extra_args="$extra_args PKGDIR=${workdir}/src"
-        fi
-
-        # If we are doing a patch build or we are building a non-push
-        # build on the waterfall, then we don't need the --release
-        # flag. Otherwise, this is potentially a build that "leaves
-        # the building", so we do want that flag. The non --release
-        # case should auto enale the faster decider when
-        # applicable. Furthermore, for the non --release cases we can
-        # accelerate the build slightly for situations where we invoke
-        # SCons multiple times on the same machine by allowing SCons
-        # to assume that implicit dependencies are cacheable across
-        # runs.
-        if [ "${is_patch|false}" = "true" ] || [ -z "${push_bucket|}" ] || [ "${compiling_for_test|false}" = "true" ]; then
-          extra_args="$extra_args --implicit-cache --build-fast-and-loose=on"
-        else
-          extra_args="$extra_args --release"
-        fi
-
-        if [ "Windows_NT" = "$OS" ]; then
-          vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
-          export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH
-        fi
-
-        ${activate_virtualenv}
-
-        ${compile_env|} $python ./buildscripts/scons.py                                     \
-            ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|}           \
-            ${scons_cache_args|} $extra_args                                                \
-            ${targets} MONGO_VERSION=${version} ${patch_compile_flags|} || exit_status=$?
-
-        # If compile fails we do not run any tests
-        if [[ $exit_status -ne 0 ]]; then
-          touch ${skip_tests}
-        fi
-        exit $exit_status
+    - command: expansions.write
+      params:
+        file: expansions.yml
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        args:
+          - "src/evergreen/scons_compile.sh"
+        env:
+          python: ${python}
+          workdir: ${workdir}
 
   "generate compile expansions":
     command: shell.exec

etc/expansions.default.yml

@@ -0,0 +1,39 @@
+# All expansions in this file must be strings.
+# Additionally, we assume that bools are "true" and "false", not "True" and
+# "False" (note leading case) throughout the codebase.
+# Always wrap the following words in quotes: "true", "false", "null"
+report_file: src/report.json
+archive_file: src/archive.json
+decompress: tar xzvf
+private_key_file: /dev/null
+aws_profile_remote: default
+resmoke_jobs_factor: "1"
+resmoke_jobs_max: "0"
+timeout_secs: "0"
+exec_timeout_secs: "0"
+python: /opt/mongodbtoolchain/v3/bin/python3
+multiversion_edition: base
+multiversion_platform: linux_x86_64
+multiversion_architecture: x86_64
+disable_unit_tests: "false"
+skip_tests: /dev/null
+resmoke_jobs: "1"
+should_shuffle: "true"
+continue_on_failure: "true"
+install_dir: dist-test/bin
+is_patch: "false"
+install_directory: /data/mongo-install-directory
+task_install_action: hardlink
+num_scons_link_jobs_available: "0.25"
+show_scons_timings: "true"
+compiling_for_test: "false"
+scons_cache_mode: nolinked
+is_commit_queue: "false"
+use_scons_cache: "false"
+npm_command: jstestfuzz
+connection_attempts: "25"
+tar: tar
+cmake_path: /opt/cmake/bin/cmake
+content_type: application/x-gzip
+jstestfuzz_concurrent_num_files: "10"
+curator_release: "latest"

evergreen/prelude.sh

@@ -0,0 +1,106 @@
+if [[ "$0" == *"/evergreen/prelude.sh" ]]; then
+  echo "ERROR: do not execute this script. source it instead. ie: . prelude.sh"
+  exit 1
+fi
+
+# path the directory that contains this script.
+evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+# bootstrapping python assumes that the user has not cd'd before the prelude.
+# Ensure that here.
+calculated_workdir=$(cd "$evergreen_dir/../.." && echo $PWD)
+if [ "$PWD" != "$calculated_workdir" ]; then
+  echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that"
+  echo "\$PWD: $PWD"
+  echo "\$calculated_workdir: $calculated_workdir"
+  exit 1
+fi
+
+function activate_venv {
+  set -e
+  # check if virtualenv is set up
+  if [ -d "${workdir}/venv" ]; then
+    if [ "Windows_NT" = "$OS" ]; then
+      # Need to quote the path on Windows to preserve the separator.
+      . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log
+    else
+      . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log
+    fi
+    if [ $? -ne 0 ]; then
+      echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
+    fi
+    python=python
+  else
+    python=${python:-/opt/mongodbtoolchain/v3/bin/python3}
+  fi
+
+  if [ "Windows_NT" = "$OS" ]; then
+    export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
+  else
+    export PYTHONPATH="$PYTHONPATH:${workdir}/src"
+  fi
+
+  echo "python set to $(which $python)"
+  set +e
+}
+
+expansions_yaml="$evergreen_dir/../../expansions.yml"
+expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
+script="$evergreen_dir/../buildscripts/evergreen_expansions2bash.py"
+if [ "Windows_NT" = "$OS" ]; then
+  expansions_yaml=$(cygpath -w "$expansions_yaml")
+  expansions_default_yaml=$(cygpath -w "$expansions_default_yaml")
+  script=$(cygpath -w "$script")
+fi
+
+eval $(activate_venv >/dev/null && $python "$script" "$expansions_yaml" "$expansions_default_yaml")
+if [ -n "$___expansions_error" ]; then
+  echo $___expansions_error
+  exit 1
+fi
+unset expansions_yaml
+unset expansions_default_yaml
+unset script
+unset evergreen_dir
+
+function add_nodejs_to_path {
+  # Add node and npm binaries to PATH
+  if [ "Windows_NT" = "$OS" ]; then
+    # An "npm" directory might not have been created in %APPDATA% by the Windows installer.
+    # Work around the issue by specifying a different %APPDATA% path.
+    # See: https://github.com/nodejs/node-v0.x-archive/issues/8141
+    export APPDATA=${workdir}/npm-app-data
+    export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location
+    # TODO: this is to work around BUILD-8652
+    cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')"
+  else
+    export PATH="$PATH:/opt/node/bin"
+  fi
+}
+
+function posix_workdir {
+  if [ "Windows_NT" = "$OS" ]; then
+    echo $(cygpath -u "${workdir}")
+  else
+    echo ${workdir}
+  fi
+}
+
+function set_sudo {
+  set -o >/tmp/settings.log
+  set +o errexit
+  grep errexit /tmp/settings.log | grep on
+  errexit_on=$?
+  # Set errexit "off".
+  set +o errexit
+  sudo=
+  # Use sudo, if it is supported.
+  sudo date >/dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    sudo=sudo
+  fi
+  # Set errexit "on", if previously enabled.
+  if [ $errexit_on -eq 0 ]; then
+    set -o errexit
+  fi
+}

evergreen/scons_compile.sh

@@ -0,0 +1,77 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+rm -rf ${install_directory}
+
+# Use hardlinks to reduce the disk space impact of installing
+# all of the binaries and associated debug info.
+
+# The expansion here is a workaround to let us set a different install-action
+# for tasks that don't support the one we set here. A better plan would be
+# to support install-action for Ninja builds directly.
+# TODO: https://jira.mongodb.org/browse/SERVER-48203
+extra_args="--install-action=${task_install_action}"
+
+# By default, limit link jobs to one quarter of our overall -j
+# concurrency unless locally overridden. We do this because in
+# static link environments, the memory consumption of each
+# link job is so high that without constraining the number of
+# links we are likely to OOM or thrash the machine. Dynamic
+# builds, where htis is not a concern, override this value.
+echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available}"
+extra_args="$extra_args --jlink=${num_scons_link_jobs_available}"
+
+if [ "${scons_cache_scope}" = "shared" ]; then
+  extra_args="$extra_args --cache-debug=scons_cache.log"
+fi
+
+# Conditionally enable scons time debugging
+if [ "${show_scons_timings}" = "true" ]; then
+  extra_args="$extra_args --debug=time"
+fi
+
+# Build packages where the upload tasks expect them
+if [ -n "${git_project_directory}" ]; then
+  extra_args="$extra_args PKGDIR='${git_project_directory}'"
+else
+  extra_args="$extra_args PKGDIR='${workdir}/src'"
+fi
+
+# If we are doing a patch build or we are building a non-push
+# build on the waterfall, then we don't need the --release
+# flag. Otherwise, this is potentially a build that "leaves
+# the building", so we do want that flag. The non --release
+# case should auto enale the faster decider when
+# applicable. Furthermore, for the non --release cases we can
+# accelerate the build slightly for situations where we invoke
+# SCons multiple times on the same machine by allowing SCons
+# to assume that implicit dependencies are cacheable across
+# runs.
+if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then
+  extra_args="$extra_args --implicit-cache --build-fast-and-loose=on"
+else
+  extra_args="$extra_args --release"
+fi
+
+if [ "Windows_NT" = "$OS" ]; then
+  vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
+  export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH
+fi
+
+activate_venv
+
+eval ${compile_env} $python ./buildscripts/scons.py \
+  ${compile_flags} ${task_compile_flags} ${task_compile_flags_extra} \
+  ${scons_cache_args} $extra_args \
+  ${targets} MONGO_VERSION=${version} ${patch_compile_flags} || exit_status=$?
+
+# If compile fails we do not run any tests
+if [[ $exit_status -ne 0 ]]; then
+  touch ${skip_tests}
+fi
+exit $exit_status

evergreen/scons_lint.sh

@@ -0,0 +1,24 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+activate_venv
+export MYPY="$(
+  if which cygpath 2>/dev/null; then
+    PATH+=":$(cypath "${workdir}")/venv_3/Scripts"
+  else
+    PATH+=":${workdir}/venv_3/bin"
+  fi
+  PATH+=':/opt/mongodbtoolchain/v3/bin'
+  which mypy
+)"
+echo "Found mypy executable at '$MYPY'"
+export extra_flags=""
+if [[ ${is_patch} == "true" ]]; then
+  extra_flags="--lint-scope=changed"
+fi
+eval ${compile_env} python3 ./buildscripts/scons.py ${compile_flags} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision}" ENTERPRISE_REV="${enterprise_rev}" ${targets}