mirror of
https://github.com/mongodb/mongo.git
synced 2024-11-29 08:42:18 +01:00
Revert "SERVER-55300 Implement expansions handling for standalone shell"
This reverts commit 24aa56a6ed.
This commit is contained in:
Robert Guo
Evergreen Agent
parent
31b4ee371c
commit
a36cc452e1
@ -1,46 +0,0 @@
|
||||
"""Convert Evergreen's expansions.yml to an eval-able shell script."""
|
||||
import sys
|
||||
import platform
|
||||
from shlex import quote
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
import click
|
||||
|
||||
|
||||
def _error(msg: str) -> None:
|
||||
print(f"___expansions_error={quote(msg)}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("expansions_file", type=str)
|
||||
@click.argument("defaults_file", type=str)
|
||||
def _main(expansions_file, defaults_file):
|
||||
try:
|
||||
with open(defaults_file) as fh:
|
||||
defaults = yaml.safe_load(fh)
|
||||
with open(expansions_file) as fh:
|
||||
expansions = yaml.safe_load(fh)
|
||||
|
||||
# inject defaults into expansions
|
||||
for key, value in defaults.items():
|
||||
if key not in expansions:
|
||||
expansions[key] = str(value)
|
||||
|
||||
if not isinstance(expansions, dict):
|
||||
_error("ERROR: expected to read a dictionary. Has the output format "
|
||||
"of expansions.write changed?")
|
||||
|
||||
if not expansions:
|
||||
_error("ERROR: found 0 expansions. This is almost certainly wrong.")
|
||||
|
||||
for key, value in expansions.items():
|
||||
print(f"{key}={quote(value)}; ", end="")
|
||||
|
||||
except Exception as ex: # pylint: disable=broad-except
|
||||
_error(ex)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_main() # pylint: disable=no-value-for-parameter
|
||||
@ -11,7 +11,6 @@ import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import shlex
|
||||
import yaml
|
||||
|
||||
VERSION_JSON = "version.json"
|
||||
@ -95,8 +94,8 @@ def generate_scons_cache_expansions():
|
||||
scons_cache_mode = "nolinked"
|
||||
|
||||
if os.getenv("USE_SCONS_CACHE") not in (None, False, "false", ""):
|
||||
expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
|
||||
scons_cache_mode, shlex.quote(default_cache_path))
|
||||
expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
|
||||
scons_cache_mode, default_cache_path)
|
||||
return expansions
|
||||
|
||||
|
||||
|
||||
@ -11,7 +11,6 @@ import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import shlex
|
||||
import yaml
|
||||
|
||||
VERSION_JSON = "version.json"
|
||||
@ -100,8 +99,8 @@ def generate_scons_cache_expansions():
|
||||
shared_mount_root = '/efs'
|
||||
default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache")
|
||||
expansions["scons_cache_path"] = default_cache_path
|
||||
expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
|
||||
scons_cache_mode, shlex.quote(default_cache_path))
|
||||
expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
|
||||
scons_cache_mode, default_cache_path)
|
||||
|
||||
# Local shared cache - host-based
|
||||
elif os.getenv("SCONS_CACHE_SCOPE") == "local":
|
||||
@ -113,8 +112,8 @@ def generate_scons_cache_expansions():
|
||||
|
||||
default_cache_path = os.path.join(default_cache_path_base, system_uuid)
|
||||
expansions["scons_cache_path"] = default_cache_path
|
||||
expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format(
|
||||
scons_cache_mode, shlex.quote(default_cache_path))
|
||||
expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(
|
||||
scons_cache_mode, default_cache_path)
|
||||
# No cache
|
||||
else:
|
||||
# Anything else is 'none'
|
||||
|
||||
@ -382,6 +382,7 @@ variables:
|
||||
#######################################
|
||||
|
||||
functions:
|
||||
|
||||
"remove files": &remove_files
|
||||
command: shell.exec
|
||||
params:
|
||||
@ -432,7 +433,7 @@ functions:
|
||||
"git get project": &git_get_project
|
||||
command: git.get_project
|
||||
params:
|
||||
directory: src
|
||||
directory: ${git_project_directory|src}
|
||||
revisions: # for each module include revision as <module_name> : ${<module_name>_rev}
|
||||
enterprise: ${enterprise_rev}
|
||||
wtdevelop: ${wtdevelop_rev}
|
||||
@ -1096,6 +1097,8 @@ functions:
|
||||
# exit immediately if virtualenv is not found
|
||||
set -o errexit
|
||||
|
||||
virtualenv_loc=$(which ${virtualenv|virtualenv})
|
||||
|
||||
python_loc=$(which ${python|/opt/mongodbtoolchain/v3/bin/python3})
|
||||
venv_dir="${workdir}/venv"
|
||||
"$python_loc" -m venv --system-site-packages "$venv_dir"
|
||||
@ -1709,32 +1712,112 @@ functions:
|
||||
- "./src/evergreen/check_run_tests_infrastructure_failure.sh"
|
||||
|
||||
"scons lint":
|
||||
- command: expansions.write
|
||||
params:
|
||||
file: expansions.yml
|
||||
- command: subprocess.exec
|
||||
- command: shell.exec
|
||||
type: test
|
||||
params:
|
||||
binary: bash
|
||||
args:
|
||||
- "src/evergreen/scons_lint.sh"
|
||||
env:
|
||||
python: ${python}
|
||||
workdir: ${workdir}
|
||||
working_dir: src
|
||||
shell: bash
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
${activate_virtualenv}
|
||||
export MYPY="$(
|
||||
if which cygpath 2>/dev/null; then
|
||||
PATH+=":$(cypath "${workdir}")/venv_3/Scripts"
|
||||
else
|
||||
PATH+=":${workdir}/venv_3/bin"
|
||||
fi
|
||||
PATH+=':/opt/mongodbtoolchain/v3/bin'
|
||||
which mypy
|
||||
)"
|
||||
echo "Found mypy executable at '$MYPY'"
|
||||
export extra_flags=""
|
||||
if [[ ${is_patch|false} == "true" ]]; then
|
||||
extra_flags="--lint-scope=changed"
|
||||
fi
|
||||
|
||||
${compile_env|} python3 ./buildscripts/scons.py ${compile_flags|} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision|}" ENTERPRISE_REV="${enterprise_rev|}" ${targets}
|
||||
|
||||
"scons compile":
|
||||
- command: expansions.write
|
||||
params:
|
||||
file: expansions.yml
|
||||
- command: subprocess.exec
|
||||
type: test
|
||||
params:
|
||||
binary: bash
|
||||
args:
|
||||
- "src/evergreen/scons_compile.sh"
|
||||
env:
|
||||
python: ${python}
|
||||
workdir: ${workdir}
|
||||
command: shell.exec
|
||||
type: test
|
||||
params:
|
||||
working_dir: src
|
||||
shell: bash
|
||||
script: |
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
rm -rf ${install_directory|/data/mongo-install-directory}
|
||||
|
||||
# Use hardlinks to reduce the disk space impact of installing
|
||||
# all of the binaries and associated debug info.
|
||||
|
||||
# The expansion here is a workaround to let us set a different install-action
|
||||
# for tasks that don't support the one we set here. A better plan would be
|
||||
# to support install-action for Ninja builds directly.
|
||||
# TODO: https://jira.mongodb.org/browse/SERVER-48203
|
||||
extra_args="--install-action=${task_install_action|hardlink}"
|
||||
|
||||
# By default, limit link jobs to one quarter of our overall -j
|
||||
# concurrency unless locally overridden. We do this because in
|
||||
# static link environments, the memory consumption of each
|
||||
# link job is so high that without constraining the number of
|
||||
# links we are likely to OOM or thrash the machine. Dynamic
|
||||
# builds, where htis is not a concern, override this value.
|
||||
echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|0.25}"
|
||||
extra_args="$extra_args --jlink=${num_scons_link_jobs_available|0.25}"
|
||||
|
||||
if [ "${scons_cache_scope|}" = "shared" ]; then
|
||||
extra_args="$extra_args --cache-debug=scons_cache.log"
|
||||
fi
|
||||
|
||||
# Conditionally enable scons time debugging
|
||||
if [ "${show_scons_timings|true}" = "true" ]; then
|
||||
extra_args="$extra_args --debug=time"
|
||||
fi
|
||||
|
||||
# Build packages where the upload tasks expect them
|
||||
if [ -n "${git_project_directory|}" ]; then
|
||||
extra_args="$extra_args PKGDIR=${git_project_directory}"
|
||||
else
|
||||
extra_args="$extra_args PKGDIR=${workdir}/src"
|
||||
fi
|
||||
|
||||
# If we are doing a patch build or we are building a non-push
|
||||
# build on the waterfall, then we don't need the --release
|
||||
# flag. Otherwise, this is potentially a build that "leaves
|
||||
# the building", so we do want that flag. The non --release
|
||||
# case should auto enale the faster decider when
|
||||
# applicable. Furthermore, for the non --release cases we can
|
||||
# accelerate the build slightly for situations where we invoke
|
||||
# SCons multiple times on the same machine by allowing SCons
|
||||
# to assume that implicit dependencies are cacheable across
|
||||
# runs.
|
||||
if [ "${is_patch|false}" = "true" ] || [ -z "${push_bucket|}" ] || [ "${compiling_for_test|false}" = "true" ]; then
|
||||
extra_args="$extra_args --implicit-cache --build-fast-and-loose=on"
|
||||
else
|
||||
extra_args="$extra_args --release"
|
||||
fi
|
||||
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
|
||||
export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH
|
||||
fi
|
||||
|
||||
${activate_virtualenv}
|
||||
|
||||
${compile_env|} $python ./buildscripts/scons.py \
|
||||
${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} \
|
||||
${scons_cache_args|} $extra_args \
|
||||
${targets} MONGO_VERSION=${version} ${patch_compile_flags|} || exit_status=$?
|
||||
|
||||
# If compile fails we do not run any tests
|
||||
if [[ $exit_status -ne 0 ]]; then
|
||||
touch ${skip_tests}
|
||||
fi
|
||||
exit $exit_status
|
||||
|
||||
"generate compile expansions":
|
||||
command: shell.exec
|
||||
|
||||
@ -1,35 +0,0 @@
|
||||
report_file: src/report.json
|
||||
archive_file: src/archive.json
|
||||
decompress: tar xzvf
|
||||
private_key_file: /dev/null
|
||||
aws_profile_remote: default
|
||||
resmoke_jobs_factor: 1
|
||||
resmoke_jobs_max: 0
|
||||
timeout_secs: 0
|
||||
exec_timeout_secs: 0
|
||||
python: /opt/mongodbtoolchain/v3/bin/python3
|
||||
multiversion_edition: base
|
||||
multiversion_platform: linux_x86_64
|
||||
multiversion_architecture: x86_64
|
||||
disable_unit_tests: false
|
||||
skip_tests: /dev/null
|
||||
resmoke_jobs: 1
|
||||
should_shuffle: true
|
||||
continue_on_failure: true
|
||||
install_dir: dist-test/bin
|
||||
is_patch: false
|
||||
install_directory: /data/mongo-install-directory
|
||||
task_install_action: hardlink
|
||||
num_scons_link_jobs_available: 0.25
|
||||
show_scons_timings: true
|
||||
compiling_for_test: false
|
||||
scons_cache_mode: nolinked
|
||||
is_commit_queue: false
|
||||
use_scons_cache: false
|
||||
npm_command: jstestfuzz
|
||||
connection_attempts: 25
|
||||
tar: tar
|
||||
cmake_path: /opt/cmake/bin/cmake
|
||||
content_type: application/x-gzip
|
||||
jstestfuzz_concurrent_num_files: 10
|
||||
curator_release: "latest"
|
||||
@ -1,106 +0,0 @@
|
||||
if [[ "$0" == *"/evergreen/prelude.sh" ]]; then
|
||||
echo "ERROR: do not execute this script. source it instead. ie: . prelude.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# path the directory that contains this script.
|
||||
evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
|
||||
# bootstrapping python assumes that the user has not cd'd before the prelude.
|
||||
# Ensure that here.
|
||||
calculated_workdir=$(cd "$evergreen_dir/../.." && echo $PWD)
|
||||
if [ "$PWD" != "$calculated_workdir" ]; then
|
||||
echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that"
|
||||
echo "\$PWD: $PWD"
|
||||
echo "\$calculated_workdir: $calculated_workdir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
function activate_venv {
|
||||
set -e
|
||||
# check if virtualenv is set up
|
||||
if [ -d "${workdir}/venv" ]; then
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
# Need to quote the path on Windows to preserve the separator.
|
||||
. "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log
|
||||
else
|
||||
. ${workdir}/venv/bin/activate 2>/tmp/activate_error.log
|
||||
fi
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
|
||||
fi
|
||||
python=python
|
||||
else
|
||||
python=${python:-/opt/mongodbtoolchain/v3/bin/python3}
|
||||
fi
|
||||
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
|
||||
else
|
||||
export PYTHONPATH="$PYTHONPATH:${workdir}/src"
|
||||
fi
|
||||
|
||||
echo "python set to $(which $python)"
|
||||
set +e
|
||||
}
|
||||
|
||||
expansions_yaml="$evergreen_dir/../../expansions.yml"
|
||||
expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
|
||||
script="$evergreen_dir/../buildscripts/evergreen_expansions2bash.py"
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
expansions_yaml=$(cygpath -w "$expansions_yaml")
|
||||
expansions_default_yaml=$(cygpath -w "$expansions_default_yaml")
|
||||
script=$(cygpath -w "$script")
|
||||
fi
|
||||
|
||||
eval $(activate_venv >/dev/null && $python "$script" "$expansions_yaml" "$expansions_default_yaml")
|
||||
if [ -n "$___expansions_error" ]; then
|
||||
echo $___expansions_error
|
||||
exit 1
|
||||
fi
|
||||
unset expansions_yaml
|
||||
unset expansions_default_yaml
|
||||
unset script
|
||||
unset evergreen_dir
|
||||
|
||||
function add_nodejs_to_path {
|
||||
# Add node and npm binaries to PATH
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
# An "npm" directory might not have been created in %APPDATA% by the Windows installer.
|
||||
# Work around the issue by specifying a different %APPDATA% path.
|
||||
# See: https://github.com/nodejs/node-v0.x-archive/issues/8141
|
||||
export APPDATA=${workdir}/npm-app-data
|
||||
export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location
|
||||
# TODO: this is to work around BUILD-8652
|
||||
cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')"
|
||||
else
|
||||
export PATH="$PATH:/opt/node/bin"
|
||||
fi
|
||||
}
|
||||
|
||||
function posix_workdir {
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
echo $(cygpath -u "${workdir}")
|
||||
else
|
||||
echo ${workdir}
|
||||
fi
|
||||
}
|
||||
|
||||
function set_sudo {
|
||||
set -o >/tmp/settings.log
|
||||
set +o errexit
|
||||
grep errexit /tmp/settings.log | grep on
|
||||
errexit_on=$?
|
||||
# Set errexit "off".
|
||||
set +o errexit
|
||||
sudo=
|
||||
# Use sudo, if it is supported.
|
||||
sudo date >/dev/null 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
sudo=sudo
|
||||
fi
|
||||
# Set errexit "on", if previously enabled.
|
||||
if [ $errexit_on -eq 0 ]; then
|
||||
set -o errexit
|
||||
fi
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
. "$DIR/prelude.sh"
|
||||
|
||||
cd src
|
||||
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
rm -rf ${install_directory}
|
||||
|
||||
# Use hardlinks to reduce the disk space impact of installing
|
||||
# all of the binaries and associated debug info.
|
||||
|
||||
# The expansion here is a workaround to let us set a different install-action
|
||||
# for tasks that don't support the one we set here. A better plan would be
|
||||
# to support install-action for Ninja builds directly.
|
||||
# TODO: https://jira.mongodb.org/browse/SERVER-48203
|
||||
extra_args="--install-action=${task_install_action}"
|
||||
|
||||
# By default, limit link jobs to one quarter of our overall -j
|
||||
# concurrency unless locally overridden. We do this because in
|
||||
# static link environments, the memory consumption of each
|
||||
# link job is so high that without constraining the number of
|
||||
# links we are likely to OOM or thrash the machine. Dynamic
|
||||
# builds, where htis is not a concern, override this value.
|
||||
echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available}"
|
||||
extra_args="$extra_args --jlink=${num_scons_link_jobs_available}"
|
||||
|
||||
if [ "${scons_cache_scope}" = "shared" ]; then
|
||||
extra_args="$extra_args --cache-debug=scons_cache.log"
|
||||
fi
|
||||
|
||||
# Conditionally enable scons time debugging
|
||||
if [ "${show_scons_timings}" = "true" ]; then
|
||||
extra_args="$extra_args --debug=time"
|
||||
fi
|
||||
|
||||
# Build packages where the upload tasks expect them
|
||||
if [ -n "${git_project_directory}" ]; then
|
||||
extra_args="$extra_args PKGDIR='${git_project_directory}'"
|
||||
else
|
||||
extra_args="$extra_args PKGDIR='${workdir}/src'"
|
||||
fi
|
||||
|
||||
# If we are doing a patch build or we are building a non-push
|
||||
# build on the waterfall, then we don't need the --release
|
||||
# flag. Otherwise, this is potentially a build that "leaves
|
||||
# the building", so we do want that flag. The non --release
|
||||
# case should auto enale the faster decider when
|
||||
# applicable. Furthermore, for the non --release cases we can
|
||||
# accelerate the build slightly for situations where we invoke
|
||||
# SCons multiple times on the same machine by allowing SCons
|
||||
# to assume that implicit dependencies are cacheable across
|
||||
# runs.
|
||||
if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then
|
||||
extra_args="$extra_args --implicit-cache --build-fast-and-loose=on"
|
||||
else
|
||||
extra_args="$extra_args --release"
|
||||
fi
|
||||
|
||||
if [ "Windows_NT" = "$OS" ]; then
|
||||
vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
|
||||
export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH
|
||||
fi
|
||||
|
||||
activate_venv
|
||||
|
||||
eval ${compile_env} $python ./buildscripts/scons.py \
|
||||
${compile_flags} ${task_compile_flags} ${task_compile_flags_extra} \
|
||||
${scons_cache_args} $extra_args \
|
||||
${targets} MONGO_VERSION=${version} ${patch_compile_flags} || exit_status=$?
|
||||
|
||||
# If compile fails we do not run any tests
|
||||
if [[ $exit_status -ne 0 ]]; then
|
||||
touch ${skip_tests}
|
||||
fi
|
||||
exit $exit_status
|
||||
@ -1,24 +0,0 @@
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||
. "$DIR/prelude.sh"
|
||||
|
||||
cd src
|
||||
|
||||
set -o errexit
|
||||
set -o verbose
|
||||
|
||||
activate_venv
|
||||
export MYPY="$(
|
||||
if which cygpath 2>/dev/null; then
|
||||
PATH+=":$(cypath "${workdir}")/venv_3/Scripts"
|
||||
else
|
||||
PATH+=":${workdir}/venv_3/bin"
|
||||
fi
|
||||
PATH+=':/opt/mongodbtoolchain/v3/bin'
|
||||
which mypy
|
||||
)"
|
||||
echo "Found mypy executable at '$MYPY'"
|
||||
export extra_flags=""
|
||||
if [[ ${is_patch} == "true" ]]; then
|
||||
extra_flags="--lint-scope=changed"
|
||||
fi
|
||||
eval ${compile_env} python3 ./buildscripts/scons.py ${compile_flags} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision}" ENTERPRISE_REV="${enterprise_rev}" ${targets}
|
||||
Loading…
Reference in New Issue
Block a user