mirrors/mongodb
mirrors/mongodb
0
0
Fork 0
mirror of https://github.com/mongodb/mongo.git synced 2024-11-28 07:59:02 +01:00
Code Issues Releases Activity
76fa3d6f89
mongodb/etc/system_perf.yml
Daniel Hill 236e3e435a SERVER-93885 no cache and keep going bazel (#26421) 
GitOrigin-RevId: 4ab7fc269d172a7fd0bc88e1297329e23bccf515
2024-08-22 22:17:08 +00:00

528 lines
17 KiB
YAML
Executable File
Raw Blame History

command_type: system
stepback: false
exec_timeout_secs: &exec_timeout_secs 21600
timeout_secs: &timeout_secs 7200
include:
- filename: etc/evergreen_yml_components/definitions.yml
- filename: etc/evergreen_yml_components/tasks/compile_tasks_shared.yml
### lint_yaml trim start ###
#
# The lines below are trimmed by yamllinters.sh so that `evergreen evaluate` works on this file.
# This allows us to test that non-dsi imports continue to work.
#
- filename: evergreen/system_perf/master/variants.yml
module: dsi
- filename: evergreen/system_perf/master/master_variants.yml
module: dsi
- filename: evergreen/system_perf/shared_tasks.yml
module: dsi
### lint_yaml trim end ###
## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
parameters:
- key: patch_compile_flags
description: "Additional SCons flags to be applied during scons compile invocations in this patch"
# see https://github.com/10gen/performance-tooling-docs/blob/main/patch_testing.md#skipping-compilation-on-sys-perf-projects
- key: reuse_compile_from
description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_ef8504eefef4c9fe2062b8424e85053b3e1ad8d2"
- key: exec_timeout_secs_override
value: *exec_timeout_secs
description: "Overrides the default exec_timeout_secs defined at the top of this file for the f_dsi_run_workload task"
- key: timeout_secs_override
value: *timeout_secs
description: "Overrides the default timeout_secs defined at the top of this file for the f_dsi_run_workload task"
variables:
_src_dir: &src_dir src/mongo
_modules: &modules
- dsi
- genny
- workloads
- linkbench
- linkbench2
- tsbs
- mongo-perf
- YCSB
- py-tpcc
- PrivateWorkloads
- flamegraph
_schedule_variant_auto_tasks_task: &schedule_variant_auto_tasks_task
name: schedule_variant_auto_tasks
activate: false
depends_on:
- name: schedule_global_auto_tasks
variant: task_generation
_schedule_patch_auto_tasks_task: &schedule_patch_auto_tasks_task
name: schedule_patch_auto_tasks
activate: false
depends_on:
- name: schedule_global_auto_tasks
variant: task_generation
_shared_compile_expansions: &shared_compile_expansions
platform: linux
project_dir: dsi
scons_cache_scope: shared
scons_cache_mode: all
has_packages: false
###
# Same in every DSI project. Ensure that this block is synchronized with
# evergreen-dsitest.yml, and src/dsi/onboarding.py (search update-repos-here)
# in the dsi repo, and etc/system_perf.yml in mongodb/mongo.
modules:
- name: dsi
owner: 10gen
repo: dsi
prefix: ${workdir}/src
branch: master
- name: genny
owner: mongodb
repo: genny
prefix: ${workdir}/src
branch: master
- name: workloads
owner: 10gen
repo: workloads
prefix: ${workdir}/src
branch: master
- name: linkbench
owner: 10gen
repo: linkbench
prefix: ${workdir}/src
branch: master
- name: linkbench2
owner: 10gen
repo: linkbench2
prefix: ${workdir}/src
branch: master
- name: tsbs
owner: mongodb-forks
repo: tsbs
prefix: ${workdir}/src
branch: main
- name: mongo-perf
owner: mongodb
repo: mongo-perf
prefix: ${workdir}/src
branch: master
- name: YCSB
owner: mongodb-labs
repo: YCSB
prefix: ${workdir}/src
branch: production
- name: py-tpcc
owner: mongodb-labs
repo: py-tpcc
prefix: ${workdir}/src
branch: production
- name: flamegraph
owner: mongodb-forks
repo: flamegraph
prefix: ${workdir}/src
branch: master
- name: PrivateWorkloads
owner: 10gen
repo: PrivateWorkloads
prefix: ${workdir}/src
branch: production
###
###
# Same in every DSI project
pre:
- func: "f_other_pre_ops"
- func: "f_dsi_pre_run"
- func: "get version expansions"
- func: "apply version expansions"
post:
- func: "f_dsi_post_run"
- func: "f_other_post_ops"
timeout:
- func: "f_dsi_timeout"
- func: "f_other_timeout"
###
functions:
f_other_post_ops:
- command: shell.exec
params:
script: |
cp ./build/Artifacts/DSIArtifacts/WorkloadOutput/reports/ycsb_100read/mongod.0/pre_post_task_output/default.profdata.tgz ./default.profdata.tgz || true
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ./default.profdata.tgz
remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/default.profdata.tgz
bucket: mciuploads
permissions: public-read
content_type: application/x-gzip
display_name: PGO profile data - Execution ${execution}
- command: shell.exec
params:
working_dir: src
script: |
# removes files from the (local) scons cache when it's over a
# threshold, to the $prune_ratio percentage. Ideally override
# these default values in the distro config in evergreen.
if [ -d "${scons_cache_path}" ]; then
/opt/mongodbtoolchain/v4/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
fi
f_other_pre_ops:
- &f_other_pre_ops
command: shell.exec
params:
silent: true
script: |
for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
pkill -9 "$PS"
done
f_other_timeout:
# Can't be empty so just `echo`.
- command: shell.exec
params: {script: "echo"}
###
# Package the supplementary DSI data (mongo tools, feature flag list,
# server params list, jstests, and buildscripts)
download mongo tools:
- command: shell.exec
params:
working_dir: src
script: |
set -o verbose
set -o errexit
arch=$(uname -m)
case $arch in
"x86_64")
if [ -f /etc/os-release ]; then
. /etc/os-release
if [ "$ID" == "amzn" ]; then
case $VERSION_ID in
"2")
binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-x86_64-100.9.4.tgz"
;;
"2023")
binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-x86_64-100.9.4.tgz"
;;
*)
echo "Unsupported Amazon Linux version: $VERSION_ID"
exit 1
;;
esac
else
echo "Unsupported Linux distribution: $ID"
exit 1
fi
else
echo "Unable to determine Linux distribution"
exit 1
fi
;;
"aarch64")
if [ -f /etc/os-release ]; then
. /etc/os-release
if [ "$ID" == "amzn" ]; then
case $VERSION_ID in
"2")
binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-aarch64-100.9.4.tgz"
;;
"2023")
binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-aarch64-100.9.4.tgz"
;;
*)
echo "Unsupported Amazon Linux version: $VERSION_ID"
exit 1
;;
esac
else
echo "Unsupported Linux distribution: $ID"
exit 1
fi
else
echo "Unable to determine Linux distribution"
exit 1
fi
;;
*)
echo "Unsupported architecture: $arch"
exit 1
;;
esac
echo "Downloading and installing MongoDB tools..."
mkdir -p mongodb/
wget "$binary_url" -O mongo-tools.tar.gz
tar -xzvf mongo-tools.tar.gz -C mongodb/ --strip-components=1 "mong*/bin"
echo "MongoDB tools installed successfully!"
generate atlas info:
- command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
uarch=$(uname -p)
os=$(uname -r)
json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }"
echo $json | jq '.' > mongodb/atlas_info.json
generate feature flag list:
- command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
pushd ..
. src/evergreen/prelude.sh
activate_venv
popd
python ./buildscripts/idl/gen_all_feature_flag_list.py
mkdir -p mongodb/feature_flags
cp ./all_feature_flags.txt mongodb/feature_flags
generate server params list:
- command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
pushd ..
. src/evergreen/prelude.sh
activate_venv
popd
python ./buildscripts/idl/gen_all_server_params_list.py
mkdir -p mongodb/server_params
cp ./all_server_params.txt mongodb/server_params
copy jstests:
- command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
mkdir -p mongodb/jstests/hooks
if [ -d jstests/hooks ]
then
echo "Fetching JS test DB correctness checks from directory jstests"
cp -a jstests/* mongodb/jstests
echo "Now adding our own special run_validate_collections.js wrapper"
mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
print("NOTE: run_validate_collections.js will skip the oplog!");
TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
await import("jstests/hooks/run_validate_collections.actual.js");
EOF
fi
copy buildscripts:
- command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
mkdir -p mongodb/buildscripts
cp -a buildscripts/* mongodb/buildscripts
package supplementary data:
- command: shell.exec
params:
working_dir: src
script: |
if [ -z "${build_patch_id}" ] || [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
tar czf supplementary-data.tgz mongodb
else
# Evergreen does not handle nested escaped expansions well
reuse_compile_from="${reuse_compile_from}"
build_patch_id="${build_patch_id}"
version_to_reuse_from=$(if [ -n "${build_patch_id}" ]; then echo "${build_patch_id}"; else echo "${reuse_compile_from}"; fi)
curl -o supplementary-data.tgz https://s3.amazonaws.com/mciuploads/${project}/${compile_variant}/$${version_to_reuse_from}/dsi/supplementary-data.tgz
fi
upload supplementary data:
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: src/supplementary-data.tgz
remote_file: ${project}/${compile_variant}/${version_id}/dsi/supplementary-data.tgz
bucket: mciuploads
permissions: public-read
content_type: application/x-gzip
display_name: supplementary-data.tgz
tasks:
- name: package_supplementary_data
depends_on:
- name: version_expansions_gen
variant: generate-tasks-for-version
commands:
- command: manifest.load
- command: git.get_project
params:
directory: src
clone_depth: 1000
- func: "set up venv"
- func: "get version expansions"
- func: "apply version expansions"
- func: "download mongo tools"
- func: "generate feature flag list"
- func: "generate atlas info"
- func: "generate server params list"
- func: "copy jstests"
- func: "copy buildscripts"
- func: "package supplementary data"
- func: "upload supplementary data"
buildvariants:
# Replacement for generate-tasks-for-version from
# evergreen_yml_components/variants/misc/task_generation.yml.
- name: generate-tasks-for-version
display_name: "~ Generate tasks for evergreen version"
activate: true
run_on:
- rhel8.8-medium
tasks:
- name: version_expansions_gen
- name: amazon2-x86-compile
display_name: "* Compile for Amazon Linux 2 x86"
expansions:
<<: *shared_compile_expansions
compile_variant: amazon2-x86-compile
compile_flags: >-
--ssl
MONGO_DISTMOD=amazon2
-j$(grep -c ^processor /proc/cpuinfo)
--release
--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
run_on:
- amazon2-xlarge
tasks:
- name: compile_and_package_serial_no_unittests_TG
- name: package_supplementary_data
- name: amazon2-arm64-compile
display_name: "* Compile for Amazon Linux 2 arm64"
expansions:
<<: *shared_compile_expansions
compile_variant: amazon2-arm64-compile
compile_flags: >-
--ssl
MONGO_DISTMOD=amazon2
-j$(grep -c ^processor /proc/cpuinfo)
--release
--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
run_on:
- amazon2-arm64-xlarge
tasks:
- name: compile_and_package_serial_no_unittests_TG
- name: package_supplementary_data
- name: amazon2-arm64-compile-pgo
display_name: "* Compile for Amazon Linux 2 arm64 PGO"
expansions:
platform: linux
project_dir: dsi
has_packages: false
use_scons_cache: false
pgo_profile_url: https://mciuploads.s3.amazonaws.com/dsi/perf-3-node-replSet.arm.aws.2024-05/b49fc707fb83a45727ad3f9f7cfa4c940a6805fe/sys_perf_perf_3_node_replSet.arm.aws.2024_05_ycsb.100read.2024_05_patch_b49fc707fb83a45727ad3f9f7cfa4c940a6805fe_66b035d75b27a40007f52c27_24_08_05_02_16_15/66b035d75b27a40007f52c27/logs/default.profdata.tgz
compile_variant: amazon2-arm64-compile-pgo
compile_flags: >-
--ssl
MONGO_DISTMOD=amazon2
BAZEL_FLAGS=--keep_going
-j$(grep -c ^processor /proc/cpuinfo)
--release
--thin-lto
--pgo
--variables-files=etc/scons/mongodbtoolchain_stable_clang.vars
run_on:
- amazon2-arm64-xlarge
tasks:
- name: compile_and_package_serial_no_unittests_TG
- name: package_supplementary_data
- name: amazon2-arm64-mongocrypt-shlib-compile
display_name: "* Compile mongo_crypt_v1.so for Amazon Linux 2 arm64"
modules: *modules
cron: "0 0 * * 4" # 00:00 on Thursday
tags: ["bazel_check"]
expansions:
<<: *shared_compile_expansions
compile_variant: amazon2-arm64-compile
use_scons_cache: true
compile_flags: >-
--separate-debug
MONGO_DISTMOD=amazon2
-j$(grep -c ^processor /proc/cpuinfo)
--release
--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
--use-diagnostic-latches=off
--allocator=system
--enterprise-features=fle
--js-engine=none
--link-model=dynamic-sdk
--enable-http-client=off
--ssl=off
SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique"
CCFLAGS="-fno-gnu-unique"
run_on:
- amazon2-arm64-xlarge
tasks:
- name: .crypt
- name: amazon2-x86-streams-compile
display_name: "* Compile Streams for Amazon Linux 2 x86"
expansions:
<<: *shared_compile_expansions
compile_variant: amazon2-x86-streams-compile
compile_flags: >-
--ssl
MONGO_DISTMOD=amazon2
-j$(grep -c ^processor /proc/cpuinfo)
--release
--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
--streams-release-build
run_on:
- amazon2-xlarge
tasks:
- name: compile_and_package_serial_no_unittests_TG
- name: package_supplementary_data
# This variant tests the reuse_compile_from feature
# by reusing compile artifacts from another compile
# in the same patch.
- name: test-reuse-compile-from
display_name: "~ Test reuse_compile_from"
cron: "0 0 * * *" # Every day at 00:00
expansions:
<<: *shared_compile_expansions
compile_variant: amazon2-arm64-compile
reuse_compile_from: ${version_id}
compile_flags: >-
--ssl
MONGO_DISTMOD=amazon2
-j$(grep -c ^processor /proc/cpuinfo)
--release
--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
run_on:
- amazon2-arm64-xlarge
tasks:
- name: compile_and_package_serial_no_unittests_TG
depends_on:
- name: archive_dist_test
variant: amazon2-arm64-compile
- name: archive_dist_test_debug
variant: amazon2-arm64-compile
View Git Blame Copy Permalink