2015-08-13 13:46:38 +02:00
|
|
|
command_type: system
|
2015-06-05 23:41:48 +02:00
|
|
|
stepback: false
|
|
|
|
|
2020-08-21 01:14:22 +02:00
|
|
|
variables:
|
|
|
|
###
|
|
|
|
# Leave this section uncommented to enable compile.
|
|
|
|
_real_remote_file: &_remote_file
|
2020-09-24 18:39:38 +02:00
|
|
|
${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile-variant|}-${version_id}.tar.gz
|
2020-08-21 01:14:22 +02:00
|
|
|
_real_compile: &_compile
|
|
|
|
- variant: linux-wt-standalone
|
|
|
|
name: compile
|
|
|
|
_real_expansions: &_expansion_updates
|
|
|
|
[]
|
|
|
|
###
|
|
|
|
|
|
|
|
###
|
|
|
|
# **Or**: Leave this section uncommented to bypass/skip compile.
|
|
|
|
# # This file ↓ came from a microbenchmarks waterfall run.
|
|
|
|
# # https://evergreen.mongodb.com/version/performance_996dcdc3d96346d71f012388eccc79c691619340
|
|
|
|
# # Artifacts eventually expire. If this fails, grab the compile artifacts url and update this.
|
|
|
|
# _skip_remote_file: &_remote_file
|
2020-09-25 18:42:52 +02:00
|
|
|
# perf/5f6ca2392fbabe40badf39c4/c39af144b2370be0537410d9bc79be66a1a5f3c7/linux/mongodb-5f6ca2392fbabe40badf39c4.tar.gz
|
2020-08-21 01:14:22 +02:00
|
|
|
# _skip_compile: &_compile
|
|
|
|
# []
|
|
|
|
# _skip_expansions: &_expansion_updates
|
2020-09-25 18:42:52 +02:00
|
|
|
# - key: mongodb_binary_archive
|
|
|
|
# value: https://mciuploads.s3.amazonaws.com/perf/5f6ca2392fbabe40badf39c4/c39af144b2370be0537410d9bc79be66a1a5f3c7/linux/mongodb-5f6ca2392fbabe40badf39c4.tar.gz
|
2020-08-21 01:14:22 +02:00
|
|
|
###
|
|
|
|
|
|
|
|
_src_dir: &src_dir src/mongo
|
|
|
|
_modules: &modules
|
|
|
|
- enterprise
|
2020-09-24 18:39:38 +02:00
|
|
|
- mongo-tools
|
|
|
|
- wtdevelop
|
2020-08-21 01:14:22 +02:00
|
|
|
|
|
|
|
- dsi
|
|
|
|
- signal-processing
|
|
|
|
- genny
|
|
|
|
- linkbench
|
|
|
|
- linkbench2
|
|
|
|
- workloads
|
|
|
|
- mongo-perf
|
|
|
|
|
|
|
|
|
|
|
|
modules:
|
|
|
|
###
|
|
|
|
# Same in every DSI project
|
|
|
|
- name: dsi
|
|
|
|
repo: git@github.com:10gen/dsi.git
|
|
|
|
prefix: ../../src
|
|
|
|
branch: master
|
|
|
|
- name: genny
|
|
|
|
repo: git@github.com:10gen/genny.git
|
|
|
|
prefix: ../../src
|
|
|
|
branch: master
|
|
|
|
- name: signal-processing
|
|
|
|
repo: git@github.com:10gen/signal-processing.git
|
|
|
|
prefix: ../../src
|
|
|
|
branch: master
|
|
|
|
- name: workloads
|
|
|
|
repo: git@github.com:10gen/workloads.git
|
|
|
|
prefix: workloads
|
|
|
|
branch: master
|
|
|
|
- name: wtdevelop
|
|
|
|
repo: git@github.com:wiredtiger/wiredtiger.git
|
|
|
|
prefix: src/third_party
|
|
|
|
branch: develop
|
|
|
|
- name: linkbench
|
|
|
|
repo: git@github.com:10gen/linkbench.git
|
2020-11-06 16:45:29 +01:00
|
|
|
prefix: ../../src
|
2020-08-21 01:14:22 +02:00
|
|
|
branch: master
|
|
|
|
- name: linkbench2
|
|
|
|
repo: git@github.com:mdcallag/linkbench.git
|
2020-11-06 16:45:29 +01:00
|
|
|
prefix: ../../src
|
2020-08-21 01:14:22 +02:00
|
|
|
branch: master
|
|
|
|
ref: 63207190657737b32eb0e81c5b81ad1b8bad0e5a
|
2020-09-25 18:42:52 +02:00
|
|
|
- name: mongo-perf
|
|
|
|
repo: git@github.com:mongodb/mongo-perf.git
|
|
|
|
prefix: ../../src
|
|
|
|
branch: master
|
2020-11-06 22:06:56 +01:00
|
|
|
- name: ycsb
|
|
|
|
repo: git@github.com:mongodb-labs/YCSB.git
|
|
|
|
prefix: ../../src
|
|
|
|
branch: master
|
|
|
|
ref: 9e7888c5009e44a185b135104dd8456eea1deb1e
|
2020-08-21 01:14:22 +02:00
|
|
|
###
|
|
|
|
|
2020-11-03 17:50:09 +01:00
|
|
|
- name: enterprise
|
|
|
|
repo: git@github.com:10gen/mongo-enterprise-modules.git
|
|
|
|
prefix: src/mongo/db/modules
|
|
|
|
branch: master
|
|
|
|
- name: mongo-tools
|
|
|
|
repo: git@github.com:mongodb/mongo-tools.git
|
|
|
|
prefix: mongo-tools/src/github.com/mongodb
|
|
|
|
branch: master
|
|
|
|
- name: mongo-perf
|
|
|
|
repo: git@github.com:mongodb/mongo-perf.git
|
|
|
|
prefix: ../../src
|
|
|
|
branch: master
|
|
|
|
|
2020-08-21 01:14:22 +02:00
|
|
|
|
|
|
|
###
|
|
|
|
# Same in every DSI project
|
2015-06-05 23:41:48 +02:00
|
|
|
pre:
|
2020-08-21 01:14:22 +02:00
|
|
|
- func: "f_other_pre_ops"
|
|
|
|
- func: "f_dsi_pre_run"
|
|
|
|
post:
|
|
|
|
- func: "f_dsi_post_run"
|
|
|
|
- func: "f_other_post_ops"
|
|
|
|
###
|
|
|
|
|
|
|
|
functions:
|
|
|
|
###
|
|
|
|
# Same in every DSI project
|
|
|
|
"f_dsi_pre_run":
|
2019-05-21 22:50:10 +02:00
|
|
|
- command: manifest.load
|
2020-08-21 01:14:22 +02:00
|
|
|
- command: expansions.update
|
|
|
|
params:
|
|
|
|
updates: *_expansion_updates
|
|
|
|
"f_run_dsi_workload":
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: |
|
|
|
|
rm -rf ./*
|
|
|
|
mkdir src
|
|
|
|
- command: manifest.load
|
|
|
|
- command: git.get_project
|
|
|
|
params:
|
|
|
|
directory: *src_dir
|
|
|
|
revisions:
|
|
|
|
dsi: ${dsi_rev}
|
|
|
|
linkbench: ${linkbench_rev}
|
|
|
|
linkbench2: ${linkbench2_rev}
|
|
|
|
genny: ${genny_rev}
|
|
|
|
workloads: ${workloads_rev}
|
|
|
|
signal-processing: ${signal-processing_rev}
|
2020-09-25 18:42:52 +02:00
|
|
|
mongo-perf: ${mongo-perf_rev}
|
2020-08-21 01:14:22 +02:00
|
|
|
- command: expansions.write
|
|
|
|
params:
|
|
|
|
file: ./expansions.yml
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: ./src/dsi/run-dsi bootstrap
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: ./src/dsi/run-dsi deploy_cluster
|
|
|
|
- command: shell.exec
|
|
|
|
type: test
|
|
|
|
params:
|
|
|
|
script: ./src/dsi/run-dsi test_control
|
|
|
|
- command: json.send
|
|
|
|
params:
|
|
|
|
name: "perf"
|
|
|
|
file: "./build/LegacyPerfJson/perf.json"
|
|
|
|
- command: shell.exec
|
|
|
|
type: test
|
|
|
|
params:
|
|
|
|
script: |
|
|
|
|
./src/dsi/run-dsi analysis
|
|
|
|
# detect outliers needs to run, so defer the post_run_check exit status to later
|
|
|
|
echo $? > post_run_check.status
|
2015-11-12 17:51:27 +01:00
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: |
|
2020-08-21 01:14:22 +02:00
|
|
|
set -o errexit
|
|
|
|
is_patch=${is_patch}
|
|
|
|
task_id=${task_id}
|
|
|
|
perf_jira_user=${perf_jira_user}
|
|
|
|
perf_jira_pw=${perf_jira_pw}
|
|
|
|
analysis_user=${dsi_analysis_atlas_user}
|
|
|
|
analysis_password=${dsi_analysis_atlas_pw}
|
|
|
|
evergreen_api_key=${evergreen_api_key}
|
|
|
|
evergreen_api_user=${evergreen_api_user}
|
|
|
|
source ./src/dsi/src/signal_processing_setup.sh
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: |
|
|
|
|
set -o verbose
|
|
|
|
source ./signal_processing_venv/bin/activate
|
|
|
|
detect-changes --config .signal-processing.yml --mongo-repo=./src/mongo
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: |
|
|
|
|
set -o verbose
|
|
|
|
source ./signal_processing_venv/bin/activate
|
|
|
|
detect-outliers --config .signal-processing.yml
|
|
|
|
- command: shell.exec
|
|
|
|
type: setup
|
|
|
|
params:
|
|
|
|
script: |
|
|
|
|
set -o verbose
|
|
|
|
filename=rejects.json
|
|
|
|
if [ -s "$filename" ]; then
|
|
|
|
echo "Rejecting task due to the following outliers:"
|
|
|
|
cat "$filename"
|
|
|
|
exit ${detected_outlier_exit_code|0}
|
|
|
|
fi
|
|
|
|
- command: shell.exec
|
|
|
|
type: test
|
|
|
|
params:
|
|
|
|
script: |
|
|
|
|
set -o verbose
|
|
|
|
exit $(cat post_run_check.status)
|
|
|
|
"f_dsi_post_run":
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: ./src/dsi/run-dsi infrastructure_teardown
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: ./src/dsi/run-dsi ./src/dsi/src/dsi/make_artifact.sh
|
|
|
|
- command: s3.put
|
|
|
|
params:
|
|
|
|
aws_key: ${aws_key}
|
|
|
|
aws_secret: ${aws_secret}
|
|
|
|
local_file: dsi-artifacts.tgz
|
|
|
|
remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz}
|
|
|
|
bucket: mciuploads
|
|
|
|
permissions: public-read
|
|
|
|
content_type: ${content_type|application/x-gzip}
|
|
|
|
display_name: Dsi Artifacts - Execution ${execution}
|
|
|
|
- command: s3.put
|
|
|
|
params:
|
|
|
|
aws_key: ${aws_key}
|
|
|
|
aws_secret: ${aws_secret}
|
|
|
|
local_file: src/mongo/workloads/workloads/jsdoc/jsdocs-redirect.html
|
|
|
|
remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html
|
|
|
|
bucket: mciuploads
|
|
|
|
permissions: public-read
|
|
|
|
content_type: text/html
|
|
|
|
display_name: workloads documentation
|
|
|
|
- command: attach.results
|
|
|
|
params:
|
|
|
|
file_location: report.json
|
|
|
|
- command: json.send
|
|
|
|
params:
|
|
|
|
name: "perf"
|
|
|
|
file: "./build/LegacyPerfJson/perf.json"
|
|
|
|
- command: s3.put
|
|
|
|
params:
|
|
|
|
aws_key: ${aws_key}
|
|
|
|
aws_secret: ${aws_secret}
|
|
|
|
local_file: pip-requirements.txt
|
|
|
|
remote_file: ${project}/${build_variant}/${revision}/pip-requirements-${task_id}-${execution}.txt
|
|
|
|
bucket: mciuploads
|
|
|
|
permissions: public-read
|
|
|
|
content_type: atext-plain
|
|
|
|
display_name: Pip Requirements
|
|
|
|
###
|
2015-06-05 23:41:48 +02:00
|
|
|
|
2020-11-03 17:50:09 +01:00
|
|
|
"f_other_post_ops":
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
working_dir: src
|
|
|
|
script: |
|
|
|
|
# removes files from the (local) scons cache when it's over a
|
|
|
|
# threshold, to the $prune_ratio percentage. Ideally override
|
|
|
|
# these default values in the distro config in evergreen.
|
|
|
|
|
|
|
|
if [ -d "${scons_cache_path}" ]; then
|
|
|
|
/opt/mongodbtoolchain/v3/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
|
|
|
|
fi
|
2020-08-21 01:14:22 +02:00
|
|
|
"f_other_pre_ops":
|
|
|
|
- &f_other_pre_ops
|
|
|
|
command: shell.exec
|
2015-09-14 03:33:21 +02:00
|
|
|
params:
|
2020-08-21 01:14:22 +02:00
|
|
|
silent: true
|
|
|
|
script: |
|
|
|
|
for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
|
|
|
|
pkill -9 "$PS"
|
|
|
|
done
|
|
|
|
|
2020-09-24 18:39:38 +02:00
|
|
|
###
|
|
|
|
# Compile
|
|
|
|
"compile mongodb":
|
|
|
|
# We create a virtual environment with the Python dependencies for compiling the server
|
|
|
|
# installed.
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
working_dir: src
|
|
|
|
script: |
|
|
|
|
set -o errexit
|
|
|
|
set -o verbose
|
|
|
|
|
|
|
|
/opt/mongodbtoolchain/v3/bin/virtualenv --python /opt/mongodbtoolchain/v3/bin/python3 "${workdir}/compile_venv"
|
|
|
|
/opt/mongodbtoolchain/v3/bin/virtualenv --python /opt/mongodbtoolchain/v3/bin/python2 "${workdir}/venv"
|
|
|
|
source "${workdir}/compile_venv/bin/activate"
|
|
|
|
|
|
|
|
python -m pip install -r etc/pip/compile-requirements.txt
|
|
|
|
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
working_dir: src
|
|
|
|
script: |
|
|
|
|
set -o errexit
|
|
|
|
set -o verbose
|
|
|
|
|
|
|
|
mkdir -p mongodb/bin
|
|
|
|
|
|
|
|
# We get the raw version string (r1.2.3-45-gabcdef) from git
|
|
|
|
MONGO_VERSION=$(git describe --abbrev=7)
|
|
|
|
|
|
|
|
# If we're going to compile the upstream wtdevelop repository for wiredtiger, add
|
|
|
|
# that githash to version string.
|
|
|
|
if [ "${compile-variant|}" = "-wtdevelop" ]; then
|
|
|
|
WT_VERSION=$(cd src/third_party/wtdevelop; git describe --abbrev=7 | cut -c 9-)
|
|
|
|
MONGO_VERSION="$MONGO_VERSION-wtdevelop-$WT_VERSION"
|
|
|
|
fi
|
|
|
|
# If this is a patch build, we add the patch version id to the version string so we know
|
|
|
|
# this build was a patch, and which evergreen task it came from
|
|
|
|
if [ "${is_patch|false}" = "true" ]; then
|
|
|
|
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
|
|
|
|
fi
|
|
|
|
|
|
|
|
# This script converts the generated version string into a sanitized version string for
|
|
|
|
# use by scons and uploading artifacts as well as information about for the scons cache.
|
|
|
|
source "${workdir}/compile_venv/bin/activate"
|
|
|
|
MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
|
|
|
|
- command: expansions.update
|
|
|
|
params:
|
|
|
|
file: src/compile_expansions.yml
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools
|
|
|
|
script: |
|
|
|
|
set -o verbose
|
|
|
|
set -o errexit
|
|
|
|
|
|
|
|
# make sure newlines in the scripts are handled correctly by windows
|
|
|
|
if [ "Windows_NT" = "$OS" ]; then
|
|
|
|
set -o igncr
|
|
|
|
fi;
|
|
|
|
|
|
|
|
# set_goenv provides set_goenv(), print_ldflags() and print_tags() used below
|
|
|
|
. ./set_goenv.sh
|
|
|
|
GOROOT="" set_goenv || exit
|
|
|
|
go version
|
|
|
|
|
|
|
|
build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
|
|
|
|
if [ "${build_mongoreplay}" = "true" ]; then
|
|
|
|
build_tools="$build_tools mongoreplay"
|
|
|
|
fi
|
|
|
|
for i in $build_tools; do
|
|
|
|
go build -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
|
|
|
|
"../../../../../mongodb/bin/$i${exe|}" --version
|
|
|
|
done
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
working_dir: src
|
|
|
|
script: |
|
|
|
|
set -o errexit
|
|
|
|
set -o verbose
|
|
|
|
source "${workdir}/compile_venv/bin/activate"
|
|
|
|
python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} install-core MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb
|
|
|
|
mkdir -p mongodb/jstests/hooks
|
|
|
|
if [ -d jstests/hooks ]
|
|
|
|
then
|
|
|
|
echo "Fetching JS test DB correctness checks from directory jstests"
|
|
|
|
cp -a jstests/* mongodb/jstests
|
|
|
|
|
|
|
|
echo "Now adding our own special run_validate_collections.js wrapper"
|
|
|
|
mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
|
|
|
|
|
|
|
|
cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
|
|
|
|
print("NOTE: run_validate_collections.js will skip the oplog!");
|
|
|
|
TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
|
|
|
|
load('jstests/hooks/run_validate_collections.actual.js');
|
|
|
|
EOF
|
|
|
|
fi
|
|
|
|
tar czf mongodb${compile-variant|}.tar.gz mongodb
|
|
|
|
- command: s3.put
|
|
|
|
params:
|
|
|
|
aws_key: ${aws_key}
|
|
|
|
aws_secret: ${aws_secret}
|
|
|
|
local_file: src/mongodb${compile-variant|}.tar.gz
|
|
|
|
remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile-variant|}-${version_id}.tar.gz
|
|
|
|
bucket: mciuploads
|
|
|
|
permissions: public-read
|
|
|
|
content_type: ${content_type|application/x-gzip}
|
|
|
|
display_name: mongodb${compile-variant|}.tar.gz
|
|
|
|
###
|
|
|
|
|
2015-06-05 23:41:48 +02:00
|
|
|
tasks:
|
2020-08-21 01:14:22 +02:00
|
|
|
###
|
|
|
|
# Same in every DSI project
|
|
|
|
- name: genny_generate_all_tasks
|
2019-05-21 22:50:10 +02:00
|
|
|
commands:
|
2020-08-21 01:14:22 +02:00
|
|
|
- command: git.get_project
|
|
|
|
params:
|
|
|
|
directory: *src_dir
|
|
|
|
revisions:
|
|
|
|
dsi: ${dsi_rev}
|
|
|
|
linkbench: ${linkbench_rev}
|
|
|
|
linkbench2: ${linkbench2_rev}
|
|
|
|
genny: ${genny_rev}
|
|
|
|
workloads: ${workloads_rev}
|
|
|
|
signal-processing: ${signal-processing_rev}
|
|
|
|
- command: expansions.write
|
|
|
|
params:
|
|
|
|
file: ./expansions.yml
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: ./src/genny/scripts/genny_auto_tasks.sh all_tasks
|
|
|
|
- command: generate.tasks
|
|
|
|
params:
|
|
|
|
files:
|
|
|
|
- build/TaskJSON/Tasks.json
|
|
|
|
- name: genny_auto_tasks
|
|
|
|
commands:
|
|
|
|
- command: git.get_project
|
|
|
|
params:
|
|
|
|
directory: *src_dir
|
|
|
|
revisions:
|
|
|
|
dsi: ${dsi_rev}
|
|
|
|
linkbench: ${linkbench_rev}
|
|
|
|
linkbench2: ${linkbench2_rev}
|
|
|
|
genny: ${genny_rev}
|
|
|
|
workloads: ${workloads_rev}
|
|
|
|
signal-processing: ${signal-processing_rev}
|
|
|
|
- command: expansions.write
|
|
|
|
params:
|
|
|
|
file: ./expansions.yml
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: ./src/genny/scripts/genny_auto_tasks.sh variant_tasks
|
|
|
|
- command: generate.tasks
|
|
|
|
params:
|
|
|
|
files:
|
|
|
|
- build/TaskJSON/Tasks.json
|
|
|
|
- name: genny_patch_tasks
|
|
|
|
commands:
|
|
|
|
- command: git.get_project
|
|
|
|
params:
|
|
|
|
directory: *src_dir
|
|
|
|
revisions:
|
|
|
|
dsi: ${dsi_rev}
|
|
|
|
linkbench: ${linkbench_rev}
|
|
|
|
linkbench2: ${linkbench2_rev}
|
|
|
|
genny: ${genny_rev}
|
|
|
|
workloads: ${workloads_rev}
|
|
|
|
signal-processing: ${signal-processing_rev}
|
|
|
|
- command: expansions.write
|
|
|
|
params:
|
|
|
|
file: ./expansions.yml
|
|
|
|
- command: shell.exec
|
|
|
|
params:
|
|
|
|
script: ./src/genny/scripts/genny_auto_tasks.sh patch_tasks
|
|
|
|
- command: generate.tasks
|
|
|
|
params:
|
|
|
|
files:
|
|
|
|
- build/TaskJSON/Tasks.json
|
|
|
|
- name: smoke_test
|
|
|
|
priority: 5
|
2020-07-20 17:13:25 +02:00
|
|
|
commands:
|
2020-08-21 01:14:22 +02:00
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: "short"
|
|
|
|
- name: smoke_test_ssl
|
|
|
|
priority: 5
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: short
|
|
|
|
mongodb_setup: replica-ssl
|
|
|
|
infrastructure_provisioning: replica
|
|
|
|
- name: dsi_integ_test_run_command_simple
|
|
|
|
priority: 5
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: "run_command_simple"
|
|
|
|
###
|
2020-07-20 17:13:25 +02:00
|
|
|
|
2015-06-05 23:41:48 +02:00
|
|
|
- name: compile
|
|
|
|
commands:
|
2020-09-24 18:39:38 +02:00
|
|
|
- command: manifest.load
|
2020-08-21 01:14:22 +02:00
|
|
|
- command: git.get_project
|
|
|
|
params:
|
|
|
|
directory: src
|
|
|
|
revisions:
|
|
|
|
enterprise: ${enterprise_rev}
|
2020-09-24 18:39:38 +02:00
|
|
|
wtdevelop: ${wtdevelop_rev}
|
|
|
|
mongo-tools: ${mongo-tools_rev}
|
2016-08-03 22:24:49 +02:00
|
|
|
- command: shell.exec
|
2015-06-05 23:41:48 +02:00
|
|
|
params:
|
2016-08-03 22:24:49 +02:00
|
|
|
working_dir: src
|
|
|
|
script: |
|
|
|
|
set -o errexit
|
|
|
|
set -o verbose
|
2020-09-24 18:39:38 +02:00
|
|
|
if [ "${compile-variant|}" = "-wtdevelop" ]; then
|
|
|
|
cd src/third_party
|
|
|
|
for wtdir in dist examples ext lang src test tools ; do
|
|
|
|
rm -rf wiredtiger/$wtdir
|
|
|
|
mv wtdevelop/$wtdir wiredtiger/
|
|
|
|
done
|
2016-08-03 22:24:49 +02:00
|
|
|
fi
|
2020-09-24 18:39:38 +02:00
|
|
|
- func: "compile mongodb"
|
2020-02-02 19:42:49 +01:00
|
|
|
|
|
|
|
|
2020-08-21 01:14:22 +02:00
|
|
|
- name: genny_execution_UserAcquisition
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
2020-10-20 16:03:35 +02:00
|
|
|
test_control: auto_genny_workload
|
2020-09-25 18:42:52 +02:00
|
|
|
auto_workload_path: execution/UserAcquisition.yml
|
2020-10-20 16:03:35 +02:00
|
|
|
- name: genny_scale_InsertRemove
|
2020-08-21 01:14:22 +02:00
|
|
|
depends_on: *_compile
|
2015-06-05 23:41:48 +02:00
|
|
|
commands:
|
2020-10-20 16:03:35 +02:00
|
|
|
- func: f_run_dsi_workload
|
2015-06-05 23:41:48 +02:00
|
|
|
vars:
|
2020-10-20 16:03:35 +02:00
|
|
|
test_control: auto_genny_workload
|
|
|
|
auto_workload_path: scale/InsertRemove.yml
|
|
|
|
- name: query
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: query,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2016-12-14 20:54:14 +01:00
|
|
|
- name: views-query
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: query_identityview,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'true'}
|
2016-12-14 20:54:14 +01:00
|
|
|
- name: views-aggregation
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: aggregation_identityview,
|
|
|
|
include_filter_2: regression,
|
|
|
|
exclude_filter: none,
|
|
|
|
threads: "1",
|
|
|
|
read_cmd: 'true'}
|
2015-08-10 21:57:22 +02:00
|
|
|
- name: where
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: where,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2015-06-05 23:41:48 +02:00
|
|
|
- name: update
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: update,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2015-06-05 23:41:48 +02:00
|
|
|
- name: insert
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: insert,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2018-10-01 21:08:41 +02:00
|
|
|
- name: wildcard-index-read
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: wildcard_read,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2018-10-01 21:08:41 +02:00
|
|
|
- name: wildcard-index-write
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: wildcard_write,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2015-06-30 17:20:39 +02:00
|
|
|
- name: geo
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: geo,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2015-06-30 17:20:39 +02:00
|
|
|
- name: misc
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: command multi remove mixed,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2020-10-23 00:18:20 +02:00
|
|
|
- name: misc_custom_filter_default
|
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
mongodb_setup: microbenchmarks_standalone_custom_filter_default
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: command multi remove mixed,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
|
|
|
- name: misc_custom_filter_slow_or_sample
|
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
mongodb_setup: microbenchmarks_standalone_custom_filter_slow_or_sample
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: command multi remove mixed,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
|
|
|
- name: misc_custom_filter_complex
|
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
mongodb_setup: microbenchmarks_standalone_custom_filter_complex
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: command multi remove mixed,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
|
|
|
- name: misc_custom_filter_whole_doc
|
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
mongodb_setup: microbenchmarks_standalone_custom_filter_whole_doc
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: command multi remove mixed,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
|
|
|
- name: misc_slowms_everything
|
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
mongodb_setup: microbenchmarks_standalone_slowms_everything
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: command multi remove mixed,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2015-06-30 17:20:39 +02:00
|
|
|
- name: singleThreaded
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: single_threaded,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: none,
|
|
|
|
threads: "1",
|
|
|
|
read_cmd: 'false'}
|
2016-08-05 19:52:55 +02:00
|
|
|
- name: aggregation
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: aggregation,
|
|
|
|
include_filter_2: regression,
|
|
|
|
exclude_filter: js,
|
|
|
|
threads: "1",
|
|
|
|
read_cmd: 'false'}
|
2017-03-09 17:09:52 +01:00
|
|
|
- name: agg-query-comparison
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: agg_query_comparison,
|
|
|
|
include_filter_2: core regression,
|
|
|
|
exclude_filter: single_threaded,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'false'}
|
2019-06-11 19:41:30 +02:00
|
|
|
- name: pipeline-updates
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: pipeline-updates,
|
|
|
|
include_filter_2: regression,
|
|
|
|
exclude_filter: none,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'true'}
|
2020-03-10 15:59:18 +01:00
|
|
|
- name: javascript
|
2020-09-25 18:42:52 +02:00
|
|
|
depends_on: *_compile
|
|
|
|
commands:
|
|
|
|
- func: f_run_dsi_workload
|
|
|
|
vars:
|
|
|
|
test_control: microbenchmarks
|
|
|
|
test_control_params: |
|
|
|
|
{include_filter_1: js,
|
|
|
|
include_filter_2: aggregation,
|
|
|
|
exclude_filter: none,
|
|
|
|
threads: "1 2 4 8",
|
|
|
|
read_cmd: 'true'}
|
2020-08-21 01:14:22 +02:00
|
|
|
|
|
|
|
|
2015-06-05 23:41:48 +02:00
|
|
|
buildvariants:
|
|
|
|
- name: linux-wt-standalone
|
2016-11-09 14:15:42 +01:00
|
|
|
display_name: Standalone Linux inMemory
|
2019-02-21 23:07:06 +01:00
|
|
|
batchtime: 90 # 1.5 hours
|
2020-08-21 01:14:22 +02:00
|
|
|
modules: *modules
|
2015-06-05 23:41:48 +02:00
|
|
|
expansions:
|
2018-03-15 14:17:37 +01:00
|
|
|
# We are explicitly tracking the rhel62 variant compile options from evergreen.yml for
|
|
|
|
# microbenchmarks, since they run on the centos6 boxes. If we can get proper artifacts directly
|
|
|
|
# from that project, we should do that and remove the compile tasks.
|
2020-07-10 16:08:36 +02:00
|
|
|
compile_flags: --ssl --separate-debug MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
|
2015-07-17 17:24:35 +02:00
|
|
|
mongod_exec_wrapper: &exec_wrapper "numactl --physcpubind=4,5,6,7 -i 1"
|
|
|
|
perf_exec_wrapper: &perf_wrapper "numactl --physcpubind=1,2,3 -i 0"
|
2018-09-10 19:50:15 +02:00
|
|
|
mongod_flags: >-
|
|
|
|
--auth
|
|
|
|
--fork
|
|
|
|
--inMemoryEngineConfigString 'eviction=(threads_min=1),'
|
|
|
|
--inMemorySizeGB 60
|
|
|
|
--networkMessageCompressors noop
|
|
|
|
--setParameter diagnosticDataCollectionEnabled=false
|
|
|
|
--setParameter enableTestCommands=1
|
|
|
|
--setParameter ttlMonitorEnabled=false
|
|
|
|
--storageEngine inMemory
|
|
|
|
--syncdelay 0
|
2016-08-03 22:24:49 +02:00
|
|
|
use_scons_cache: true
|
2020-09-24 18:39:38 +02:00
|
|
|
platform: linux
|
2020-09-25 18:42:52 +02:00
|
|
|
infrastructure_provisioning: microbenchmarks
|
|
|
|
mongodb_setup: microbenchmarks_standalone
|
2020-09-30 21:54:37 +02:00
|
|
|
canaries: none
|
2020-09-25 18:42:52 +02:00
|
|
|
storageEngine: inMemory
|
2020-09-24 18:39:38 +02:00
|
|
|
project_dir: &project_dir perf
|
2015-06-05 23:41:48 +02:00
|
|
|
run_on:
|
2015-06-09 22:32:13 +02:00
|
|
|
- "centos6-perf"
|
2015-06-05 23:41:48 +02:00
|
|
|
tasks:
|
|
|
|
- name: compile
|
|
|
|
distros:
|
2016-06-23 18:56:05 +02:00
|
|
|
- rhel62-large
|
2019-05-21 22:50:10 +02:00
|
|
|
- name: genny_scale_InsertRemove
|
2020-07-20 17:13:25 +02:00
|
|
|
- name: genny_execution_UserAcquisition
|
2016-08-05 19:52:55 +02:00
|
|
|
- name: aggregation
|
2017-03-09 17:09:52 +01:00
|
|
|
- name: agg-query-comparison
|
2015-06-05 23:41:48 +02:00
|
|
|
- name: query
|
2016-12-14 20:54:14 +01:00
|
|
|
- name: views-aggregation
|
|
|
|
- name: views-query
|
2015-08-10 21:57:22 +02:00
|
|
|
- name: where
|
2015-06-05 23:41:48 +02:00
|
|
|
- name: update
|
|
|
|
- name: insert
|
2018-10-01 21:08:41 +02:00
|
|
|
- name: wildcard-index-read
|
|
|
|
- name: wildcard-index-write
|
2015-06-30 17:20:39 +02:00
|
|
|
- name: geo
|
|
|
|
- name: misc
|
2020-10-23 00:18:20 +02:00
|
|
|
- name: misc_custom_filter_default
|
|
|
|
- name: misc_custom_filter_slow_or_sample
|
|
|
|
- name: misc_custom_filter_complex
|
|
|
|
- name: misc_custom_filter_whole_doc
|
|
|
|
- name: misc_slowms_everything
|
2015-06-30 17:20:39 +02:00
|
|
|
- name: singleThreaded
|
2019-06-11 19:41:30 +02:00
|
|
|
- name: pipeline-updates
|
2020-03-10 15:59:18 +01:00
|
|
|
- name: javascript
|
2015-06-18 01:09:59 +02:00
|
|
|
|
|
|
|
- name: linux-wt-repl
|
2016-11-09 14:15:42 +01:00
|
|
|
display_name: 1-Node ReplSet Linux inMemory
|
2019-02-21 23:07:06 +01:00
|
|
|
batchtime: 90 # 1.5 hours
|
2020-08-21 01:14:22 +02:00
|
|
|
modules: *modules
|
2015-06-18 01:09:59 +02:00
|
|
|
expansions:
|
2015-07-17 17:24:35 +02:00
|
|
|
mongod_exec_wrapper: *exec_wrapper
|
|
|
|
perf_exec_wrapper: *perf_wrapper
|
2018-09-10 19:50:15 +02:00
|
|
|
mongod_flags: >-
|
|
|
|
--auth
|
|
|
|
--fork
|
|
|
|
--inMemoryEngineConfigString 'eviction=(threads_min=1),'
|
|
|
|
--inMemorySizeGB 60
|
|
|
|
--networkMessageCompressors noop
|
|
|
|
--oplogSize 30000
|
|
|
|
--replSet test
|
|
|
|
--setParameter diagnosticDataCollectionEnabled=false
|
|
|
|
--setParameter enableTestCommands=1
|
|
|
|
--setParameter ttlMonitorEnabled=false
|
|
|
|
--storageEngine inMemory
|
|
|
|
--syncdelay 0
|
2020-09-24 18:39:38 +02:00
|
|
|
platform: linux
|
2020-09-25 18:42:52 +02:00
|
|
|
infrastructure_provisioning: microbenchmarks
|
|
|
|
mongodb_setup: microbenchmarks_replica
|
2020-09-30 21:54:37 +02:00
|
|
|
canaries: none
|
2020-09-25 18:42:52 +02:00
|
|
|
storageEngine: inMemory
|
2020-09-24 18:39:38 +02:00
|
|
|
project_dir: *project_dir
|
2015-06-18 01:09:59 +02:00
|
|
|
run_on:
|
|
|
|
- "centos6-perf"
|
|
|
|
tasks:
|
2019-05-21 22:50:10 +02:00
|
|
|
- name: genny_scale_InsertRemove
|
2015-06-18 01:09:59 +02:00
|
|
|
- name: update
|
|
|
|
- name: insert
|
2015-06-30 17:20:39 +02:00
|
|
|
- name: misc
|
|
|
|
- name: singleThreaded
|
2018-10-01 21:08:41 +02:00
|
|
|
- name: wildcard-index-write
|
2019-06-11 19:41:30 +02:00
|
|
|
- name: pipeline-updates
|