mirror of
https://github.com/mongodb/mongo.git
synced 2024-11-28 16:24:56 +01:00
300 lines
8.2 KiB
YAML
300 lines
8.2 KiB
YAML
stepback: false
|
|
command_type: system
|
|
|
|
pre:
|
|
- command: shell.track
|
|
|
|
post:
|
|
- command: attach.results
|
|
params:
|
|
file_location: src/report.json
|
|
- command: shell.cleanup
|
|
- command: shell.exec
|
|
# destroy the cluster
|
|
params:
|
|
working_dir: dsi
|
|
script: |
|
|
set -e
|
|
set -o verbose
|
|
cd ./clusters/${cluster}
|
|
# call terraform destroy twice to avoid AWS timeout
|
|
yes yes | ./terraform destroy
|
|
yes yes | ./terraform destroy
|
|
echo "Cluster DESTROYED."
|
|
|
|
functions:
|
|
"prepare environment":
|
|
- command: shell.exec
|
|
params:
|
|
script: |
|
|
rm -rf ./*
|
|
mkdir src
|
|
- command: shell.exec
|
|
# checkout and setup DSI environment
|
|
params:
|
|
script: |
|
|
set -e
|
|
set -v
|
|
git clone git@github.com:10gen/dsi.git
|
|
cd dsi
|
|
# setup execution environment
|
|
./bin/setup-dsi-env.sh
|
|
- command: shell.exec
|
|
# configure environment, has private information, no logging
|
|
params:
|
|
working_dir: dsi
|
|
silent: true
|
|
script: |
|
|
cd ./clusters/${cluster}
|
|
# stage aws credential for terraform
|
|
../../bin/make_terraform_env.sh ${terraform_key} ${terraform_secret} https://s3.amazonaws.com/mciuploads/longevity/${build_variant}/${revision}/mongodb-${build_id}.tar.gz
|
|
# generate aws private key file
|
|
echo "${ec2_pem}" > ../../keys/aws.pem
|
|
chmod 400 ../../keys/aws.pem
|
|
|
|
"bring up 3 shard cluster":
|
|
- command: shell.exec
|
|
# bring up the cluster
|
|
params:
|
|
working_dir: dsi
|
|
silent: true
|
|
script: |
|
|
# to create a mongod EC2 cluster
|
|
set -e
|
|
set -o verbose
|
|
cd ./clusters/${cluster}
|
|
../../bin/setup-cluster.sh ${cluster}
|
|
echo "EC2 Cluster CREATED."
|
|
|
|
"configure mongodb cluster":
|
|
- command: shell.exec
|
|
# bring up the mongod
|
|
params:
|
|
working_dir: dsi
|
|
script: |
|
|
set -e
|
|
set -o verbose
|
|
cd ./clusters/${cluster}
|
|
# configure mongodb cluster with wiredTiger
|
|
USE_CSRS=${use_csrs} ../../bin/config-${setup}.sh mongodb ${storageEngine} ${numShard}
|
|
echo "${cluster} MongoDB Cluster STARTED."
|
|
|
|
"run test":
|
|
- command: shell.exec
|
|
params:
|
|
working_dir: dsi
|
|
script: |
|
|
set -e
|
|
set -v
|
|
cd ./clusters/${cluster}
|
|
# show cluster details
|
|
cat ips.sh
|
|
./update_run_config.sh
|
|
cat run-${test}.json
|
|
# clean up old artifact from the last test
|
|
rm -f ../perf.json
|
|
rm -rf ./reports
|
|
rm -f ../../reports.tgz
|
|
echo "Run test for ${test} with setup ${setup}"
|
|
# run test
|
|
../../bin/run-${test}.sh ${storageEngine} ${setup} ${cluster}
|
|
mv perf.json ..
|
|
cd ..
|
|
cat perf.json >> perf_all.json
|
|
echo "Complete test for ${test} with setup ${setup}!"
|
|
- command: "json.send"
|
|
params:
|
|
name: "perf"
|
|
file: "dsi/clusters/perf.json"
|
|
|
|
"destroy cluster":
|
|
- command: shell.exec
|
|
# destroy the cluster
|
|
params:
|
|
working_dir: dsi
|
|
script: |
|
|
set -e
|
|
set -o verbose
|
|
cd ./clusters/${cluster}
|
|
# destroy the EC2 cluster
|
|
yes yes | ./terraform destroy
|
|
yes yes | ./terraform destroy
|
|
# clean all file to be safe
|
|
rm -rf *
|
|
echo "Cluster DESTROYED."
|
|
echo "All perf results"
|
|
cd ..
|
|
cat perf_all.json | egrep "name|ops_per_sec"
|
|
|
|
"make test log artifact":
|
|
- command: shell.exec
|
|
# destroy the cluster
|
|
params:
|
|
working_dir: dsi
|
|
script: |
|
|
set -e
|
|
set -o verbose
|
|
cd ./clusters/${cluster}/reports
|
|
# move additional file here
|
|
cp ../ips.sh .
|
|
cp ../../perf_all.json .
|
|
cd ..
|
|
rm -rf ../reports
|
|
mv ./reports ..
|
|
|
|
- command: archive.targz_pack
|
|
params:
|
|
target: "reports.tgz"
|
|
source_dir: "dsi/clusters"
|
|
include:
|
|
- "reports/**"
|
|
|
|
"upload log file":
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: reports.tgz
|
|
remote_file: longevity/${build_variant}/${revision}/logs/${test}-${build_id}.${ext|tgz}
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/x-gzip}
|
|
display_name: ${test}-longevity-testlog
|
|
|
|
"analyze":
|
|
- command: json.get_history
|
|
params:
|
|
task: ${task_name}
|
|
file: "src/history.json"
|
|
name: "perf"
|
|
- command: json.get_history
|
|
params:
|
|
tags: true
|
|
task: ${task_name}
|
|
file: "src/tags.json"
|
|
name: "perf"
|
|
- command: shell.exec
|
|
# post_run_check.py and longevity_override.json for DSI tests are part of dsi repo
|
|
type : test
|
|
params:
|
|
working_dir: src
|
|
script: |
|
|
set -o errexit
|
|
set -o verbose
|
|
python -u ../dsi/analysis/post_run_check.py --rev ${revision} -f history.json -t tags.json --refTag 3.2.1-Baseline --overrideFile ../dsi/analysis/master/longevity_override.json --project_id mongo-longevity --task_name ${task_name} --variant ${build_variant}
|
|
|
|
|
|
tasks:
|
|
- name: compile
|
|
commands:
|
|
- command: git.get_project
|
|
params:
|
|
directory: src
|
|
- command: git.apply_patch
|
|
params:
|
|
directory: src
|
|
- command: shell.exec
|
|
params:
|
|
working_dir: src
|
|
script: |
|
|
set -o errexit
|
|
set -o verbose
|
|
${scons|scons} ${compile_flags|} mongo mongod mongos
|
|
# strip binaries here
|
|
strip mongo
|
|
strip mongos
|
|
strip mongod
|
|
mkdir -p mongodb/bin
|
|
mv mongo mongodb/bin
|
|
mv mongod mongodb/bin
|
|
mv mongos mongodb/bin
|
|
tar cvf mongodb.tar mongodb
|
|
gzip mongodb.tar
|
|
- command: s3.put
|
|
params:
|
|
aws_key: ${aws_key}
|
|
aws_secret: ${aws_secret}
|
|
local_file: src/mongodb.tar.gz
|
|
remote_file: longevity/${build_variant}/${revision}/mongodb-${build_id}.tar.gz
|
|
bucket: mciuploads
|
|
permissions: public-read
|
|
content_type: ${content_type|application/x-gzip}
|
|
display_name: mongodb.tar.gz
|
|
|
|
- name: shard_cluster_test
|
|
depends_on:
|
|
- name: compile
|
|
exec_timeout_secs: 216000
|
|
commands:
|
|
- func: "prepare environment"
|
|
- func: "bring up 3 shard cluster"
|
|
- func: "configure mongodb cluster"
|
|
vars:
|
|
use_csrs: "true"
|
|
- func: "run test"
|
|
vars:
|
|
test: "ycsb"
|
|
- func: "make test log artifact"
|
|
- func: "upload log file"
|
|
vars:
|
|
test: "ycsb"
|
|
- func: "destroy cluster"
|
|
- func: "analyze"
|
|
|
|
- name: shard_cluster_MMAPv1_test
|
|
depends_on:
|
|
- name: compile
|
|
exec_timeout_secs: 28800
|
|
commands:
|
|
- func: "prepare environment"
|
|
- func: "bring up 3 shard cluster"
|
|
- func: "configure mongodb cluster"
|
|
vars:
|
|
use_csrs: "true"
|
|
- func: "run test"
|
|
vars:
|
|
test: "ycsb-mmap"
|
|
- func: "make test log artifact"
|
|
- func: "upload log file"
|
|
vars:
|
|
test: "ycsb"
|
|
- func: "destroy cluster"
|
|
- func: "analyze"
|
|
|
|
buildvariants:
|
|
- name: linux-wt-shard
|
|
display_name: Linux WT Shard
|
|
batchtime: 40320 # 4 weeks
|
|
expansions:
|
|
compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
|
|
setup: shard
|
|
cluster: longevity
|
|
numShard: 3
|
|
storageEngine: wiredTiger
|
|
|
|
run_on:
|
|
- "linux-64-amzn-perf-longevity"
|
|
tasks:
|
|
- name: compile
|
|
distros:
|
|
- rhel55
|
|
- name: shard_cluster_test
|
|
|
|
- name: linux-mmapv1-shard
|
|
display_name: Linux MMAPv1 Shard
|
|
batchtime: 40320 # 4 week
|
|
expansions:
|
|
compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
|
|
setup: shard
|
|
cluster: longevity
|
|
numShard: 3
|
|
storageEngine: mmapv1
|
|
|
|
run_on:
|
|
- "linux-64-amzn-perf-longevity"
|
|
tasks:
|
|
- name: compile
|
|
distros:
|
|
- rhel55
|
|
- name: shard_cluster_MMAPv1_test
|