mongodb/etc/perf.yml

command_type: system
stepback: false

pre:
    - command: shell.track

post:
    - command: "json.send"
      params:
        name: "perf"
        file: "src/perf.json"
    - command: attach.results
      params:
        file_location: src/report.json
    - command: shell.cleanup

functions:
  "start server":
    - command: shell.exec
      params:
        script: |
          rm -rf ./*
          mkdir src          
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: perf/${version_id}/${revision}/mongod-${version_id}
        bucket: mciuploads
        local_file: src/mongod
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: perf/${version_id}/${revision}/mongo-${version_id}
        bucket: mciuploads
        local_file: src/mongo
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e 
          set -v
          chmod +x mongod
          chmod +x mongo
          git clone https://github.com/mongodb/mongo-perf perf
          cd perf
          git checkout r20150814          
    - command: shell.exec
      params:
        background: true
        working_dir: src
        script: |
          set -e 
          set -o verbose
          mkdir -p ./dbdata
          ${mongod_exec_wrapper} ./mongod --dbpath ./dbdata ${mongod_flags}
          sleep 5
          # if we started a replset, initiate it and wait for it to become primary
          ./mongo --eval "if(db.isMaster().isreplicaset){rs.initiate(); assert.soon(function(){return db.isMaster().ismaster}, 'no primary')}"
          echo "MONGOD STARTED."          
  "compare":
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: perf/${version_id}/${revision}/compare.py-${version_id}
        bucket: mciuploads
        local_file: src/compare.py
    - command: json.get
      params:
        task: ${compare_task}
        variant : ${variant1}
        file: "src/stand.json"
        name: "perf"
    - command: json.get
      params:
        task: ${compare_task}
        variant : ${variant2}
        file: "src/node.json"
        name: "perf"
    - command: shell.exec
      type : test
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          virtualenv ./venv
          source ./venv/bin/activate
          python compare.py -b stand.json -c node.json -t ${threshold}          
  "analyze":
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: perf/${version_id}/${revision}/perf_regression_check.py-${version_id}
        bucket: mciuploads
        local_file: src/perf_regression_check.py
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: perf/${version_id}/${revision}/override.json-${version_id}
        bucket: mciuploads
        local_file: src/override.json
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -v
          virtualenv ./venv
          source ./venv/bin/activate
          pip install argparse python-dateutil          
    - command: json.get_history
      params:
        task: ${task_name}
        file: "src/history.json"
        name: "perf"
    - command: json.get_history
      params:
        tags: true
        task: ${task_name}
        file: "src/tags.json"
        name: "perf"
    - command: shell.exec
      type : test
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          source ./venv/bin/activate
          python perf_regression_check.py -f history.json --rev ${revision} -t tags.json --refTag 3.0.6-Baseline --overrideFile override.json --variant ${build_variant} --threshold 0.10 --threadThreshold 0.15          
  "run perf tests":
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -v
          virtualenv ./venv
          source ./venv/bin/activate
          pip install argparse          
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -v
          source ./venv/bin/activate
          cd perf
          #  give mongod a few seconds to start up so that we can connect.
          sleep 5
          ${perf_exec_wrapper} python benchrun.py --shell ../mongo -t ${threads} --trialCount 5 -f testcases/*.js --includeFilter ${includeFilter1}  --includeFilter ${includeFilter2} --excludeFilter ${excludeFilter} --out perf.json --exclude-testbed          
    - command: "json.send"
      params:
        name: "perf"
        file: "src/perf/perf.json"

tasks:
- name: compile
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          ${scons|scons} ${compile_flags|} mongo
          ${scons|scons} ${compile_flags|} mongod          
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongod
        remote_file: perf/${version_id}/${revision}/mongod-${version_id}
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/octet-stream}
        display_name: mongod
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongo
        remote_file: perf/${version_id}/${revision}/mongo-${version_id}
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/octet-stream}
        display_name: mongo
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/buildscripts/perf_regression_check.py
        remote_file: perf/${version_id}/${revision}/perf_regression_check.py-${version_id}
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/octet-stream}
        display_name: perf_regression_check.py
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/buildscripts/compare.py
        remote_file: perf/${version_id}/${revision}/compare.py-${version_id}
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/octet-stream}
        display_name: compare.py
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/etc/override.json
        remote_file: perf/${version_id}/${revision}/override.json-${version_id}
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/octet-stream}
        display_name: override.json

- name: query
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "query"
        includeFilter2 : "core regression" 
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "analyze"
- name: where
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "where"
        includeFilter2 : "core regression" 
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "analyze"
- name: update
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "update"
        includeFilter2 : "core regression" 
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "analyze"
- name: insert
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "insert"
        includeFilter2 : "core regression" 
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "analyze"
- name: geo
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "geo"
        includeFilter2 : "core regression" 
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "analyze"
- name: misc
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "command multi remove mixed"
        includeFilter2 : "core regression" 
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "analyze"
- name: singleThreaded
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "single_threaded"
        includeFilter2 : "core regression" 
        excludeFilter : "none"
        threads : "1"
    - func: "analyze"
- name: singleThreaded-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : singleThreaded
      variant : linux-wt-standalone
    - name : singleThreaded
      variant :  linux-wt-repl
  commands:
    - func: "compare"
      vars:
        compare_task : "singleThreaded"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
        threshold : 75
- name: insert-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : insert
      variant : linux-wt-standalone
      status : "*"
    - name : insert
      variant :  linux-wt-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "insert"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
        threshold : 75
- name: update-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : update
      variant : linux-wt-standalone
      status : "*"
    - name : update
      variant :  linux-wt-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "update"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
        threshold : 75
- name: misc-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : misc
      variant : linux-wt-standalone
      status : "*"
    - name : misc
      variant :  linux-wt-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "misc"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
        threshold : 75
- name: singleThreaded-wt-mmap-standalone-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : singleThreaded
      variant : linux-wt-standalone
      status : "*"
    - name : singleThreaded
      variant :  linux-mmap-standalone
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "singleThreaded"
        variant2 : "linux-wt-standalone"
        variant1 : "linux-mmap-standalone"
        threshold : 85
- name: query-wt-mmap-standalone-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : query
      variant : linux-wt-standalone
      status : "*"
    - name : query
      variant :  linux-mmap-standalone
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "query"
        variant2 : "linux-wt-standalone"
        variant1 : "linux-mmap-standalone"
        threshold : 85
- name: where-wt-mmap-standalone-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : where
      variant : linux-wt-standalone
      status : "*"
    - name : where
      variant :  linux-mmap-standalone
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "where"
        variant2 : "linux-wt-standalone"
        variant1 : "linux-mmap-standalone"
        threshold : 85
- name: geo-wt-mmap-standalone-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : geo
      variant : linux-wt-standalone
      status : "*"
    - name : geo
      variant :  linux-mmap-standalone
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "geo"
        variant2 : "linux-wt-standalone"
        variant1 : "linux-mmap-standalone"
        threshold : 85
- name: insert-wt-mmap-standalone-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : insert
      variant : linux-wt-standalone
      status : "*"
    - name : insert
      variant :  linux-mmap-standalone
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "insert"
        variant2 : "linux-wt-standalone"
        variant1 : "linux-mmap-standalone"
        threshold : 85
- name: update-wt-mmap-standalone-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : update
      variant : linux-wt-standalone
      status : "*"
    - name : update
      variant :  linux-mmap-standalone
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "update"
        variant2 : "linux-wt-standalone"
        variant1 : "linux-mmap-standalone"
        threshold : 85
- name: misc-wt-mmap-standalone-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : misc
      variant : linux-wt-standalone
      status : "*"
    - name : misc
      variant :  linux-mmap-standalone
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "misc"
        variant2 : "linux-wt-standalone"
        variant1 : "linux-mmap-standalone"
        threshold : 85
- name: singleThreaded-wt-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : singleThreaded
      variant : linux-wt-repl
      status : "*"
    - name : singleThreaded
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "singleThreaded"
        variant2 : "linux-wt-repl"
        variant1 : "linux-mmap-repl"
        threshold : 85
- name: insert-wt-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : insert
      variant : linux-wt-repl
      status : "*"
    - name : insert
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "insert"
        variant2 : "linux-wt-repl"
        variant1 : "linux-mmap-repl"
        threshold : 85
- name: update-wt-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : update
      variant : linux-wt-repl
      status : "*"
    - name : update
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "update"
        variant2 : "linux-wt-repl"
        variant1 : "linux-mmap-repl"
        threshold : 85
- name: misc-wt-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
      status : "*"
    - name : misc
      variant : linux-wt-repl
    - name : misc
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "misc"
        variant2 : "linux-wt-repl"
        variant1 : "linux-mmap-repl"
        threshold : 85
- name: singleThreaded-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : singleThreaded
      variant : linux-mmap-standalone
      status : "*"
    - name : singleThreaded
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "singleThreaded"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
        threshold : 75
- name: insert-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : insert
      variant : linux-mmap-standalone
      status : "*"
    - name : insert
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "insert"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
        threshold : 75
- name: update-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : update
      variant : linux-mmap-standalone
      status : "*"
    - name : update
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "update"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
        threshold : 75
- name: misc-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : misc
      variant : linux-mmap-standalone
      status : "*"
    - name : misc
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "compare"
      vars:
        compare_task : "misc"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
        threshold : 75

buildvariants:
- name: linux-wt-standalone
  display_name: Linux WT Standalone
  batchtime: 90 # 1.5 hours
  expansions:
    compile_flags: &compile_flags -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
    mongod_exec_wrapper: &exec_wrapper "numactl --physcpubind=4,5,6,7 -i 1"
    perf_exec_wrapper: &perf_wrapper "numactl --physcpubind=1,2,3 -i 0"
    mongod_flags: "--storageEngine=wiredTiger --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --wiredTigerCacheSizeGB 16"

  run_on:
  - "centos6-perf"
  tasks:
  - name: compile
    distros:
    - rhel55
  - name: query
  - name: where
  - name: update
  - name: insert
  - name: geo
  - name: misc
  - name: singleThreaded

- name: linux-mmap-standalone
  display_name: Linux MMAPv1 Standalone
  batchtime: 90 # 1.5 hours
  expansions:
    compile_flags: *compile_flags
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false"

  run_on:
  - "centos6-perf"
  tasks:
  - name: query
  - name: where
  - name: update
  - name: insert
  - name: geo
  - name: misc
  - name: singleThreaded

- name: linux-wt-repl
  display_name: Linux WT 1-Node ReplSet
  batchtime: 360 # 6 hours
  expansions:
    compile_flags: *compile_flags 
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=wiredTiger --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --wiredTigerCacheSizeGB 16 --oplogSize 100000"

  run_on:
  - "centos6-perf"
  tasks:
  - name: update
  - name: insert
  - name: misc
  - name: singleThreaded

- name: linux-mmap-repl
  display_name: Linux MMAPv1 1-Node ReplSet
  batchtime: 360 # 6 hours
  expansions:
    compile_flags: *compile_flags 
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --oplogSize 100000"

  run_on:
  - "centos6-perf"
  tasks:
  - name: update
  - name: insert
  - name: misc
  - name: singleThreaded

- name: linux-wt-repl-compare
  display_name: Linux WT 1-Node ReplSet Compare
  batchtime: 360 # 6 hours
  expansions:
    compile_flags: *compile_flags 
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=wiredTiger --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --oplogSize 100000"
  run_on:
  - "centos6-perf"
#    - "ubuntu1404-docker"
  tasks:
  - name: singleThreaded-wt-repl-comp
  - name: insert-wt-repl-comp
  - name: update-wt-repl-comp
  - name: misc-wt-repl-comp

- name: linux-mmap-repl-compare
  display_name: Linux MMAP 1-Node ReplSet Compare
  batchtime: 360 # 6 hours
  expansions:
    compile_flags: *compile_flags 
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --oplogSize 100000"
  run_on:
  - "centos6-perf"
#    - "ubuntu1404-docker"
  tasks:
  - name: singleThreaded-mmap-repl-comp
  - name: insert-mmap-repl-comp
  - name: update-mmap-repl-comp
  - name: misc-mmap-repl-comp

- name: linux-wt-mmap-standalone-compare
  display_name: Linux WT-MMAP Standalone Compare
  batchtime: 90 # 1.5 hours
  expansions:
    compile_flags: *compile_flags 
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --oplogSize 100000"
  run_on:
  - "centos6-perf"
#    - "ubuntu1404-docker"
  tasks:
  - name: singleThreaded-wt-mmap-standalone-comp
  - name: insert-wt-mmap-standalone-comp
  - name: update-wt-mmap-standalone-comp
  - name: misc-wt-mmap-standalone-comp
  - name: query-wt-mmap-standalone-comp
  - name: where-wt-mmap-standalone-comp
  - name: geo-wt-mmap-standalone-comp

- name: linux-wt-mmap-repl-compare
  display_name: Linux WT-MMAP 1-Node ReplSet Compare
  batchtime: 360 # 6 hours
  expansions:
    compile_flags: *compile_flags 
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --oplogSize 100000"
  run_on:
  - "centos6-perf"
#    - "ubuntu1404-docker"
  tasks:
  - name: singleThreaded-wt-mmap-repl-comp
  - name: insert-wt-mmap-repl-comp
  - name: update-wt-mmap-repl-comp
  - name: misc-wt-mmap-repl-comp