mongodb/etc/system_perf.yml

command_type: system
stepback: false
exec_timeout_secs: &exec_timeout_secs 36000
timeout_secs: &timeout_secs 7200

include:
  - filename: etc/evergreen_yml_components/definitions.yml
  ### lint_yaml trim start ###
  #
  # The lines below are trimmed by yamllinters.sh so that `evergreen evaluate` works on this file.
  # This allows us to test that non-dsi imports continue to work.
  #
  - filename: evergreen/system_perf/master/variants.yml
    module: dsi
  - filename: evergreen/system_perf/master/master_variants.yml
    module: dsi
  - filename: evergreen/system_perf/shared_tasks.yml
    module: dsi
  - filename: evergreen/system_perf/master/genny_tasks.yml
    module: genny
  ### lint_yaml trim end ###

## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
parameters:
  - key: patch_compile_flags
    description: "Additional SCons flags to be applied during scons compile invocations in this patch"
  # see https://github.com/10gen/performance-tooling-docs/blob/main/patch_testing.md#skipping-compilation-on-sys-perf-projects
  - key: reuse_compile_from
    description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_ef8504eefef4c9fe2062b8424e85053b3e1ad8d2"
  - key: exec_timeout_secs_override
    value: *exec_timeout_secs
    description: "Overrides the default exec_timeout_secs defined at the top of this file for the f_dsi_run_workload task"
  - key: timeout_secs_override
    value: *timeout_secs
    description: "Overrides the default timeout_secs defined at the top of this file for the f_dsi_run_workload task"

variables:
  _modules: &modules
    - dsi
    - genny
    - workloads
    - linkbench
    - linkbench2
    - tsbs
    - mongo-perf
    - YCSB
    - py-tpcc
    - PrivateWorkloads
    - flamegraph

###
# Same in every DSI project. Ensure that this block is synchronized with
# evergreen-dsitest.yml, and src/dsi/onboarding.py (search update-repos-here)
# in the dsi repo, and etc/system_perf.yml in mongodb/mongo.
modules:
  - name: dsi
    owner: 10gen
    repo: dsi
    prefix: ${workdir}/src
    branch: master
  - name: genny
    owner: mongodb
    repo: genny
    prefix: ${workdir}/src
    branch: master
  - name: workloads
    owner: 10gen
    repo: workloads
    prefix: ${workdir}/src
    branch: master
  - name: linkbench
    owner: 10gen
    repo: linkbench
    prefix: ${workdir}/src
    branch: master
  - name: linkbench2
    owner: 10gen
    repo: linkbench2
    prefix: ${workdir}/src
    branch: master
  - name: tsbs
    owner: mongodb-forks
    repo: tsbs
    prefix: ${workdir}/src
    branch: main
  - name: mongo-perf
    owner: mongodb
    repo: mongo-perf
    prefix: ${workdir}/src
    branch: master
  - name: YCSB
    owner: mongodb-labs
    repo: YCSB
    prefix: ${workdir}/src
    branch: main
  - name: py-tpcc
    owner: mongodb-labs
    repo: py-tpcc
    prefix: ${workdir}/src
    branch: production
  - name: flamegraph
    owner: mongodb-forks
    repo: flamegraph
    prefix: ${workdir}/src
    branch: master
  - name: PrivateWorkloads
    owner: 10gen
    repo: PrivateWorkloads
    prefix: ${workdir}/src
    branch: production
###

###
# Same in every DSI project
pre:
  - func: "f_other_pre_ops"
  - func: "f_dsi_pre_run"
post:
  - func: "f_dsi_post_run"
  - func: "f_other_post_ops"
timeout:
  - func: "f_dsi_timeout"
  - func: "f_other_timeout"
###

functions:
  f_other_post_ops:
    - command: shell.exec
      params:
        script: |
          cp ./build/Artifacts/DSIArtifacts/WorkloadOutput/reports/ycsb_100read/mongod.0/pre_post_task_output/default.profdata.tgz ./default.profdata.tgz || true          
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: ./default.profdata.tgz
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/default.profdata.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/x-gzip
        display_name: PGO profile data - Execution ${execution}
    - command: shell.exec
      params:
        working_dir: src
        script: |
          # removes files from the (local) scons cache when it's over a
          # threshold, to the $prune_ratio percentage. Ideally override
          # these default values in the distro config in evergreen.

          if [ -d "${scons_cache_path}" ]; then
              /opt/mongodbtoolchain/v4/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
          fi          
  f_other_pre_ops:
    - command: shell.exec
      params:
        silent: true
        script: |
          for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
              pkill -9 "$PS"
          done          
  f_other_timeout:
    # Can't be empty so just `echo`.
    - command: shell.exec
      params: {script: "echo"}

  ###
  # Package the supplementary DSI data (mongo tools, feature flag list,
  # server params list, jstests, and buildscripts)
  download mongo tools:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o verbose
          set -o errexit

          arch=$(uname -m)
          case $arch in
            "x86_64")
              if [ -f /etc/os-release ]; then
                . /etc/os-release
                if [ "$ID" == "amzn" ]; then
                  case $VERSION_ID in
                    "2")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-x86_64-100.9.4.tgz"
                      ;;
                    "2023")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-x86_64-100.9.4.tgz"
                      ;;
                    *)
                      echo "Unsupported Amazon Linux version: $VERSION_ID"
                      exit 1
                      ;;
                  esac
                else
                  echo "Unsupported Linux distribution: $ID"
                  exit 1
                fi
              else
                echo "Unable to determine Linux distribution"
                exit 1
              fi
              ;;
            "aarch64")
              if [ -f /etc/os-release ]; then
                . /etc/os-release
                if [ "$ID" == "amzn" ]; then
                  case $VERSION_ID in
                    "2")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-aarch64-100.9.4.tgz"
                      ;;
                    "2023")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-aarch64-100.9.4.tgz"
                      ;;
                    *)
                      echo "Unsupported Amazon Linux version: $VERSION_ID"
                      exit 1
                      ;;
                  esac
                else
                  echo "Unsupported Linux distribution: $ID"
                  exit 1
                fi
              else
                echo "Unable to determine Linux distribution"
                exit 1
              fi
              ;;
            *)
              echo "Unsupported architecture: $arch"
              exit 1
              ;;
          esac

          echo "Downloading and installing MongoDB tools..."
          mkdir -p mongodb/
          wget "$binary_url" -O mongo-tools.tar.gz
          tar -xzvf mongo-tools.tar.gz -C mongodb/ --strip-components=1 "mong*/bin"
          echo "MongoDB tools installed successfully!"          
  generate atlas info:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose

          uarch=$(uname -p)
          os=$(uname -r)
          json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }"
          echo $json | jq '.' > mongodb/atlas_info.json          
  generate feature flag list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          pushd ..
          . src/evergreen/prelude.sh
          activate_venv
          popd

          python ./buildscripts/idl/gen_all_feature_flag_list.py
          mkdir -p mongodb/feature_flags
          cp ./all_feature_flags.txt mongodb/feature_flags          
  generate server params list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          pushd ..
          . src/evergreen/prelude.sh
          activate_venv
          popd

          python ./buildscripts/idl/gen_all_server_params_list.py
          mkdir -p mongodb/server_params
          cp ./all_server_params.txt mongodb/server_params          
  copy jstests:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          mkdir -p mongodb/jstests/hooks
          if [ -d jstests/hooks ]
          then
            echo "Fetching JS test DB correctness checks from directory jstests"
            cp -a jstests/* mongodb/jstests

            echo "Now adding our own special run_validate_collections.js wrapper"
            mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js

            cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
            print("NOTE: run_validate_collections.js will skip the oplog!");
            TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
            await import("jstests/hooks/run_validate_collections.actual.js");
          EOF
          fi          
  copy buildscripts:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          mkdir -p mongodb/buildscripts
          cp -a buildscripts/* mongodb/buildscripts          
  package supplementary data:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${build_patch_id}" ] || [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            tar czf supplementary-data.tgz mongodb
          else
            # Evergreen does not handle nested escaped expansions well
            reuse_compile_from="${reuse_compile_from}"
            build_patch_id="${build_patch_id}"
            version_to_reuse_from=$(if [ -n "${build_patch_id}" ]; then echo "${build_patch_id}"; else echo "${reuse_compile_from}"; fi)

            curl -o supplementary-data.tgz https://s3.amazonaws.com/mciuploads/${project}/${compile_variant}/$${version_to_reuse_from}/dsi/supplementary-data.tgz
          fi          
  upload supplementary data:
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/supplementary-data.tgz
        remote_file: ${project}/${compile_variant}/${version_id}/dsi/supplementary-data.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/x-gzip
        display_name: supplementary-data.tgz

  f_schedule_compile:
    - command: shell.exec
      params:
        working_dir: src
        script: ./dsi/run-dsi selftest scripts/schedule_compile.py
        include_expansions_in_env:
          - version_id
          - evergreen_api_user
          - evergreen_api_key
          - compile_variant
          - compile_task_name
          - binary_artifact_name
          - reuse_compile_from
          - compile_project

tasks:
  - name: schedule_compile
    commands:
      - command: git.get_project
        params:
          directory: src
          clone_depth: 1000
      - func: f_schedule_compile
      - command: s3.put
        params:
          aws_key: ${aws_key}
          aws_secret: ${aws_secret}
          local_file: src/binary-url.txt
          remote_file: ${project}/${version_id}/binary-urls/${compile_variant}-binary-url.txt
          bucket: mciuploads
          permissions: public-read
          content_type: text/plain
          display_name: binary-url.txt
  - name: package_supplementary_data
    commands:
      - command: manifest.load
      - func: "git get shallow project"
      - func: "f_expansions_write"
      - func: "restore git history and tags"
      - func: "add git tag"
      - func: "kill processes"
      - func: "cleanup environment"
      - func: "set up venv"
      - func: "upload pip requirements"
      - func: "generate version expansions"
      - func: "apply version expansions"
      - func: "download mongo tools"
      - func: "generate feature flag list"
      - func: "generate atlas info"
      - func: "generate server params list"
      - func: "copy jstests"
      - func: "copy buildscripts"
      - func: "package supplementary data"
      - func: "upload supplementary data"

buildvariants:
  - name: schedule-compile-amazon2-arm64
    display_name: "* Schedule Compile for Amazon Linux 2 arm64"
    expansions:
      compile_variant: amazon2-arm64-compile
      compile_task_name: archive_dist_test
      binary_artifact_name: Binaries
    modules: *modules
    run_on:
      - amazon2-arm64-latest-small
    tasks:
      - name: schedule_compile
      - name: package_supplementary_data

  - name: schedule-compile-amazon2023-arm64
    display_name: "* Schedule Compile for Amazon Linux 2023 arm64"
    expansions:
      compile_variant: amazon2023-arm64-compile
      compile_task_name: archive_dist_test
      binary_artifact_name: Binaries
    modules: *modules
    run_on:
      - amazon2023-arm64-latest-small
    tasks:
      - name: schedule_compile
      - name: package_supplementary_data

  - name: schedule-compile-amazon2-x86
    display_name: "* Schedule Compile for Amazon Linux 2 x86"
    expansions:
      compile_variant: amazon2-x86-compile
      compile_task_name: archive_dist_test
      binary_artifact_name: Binaries
    modules: *modules
    run_on:
      - amazon2-latest-small
    tasks:
      - name: schedule_compile
      - name: package_supplementary_data

  - name: schedule-compile-amazon2-arm64-pgo
    display_name: "* Schedule Compile for Amazon Linux 2 arm64 PGO"
    expansions:
      compile_variant: amazon2-arm64-compile-pgo
      compile_task_name: archive_dist_test
      binary_artifact_name: Binaries
    modules: *modules
    run_on:
      - amazon2-arm64-latest-small
    tasks:
      - name: schedule_compile
      - name: package_supplementary_data

  - name: schedule-compile-amazon2-arm64-mongocrypt-shlib
    display_name: "* Schedule mongo_crypt_v1.so Compile for Amazon Linux 2 arm64"
    expansions:
      compile_variant: amazon2-arm64-mongocrypt-shlib-compile
      compile_task_name: crypt_create_lib
      binary_artifact_name: Mongo crypt Library
    modules: *modules
    run_on:
      - amazon2-arm64-latest-small
    tasks:
      - name: schedule_compile

  - name: schedule-compile-amazon2-x86-streams
    display_name: "* Compile Streams for Amazon Linux 2 x86"
    expansions:
      compile_variant: amazon2-x86-streams-compile
      compile_task_name: archive_dist_test
      binary_artifact_name: Binaries
    modules: *modules
    run_on:
      - amazon2-arm64-latest-small
    tasks:
      - name: schedule_compile
      - name: package_supplementary_data