mongodb/etc/system_perf.yml

command_type: system
stepback: false

include:
  - filename: etc/evergreen_yml_components/variants/misc/task_generation.yml
  - filename: etc/evergreen_yml_components/definitions.yml
  - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
  - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
  - filename: etc/evergreen_yml_components/tasks/compile_tasks.yml
  - filename: etc/evergreen_yml_components/tasks/misc_tasks.yml
  - filename: etc/evergreen_yml_components/perf_tasks.yml
  - filename: etc/evergreen_yml_components/variants/perf/perf.yml
  - filename: etc/evergreen_yml_components/variants/perf/perf_branching.yml

## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
parameters:
  - key: patch_compile_flags
    description: "Additional SCons flags to be applied during scons compile invocations in this patch"
  # see https://github.com/10gen/performance-tooling-docs/blob/main/patch_testing.md#skipping-compilation-on-sys-perf-projects
  - key: reuse_compile_from
    description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_ef8504eefef4c9fe2062b8424e85053b3e1ad8d2"

variables:
  ###
  # Leave this section uncommented to enable compile.
  _real_compile_amazon2: &_compile_amazon2
    - name: compile
      variant: compile-amazon2
  _real_compile_rhel70: &_compile_rhel70
    - name: compile
      variant: compile-rhel70
  _real_compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64
    - name: compile
      variant: compile-amazon-linux2-arm64
  _real_compile_amazon_linux2_arm64_with_mongocrypt_shlib: &_compile_amazon_linux2_arm64_with_mongocrypt_shlib
    - name: compile
      variant: compile-amazon-linux2-arm64
    - name: compile_mongocrypt_shlib
      variant: compile-amazon-linux2-arm64-mongocrypt-shlib
  _real_compile_amazon2_streams: &_compile_amazon2_streams
    - name: compile
      variant: compile-amazon2-streams
  _real_compile_al2023_arm64: &_compile_al2023_arm64
    - name: compile
      variant: compile-al2023-arm64
  _real_compile_al2023_arm64_with_mongocrypt_shlib: &_compile_al2023_arm64_with_mongocrypt_shlib
    - name: compile
      variant: compile-al2023-arm64
    - name: compile_mongocrypt_shlib
      variant: compile-al2023-arm64-mongocrypt-shlib
  _real_compile_mixed_amazon_linux2_al2023_arm64: &_compile_mixed_amazon_linux2_al2023_arm64
    - name: compile
      variant: compile-amazon-linux2-arm64
    - name: compile
      variant: compile-al2023-arm64

  _real_expansions: &_expansion_updates []
  ###

  ###
  # **Or**: Leave this section uncommented to bypass/skip compile.
  #  _skip_compile_amazon2: &_compile_amazon2
  #      []
  #  _skip_compile_rhel70: &_compile_rhel70
  #      []
  #  _skip_compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64
  #     []
  #  _skip_compile_amazon_linux2_arm64_with_mongocrypt_shlib: &_compile_amazon_linux2_arm64_with_mongocrypt_shlib
  #     []
  #  _skip_compile_amazon2_streams: &_compile_amazon2_streams
  #     []
  #  _skip_compile_al2023_arm64: &_compile_al2023_arm64
  #     []
  #  _skip_expansions: &_expansion_updates
  #      # This is the normal (amazon2) "compile" artifact from https://evergreen.mongodb.com/version/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9
  #      - key: mdb_binary_for_client
  #        value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz
  #      - key: mdb_binary_for_server
  #        value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9/97c6a9e443ff7e171b7310a1fa5c05d0768faff9/linux/mongodb-sys_perf_97c6a9e443ff7e171b7310a1fa5c05d0768faff9.tar.gz
  ###

  _src_dir: &src_dir src/mongo
  _modules: &modules
    - mongo-tools
    # - mongo
    - dsi
    - genny
    - workloads
    - linkbench
    - linkbench2
    - tsbs
    - mongo-perf
    - YCSB
    - py-tpcc
    - PrivateWorkloads
    - flamegraph

  _schedule_variant_auto_tasks_task: &schedule_variant_auto_tasks_task
    name: schedule_variant_auto_tasks
    activate: false
    depends_on:
      - name: schedule_global_auto_tasks
        variant: task_generation

  _schedule_patch_auto_tasks_task: &schedule_patch_auto_tasks_task
    name: schedule_patch_auto_tasks
    activate: false
    depends_on:
      - name: schedule_global_auto_tasks
        variant: task_generation

modules:
  ###
  # Same in every DSI project. Ensure that this block is synchronized with
  # evergreen-dsitest.yml, atlas/system_perf_atlas.yml, and src/dsi/onboarding.py
  # (search update-repos-here) in this repo, and etc/system_perf.yml and
  # etc/perf.yml in mongodb/mongo
  - name: dsi
    owner: 10gen
    repo: dsi
    prefix: ${workdir}/src
    branch: master
  - name: genny
    owner: mongodb
    repo: genny
    prefix: ${workdir}/src
    branch: master
  - name: workloads
    owner: 10gen
    repo: workloads
    prefix: ${workdir}/src
    branch: master
  - name: linkbench
    owner: 10gen
    repo: linkbench
    prefix: ${workdir}/src
    branch: master
  - name: linkbench2
    owner: 10gen
    repo: linkbench2
    prefix: ${workdir}/src
    branch: master
  - name: tsbs
    owner: mongodb-forks
    repo: tsbs
    prefix: ${workdir}/src
    branch: main
  - name: mongo-perf
    owner: mongodb
    repo: mongo-perf
    prefix: ${workdir}/src
    branch: master
  - name: YCSB
    owner: mongodb-labs
    repo: YCSB
    prefix: ${workdir}/src
    branch: production
  - name: py-tpcc
    owner: mongodb-labs
    repo: py-tpcc
    prefix: ${workdir}/src
    branch: production
  - name: flamegraph
    owner: mongodb-forks
    repo: flamegraph
    prefix: ${workdir}/src
    branch: master
  #  - name: mongo
  #    owner: mongodb
  #    repo: mongo
  #    prefix: ${workdir}/src
  #    branch: master
  ###
  # Pinned to version 100.7.2
  - name: mongo-tools
    owner: mongodb
    repo: mongo-tools
    prefix: mongo-tools/src/github.com/mongodb
    branch: db8c5c4
  - name: PrivateWorkloads
    owner: 10gen
    repo: PrivateWorkloads
    prefix: ${workdir}/src
    branch: production

###
# Same in every DSI project
pre:
  - func: "f_other_pre_ops"
  - func: "f_dsi_pre_run"
post:
  - func: "f_dsi_post_run"
  - func: "f_other_post_ops"
timeout:
  - func: "f_dsi_timeout"
  - func: "f_other_timeout"
###

functions:
  ###
  # Same in every DSI project
  f_dsi_pre_run:
    - command: manifest.load
    - command: expansions.update
      params:
        updates: *_expansion_updates
  f_run_dsi_workload:
    - command: git.get_project
      params:
        directory: *src_dir
        clone_depth: 1000
        revisions:
          dsi: ${dsi_rev}
          genny: ${genny_rev}
          linkbench: ${linkbench_rev}
          linkbench2: ${linkbench2_rev}
          tsbs: ${tsbs_rev}
          workloads: ${workloads_rev}
          mongo-perf: ${mongo-perf_rev}
          YCSB: ${YCSB_rev}
          py-tpcc: ${py-tpcc_rev}
          flamegraph: ${flamegraph_rev}
          # mongo: ${mongo_rev}
          PrivateWorkloads: ${PrivateWorkloads_rev}
    - command: expansions.write
      params:
        file: ./expansions.yml
        redacted: true
    - command: shell.exec
      params:
        script: ./src/dsi/run-dsi run_workload
    - command: shell.exec
      type: system
      params:
        script: ./src/dsi/run-dsi determine_failure -m SYSTEM
    - command: shell.exec
      type: setup
      params:
        script: ./src/dsi/run-dsi determine_failure -m SETUP
    - command: shell.exec
      type: test
      params:
        script: ./src/dsi/run-dsi determine_failure -m TEST
  f_dsi_post_run:
    - command: shell.exec
      params:
        script: ./src/dsi/run-dsi post_run
    - command: perf.send
      params:
        file: ./build/CedarReports/cedar_report.json
        aws_key: ${terraform_key}
        aws_secret: ${terraform_secret}
        bucket: genny-metrics
        region: us-east-1
        prefix: ${task_id}_${execution}
    - command: attach.results
      params:
        file_location: ./build/EvergreenResultsJson/results.json
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: ./build/Artifacts/DSIArtifacts.tgz
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/x-gzip
        display_name: DSI Artifacts - Execution ${execution}
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: ./build/Documentation/index.html
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html
        bucket: mciuploads
        permissions: public-read
        content_type: text/html
        display_name: Documentation
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: bootstrap.yml
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/bootstrap-${task_name}-${build_id}-${execution}.yml
        bucket: mciuploads
        permissions: public-read
        content_type: text/plain
        display_name: Task Bootstrap Config
  f_dsi_timeout:
    - command: shell.exec
      params:
        script: ./src/dsi/run-dsi on_timeout
  ###

  f_other_post_ops:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          # removes files from the (local) scons cache when it's over a
          # threshold, to the $prune_ratio percentage. Ideally override
          # these default values in the distro config in evergreen.

          if [ -d "${scons_cache_path}" ]; then
              /opt/mongodbtoolchain/v4/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
          fi          
  f_other_pre_ops:
    - &f_other_pre_ops
      command: shell.exec
      params:
        silent: true
        script: |
          for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
              pkill -9 "$PS"
          done          
  f_other_timeout:
    # Can't be empty so just `echo`.
    - command: shell.exec
      params: {script: "echo"}

  f_generate_all_variant_auto_tasks:
    - command: git.get_project
      params:
        directory: *src_dir
        revisions:
          dsi: ${dsi_rev}
          genny: ${genny_rev}
          linkbench: ${linkbench_rev}
          linkbench2: ${linkbench2_rev}
          tsbs: ${tsbs_rev}
          workloads: ${workloads_rev}
          mongo-perf: ${mongo-perf_rev}
          YCSB: ${YCSB_rev}
          py-tpcc: ${py-tpcc_rev}
          PrivateWorkloads: ${PrivateWorkloads_rev}
    - command: expansions.write
      params:
        file: ./expansions.yml
    - command: shell.exec
      params:
        # The script below makes sure that if we generate all tasks in a patch, we will only generate
        # them and not run all of them, since activating all tasks is the default behavior for task
        # generation in a patch.
        script: |
          if [ "${is_patch|false}" = "true" ]; then
            ./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/etc/system_perf.yml --project-file ./src/mongo/etc/evergreen_yml_components/variants/perf/perf.yml --project-file ./src/mongo/etc/evergreen_yml_components/variants/perf/perf_branching.yml --no-activate
          else
            ./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/etc/system_perf.yml --project-file ./src/mongo/etc/evergreen_yml_components/variants/perf/perf.yml --project-file ./src/mongo/etc/evergreen_yml_components/variants/perf/perf_branching.yml
          fi          
    - command: generate.tasks
      params:
        files:
          - build/TaskJSON/Tasks.json

  ###
  # Prepares the environment before compiling the binaries
  compile prep:
    # We create a virtual environment with the Python dependencies for compiling the server
    # installed.
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose

          mkdir -p mongodb/bin

          # This is copied from evergreen/functions/venv_setup.sh
          # Consider consolidating this in the future
          /opt/mongodbtoolchain/v4/bin/python3 -m venv "${workdir}/compile_venv"
          source "${workdir}/compile_venv/bin/activate"

          python -m pip install "poetry==1.5.1"
          # See BF-29897 / https://github.com/python-poetry/poetry/issues/7584
          poetry config installer.max-workers 10
          # Trying poetry install due to https://github.com/python-poetry/poetry/issues/7611
          python -m poetry install --no-root --sync || python -m poetry install --no-root --sync          
    - command: expansions.write
      params:
        file: expansions.yml
        redacted: true
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose

          source "${workdir}/compile_venv/bin/activate"

          # We get the raw version string (r1.2.3-45-gabcdef) from git
          # And append "-sys-perf" at the end
          export MONGO_VERSION=$(git describe --abbrev=7)
          MONGO_VERSION="$MONGO_VERSION-sys-perf"

          # If this is a patch build, we add the patch version id to the version string so we know
          # this build was a patch, and which evergreen task it came from
          if [ "${is_patch|false}" = "true" ]; then
            MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
          fi

          # This script handles sanitizing the version string for use during SCons build
          # and when pushing artifacts up to S3.
          IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
            buildscripts/generate_version_expansions.py --out version_expansions.yml          
    - command: expansions.update
      params:
        file: src/version_expansions.yml
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose

          # This script handles whether the SCons cache should be used
          source "${workdir}/compile_venv/bin/activate"
          SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
            IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
            python buildscripts/generate_compile_expansions.py --out compile_expansions.yml          
    - command: expansions.update
      params:
        file: src/compile_expansions.yml
  get engflow creds perf:
  - command: s3.get
    display_name: "get engflow key"
    params:
      aws_key: ${engflow_key}
      aws_secret: ${engflow_secret}
      remote_file: engflow/engflow.key
      bucket: serverengflow
      local_file: "src/engflow.key"
  - command: s3.get
    display_name: "get engflow cert"
    params:
      aws_key: ${engflow_key}
      aws_secret: ${engflow_secret}
      remote_file: engflow/engflow.cert
      bucket: serverengflow
      local_file: "src/engflow.cert"
  - command: shell.exec
    params:
      display_name: "generate evergreen engflow bazelrc"
      shell: bash
      working_dir: src
      script: |
        set -o errexit
        set -o verbose

        # Pulled from evergreen/generate_evergreen_engflow_bazelrc.sh consider
        # consolidating once prelude.sh is runnable in the perf project.

        source ./evergreen/bazel_RBE_supported.sh

        if bazel_rbe_supported; then

          uri="https://spruce.mongodb.com/task/${task_id}?execution=${execution}"

          echo "build --tls_client_certificate=./engflow.cert" > .bazelrc.evergreen_engflow_creds
          echo "build --tls_client_key=./engflow.key" >> .bazelrc.evergreen_engflow_creds
          echo "build --bes_keywords=engflow:CiCdPipelineName=${build_variant}" >> .bazelrc.evergreen_engflow_creds
          echo "build --bes_keywords=engflow:CiCdJobName=${task_name}" >> .bazelrc.evergreen_engflow_creds
          echo "build --bes_keywords=engflow:CiCdUri=$uri" >> .bazelrc.evergreen_engflow_creds
          echo "build --bes_keywords=evg:project=${project}" >> .bazelrc.evergreen_engflow_creds
          echo "build --workspace_status_command=./evergreen/engflow_workspace_status.sh" >> .bazelrc.evergreen_engflow_creds
        fi        
  compile prep old:
    # We create a virtual environment with the Python dependencies for compiling the server
    # installed.
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose

          mkdir -p mongodb/bin

          # This is copied from evergreen/functions/venv_setup.sh
          # Consider consolidating this in the future
          /opt/mongodbtoolchain/v4/bin/python3 -m venv "${workdir}/compile_venv"
          source "${workdir}/compile_venv/bin/activate"

          python -m pip install "poetry==1.5.1"
          # See BF-29897 / https://github.com/python-poetry/poetry/issues/7584
          poetry config installer.max-workers 10
          # Trying poetry install due to https://github.com/python-poetry/poetry/issues/7611
          python -m poetry install --no-root --sync || python -m poetry install --no-root --sync          
    - command: expansions.write
      params:
        file: expansions.yml
        redacted: true
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose

            source "${workdir}/compile_venv/bin/activate"

            # We get the raw version string (r1.2.3-45-gabcdef) from git
            # And append "-sys-perf" at the end
            export MONGO_VERSION=$(git describe --abbrev=7)
            MONGO_VERSION="$MONGO_VERSION-sys-perf"

            # If this is a patch build, we add the patch version id to the version string so we know
            # this build was a patch, and which evergreen task it came from
            if [ "${is_patch|false}" = "true" ]; then
              MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
            fi

            # This script handles sanitizing the version string for use during SCons build
            # and when pushing artifacts up to S3.
            IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
              buildscripts/generate_version_expansions.py --out version_expansions.yml
          else
            touch version_expansions.yml
          fi          
    - command: expansions.update
      params:
        file: src/version_expansions.yml
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose

            # This script handles whether the SCons cache should be used
            source "${workdir}/compile_venv/bin/activate"
            SCONS_CACHE_MODE=${scons_cache_mode|} USE_SCONS_CACHE=${use_scons_cache|false} \
              IS_PATCH=${is_patch|false} IS_COMMIT_QUEUE=${is_commit_queue|false} \
              python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
          else
            touch compile_expansions.yml
          fi          
    - command: expansions.update
      params:
        file: src/compile_expansions.yml
  ###

  ###
  # Compile mongodb
  compile mongo tools:
    - command: shell.exec
      params:
        working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools
        script: |
          set -o verbose
          set -o errexit

          # make sure newlines in the scripts are handled correctly by windows
          if [ "Windows_NT" = "$OS" ]; then
            set -o igncr
          fi;

          export cleanup=false

          # install go for tools if required (al2023 for now).
          if [ ! -d "/opt/golang/go1.19" ]; then
            cleanup=true
            echo -n "go-lang missing downloading ..."
            go_file=./go1.19.8.tar.gz
            wget --quiet https://go.dev/dl/go1.19.8.linux$(echo ${compile_variant} | sed 's/^-al202.//').tar.gz -O "$go_file"
            echo -n " installing ..."
            sudo mkdir -p /opt/golang/go1.19
            sudo tar -C /opt/golang/go1.19 -xzf "$go_file" --strip-components=1
            [ -e "$go_file" ] && rm -f "$go_file"
            echo  "done"
          fi

          # set_goenv provides set_goenv()
          . ./set_goenv.sh
          GOROOT="" set_goenv || exit
          go version

          build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
          if [ "${build_mongoreplay}" = "true" ]; then
              build_tools="$build_tools mongoreplay"
          fi
          for i in $build_tools; do
              go build -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
              "../../../../../mongodb/bin/$i${exe|}" --version
          done
          if "$cleanup"; then
            sudo rm -rf "/opt/golang/go1.19"
          fi          
  compile mongo tools old:
    - command: shell.exec
      params:
        working_dir: src/mongo-tools/src/github.com/mongodb/mongo-tools
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o verbose
            set -o errexit

            # make sure newlines in the scripts are handled correctly by windows
            if [ "Windows_NT" = "$OS" ]; then
              set -o igncr
            fi;

            export cleanup=false

            # install go for tools if required (al2023 for now).
            if [ ! -d "/opt/golang/go1.19" ]; then
              cleanup=true
              echo -n "go-lang missing downloading ..."
              go_file=./go1.19.8.tar.gz
              wget --quiet https://go.dev/dl/go1.19.8.linux$(echo ${compile_variant} | sed 's/^-al202.//').tar.gz -O "$go_file"
              echo -n " installing ..."
              sudo mkdir -p /opt/golang/go1.19
              sudo tar -C /opt/golang/go1.19 -xzf "$go_file" --strip-components=1
              [ -e "$go_file" ] && rm -f "$go_file"
              echo  "done"
            fi

            # set_goenv provides set_goenv()
            . ./set_goenv.sh
            GOROOT="" set_goenv || exit
            go version

            build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
            if [ "${build_mongoreplay}" = "true" ]; then
                build_tools="$build_tools mongoreplay"
            fi
            for i in $build_tools; do
                go build -o "../../../../../mongodb/bin/$i${exe|}" $i/main/$i.go
                "../../../../../mongodb/bin/$i${exe|}" --version
            done
            if "$cleanup"; then
              sudo rm -rf "/opt/golang/go1.19"
            fi
          fi          
  generate feature flag list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          source "${workdir}/compile_venv/bin/activate"
          python ./buildscripts/idl/gen_all_feature_flag_list.py
          mkdir -p mongodb/feature_flags
          cp ./all_feature_flags.txt mongodb/feature_flags          
  generate feature flag list old:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose
            source "${workdir}/compile_venv/bin/activate"
            python ./buildscripts/idl/gen_all_feature_flag_list.py
            mkdir -p mongodb/feature_flags
            cp ./all_feature_flags.txt mongodb/feature_flags
          fi          
  generate server params list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          source "${workdir}/compile_venv/bin/activate"
          python ./buildscripts/idl/gen_all_server_params_list.py
          mkdir -p mongodb/server_params
          cp ./all_server_params.txt mongodb/server_params          
  generate server params list old:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose
            source "${workdir}/compile_venv/bin/activate"
            python ./buildscripts/idl/gen_all_server_params_list.py
            mkdir -p mongodb/server_params
            cp ./all_server_params.txt mongodb/server_params
          fi          
  compile mongodb:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose
            source "${workdir}/compile_venv/bin/activate"
            python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} $extra_args install-core install-jstestshell SPLIT_DWARF=0 MONGO_VERSION=${version} DESTDIR=$(pwd)/mongodb ${patch_compile_flags|}
          fi          
  download mongodb:
    - command: shell.exec
      params:
        working_dir: src
        script: |
            set -o errexit
            set -o verbose
            wget --quiet https://s3.amazonaws.com/mciuploads/${project}/${compile_variant}/${version_id}/binaries/mongo-${revision_order_id}.${ext|tgz}
            tar -xvf mongo-${revision_order_id}.${ext|tgz}
            mv ./dist-test/bin/mongo mongodb/bin
            mv ./dist-test/bin/mongos mongodb/bin
            mv ./dist-test/bin/mongod mongodb/bin
            mv ./dist-test/bin/mongocryptd mongodb/bin
            # Copy libs folder if it exists
            [ ! -d ./dist-test/lib ] || mv ./dist-test/lib mongodb            
  download mongodb old:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose
            wget --quiet https://s3.amazonaws.com/mciuploads/${project}/${compile_variant}/${version_id}/binaries/mongo-${revision_order_id}.${ext|tgz}
            tar -xvf mongo-${revision_order_id}.${ext|tgz}
            mv ./dist-test/bin/mongo mongodb/bin
            mv ./dist-test/bin/mongos mongodb/bin
            mv ./dist-test/bin/mongod mongodb/bin
            mv ./dist-test/bin/mongocryptd mongodb/bin
          fi          
  download jstests:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          mkdir -p mongodb/jstests/hooks
          if [ -d jstests/hooks ]
          then
            echo "Fetching JS test DB correctness checks from directory jstests"
            cp -a jstests/* mongodb/jstests

            echo "Now adding our own special run_validate_collections.js wrapper"
            mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js

            cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
            print("NOTE: run_validate_collections.js will skip the oplog!");
            TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
            await import("jstests/hooks/run_validate_collections.actual.js");
          EOF
          fi          
  download jstests old:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose
            mkdir -p mongodb/jstests/hooks
            if [ -d jstests/hooks ]
            then
              echo "Fetching JS test DB correctness checks from directory jstests"
              cp -a jstests/* mongodb/jstests

              echo "Now adding our own special run_validate_collections.js wrapper"
              mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js

              cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
              print("NOTE: run_validate_collections.js will skip the oplog!");
              TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
              await import("jstests/hooks/run_validate_collections.actual.js");
          EOF
            fi
          fi          
  package binary:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          tar czf mongodb${compile_variant|}.tar.gz mongodb          
  package binary old:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            tar czf mongodb${compile_variant|}.tar.gz mongodb
          fi          
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -n "${reuse_compile_from}" ] && [ "${is_patch|false}" = "true" ]; then
            set -o errexit
            set -o verbose

            source "${workdir}/compile_venv/bin/activate"
            python buildscripts/download_sys_perf_binaries.py -v ${reuse_compile_from} -b ${build_variant} -u ${evergreen_api_user} -k ${evergreen_api_key}
            mv binary.tar.gz mongodb${compile_variant|}.tar.gz
          fi          
  package old debugsymbols:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            set -o errexit
            set -o verbose
            # Put all matching mongo debug from the build directory in an archive in the same location
            # as the library archive (i.e. mongodb/bin).
            tar czvf mongodb${compile_variant|}-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:mongodb/bin/:'
          fi          
  upload binary:
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongodb${compile_variant|}.tar.gz
        remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}.tar.gz
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/x-gzip}
        display_name: mongodb${compile_variant|}.tar.gz
  upload old debugsymbols:
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongodb${compile_variant|}-debugsymbols.tar.gz
        remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${compile_variant|}-${version_id}-debugsymbols.tar.gz
        bucket: mciuploads
        permissions: public-read
        optional: true
        content_type: ${content_type|application/x-gzip}
        display_name: mongo-debugsymbols.tgz
  ###

  ###
  # Compile mongo_crypt_v1 shared library
  compile mongocrypt shlib:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          source "${workdir}/compile_venv/bin/activate"
          if [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            python ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} $extra_args SPLIT_DWARF=0 archive-mongo-crypt-dev MONGO_VERSION=${version} DESTDIR=$(pwd)/crypt-lib-${version} PKGDIR=$(pwd) ${patch_compile_flags|}
            # Put all matching mongo .debug from the build directory in an archive in the same location
            # as the library archive (i.e. lib).
            tar czvf mongo-crypt-dev-debugsymbols.tar.gz $(find ./build/cached -name mongo\*.debug -type f) --xform 's:^.*/:lib/:'
          else
            python buildscripts/download_sys_perf_binaries.py -v ${reuse_compile_from} -b ${build_variant} -u ${evergreen_api_user} -k ${evergreen_api_key}
            mv binary.tar.gz mongo-crypt-dev.tgz
          fi          
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongo-crypt-dev.tgz
        remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongo_crypt_shared_v1-${compile_variant|}-${version_id}.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/x-gzip}
        display_name: mongo_crypt_shared_v1-${version|}-${compile_variant|}.tgz
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongo-crypt-dev-debugsymbols.tar.gz
        remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongo_crypt_shared_v1-${compile_variant|}-${version_id}-debugsymbols.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/x-gzip}
        optional: true
        display_name: mongo_crypt_shared_v1-debugsymbols.tgz
  ###

tasks:
  - name: compile
    commands:
      - command: manifest.load
      - command: git.get_project
        params:
          directory: src
          revisions:
            mongo-tools: ${mongo-tools_rev}
      - func: "compile prep old"
      - func: "get engflow creds perf"
      - func: "compile mongo tools old"
      - func: "generate feature flag list old"
      - func: "generate server params list old"
      - func: "compile mongodb"
      - func: "download jstests old"
      - func: "package binary old"
      - func: "upload binary"
      - func: "package old debugsymbols"
      - func: "upload old debugsymbols"

  - name: package_new_compile
    commands:
      - command: manifest.load
      - command: git.get_project
        params:
          directory: src
          revisions:
            mongo-tools: ${mongo-tools_rev}
      - func: "compile prep"
      - func: "get engflow creds perf"
      - func: "compile mongo tools"
      - func: "generate feature flag list"
      - func: "generate server params list"
      - func: "download mongodb"
      - func: "download jstests"
      - func: "package binary"
      - func: "upload binary"

  - name: compile_mongocrypt_shlib
    commands:
      - command: manifest.load
      - command: git.get_project
        params:
          directory: src
      - func: "compile prep"
      - func: "get engflow creds perf"
      - func: "compile mongocrypt shlib"

  - name: industry_benchmarks
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb"

  - name: ycsb_60GB
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb-60GB"

  - name: ycsb.load
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb.load"

  - name: ycsb_60GB.long
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb-60GB.long"

  - name: industry_benchmarks_secondary_reads
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb-secondary-reads"

  - name: industry_benchmarks_w1
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb-w1.2023-02"

  - name: industry_benchmarks_stepdowns
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb_stepdowns"

  - name: industry_benchmarks_rolling_restarts
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb_rolling_restarts"

  - name: industry_benchmarks_non_retryable_writes_stepdowns
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb_non_retryable_writes_stepdowns"

  - name: industry_benchmarks_non_retryable_writes_rolling_restarts
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "ycsb_non_retryable_writes_rolling_restarts"

  - name: genny_canaries
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "genny_canaries"

  - name: retryable_writes_workloads
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          test_control: "retryable_writes"

  # Named Pipes single concurrent query benchmarks
  - name: external_data_source_1
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          mongodb_setup: "external_data_source"
          test_control: "external_data_source_1"

  # Named Pipes four concurrent query benchmarks
  - name: external_data_source_4
    priority: 5
    commands:
      - func: f_run_dsi_workload
        vars:
          mongodb_setup: "external_data_source"
          test_control: "external_data_source_4"

buildvariants:
  - name: task_generation
    display_name: " * Task Generation"
    cron: "0 0 1 1 *" # Every year starting 1/1 at 00:00
    modules: *modules
    expansions:
      platform: linux
      project_dir: &project_dir dsi
    run_on:
      - amazon2-build
    tasks:
      - name: generate_all_variant_auto_tasks
        activate: true
      - name: schedule_global_auto_tasks

  ###
  # Old compiles
  # - name: compile-amazon2
  #   display_name: Compile
  #   modules: *modules
  #   cron: "0 0 * * *" # Everyday at 00:00
  #   expansions: &compile-expansions
  #     platform: linux
  #     project_dir: *project_dir
  #     tooltags: ""
  #     use_scons_cache: true
  #     compile_flags: >-
  #       --ssl
  #       --separate-debug
  #       MONGO_DISTMOD=amazon2
  #       -j$(grep -c ^processor /proc/cpuinfo)
  #       --release
  #       --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
  #       --use-diagnostic-latches=off
  #       install-mongocryptd
  #   run_on:
  #     - "amazon2-xlarge"
  #   tasks:
  #     - name: compile

  # - name: compile-amazon2-streams
  #   display_name: Compile Mongo Streams for Amazon2
  #   modules: *modules
  #   cron: "0 0 * * *" # Everyday at 00:00
  #   expansions:
  #     <<: *compile-expansions
  #     compile_variant: -stream-x86_64
  #     compile_flags: >-
  #       --ssl
  #       --separate-debug
  #       MONGO_DISTMOD=amazon2
  #       -j$(grep -c ^processor /proc/cpuinfo)
  #       --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
  #       --streams-release-build
  #   run_on:
  #     - "amazon2-xlarge"
  #   tasks:
  #     - name: compile

  # - name: compile-amazon-linux2-arm64
  #   display_name: Compile for Amazon Linux 2 arm64
  #   modules: *modules
  #   cron: "0 0 * * *" # Everyday at 00:00
  #   expansions:
  #     <<: *compile-expansions
  #     compile_variant: -arm64
  #   run_on:
  #     - "amazon2-arm64-xlarge"
  #   tasks:
  #     - name: compile

  # - name: compile-amazon-linux2-arm64-mongocrypt-shlib
  #   display_name: Compile mongo_crypt_v1.so for Amazon Linux 2 arm64
  #   modules: *modules
  #   cron: "0 0 * * 4" # 00:00 on Thursday
  #   expansions:
  #     <<: *compile-expansions
  #     compile_variant: -arm64
  #     compile_flags: >-
  #       --separate-debug
  #       MONGO_DISTMOD=amazon2
  #       -j$(grep -c ^processor /proc/cpuinfo)
  #       --release
  #       --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
  #       --use-diagnostic-latches=off
  #       --allocator=system
  #       --enterprise-features=fle
  #       --js-engine=none
  #       --link-model=dynamic-sdk
  #       --enable-http-client=off
  #       --ssl=off
  #       SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique"
  #       CCFLAGS="-fno-gnu-unique"
  #   run_on:
  #     - "amazon2-arm64-large"
  #   tasks:
  #     - name: compile_mongocrypt_shlib

  # - &compile-al2023-arm64
  #   name: compile-al2023-arm64
  #   display_name: Compile for Al2023 arm64
  #   modules: *modules
  #   cron: "0 0 * * *" # Everyday at 00:00
  #   expansions: &compile-al2023-expansions
  #     platform: linux
  #     project_dir: *project_dir
  #     tooltags: ""
  #     use_scons_cache: true
  #     compile_flags: >-
  #       --ssl
  #       --separate-debug
  #       MONGO_DISTMOD=rhel82
  #       -j$(grep -c ^processor /proc/cpuinfo)
  #       --release
  #       --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
  #       --use-diagnostic-latches=off
  #       install-mongocryptd
  #     compile_variant: -al2023-arm64
  #   run_on:
  #     - "amazon2023.0-arm64-large"
  #   tasks:
  #     - name: compile

  # - name: compile-al2023-arm64-mongocrypt-shlib
  #   display_name: Compile mongo_crypt_v1.so for Al2023 arm64
  #   modules: *modules
  #   cron: "0 0 * * 4" # 00:00 on Thursday
  #   expansions:
  #     <<: *compile-al2023-expansions
  #     compile_variant: -al2023-arm64
  #     compile_flags: >-
  #       --separate-debug
  #       MONGO_DISTMOD=rhel82
  #       -j$(grep -c ^processor /proc/cpuinfo)
  #       --release
  #       --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
  #       --use-diagnostic-latches=off
  #       --allocator=system
  #       --enterprise-features=fle
  #       --js-engine=none
  #       --link-model=dynamic-sdk
  #       --enable-http-client=off
  #       --ssl=off
  #       SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique"
  #       CCFLAGS="-fno-gnu-unique"
  #   run_on:
  #     - "amazon2023.0-arm64-large"
  #   tasks:
  #     - name: compile_mongocrypt_shlib
  ###

  ###
  # New compiles
  - name: amazon2-x86-compile
    display_name: "* Compile for Amazon Linux 2 x86 NEW"
    modules: *modules
    expansions:
      platform: linux
      project_dir: *project_dir
      compile_variant: amazon2-x86-compile
      scons_cache_scope: shared
      scons_cache_mode: all
      has_packages: false
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars        
    run_on:
      - amazon2-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_new_compile
        depends_on:
          - name: archive_dist_test
            variant: amazon2-x86-compile

  - name: amazon2-x86-openssl111-compile
    display_name: "* Compile for Amazon Linux 2 x86 OpenSSL 1.1.1"
    modules: *modules
    expansions:
      platform: linux
      project_dir: *project_dir
      compile_variant: amazon2-x86-openssl111-compile
      scons_cache_scope: shared
      scons_cache_mode: all
      has_packages: false
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars        
    run_on:
      - amazon2-openssl111-large
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_new_compile
        depends_on:
          - name: archive_dist_test
            variant: amazon2-x86-openssl111-compile

  - name: amazon2-arm64-compile
    display_name: "* Compile for Amazon Linux 2 arm64 NEW"
    modules: *modules
    expansions:
      platform: linux
      project_dir: *project_dir
      compile_variant: amazon2-arm64-compile
      scons_cache_scope: shared
      scons_cache_mode: all
      has_packages: false
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars        
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_new_compile
        depends_on:
          - name: archive_dist_test
            variant: amazon2-arm64-compile

  - name: amazon2-arm64-mongocrypt-shlib-compile
    display_name: "* Compile mongo_crypt_v1.so for Amazon Linux 2 arm64 NEW"
    modules: *modules
    cron: "0 0 * * 4" # 00:00 on Thursday
    tags: ["bazel_check"]
    expansions:
      platform: linux
      project_dir: *project_dir
      use_scons_cache: true
      compile_variant: amazon2-arm64-compile
      scons_cache_scope: shared
      scons_cache_mode: all
      has_packages: false
      compile_flags: >-
        --separate-debug
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
        --use-diagnostic-latches=off
        --allocator=system
        --enterprise-features=fle
        --js-engine=none
        --link-model=dynamic-sdk
        --enable-http-client=off
        --ssl=off
        SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique"
        CCFLAGS="-fno-gnu-unique"        
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: compile_mongocrypt_shlib

  - name: amazon2-x86-streams-compile
    display_name: "* Compile Streams for Amazon Linux 2 x86 NEW"
    modules: *modules
    expansions:
      platform: linux
      project_dir: *project_dir
      compile_variant: amazon2-x86-streams-compile
      scons_cache_scope: shared
      scons_cache_mode: all
      has_packages: false
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
        --streams-release-build        
    run_on:
      - amazon2-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_new_compile
        depends_on:
          - name: archive_dist_test
            variant: amazon2-x86-streams-compile

  - name: al2023-arm64-compile
    display_name: "* Compile for Al2023 arm64 NEW"
    modules: *modules
    expansions:
      platform: linux
      project_dir: *project_dir
      compile_variant: al2023-arm64-compile
      scons_cache_scope: shared
      scons_cache_mode: all
      has_packages: false
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=rhel82
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars        
    run_on:
      - amazon2023.0-arm64-large
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_new_compile
        depends_on:
          - name: archive_dist_test
            variant: al2023-arm64-compile
  ###