SERVER-94497: Support code coverage from bazel test targets (#27116)

GitOrigin-RevId: fc4d8d3898d3afbcaa20e7b596e4ed7da7d464b2

etc/evergreen_yml_components/definitions.yml

@@ -1350,6 +1350,22 @@ functions:
     - *f_expansions_write
     - *bazel_run_sh
 
+  # TODO (SERVER-94776): reuse exposed bazel commands
+  "bazel coverage sh": &bazel_coverage_sh
+    command: subprocess.exec
+    display_name: "bazel coverage sh"
+    type: test
+    params:
+      binary: bash
+      args:
+        - "src/evergreen/bazel_coverage.sh"
+
+  "bazel coverage":
+    - *get_version_expansions
+    - *apply_version_expansions
+    - *f_expansions_write
+    - *bazel_coverage_sh
+
   "ninja compile sh": &ninja_compile_sh
     command: subprocess.exec
     display_name: "ninja compile sh"
@@ -2694,11 +2710,6 @@ functions:
       binary: bash
       args:
         - "./src/evergreen/functions/code_coverage_data_process.sh"
-      env:
-        GCOV_TOOL: ${gcov_tool}
-        COVERALLS_REPO_TOKEN: ${coveralls_token}
-        TRAVIS_JOB_ID: ${revision_order_id}
-        TRAVIS_PULL_REQUEST: ${github_pr_number}
 
   "save code coverage data":
     - *f_expansions_write

etc/evergreen_yml_components/tasks/misc_tasks.yml

@@ -1075,6 +1075,41 @@ tasks:
           args:
             - "src/evergreen/lint_yaml.sh"
 
+  - name: bazel_coverage
+    tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+    commands:
+      - command: timeout.update
+        params:
+          exec_timeout_secs: 2400 # 40 minutes
+      - func: "f_expansions_write"
+      - command: manifest.load
+      - func: "git get project and add git tag"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - func: "get engflow creds"
+      # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted.
+      - func: "fetch bazel"
+      - func: "bazel coverage"
+        vars:
+          # Running something like "bazel test //src/..." has to do a lot of preprocessing on *all*
+          # targets before it actually runs the much smaller fraction of test targets, unless we're
+          # certain that a full build has already been cached.
+          # It might be optimized by doing a query upfront, which uses minimal analysis:
+          #   targets=$(bazel query 'kind(".*_test", //src/...)')
+          #   bazel test $targets
+          target: >-
+            //src/...
+          args: >-
+            --test_output=all
+            --config=local
+            --combined_report=lcov
+
   - name: monitor_build_status
     tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"]
     commands:

etc/evergreen_yml_components/variants/codecoverage/test_dev.yml

@@ -41,6 +41,7 @@ buildvariants:
       - name: compile_test_parallel_unittest_stream_TG
         distros:
           - amazon2023.3-arm64-large
+      - name: bazel_coverage
 
   # Variant to support Code Coverage on amd64/x86_64
   - name: &rhel-93-64-bit-coverage rhel-93-64-bit-coverage
@@ -82,3 +83,4 @@ buildvariants:
       - name: compile_test_parallel_unittest_stream_TG
         distros:
           - rhel93-large
+      - name: bazel_coverage

evergreen/bazel_coverage.sh

@@ -0,0 +1,40 @@
+# TODO (SERVER-94776): Expose a reusable bazel command processor
+
+# Usage:
+#   bazel_coverage [arguments]
+#
+# Required environment variables:
+# * ${target} - Build target
+# * ${args} - Extra command line args to pass to "bazel coverage"
+
+# Needed for evergreen scripts that use evergreen expansions and utility methods.
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+# Use `eval` to force evaluation of the environment variables in the echo statement:
+eval echo "Execution environment: Args: ${args} Target: ${target}"
+
+source ./evergreen/bazel_RBE_supported.sh
+
+if bazel_rbe_supported; then
+  LOCAL_ARG=""
+else
+  LOCAL_ARG="--config=local"
+fi
+
+# We only support explicitly limited arch for code coverage, so there
+# are fewer conditionals here than elsewhere in more general utilities.
+BAZEL_BINARY=bazel
+
+# Print command being run to file that can be uploaded
+echo "python buildscripts/install_bazel.py" > bazel-invocation.txt
+echo "bazel coverage --verbose_failures $LOCAL_ARG ${args} ${target}" >> bazel-invocation.txt
+
+eval $BAZEL_BINARY coverage --verbose_failures $LOCAL_ARG ${args} ${target}
+
+exit $RET

evergreen/functions/code_coverage_data_process.sh

@@ -5,18 +5,61 @@ set +o errexit
 
 cd src
 
+ARCH=$(uname -m)
+if [[ "$ARCH" == "s390x" || "$ARCH" == "s390" || "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" || "$ARCH" == "ppcle" ]]; then
+  echo "Code coverage not supported on this architecture"
+  exit 0
+fi
+
+# TODO (SERVER-94775): generalize the bazel command
+BAZEL_BINARY=$TMPDIR/bazelisk
+BAZEL_OUTPUT_PATH=$($BAZEL_BINARY info output_path)
+
+gcno_files=$(find $BAZEL_OUTPUT_PATH -type f -name "*.gcno")
+if [ -n "$gcno_files" ]; then
+  echo "Found code coverage files:"
+  find $BAZEL_OUTPUT_PATH -type f -name "*.gcno"
+
+  # Use coverage-reporter to scrape and upload data left by Bazel test targets.
+  # This implicitly uploads the data.
+  # https://github.com/coverallsapp/coverage-reporter
+  if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then
+    # arm64
+    echo "Coveralls coverage reporter is not yet supported on arm64."
+    echo "https://github.com/coverallsapp/coverage-reporter/issues/137"
+    # don't exit, there may be more data from SCons-produced builds
+  else
+    # amd64
+    curl -L https://github.com/coverallsapp/coverage-reporter/releases/download/v0.6.14/coveralls-linux.tar.gz | tar -xz
+    # checksum from their release artifacts
+    echo 'a7e3d9d7f41a89c989a5719f52d29c84d461e0d5c07085598a2f28e4dba6f57b coveralls' | sha256sum --check
+
+    # There are still some "travis" related namings, which get different UI affordances on the
+    # Coveralls dashboard that otherwise are not shown (like the build/job links)
+    ./coveralls report \
+      --repo-token=${coveralls_token} \
+      --pull-request=${github_pr_number} \
+      --service-name="travis-ci" \
+      --build-url="https://spruce.mongodb.com/version/${version_id}/" \
+      --job-id=${revision_order_id} \
+      --job-url="https://spruce.mongodb.com/task/${task_id}/" \
+      "$BAZEL_OUTPUT_PATH/_coverage/_coverage_report.dat"
+  fi
+fi
+
+# Look for evidence of SCons-produced artifacts
 if [ ! -d "./build/debug" ]; then
   echo "No code coverage to process - no 'build/debug' directory found."
   exit 0
 fi
 
-file_list=$(find ./build/debug -type f -name "*.gcno")
-if [ ! -n "$file_list" ]; then
+gcno_files=$(find ./build/debug -type f -name "*.gcno")
+if [ ! -n "$gcno_files" ]; then
   echo "No code coverage to process - no '.gcno' files found."
   exit 0
 fi
 
-if [ -z "${GCOV_TOOL:-}" ]; then
+if [ -z "${gcov_tool}" ]; then
   echo "No coverage tool defined. Set the gcov_tool expansion in evergreen.yml" >&2
   exit 0
 fi
@@ -26,10 +69,16 @@ find ./build/debug -type f -name "*.gcno"
 
 activate_venv
 
+# Use gcovr to process code coverage files (.gcno/.gcda) from SCons targets.
+# This produces a coveralls-formatted json file that we have to explicitly upload.
+# https://gcovr.com/en/stable/index.html
 pipx install "gcovr==7.2" || exit 1
 
-# Process code coverage files (.gcno/.gcda) directly into coveralls format
-# https://gcovr.com/en/stable/index.html
+# gcovr relies on these env vars
+export COVERALLS_REPO_TOKEN=${coveralls_token}
+export TRAVIS_PULL_REQUEST=${github_pr_number}
+export TRAVIS_JOB_ID=${revision_order_id}
+
 gcovr \
   --output gcovr-coveralls.json \
   --coveralls-pretty \
@@ -48,7 +97,7 @@ gcovr \
   --gcov-exclude-directories build/debug/mongo/idl/ \
   --gcov-exclude-directories build/debug/mongo/unittest/ \
   --gcov-exclude-directories build/debug/third_party/ \
-  --gcov-executable ${GCOV_TOOL[@]} \
+  --gcov-executable ${gcov_tool} \
   build/debug
 
 # Upload json data; view at https://coveralls.io/github/10gen/mongo

src/mongo/platform/BUILD.bazel

@@ -46,6 +46,21 @@ mongo_cc_library(
     ],
 )
 
+# Run it as an executable:
+#   bazel run //src/mongo/platform:visibility_test1
+#
+# Execute as a test (this will parallelize the runs and accept test-specific options):
+#   bazel test //src/mongo/platform:visibility_test1 --test_output=all
+#
+# Test and collect code coverage (notice the "_with_debug" suffix):
+#   bazel coverage //src/mongo/platform:visibility_test1_with_debug --test_output=all --config=local --combined_report=lcov
+#
+# View coverage reports locally
+#   genhtml --branch-coverage --output coverage "$(bazel info output_path)/_coverage/_coverage_report.dat"
+# That creates a file-tree inside a top-level “coverage” folder. To view the rendered HTML report
+# in VSCode, install the [Live HTML Previewer](https://marketplace.visualstudio.com/items?itemName=hdg.live-html-previewer)
+# VSCode extension. Once installed, open the coverage/index.html file and click the “Show Preview”
+# widget that comes from the extension.
 mongo_cc_test(
     name = "visibility_test1",
     srcs = [