From e5b772bc36621218bd03ad0908b81c4597dc4497 Mon Sep 17 00:00:00 2001 From: Steve McClure Date: Wed, 18 Sep 2024 20:09:29 -0400 Subject: [PATCH] SERVER-94497: Support code coverage from bazel test targets (#27116) GitOrigin-RevId: fc4d8d3898d3afbcaa20e7b596e4ed7da7d464b2 --- etc/evergreen_yml_components/definitions.yml | 21 +++++-- .../tasks/misc_tasks.yml | 35 +++++++++++ .../variants/codecoverage/test_dev.yml | 2 + evergreen/bazel_coverage.sh | 40 ++++++++++++ .../functions/code_coverage_data_process.sh | 61 +++++++++++++++++-- src/mongo/platform/BUILD.bazel | 15 +++++ 6 files changed, 163 insertions(+), 11 deletions(-) create mode 100644 evergreen/bazel_coverage.sh diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 1cb65498274..91b2df21363 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1350,6 +1350,22 @@ functions: - *f_expansions_write - *bazel_run_sh + # TODO (SERVER-94776): reuse exposed bazel commands + "bazel coverage sh": &bazel_coverage_sh + command: subprocess.exec + display_name: "bazel coverage sh" + type: test + params: + binary: bash + args: + - "src/evergreen/bazel_coverage.sh" + + "bazel coverage": + - *get_version_expansions + - *apply_version_expansions + - *f_expansions_write + - *bazel_coverage_sh + "ninja compile sh": &ninja_compile_sh command: subprocess.exec display_name: "ninja compile sh" @@ -2694,11 +2710,6 @@ functions: binary: bash args: - "./src/evergreen/functions/code_coverage_data_process.sh" - env: - GCOV_TOOL: ${gcov_tool} - COVERALLS_REPO_TOKEN: ${coveralls_token} - TRAVIS_JOB_ID: ${revision_order_id} - TRAVIS_PULL_REQUEST: ${github_pr_number} "save code coverage data": - *f_expansions_write diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index cd21d528534..23b03758eb8 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -1075,6 +1075,41 @@ tasks: args: - "src/evergreen/lint_yaml.sh" + - name: bazel_coverage + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - command: timeout.update + params: + exec_timeout_secs: 2400 # 40 minutes + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. + - func: "fetch bazel" + - func: "bazel coverage" + vars: + # Running something like "bazel test //src/..." has to do a lot of preprocessing on *all* + # targets before it actually runs the much smaller fraction of test targets, unless we're + # certain that a full build has already been cached. + # It might be optimized by doing a query upfront, which uses minimal analysis: + # targets=$(bazel query 'kind(".*_test", //src/...)') + # bazel test $targets + target: >- + //src/... + args: >- + --test_output=all + --config=local + --combined_report=lcov + - name: monitor_build_status tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] commands: diff --git a/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml b/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml index e4604817026..3cc0044baa0 100644 --- a/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml +++ b/etc/evergreen_yml_components/variants/codecoverage/test_dev.yml @@ -41,6 +41,7 @@ buildvariants: - name: compile_test_parallel_unittest_stream_TG distros: - amazon2023.3-arm64-large + - name: bazel_coverage # Variant to support Code Coverage on amd64/x86_64 - name: &rhel-93-64-bit-coverage rhel-93-64-bit-coverage @@ -82,3 +83,4 @@ buildvariants: - name: compile_test_parallel_unittest_stream_TG distros: - rhel93-large + - name: bazel_coverage diff --git a/evergreen/bazel_coverage.sh b/evergreen/bazel_coverage.sh new file mode 100644 index 00000000000..a54e5de2186 --- /dev/null +++ b/evergreen/bazel_coverage.sh @@ -0,0 +1,40 @@ +# TODO (SERVER-94776): Expose a reusable bazel command processor + +# Usage: +# bazel_coverage [arguments] +# +# Required environment variables: +# * ${target} - Build target +# * ${args} - Extra command line args to pass to "bazel coverage" + +# Needed for evergreen scripts that use evergreen expansions and utility methods. +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +# Use `eval` to force evaluation of the environment variables in the echo statement: +eval echo "Execution environment: Args: ${args} Target: ${target}" + +source ./evergreen/bazel_RBE_supported.sh + +if bazel_rbe_supported; then + LOCAL_ARG="" +else + LOCAL_ARG="--config=local" +fi + +# We only support explicitly limited arch for code coverage, so there +# are fewer conditionals here than elsewhere in more general utilities. +BAZEL_BINARY=bazel + +# Print command being run to file that can be uploaded +echo "python buildscripts/install_bazel.py" > bazel-invocation.txt +echo "bazel coverage --verbose_failures $LOCAL_ARG ${args} ${target}" >> bazel-invocation.txt + +eval $BAZEL_BINARY coverage --verbose_failures $LOCAL_ARG ${args} ${target} + +exit $RET diff --git a/evergreen/functions/code_coverage_data_process.sh b/evergreen/functions/code_coverage_data_process.sh index 4e3c2c4897f..a24c2a6da9f 100755 --- a/evergreen/functions/code_coverage_data_process.sh +++ b/evergreen/functions/code_coverage_data_process.sh @@ -5,18 +5,61 @@ set +o errexit cd src +ARCH=$(uname -m) +if [[ "$ARCH" == "s390x" || "$ARCH" == "s390" || "$ARCH" == "ppc64le" || "$ARCH" == "ppc64" || "$ARCH" == "ppc" || "$ARCH" == "ppcle" ]]; then + echo "Code coverage not supported on this architecture" + exit 0 +fi + +# TODO (SERVER-94775): generalize the bazel command +BAZEL_BINARY=$TMPDIR/bazelisk +BAZEL_OUTPUT_PATH=$($BAZEL_BINARY info output_path) + +gcno_files=$(find $BAZEL_OUTPUT_PATH -type f -name "*.gcno") +if [ -n "$gcno_files" ]; then + echo "Found code coverage files:" + find $BAZEL_OUTPUT_PATH -type f -name "*.gcno" + + # Use coverage-reporter to scrape and upload data left by Bazel test targets. + # This implicitly uploads the data. + # https://github.com/coverallsapp/coverage-reporter + if [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then + # arm64 + echo "Coveralls coverage reporter is not yet supported on arm64." + echo "https://github.com/coverallsapp/coverage-reporter/issues/137" + # don't exit, there may be more data from SCons-produced builds + else + # amd64 + curl -L https://github.com/coverallsapp/coverage-reporter/releases/download/v0.6.14/coveralls-linux.tar.gz | tar -xz + # checksum from their release artifacts + echo 'a7e3d9d7f41a89c989a5719f52d29c84d461e0d5c07085598a2f28e4dba6f57b coveralls' | sha256sum --check + + # There are still some "travis" related namings, which get different UI affordances on the + # Coveralls dashboard that otherwise are not shown (like the build/job links) + ./coveralls report \ + --repo-token=${coveralls_token} \ + --pull-request=${github_pr_number} \ + --service-name="travis-ci" \ + --build-url="https://spruce.mongodb.com/version/${version_id}/" \ + --job-id=${revision_order_id} \ + --job-url="https://spruce.mongodb.com/task/${task_id}/" \ + "$BAZEL_OUTPUT_PATH/_coverage/_coverage_report.dat" + fi +fi + +# Look for evidence of SCons-produced artifacts if [ ! -d "./build/debug" ]; then echo "No code coverage to process - no 'build/debug' directory found." exit 0 fi -file_list=$(find ./build/debug -type f -name "*.gcno") -if [ ! -n "$file_list" ]; then +gcno_files=$(find ./build/debug -type f -name "*.gcno") +if [ ! -n "$gcno_files" ]; then echo "No code coverage to process - no '.gcno' files found." exit 0 fi -if [ -z "${GCOV_TOOL:-}" ]; then +if [ -z "${gcov_tool}" ]; then echo "No coverage tool defined. Set the gcov_tool expansion in evergreen.yml" >&2 exit 0 fi @@ -26,10 +69,16 @@ find ./build/debug -type f -name "*.gcno" activate_venv +# Use gcovr to process code coverage files (.gcno/.gcda) from SCons targets. +# This produces a coveralls-formatted json file that we have to explicitly upload. +# https://gcovr.com/en/stable/index.html pipx install "gcovr==7.2" || exit 1 -# Process code coverage files (.gcno/.gcda) directly into coveralls format -# https://gcovr.com/en/stable/index.html +# gcovr relies on these env vars +export COVERALLS_REPO_TOKEN=${coveralls_token} +export TRAVIS_PULL_REQUEST=${github_pr_number} +export TRAVIS_JOB_ID=${revision_order_id} + gcovr \ --output gcovr-coveralls.json \ --coveralls-pretty \ @@ -48,7 +97,7 @@ gcovr \ --gcov-exclude-directories build/debug/mongo/idl/ \ --gcov-exclude-directories build/debug/mongo/unittest/ \ --gcov-exclude-directories build/debug/third_party/ \ - --gcov-executable ${GCOV_TOOL[@]} \ + --gcov-executable ${gcov_tool} \ build/debug # Upload json data; view at https://coveralls.io/github/10gen/mongo diff --git a/src/mongo/platform/BUILD.bazel b/src/mongo/platform/BUILD.bazel index 671f16b6dd5..2000988c893 100644 --- a/src/mongo/platform/BUILD.bazel +++ b/src/mongo/platform/BUILD.bazel @@ -46,6 +46,21 @@ mongo_cc_library( ], ) +# Run it as an executable: +# bazel run //src/mongo/platform:visibility_test1 +# +# Execute as a test (this will parallelize the runs and accept test-specific options): +# bazel test //src/mongo/platform:visibility_test1 --test_output=all +# +# Test and collect code coverage (notice the "_with_debug" suffix): +# bazel coverage //src/mongo/platform:visibility_test1_with_debug --test_output=all --config=local --combined_report=lcov +# +# View coverage reports locally +# genhtml --branch-coverage --output coverage "$(bazel info output_path)/_coverage/_coverage_report.dat" +# That creates a file-tree inside a top-level “coverage” folder. To view the rendered HTML report +# in VSCode, install the [Live HTML Previewer](https://marketplace.visualstudio.com/items?itemName=hdg.live-html-previewer) +# VSCode extension. Once installed, open the coverage/index.html file and click the “Show Preview” +# widget that comes from the extension. mongo_cc_test( name = "visibility_test1", srcs = [