From 3220129aea40bd6ecb880646e54adcb20d7c4fbd Mon Sep 17 00:00:00 2001 From: Andrew Bradshaw Date: Fri, 1 Nov 2024 13:26:33 -0700 Subject: [PATCH] SERVER-91979 Upgrade to Bazel 7.2 (#28749) GitOrigin-RevId: 9139373c72c251a5474d080cbccba8ef221e1386 --- .bazelrc | 16 +++-- .bazelversion | 4 +- .gitignore | 3 + WORKSPACE.bazel | 21 +++++- bazel/config/BUILD.bazel | 6 +- .../architecture/ppc64le_build_from_source.md | 10 +-- .../architecture/s390x_build_from_source.md | 10 +-- bazel/mongo_src_rules.bzl | 4 +- bazel/platforms/BUILD.bazel | 4 +- bazel/platforms/local_config_platform.bzl | 18 +++-- .../local_config_platform_extension.bzl | 66 +++++++++++++++++++ bazel/separate_debug.bzl | 13 +++- bazel/toolchains/mongo_toolchain.BUILD | 8 +-- bazel/toolchains/mongo_toolchain.bzl | 4 +- bazel/toolchains/python_toolchain.bzl | 2 +- buildscripts/bazel_scons_diff.py | 4 +- buildscripts/install_bazel.py | 6 +- evergreen/bazel_compile.sh | 4 +- evergreen/bazel_run.sh | 4 +- evergreen/bazel_scons_diff.sh | 4 +- evergreen/get_bazelisk.sh | 2 +- site_scons/site_tools/integrate_bazel.py | 24 +++++-- src/mongo/db/fts/unicode/BUILD.bazel | 2 +- .../IntelRDFPMathLib20U1/BUILD.bazel | 6 ++ src/third_party/mozjs/BUILD.bazel | 7 +- src/third_party/unwind/BUILD.bazel | 18 ++--- src/third_party/wiredtiger/BUILD.bazel | 2 +- tools/bazel | 22 ++++++- 28 files changed, 218 insertions(+), 76 deletions(-) create mode 100644 bazel/platforms/local_config_platform_extension.bzl diff --git a/.bazelrc b/.bazelrc index e34eb6beff1..8705e555f35 100644 --- a/.bazelrc +++ b/.bazelrc @@ -129,7 +129,7 @@ build:linux --jobs=200 build --remote_cache=grpcs://sodalite.cluster.engflow.com build --bes_backend=grpcs://sodalite.cluster.engflow.com build --bes_results_url=https://sodalite.cluster.engflow.com/invocation/ -build --experimental_remote_cache_compression=true +build --remote_cache_compression=true build --grpc_keepalive_time=30s build --nolegacy_important_outputs build --bes_keywords=repo:mongo @@ -156,7 +156,7 @@ build:local --bes_backend= build:local --bes_results_url= build:local --tls_client_certificate= build:local --tls_client_key= -build:local --experimental_remote_cache_compression=false +build:local --remote_cache_compression=false build:local --grpc_keepalive_time=0s build:local --legacy_important_outputs build:local --jobs=auto @@ -174,18 +174,17 @@ build:public-release --bes_backend= build:public-release --bes_results_url= build:public-release --tls_client_certificate= build:public-release --tls_client_key= -build:public-release --experimental_remote_cache_compression=false +build:public-release --remote_cache_compression=false build:public-release --grpc_keepalive_time=0s build:public-release --legacy_important_outputs # Avoid failing builds when BES metadata fails to upload. common --bes_upload_mode=fully_async -# TODO SERVER-89593 -# Renable after bazel 7.0 due to query bug: -# https://github.com/bazelbuild/bazel/issues/10902 # Enable Build without the Bytes -# common --remote_download_outputs=toplevel +common --remote_download_outputs=toplevel + +common --enable_bzlmod=false # Default Mongo Version if a version is not specified. build --define=MONGO_VERSION=8.1.0-alpha @@ -205,6 +204,9 @@ try-import %workspace%/.bazelrc.evergreen_engflow_creds # Evergreen settings, ex. in evergreen this will set the key/cert options for access to the cluster try-import %workspace%/.bazelrc.evergreen +# Import settings for selecting the right xcode version on mac +try-import %workspace%/.bazelrc.xcode + # local default dev settings try-import %workspace%/.bazelrc.workstation diff --git a/.bazelversion b/.bazelversion index 686330a9ff6..253fd3f4f8a 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1,3 +1,3 @@ -6.4.0 -# TODO(SERVER-86050): s390x and ppc64le are using custom built Bazel v6.4.0 binaries +7.2.1 +# TODO(SERVER-86050): s390x and ppc64le are using custom built Bazel v7.2.1 binaries # Do not update this without updating the custom built binaries until Bazelisk is supported on s390x & ppc64le. diff --git a/.gitignore b/.gitignore index b0e16220ded..6f7ba49eaa8 100644 --- a/.gitignore +++ b/.gitignore @@ -279,11 +279,14 @@ buildifier .bazelrc.local .bazelrc.gitinfo .bazelrc.workstation +.bazelrc.xcode .bazel_info_for_ninja.txt .ninja_last_command_line_targets.txt bazel/coverity/analysis/BUILD.bazel .bazel_include_info.json .bazel_header_list_cache +MODULE.bazel +MODULE.bazel.lock # generated configs for external fixture suites docker_compose/ diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 37f8109d7b6..800a881f63d 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -1,4 +1,23 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# Apple cc toolchain needs to be loaded before regular cc toolchain or else Apple will just use regular cc toolchain +http_archive( + name = "build_bazel_apple_support", + sha256 = "b53f6491e742549f13866628ddffcc75d1f3b2d6987dc4f14a16b242113c890b", + url = "https://github.com/bazelbuild/apple_support/releases/download/1.17.1/apple_support.1.17.1.tar.gz", +) + +load( + "@build_bazel_apple_support//lib:repositories.bzl", + "apple_support_dependencies", +) + +apple_support_dependencies() + +load("@bazel_features//:deps.bzl", "bazel_features_deps") + +bazel_features_deps() + load("//bazel/platforms:local_config_platform.bzl", "setup_local_config_platform") load("//bazel/toolchains:mongo_toolchain.bzl", "toolchain_download") @@ -17,7 +36,7 @@ http_archive( ], ) -setup_local_config_platform(name = "local_config_platform") +setup_local_config_platform(name = "internal_platforms_do_not_use") toolchain_download(name = "mongo_toolchain") diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel index b3743153b7e..26adab4e1f4 100644 --- a/bazel/config/BUILD.bazel +++ b/bazel/config/BUILD.bazel @@ -194,7 +194,7 @@ config_setting( name = "linux_ppc64le", constraint_values = [ "@platforms//os:linux", - "@platforms//cpu:ppc64le", + "@platforms//cpu:ppc", ], ) @@ -259,7 +259,7 @@ selects.config_setting_group( match_any = [ "@platforms//cpu:aarch64", "@platforms//cpu:x86_64", - "@platforms//cpu:ppc64le", + "@platforms//cpu:ppc", ], ) @@ -848,7 +848,7 @@ selects.config_setting_group( name = "_tcmalloc_gperf_auto_supported", match_any = [ "@platforms//os:windows", - "@platforms//cpu:ppc64le", + "@platforms//cpu:ppc", "@platforms//cpu:s390x", ], ) diff --git a/bazel/docs/architecture/ppc64le_build_from_source.md b/bazel/docs/architecture/ppc64le_build_from_source.md index 6f008409815..58665e56649 100644 --- a/bazel/docs/architecture/ppc64le_build_from_source.md +++ b/bazel/docs/architecture/ppc64le_build_from_source.md @@ -8,12 +8,12 @@ Bazel usually comes with a built-in JDK. However, the tooling used to build the On the MongoDB PPC64LE Evergreen static hosts and dev hosts, the OpenJDK 11 installation exists at: -/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le +/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.ppc64le To compile with on these platforms, the developer must set JAVA_HOME before invoking Bazel. -# Bazel v6.4.0 Compilation Steps +# Bazel v7.2.1 Compilation Steps - curl -O -L https://github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-dist.zip - unzip bazel-6.4.0-dist.zip - JAVA_HOME=/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le ./compile.sh + curl -O -L https://github.com/bazelbuild/bazel/releases/download/7.2.1/bazel-7.2.1-dist.zip + unzip bazel-7.2.1-dist.zip + JAVA_HOME=/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.ppc64le ./compile.sh diff --git a/bazel/docs/architecture/s390x_build_from_source.md b/bazel/docs/architecture/s390x_build_from_source.md index 41e0344c2e2..e9c88ef5051 100644 --- a/bazel/docs/architecture/s390x_build_from_source.md +++ b/bazel/docs/architecture/s390x_build_from_source.md @@ -8,12 +8,12 @@ Bazel usually comes with a built-in JDK. However, the tooling used to build the On the MongoDB S390X Evergreen static hosts and dev hosts, the OpenJDK 11 installation exists at: -/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x +/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.s390x To compile with on these platforms, the developer must set JAVA_HOME before invoking Bazel. -# Bazel v6.4.0 Compilation Steps +# Bazel v7.2.1 Compilation Steps - curl -O -L https://github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-dist.zip - unzip bazel-6.4.0-dist.zip - JAVA_HOME=/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x ./compile.sh + curl -O -L https://github.com/bazelbuild/bazel/releases/download/7.2.1/bazel-7.2.1-dist.zip + unzip bazel-7.2.1-dist.zip + JAVA_HOME=/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.s390x ./compile.sh diff --git a/bazel/mongo_src_rules.bzl b/bazel/mongo_src_rules.bzl index 9477a654baa..f2a2d49a043 100644 --- a/bazel/mongo_src_rules.bzl +++ b/bazel/mongo_src_rules.bzl @@ -1329,7 +1329,7 @@ def force_includes_copt(package_name, name): if package_name.startswith("src/mongo"): basic_h = "mongo/platform/basic.h" return select({ - "@platforms//os:windows": ["/FI", basic_h], + "@platforms//os:windows": ["/FI" + basic_h], "//conditions:default": ["-include", basic_h], }) @@ -1341,7 +1341,7 @@ def force_includes_copt(package_name, name): "//bazel/config:linux_x86_64": ["-include", "third_party/mozjs/platform/x86_64/linux/build/js-config.h"], "//bazel/config:macos_aarch64": ["-include", "third_party/mozjs/platform/aarch64/macOS/build/js-config.h"], "//bazel/config:macos_x86_64": ["-include", "third_party/mozjs/platform/x86_64/macOS/build/js-config.h"], - "//bazel/config:windows_x86_64": ["/FI", "third_party/mozjs/platform/x86_64/windows/build/js-config.h"], + "//bazel/config:windows_x86_64": ["/FI" + "third_party/mozjs/platform/x86_64/windows/build/js-config.h"], }) return [] diff --git a/bazel/platforms/BUILD.bazel b/bazel/platforms/BUILD.bazel index b36665e7a8a..b6b06f97c05 100644 --- a/bazel/platforms/BUILD.bazel +++ b/bazel/platforms/BUILD.bazel @@ -98,7 +98,7 @@ platform( platform( name = "linux_ppc64le", constraint_values = [ - "@platforms//cpu:ppc64le", + "@platforms//cpu:ppc", "@platforms//os:linux", ], ) @@ -114,7 +114,7 @@ platform( platform( name = "rhel8_ppc64le", constraint_values = [ - "@platforms//cpu:ppc64le", + "@platforms//cpu:ppc", "@platforms//os:linux", ":use_mongo_toolchain", ], diff --git a/bazel/platforms/local_config_platform.bzl b/bazel/platforms/local_config_platform.bzl index 300fd1110f6..42b262bed3b 100644 --- a/bazel/platforms/local_config_platform.bzl +++ b/bazel/platforms/local_config_platform.bzl @@ -12,7 +12,7 @@ _ARCH_MAP = { "amd64": "@platforms//cpu:x86_64", "aarch64": "@platforms//cpu:arm64", "x86_64": "@platforms//cpu:x86_64", - "ppc64le": "@platforms//cpu:ppc64le", + "ppc64le": "@platforms//cpu:ppc", "s390x": "@platforms//cpu:s390x", } @@ -85,18 +85,24 @@ def _setup_local_config_platform(ctx): } ctx.template( - "BUILD.bazel", + "host/BUILD.bazel", ctx.attr.build_tpl, substitutions = substitutions, ) ctx.template( - "constraints.bzl", + "host/constraints.bzl", ctx.attr.constraints_tpl, substitutions = substitutions, ) - return result + ctx.template( + "host/extension.bzl", + ctx.attr.extension_tpl, + substitutions = substitutions, + ) + + return None setup_local_config_platform = repository_rule( implementation = _setup_local_config_platform, @@ -109,6 +115,10 @@ setup_local_config_platform = repository_rule( default = "//bazel/platforms:local_config_platform_constraints.bzl", doc = "Template modeling the builtin local config platform constraints file.", ), + "extension_tpl": attr.label( + default = "//bazel/platforms:local_config_platform_extension.bzl", + doc = "Template modeling the builtin local config platform constraints file.", + ), }, environ = ["USE_NATIVE_TOOLCHAIN"], ) diff --git a/bazel/platforms/local_config_platform_extension.bzl b/bazel/platforms/local_config_platform_extension.bzl new file mode 100644 index 00000000000..6bdc4cfb292 --- /dev/null +++ b/bazel/platforms/local_config_platform_extension.bzl @@ -0,0 +1,66 @@ +def _translate_cpu(arch): + if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]: + return "x86_32" + if arch in ["amd64", "x86_64", "x64"]: + return "x86_64" + if arch in ["ppc", "ppc64", "ppc64le"]: + return "ppc" + if arch in ["arm", "armv7l"]: + return "arm" + if arch in ["aarch64"]: + return "aarch64" + if arch in ["s390x", "s390"]: + return "s390x" + if arch in ["mips64el", "mips64"]: + return "mips64" + if arch in ["riscv64"]: + return "riscv64" + return None + +def _translate_os(os): + if os.startswith("mac os"): + return "osx" + if os.startswith("freebsd"): + return "freebsd" + if os.startswith("openbsd"): + return "openbsd" + if os.startswith("linux"): + return "linux" + if os.startswith("windows"): + return "windows" + return None + +def _host_platform_repo_impl(rctx): + cpu = _translate_cpu(rctx.os.arch) + os = _translate_os(rctx.os.name) + + cpu = "" if cpu == None else " '@platforms//cpu:%s',\n" % cpu + os = "" if os == None else " '@platforms//os:%s',\n" % os + + rctx.file("BUILD.bazel", """ +# DO NOT EDIT: automatically generated BUILD file +exports_files(["constraints.bzl"]) +""") + + rctx.file("constraints.bzl", """ +# DO NOT EDIT: automatically generated constraints list +HOST_CONSTRAINTS = [ +%s%s] +""" % (cpu, os)) + +host_platform_repo = repository_rule( + implementation = _host_platform_repo_impl, + doc = """Generates constraints for the host platform. The constraints.bzl +file contains a single HOST_CONSTRAINTS variable, which is a +list of strings, each of which is a label to a constraint_value +for the host platform.""", +) + +def _host_platform_impl(_mctx): + host_platform_repo(name = "host_platform") + +host_platform = module_extension( + implementation = _host_platform_impl, + doc = """Generates a host_platform_repo repo named +host_platform, containing constraints for the host platform.""", +) diff --git a/bazel/separate_debug.bzl b/bazel/separate_debug.bzl index de03b73722b..efd52b3f618 100644 --- a/bazel/separate_debug.bzl +++ b/bazel/separate_debug.bzl @@ -3,6 +3,7 @@ load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") WITH_DEBUG_SUFFIX = "_with_debug" CC_SHARED_LIBRARY_SUFFIX = "_shared" SHARED_ARCHIVE_SUFFIX = "_shared_archive" +MAC_DEBUG_FOLDER_EXTENSION = ".dSYM" def get_inputs_and_outputs(ctx, shared_ext, static_ext, debug_ext): """ @@ -35,7 +36,10 @@ def get_inputs_and_outputs(ctx, shared_ext, static_ext, debug_ext): if shared_lib: basename = shared_lib.basename[:-len(WITH_DEBUG_SUFFIX + shared_ext + CC_SHARED_LIBRARY_SUFFIX)] if ctx.attr.enabled: - debug_info = ctx.actions.declare_file(basename + shared_ext + debug_ext) + if debug_ext == MAC_DEBUG_FOLDER_EXTENSION: + debug_info = ctx.actions.declare_directory(basename + shared_ext + debug_ext) + else: + debug_info = ctx.actions.declare_file(basename + shared_ext + debug_ext) else: debug_info = None output_bin = ctx.actions.declare_file(basename + shared_ext) @@ -49,7 +53,10 @@ def get_inputs_and_outputs(ctx, shared_ext, static_ext, debug_ext): basename = program_bin.basename[:-len(WITH_DEBUG_SUFFIX)] if ctx.attr.enabled: - debug_info = ctx.actions.declare_file(basename + debug_ext) + if debug_ext == MAC_DEBUG_FOLDER_EXTENSION: + debug_info = ctx.actions.declare_directory(basename + debug_ext) + else: + debug_info = ctx.actions.declare_file(basename + debug_ext) else: debug_info = None output_bin = ctx.actions.declare_file(basename) @@ -292,7 +299,7 @@ def linux_extraction(ctx, cc_toolchain, inputs): def macos_extraction(ctx, cc_toolchain, inputs): outputs = [] unstripped_static_bin = None - input_bin, output_bin, debug_info, static_lib = get_inputs_and_outputs(ctx, ".dylib", ".a", ".dSYM") + input_bin, output_bin, debug_info, static_lib = get_inputs_and_outputs(ctx, ".dylib", ".a", MAC_DEBUG_FOLDER_EXTENSION) input_file = ctx.attr.binary_with_debug.files.to_list() if input_bin: diff --git a/bazel/toolchains/mongo_toolchain.BUILD b/bazel/toolchains/mongo_toolchain.BUILD index f59bf72540b..19c5c80d846 100644 --- a/bazel/toolchains/mongo_toolchain.BUILD +++ b/bazel/toolchains/mongo_toolchain.BUILD @@ -36,7 +36,7 @@ mongo_cc_toolchain_config( name = "cc_gcc_toolchain_config", bin_dirs = COMMON_BINDIRS, compiler = "gcc", - cpu = "{platforms_arch}", + cpu = "{bazel_toolchain_cpu}", cxx_builtin_include_directories = COMMON_BUILTIN_INCLUDE_DIRECTORIES, extra_ldflags = LINK_FLAGS, includes = GCC_INCLUDE_DIRS + COMMON_INCLUDE_DIRECTORIES + COMMON_BUILTIN_INCLUDE_DIRECTORIES, @@ -65,7 +65,7 @@ mongo_cc_toolchain_config( name = "cc_clang_toolchain_config", bin_dirs = COMMON_BINDIRS, compiler = "clang", - cpu = "{platforms_arch}", + cpu = "{bazel_toolchain_cpu}", cxx_builtin_include_directories = COMMON_BUILTIN_INCLUDE_DIRECTORIES, extra_ldflags = LINK_FLAGS, includes = CLANG_INCLUDE_DIRS + COMMON_INCLUDE_DIRECTORIES + COMMON_BUILTIN_INCLUDE_DIRECTORIES, @@ -112,12 +112,12 @@ toolchain( name = "mongo_toolchain", exec_compatible_with = [ "@platforms//os:linux", - "@platforms//cpu:{platforms_arch}", + "@platforms//cpu:{bazel_toolchain_cpu}", "@//bazel/platforms:use_mongo_toolchain", ], target_compatible_with = [ "@platforms//os:linux", - "@platforms//cpu:{platforms_arch}", + "@platforms//cpu:{bazel_toolchain_cpu}", "@//bazel/platforms:use_mongo_toolchain", ], toolchain = ":cc_mongo_toolchain", diff --git a/bazel/toolchains/mongo_toolchain.bzl b/bazel/toolchains/mongo_toolchain.bzl index 2c65b0508c8..dd05fdc98a9 100644 --- a/bazel/toolchains/mongo_toolchain.bzl +++ b/bazel/toolchains/mongo_toolchain.bzl @@ -11,7 +11,7 @@ _OS_MAP = { _ARCH_MAP = { "aarch64": "@platforms//cpu:arm64", "x86_64": "@platforms//cpu:x86_64", - "ppc64le": "@platforms//cpu:ppc64le", + "ppc64le": "@platforms//cpu:ppc", "s390x": "@platforms//cpu:s390x", } @@ -57,7 +57,7 @@ def _toolchain_download(ctx): elif arch == "x86_64": substitutions = { "{platforms_arch}": "x86_64", - "{bazel_toolchain_cpu}": "k8", + "{bazel_toolchain_cpu}": "x86_64", "{arch}": arch, } elif arch == "ppc64le": diff --git a/bazel/toolchains/python_toolchain.bzl b/bazel/toolchains/python_toolchain.bzl index 0f12c4cc231..389f759b684 100644 --- a/bazel/toolchains/python_toolchain.bzl +++ b/bazel/toolchains/python_toolchain.bzl @@ -12,7 +12,7 @@ _ARCH_MAP = { "amd64": "@platforms//cpu:x86_64", "aarch64": "@platforms//cpu:arm64", "x86_64": "@platforms//cpu:x86_64", - "ppc64le": "@platforms//cpu:ppc64le", + "ppc64le": "@platforms//cpu:ppc", "s390x": "@platforms//cpu:s390x", } diff --git a/buildscripts/bazel_scons_diff.py b/buildscripts/bazel_scons_diff.py index 3e76af8fbc5..acb1f3ba1b8 100644 --- a/buildscripts/bazel_scons_diff.py +++ b/buildscripts/bazel_scons_diff.py @@ -405,9 +405,9 @@ def main(): # Set JAVA_HOME on ppc & s390x architectures global bazel_env_settings if platform.machine().lower() == "ppc64le": - bazel_env_settings["JAVA_HOME"] = "/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le" + bazel_env_settings["JAVA_HOME"] = "/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.ppc64le" elif platform.machine().lower() == "s390x": - bazel_env_settings["JAVA_HOME"] = "/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x" + bazel_env_settings["JAVA_HOME"] = "/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.s390x" target_type_map = {} for scons_target in args.scons_targets: diff --git a/buildscripts/install_bazel.py b/buildscripts/install_bazel.py index df7f01d6005..07548694d46 100644 --- a/buildscripts/install_bazel.py +++ b/buildscripts/install_bazel.py @@ -10,8 +10,8 @@ import urllib.request from retry import retry _S3_HASH_MAPPING = { - "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-ppc64le": "dd21c75817533ff601bf797e64f0eb2f7f6b813af26c829f0bda30e328caef46", - "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-s390x": "6d72eabc1789b041bbe4cfc033bbac4491ec9938ef6da9899c0188ecf270a7f4", + "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-7.2.1-ppc64le": "4ecc7f1396b8d921c6468b34cc8ed356c4f2dbe8a154c25d681a61ccb5dfc9cb", + "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-7.2.1-s390x": "2f5f7fd747620d96e885766a4027347c75c0f455c68219211a00e72fc6413be9", "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-amd64": "f2ba5f721a995b54bab68c6b76a340719888aa740310e634771086b6d1528ecd", "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-arm64": "69fa21cd2ccffc2f0970c21aa3615484ba89e3553ecce1233a9d8ad9570d170e", "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-amd64": "d28b588ac0916abd6bf02defb5433f6eddf7cba35ffa808eabb65a44aab226f7", @@ -72,7 +72,7 @@ def install_bazel(binary_directory: str) -> str: print( "Warning: Bazelisk is not supported on this platform. Installing Bazel directly instead." ) - s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-{normalized_arch}" + s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-7.2.1-{normalized_arch}" _download_path_with_retry(s3_path, binary_path) _verify_s3_hash(s3_path, binary_path) diff --git a/evergreen/bazel_compile.sh b/evergreen/bazel_compile.sh index 56d78b0eeb7..4e7960115d3 100755 --- a/evergreen/bazel_compile.sh +++ b/evergreen/bazel_compile.sh @@ -42,9 +42,9 @@ fi # Set the JAVA_HOME directories for ppc64le and s390x since their bazel binaries are not compiled with a built-in JDK. # TODO(SERVER-86050): remove the branch once bazelisk is built on s390x & ppc64le if [[ $ARCH == "ppc64le" ]]; then - export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le" + export JAVA_HOME="/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.ppc64le" elif [[ $ARCH == "s390x" ]]; then - export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x" + export JAVA_HOME="/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.s390x" fi # TODO(SERVER-86050): remove the branch once bazelisk is built on s390x & ppc64le diff --git a/evergreen/bazel_run.sh b/evergreen/bazel_run.sh index e5064dfc41e..7f7adc506e9 100644 --- a/evergreen/bazel_run.sh +++ b/evergreen/bazel_run.sh @@ -45,9 +45,9 @@ fi # Set the JAVA_HOME directories for ppc64le and s390x since their bazel binaries are not compiled with a built-in JDK. if [[ $ARCH == "ppc64le" ]]; then - export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le" + export JAVA_HOME="/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.ppc64le" elif [[ $ARCH == "s390x" ]]; then - export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x" + export JAVA_HOME="/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.s390x" fi # Print command being run to file that can be uploaded diff --git a/evergreen/bazel_scons_diff.sh b/evergreen/bazel_scons_diff.sh index ad91db15cb2..78e7630ccaa 100644 --- a/evergreen/bazel_scons_diff.sh +++ b/evergreen/bazel_scons_diff.sh @@ -36,9 +36,9 @@ fi # Set the JAVA_HOME directories for ppc64le and s390x since their bazel binaries are not compiled with a built-in JDK. if [[ $ARCH == "ppc64le" ]]; then - export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le" + export JAVA_HOME="/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.ppc64le" elif [[ $ARCH == "s390x" ]]; then - export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x" + export JAVA_HOME="/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.s390x" fi eval ${compile_env} $python ./buildscripts/bazel_scons_diff.py \ diff --git a/evergreen/get_bazelisk.sh b/evergreen/get_bazelisk.sh index ccaf4bb8fd8..0cf8478137a 100755 --- a/evergreen/get_bazelisk.sh +++ b/evergreen/get_bazelisk.sh @@ -36,7 +36,7 @@ mkdir -p $TMPDIR # TODO(SERVER-86050): remove the branch once bazelisk is built on s390x & ppc64le if [[ $ARCH == "ppc64le" ]] || [[ $ARCH == "s390x" ]]; then - REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-${ARCH} + REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-7.2.1-${ARCH} LOCAL_PATH=$TMPDIR/bazel else REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-${OS}-${ARCH}${EXT} diff --git a/site_scons/site_tools/integrate_bazel.py b/site_scons/site_tools/integrate_bazel.py index 5ad0f69771c..9044cec9d7d 100644 --- a/site_scons/site_tools/integrate_bazel.py +++ b/site_scons/site_tools/integrate_bazel.py @@ -74,8 +74,8 @@ _DISTRO_PATTERN_MAP = { } _S3_HASH_MAPPING = { - "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-ppc64le": "dd21c75817533ff601bf797e64f0eb2f7f6b813af26c829f0bda30e328caef46", - "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-s390x": "6d72eabc1789b041bbe4cfc033bbac4491ec9938ef6da9899c0188ecf270a7f4", + "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-7.2.1-ppc64le": "4ecc7f1396b8d921c6468b34cc8ed356c4f2dbe8a154c25d681a61ccb5dfc9cb", + "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-7.2.1-s390x": "2f5f7fd747620d96e885766a4027347c75c0f455c68219211a00e72fc6413be9", "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-amd64": "f2ba5f721a995b54bab68c6b76a340719888aa740310e634771086b6d1528ecd", "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-arm64": "69fa21cd2ccffc2f0970c21aa3615484ba89e3553ecce1233a9d8ad9570d170e", "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-amd64": "d28b588ac0916abd6bf02defb5433f6eddf7cba35ffa808eabb65a44aab226f7", @@ -172,7 +172,9 @@ def bazel_builder_action( # So we will do the renaming of dwarf file to what scons expects here, before we copy to scons tree substring_end = str(t).find(".dSYM/") + 5 t = str(t)[:substring_end] - s = Globals.bazel_output(t) + # This is declared as an output folder, so bazel appends (TreeArtifact) to it + s = Globals.bazel_output(t + " (TreeArtifact)") + s = str(s).removesuffix(" (TreeArtifact)") dwarf_info_base = os.path.splitext(os.path.splitext(os.path.basename(t))[0])[0] dwarf_sym_with_debug = os.path.join( s, f"Contents/Resources/DWARF/{dwarf_info_base}_shared_with_debug.dylib" @@ -186,12 +188,20 @@ def bazel_builder_action( s, f"Contents/Resources/DWARF/{dwarf_info_base}_with_debug" ) dwarf_sym = os.path.join(s, f"Contents/Resources/DWARF/{dwarf_info_base}") - # rename the file for scons - shutil.copy(dwarf_sym_with_debug, dwarf_sym) # copy the whole dSYM in one operation. Clean any existing files that might be in the way. + print(f"Moving .dSYM from {s} over to {t}.") shutil.rmtree(str(t), ignore_errors=True) shutil.copytree(s, str(t)) + # we want to change the permissions back to normal permissions on the folders copied rather than read only + os.chmod(t, 0o755) + for root, dirs, files in os.walk(t): + for name in files: + os.chmod(os.path.join(root, name), 0o755) + for name in dirs: + os.chmod(os.path.join(root, name), 0o755) + # shouldn't write our own files to the bazel directory, renaming file for scons + shutil.copy(dwarf_sym_with_debug.replace(s, t), dwarf_sym.replace(s, t)) else: s = Globals.bazel_output(t) shutil.copy(s, str(t)) @@ -898,11 +908,11 @@ def setup_bazel_env_vars() -> None: # Set the JAVA_HOME directories for ppc64le and s390x since their bazel binaries are not compiled with a built-in JDK. if platform.machine().lower() == "ppc64le": Globals.bazel_env_variables["JAVA_HOME"] = ( - "/usr/lib/jvm/java-11-openjdk-11.0.4.11-2.el8.ppc64le" + "/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.ppc64le" ) elif platform.machine().lower() == "s390x": Globals.bazel_env_variables["JAVA_HOME"] = ( - "/usr/lib/jvm/java-11-openjdk-11.0.11.0.9-0.el8_3.s390x" + "/usr/lib/jvm/java-21-openjdk-21.0.4.0.7-1.el8.s390x" ) diff --git a/src/mongo/db/fts/unicode/BUILD.bazel b/src/mongo/db/fts/unicode/BUILD.bazel index 2421e14d0ba..5f2345aec9c 100644 --- a/src/mongo/db/fts/unicode/BUILD.bazel +++ b/src/mongo/db/fts/unicode/BUILD.bazel @@ -71,7 +71,7 @@ mongo_cc_library( ] + select({ "@platforms//cpu:x86_64": ["byte_vector_sse2.h"], "@platforms//cpu:aarch64": ["byte_vector_neon.h"], - "@platforms//cpu:ppc64le": ["byte_vector_altivec.h"], + "@platforms//cpu:ppc": ["byte_vector_altivec.h"], "//conditions:default": [], }), deps = [ diff --git a/src/third_party/IntelRDFPMathLib20U1/BUILD.bazel b/src/third_party/IntelRDFPMathLib20U1/BUILD.bazel index 18ec68e798b..5e9adc29d01 100644 --- a/src/third_party/IntelRDFPMathLib20U1/BUILD.bazel +++ b/src/third_party/IntelRDFPMathLib20U1/BUILD.bazel @@ -74,6 +74,11 @@ IGNORE_WARNINGS_COPT = select({ "//conditions:default": ["-w"], }) +DYNAMIC_LOOKUP_LINKOPT = select({ + "@platforms//os:macos": ["-Wl,-undefined,dynamic_lookup"], + "//conditions:default": [""], +}) + INTEL_RDFP_FLOAT128_INTERNAL_HEADERS = [ "LIBRARY/float128/architecture.h", "LIBRARY/float128/assert.h", @@ -132,6 +137,7 @@ mongo_cc_library( "LIBRARY/float128/dpml_log.c", ] + INTEL_RDFP_FLOAT128_INTERNAL_HEADERS, copts = IGNORE_WARNINGS_COPT, + linkopts = DYNAMIC_LOOKUP_LINKOPT, local_defines = INTEL_RDFP_MATHLIB_DEFINES + [ "LOG1P=1", "efi2=1", diff --git a/src/third_party/mozjs/BUILD.bazel b/src/third_party/mozjs/BUILD.bazel index babb383b9b9..64cbff2a05f 100644 --- a/src/third_party/mozjs/BUILD.bazel +++ b/src/third_party/mozjs/BUILD.bazel @@ -47,8 +47,7 @@ PLATFORM_INCLUDES = select({ PLATFORM_COPTS = select({ config: [ - "/FI" if os == "windows" else "-include", - "$(location platform/{}/{}/build/js-confdefs.h)".format(arch, os), + ("/FI" if os == "windows" else "-include") + "$(location platform/{}/{}/build/js-confdefs.h)".format(arch, os), ] + (["-mavx2"] if arch == "x86_64" and os != "windows" else []) for config, (arch, os) in PLATFORMS.items() } | { @@ -297,6 +296,10 @@ mongo_cc_library( "include", "mongo_sources", ] + PLATFORM_INCLUDES, + linkopts = select({ + "@platforms//os:macos": ["-Wl,-undefined,dynamic_lookup"], + "//conditions:default": [""], + }), local_defines = [ "IMPL_MFBT=1", "U_NO_DEFAULT_INCLUDE_UTF_HEADERS=1", diff --git a/src/third_party/unwind/BUILD.bazel b/src/third_party/unwind/BUILD.bazel index 198385625f1..8ba65d25d90 100644 --- a/src/third_party/unwind/BUILD.bazel +++ b/src/third_party/unwind/BUILD.bazel @@ -98,7 +98,7 @@ mongo_cc_library( "dist/src/x86_64/is_fpreg.c", "dist/src/x86_64/regname.c", ], - "@platforms//cpu:ppc64le": [ + "@platforms//cpu:ppc": [ "dist/src/ppc/Lget_proc_info.c", "dist/src/ppc/Lget_save_loc.c", "dist/src/ppc/Linit_local.c", @@ -142,7 +142,7 @@ mongo_cc_library( "dist/src/x86_64/getcontext.S", "dist/src/x86_64/setcontext.S", ], - "@platforms//cpu:ppc64le": [], + "@platforms//cpu:ppc": [], "@platforms//cpu:s390x": [ "dist/src/s390x/getcontext.S", "dist/src/s390x/setcontext.S", @@ -171,7 +171,7 @@ mongo_cc_library( "-Isrc/third_party/unwind/platform/linux_x86_64/build/include", "-Isrc/third_party/unwind/dist/include/tdep-x86_64", ], - "@platforms//cpu:ppc64le": [ + "@platforms//cpu:ppc": [ "-Isrc/third_party/unwind/platform/linux_ppc64le/build/include", "-Isrc/third_party/unwind/dist/include/tdep-ppc64", ], @@ -204,14 +204,10 @@ mongo_cc_library( "no_warn_non_virtual_detour", ], includes = select({ - "@platforms//cpu:{}".format(cpu): ["platform/linux_{}/install/include".format(cpu)] - for cpu in [ - "aarch64", - "x86_64", - "ppc64le", - "s390x", - ] - } | { + "@platforms//cpu:aarch64": ["platform/linux_aarch64/install/include"], + "@platforms//cpu:x86_64": ["platform/linux_x86_64/install/include"], + "@platforms//cpu:ppc": ["platform/linux_ppc64le/install/include"], + "@platforms//cpu:s390x": ["platform/linux_s390x/install/include"], "//conditions:default": [], }), local_defines = [ diff --git a/src/third_party/wiredtiger/BUILD.bazel b/src/third_party/wiredtiger/BUILD.bazel index e38d52f8326..70fba3deb40 100644 --- a/src/third_party/wiredtiger/BUILD.bazel +++ b/src/third_party/wiredtiger/BUILD.bazel @@ -102,7 +102,7 @@ mongo_cc_library( ] + select({ "@platforms//cpu:aarch64": WT_FILELIST_ARM64_HOST, "@platforms//cpu:x86_64": WT_FILELIST_X86_HOST, - "@platforms//cpu:ppc64le": WT_FILELIST_POWERPC_HOST, + "@platforms//cpu:ppc": WT_FILELIST_POWERPC_HOST, "@platforms//cpu:s390x": [ WT_FILELIST_ZSERIES_HOST[0], ":wiredtiger_s390x_checksum_index_1", diff --git a/tools/bazel b/tools/bazel index 405997ce1c2..04844903ae5 100755 --- a/tools/bazel +++ b/tools/bazel @@ -3,6 +3,12 @@ # Whenever Bazel is invoked, it first calls this script setting "BAZEL_REAL" to the path of the real Bazel binary. # Use this file as a wrapper for any logic that should run before bazel itself is executed. +# WARNING : If you run //:compiledb target, you can not print to stdout in this file as it will fail with +# "Bazel aquery failed." because it is reading this files stdout as aquery output + +bazel_real="$BAZEL_REAL" +bazelrc_xcode_lines=() + if [[ -z "${BAZELISK_SKIP_WRAPPER}" ]]; then echo "You're not using Bazelisk, which is recommended for a consistent build environment." >&2 echo "Your version of Bazel may be mismatched with the version intended to be used to build MongoDB." >&2 @@ -18,4 +24,18 @@ if [ -d .git ]; then echo "build --define GIT_COMMIT_HASH=$(git rev-parse HEAD)" > .bazelrc.gitinfo fi -$BAZEL_REAL "$@" +if [[ $OSTYPE == darwin* ]]; then + echo "Running on Apple (darwin), creating .bazelrc for xcode settings." >&2 + xcode_path=$(xcode-select -p) >&2 + xcode_version=$(xcodebuild -version | tail -1 | cut -d " " -f3) >&2 + xcode_build_number=$(/usr/bin/xcodebuild -version 2>/dev/null | tail -1 | cut -d " " -f3) >&2 + + bazelrc_lines+=("startup --host_jvm_args=-Xdock:name=$xcode_path") >&2 + bazelrc_lines+=("build --xcode_version=$xcode_version") >&2 + bazelrc_lines+=("build --repo_env=USE_CLANG_CL=$xcode_version") >&2 + bazelrc_lines+=("build --repo_env=DEVELOPER_DIR=$xcode_path") >&2 +fi + +printf '%s\n' "${bazelrc_xcode_lines[@]}" > .bazelrc.xcode + +exec "$bazel_real" "$@"