From ae7400c2f3d5818845359665d9f55c903c6695ae Mon Sep 17 00:00:00 2001 From: Zack Winter <3457246+zackwintermdb@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:17:15 -0700 Subject: [PATCH] SERVER-93301 Port version_impl from SCons to Bazel (#27348) GitOrigin-RevId: a4b73b807c55e5c7e2dc9927a4c231cd56da3aff --- .bazelrc | 7 +- .github/CODEOWNERS | 1 + BUILD.bazel | 7 + bazel/config/generate_config_header.bzl | 30 ++- site_scons/libdeps_tool.py | 5 +- site_scons/site_tools/integrate_bazel.py | 2 + src/mongo/util/BUILD.bazel | 53 +++-- src/mongo/util/OWNERS.yml | 3 + src/mongo/util/SConscript | 55 ----- src/mongo/util/version_constants_gen.py | 267 +++++++++++++++++++++++ 10 files changed, 358 insertions(+), 72 deletions(-) create mode 100644 src/mongo/util/version_constants_gen.py diff --git a/.bazelrc b/.bazelrc index ea3fa302582..a6b37a06f10 100644 --- a/.bazelrc +++ b/.bazelrc @@ -192,7 +192,10 @@ common --bes_upload_mode=fully_async # common --remote_download_outputs=toplevel # Default Mongo Version if a version is not specified. -build --define=MONGO_VERSION=8.1.0 +build --define=MONGO_VERSION=8.1.0-alpha + +# Default distmod if not specified. +build --define=MONGO_DISTMOD="" # try to import the bazelrc files if available try-import %workspace%/.bazelrc.local @@ -204,4 +207,4 @@ try-import %workspace%/.bazelrc.evergreen_engflow_creds try-import %workspace%/.bazelrc.evergreen # local default dev settings -try-import %workspace%/.bazelrc.workstation \ No newline at end of file +try-import %workspace%/.bazelrc.workstation diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f57899546d8..71f4a33db2d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1513,6 +1513,7 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot /src/mongo/util/**/tcmalloc* @10gen/server-workload-scheduling @svc-auto-approve-bot /src/mongo/util/**/log_and_backoff* @10gen/server-networking-and-observability @svc-auto-approve-bot /src/mongo/util/**/read_through_cache* @10gen/server-catalog-and-routing @svc-auto-approve-bot +/src/mongo/util/**/version_constants_gen.py @10gen/devprod-build @svc-auto-approve-bot # The following patterns are parsed from ./src/mongo/util/cmdline_utils/OWNERS.yml /src/mongo/util/cmdline_utils/**/* @10gen/server-security @svc-auto-approve-bot diff --git a/BUILD.bazel b/BUILD.bazel index 0bd697457af..8718e593649 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -7,8 +7,15 @@ exports_files([ "pyproject.toml", "poetry.lock", "symbols.orderfile", + ".git/HEAD", ]) +# glob to allow this to be empty +filegroup( + name = "git_head_optional", + srcs = glob([".git/HEAD"]), +) + npm_link_all_packages(name = "node_modules") alias( diff --git a/bazel/config/generate_config_header.bzl b/bazel/config/generate_config_header.bzl index b98e1e58833..d878a3e6d63 100644 --- a/bazel/config/generate_config_header.bzl +++ b/bazel/config/generate_config_header.bzl @@ -23,12 +23,31 @@ def generate_config_header_impl(ctx): action_name = ACTION_NAMES.cpp_compile, variables = compile_variables, ) + link_flags = cc_common.get_memory_inefficient_command_line( + feature_configuration = feature_configuration, + action_name = ACTION_NAMES.cpp_link_executable, + variables = compile_variables, + ) env_flags = cc_common.get_environment_variables( feature_configuration = feature_configuration, action_name = ACTION_NAMES.cpp_compile, variables = compile_variables, ) + expanded_extra_definitions = {} + for key, val in ctx.attr.extra_definitions.items(): + # Bazel throws an error if you try to call this on a location var + if "$(location" not in val: + expanded_extra_definitions |= { + key: ctx.expand_make_variables("generate_config_header_expand", val, ctx.var), + } + + expanded_extra_definitions |= { + "compile_variables": " ".join(compiler_flags + ctx.attr.cpp_opts), + "linkflags": " ".join(link_flags + ctx.attr.cpp_linkflags), + "cpp_defines": " ".join(ctx.attr.cpp_defines), + } + python = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"].py3_runtime generator_script = ctx.attr.generator_script.files.to_list()[0].path @@ -66,7 +85,7 @@ def generate_config_header_impl(ctx): "--compiler-path", compiler_bin, "--extra-definitions", - json.encode(ctx.attr.extra_definitions), + json.encode(expanded_extra_definitions), ] + additional_inputs + [ @@ -106,6 +125,15 @@ generate_config_header = rule( doc = "Additional inputs to this rule.", allow_files = True, ), + "cpp_linkflags": attr.string_list( + doc = "C++ linkflags.", + ), + "cpp_opts": attr.string_list( + doc = "C++ opts.", + ), + "cpp_defines": attr.string_list( + doc = "C++ defines.", + ), "generator_script": attr.label( doc = "The python generator script to use.", default = "//bazel/config:generate_config_header.py", diff --git a/site_scons/libdeps_tool.py b/site_scons/libdeps_tool.py index 3536f3b46c0..4315eb83b47 100644 --- a/site_scons/libdeps_tool.py +++ b/site_scons/libdeps_tool.py @@ -1496,7 +1496,10 @@ def generate_libdeps_graph(env): graph_hash, [env.File("#SConstruct")] + glob.glob("**/SConscript", recursive=True) - + [os.path.abspath(__file__), env.File("$BUILD_DIR/mongo/util/version_constants.h")], + + [ + os.path.abspath(__file__), + env.File("$BAZEL_OUT_DIR/src/mongo/util/version_constants.h"), + ], ) graph_node = env.Command( diff --git a/site_scons/site_tools/integrate_bazel.py b/site_scons/site_tools/integrate_bazel.py index abbd37a4525..8aa0f3acb5d 100644 --- a/site_scons/site_tools/integrate_bazel.py +++ b/site_scons/site_tools/integrate_bazel.py @@ -1004,6 +1004,8 @@ def generate(env: SCons.Environment.Environment) -> None: f'--use_sasl_client={env.GetOption("use-sasl-client") is not None}', "--define", f"MONGO_VERSION={env['MONGO_VERSION']}", + "--define", + f"MONGO_DISTMOD={env['MONGO_DISTMOD']}", "--compilation_mode=dbg", # always build this compilation mode as we always build with -g ] diff --git a/src/mongo/util/BUILD.bazel b/src/mongo/util/BUILD.bazel index ade86cf8de2..704809b8622 100644 --- a/src/mongo/util/BUILD.bazel +++ b/src/mongo/util/BUILD.bazel @@ -1,5 +1,6 @@ -load("//bazel:mongo_src_rules.bzl", "idl_generator", "mongo_cc_library") +load("//bazel:mongo_src_rules.bzl", "MONGO_GLOBAL_COPTS", "MONGO_GLOBAL_DEFINES", "MONGO_GLOBAL_LINKFLAGS", "idl_generator", "mongo_cc_library") load("//bazel/config:render_template.bzl", "render_template") +load("//bazel/config:generate_config_header.bzl", "generate_config_header") package(default_visibility = ["//visibility:public"]) @@ -680,18 +681,18 @@ mongo_cc_library( ], ) -# mongo_cc_library( -# name = "version_impl", -# srcs = [ -# "version_impl.cpp", -# ], -# hdrs = [ -# "version_constants.h", -# ], -# deps = [ -# "//src/mongo:base", -# ], -# ) +mongo_cc_library( + name = "version_impl", + srcs = [ + "version_impl.cpp", + ], + hdrs = [ + "version_constants.h", + ], + deps = [ + "//src/mongo:base", + ], +) mongo_cc_library( name = "tcmalloc_set_parameter", @@ -770,3 +771,29 @@ mongo_cc_library( ], }), ) + +generate_config_header( + name = "version_constants_gen", + additional_inputs = ["//:git_head_optional"], + checks = ":version_constants_gen.py", + cpp_defines = MONGO_GLOBAL_DEFINES, + cpp_linkflags = MONGO_GLOBAL_LINKFLAGS, + cpp_opts = MONGO_GLOBAL_COPTS, + extra_definitions = { + "MONGO_DISTMOD": "$(MONGO_DISTMOD)", + "MONGO_VERSION": "$(MONGO_VERSION)", + } | select({ + "//bazel/config:js_engine_mozjs": {"js_engine_ver": "mozjs"}, + "//conditions:default": {"js_engine_ver": "none"}, + }) | select({ + "//bazel/config:tcmalloc_google_enabled": {"MONGO_ALLOCATOR": "tcmalloc-google"}, + "//bazel/config:tcmalloc_gperf_enabled": {"MONGO_ALLOCATOR": "tcmalloc-gperf"}, + "//conditions:default": {"MONGO_ALLOCATOR": "system"}, + }) | select({ + "//bazel/config:build_enterprise_enabled": {"build_enterprise_enabled": "1"}, + "//conditions:default": {}, + }), + logfile = "version_constants_gen.log", + output = "version_constants.h", + template = "version_constants.h.in", +) diff --git a/src/mongo/util/OWNERS.yml b/src/mongo/util/OWNERS.yml index b4bf1d64c80..21f85b5392b 100644 --- a/src/mongo/util/OWNERS.yml +++ b/src/mongo/util/OWNERS.yml @@ -54,3 +54,6 @@ filters: - "read_through_cache*": approvers: - 10gen/server-catalog-and-routing + - "version_constants_gen.py": + approvers: + - 10gen/devprod-build diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index 156c5d52062..ffdbbc363ec 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -19,51 +19,6 @@ env = env.Clone() env.InjectThirdParty("asio") -js_engine_ver = get_option("js-engine") if get_option("server-js") == "on" else "none" - -module_list = ",\n".join(['"{0}"_sd'.format(x) for x in env["MONGO_MODULES"]]) - - -# Render the MONGO_BUILDINFO_ENVIRONMENT_DATA dict into an initializer for a -# `std::vector`. -def fmtBuildInfo(data): - def fmtBool(val): - return "true" if val else "false" - - def fmtStr(val): - return 'R"({0})"_sd'.format(val.replace("\\", r"\\")) - - def fmtObj(obj): - return "{{{}, {}, {}, {}}}".format( - fmtStr(obj["key"]), - fmtStr(env.subst(obj["value"])), - fmtBool(obj["inBuildInfo"]), - fmtBool(obj["inVersion"]), - ) - - return ",\n".join([fmtObj(obj) for _, obj in data.items()]) - - -buildInfoInitializer = fmtBuildInfo(env["MONGO_BUILDINFO_ENVIRONMENT_DATA"]) - -generatedVersionFile = env.Substfile( - "version_constants.h.in", - SUBST_DICT=[ - ("@mongo_version@", env["MONGO_VERSION"]), - ("@mongo_version_major@", version_parts[0]), - ("@mongo_version_minor@", version_parts[1]), - ("@mongo_version_patch@", version_parts[2]), - ("@mongo_version_extra@", version_parts[3]), - ("@mongo_version_extra_str@", version_extra), - ("@mongo_git_hash@", env["MONGO_GIT_HASH"]), - ("@buildinfo_js_engine@", js_engine_ver), - ("@buildinfo_allocator@", env["MONGO_ALLOCATOR"]), - ("@buildinfo_modules@", module_list), - ("@buildinfo_environment_data@", buildInfoInitializer), - ], -) -env.Alias("generated-sources", generatedVersionFile) - if env.TargetOSIs("windows"): enterpriseEnv = env.Clone().InjectModule("enterprise") generatedResourceConstantFile = enterpriseEnv.Substfile( @@ -95,16 +50,6 @@ env.SConscript( ], ) -env.Library( - target="version_impl", - source=[ - "version_impl.cpp", - ], - LIBDEPS=[ - "$BUILD_DIR/mongo/base", - ], -) - env.Benchmark( target="fail_point_bm", source=[ diff --git a/src/mongo/util/version_constants_gen.py b/src/mongo/util/version_constants_gen.py new file mode 100644 index 00000000000..284f91b8f54 --- /dev/null +++ b/src/mongo/util/version_constants_gen.py @@ -0,0 +1,267 @@ +# Generates "version_constants.h" config header file containing feature flags generated by checking for the availability of certain compiler features. +# This script is invoked by the Bazel build system to generate the "version_constants.h" file automatically as part of the build. +import json +import os +import platform +import re +import subprocess +import threading +from typing import Dict + + +# This function gets the running OS as identified by Python +# It should only be used to set up defaults for options/variables, because +# its value could potentially be overridden by setting TARGET_OS on the +# command-line. Treat this output as the value of HOST_OS +def get_running_os_name(): + running_os = os.sys.platform + if running_os.startswith("linux"): + running_os = "linux" + elif running_os.startswith("freebsd"): + running_os = "freebsd" + elif running_os.startswith("openbsd"): + running_os = "openbsd" + elif running_os == "sunos5": + running_os = "solaris" + elif running_os == "win32": + running_os = "windows" + elif running_os == "darwin": + running_os = "macOS" + else: + running_os = "unknown" + return running_os + + +def tool_to_path(tool, compiler_path): + if tool == "CC": + return compiler_path + elif tool == "CXX": + if os.path.basename(compiler_path) == "gcc": + return os.path.join(os.path.dirname(compiler_path), "g++") + elif os.path.basename(compiler_path) == "clang": + return os.path.join(os.path.dirname(compiler_path), "clang++") + else: + return + + +def get_toolchain_ver(tool, compiler_path): + # By default we don't know the version of each tool, and only report what + # command gets executed (gcc vs /opt/mongodbtoolchain/bin/gcc). + verstr = "version unknown" + proc = None + tool = tool_to_path(tool, compiler_path) + if compiler_path.endswith("gcc") or compiler_path.endswith("clang"): + proc = subprocess.run( + f"{tool} --version", + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + universal_newlines=True, + check=True, + shell=True, + text=True, + ) + verstr = proc.stdout + elif compiler_path.endswith("cl"): + proc = subprocess.run( + tool, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + stdin=subprocess.DEVNULL, + universal_newlines=True, + check=True, + shell=True, + text=True, + ) + verstr = proc.stderr + + return f"{tool}: {verstr}" + + +# This is the key/value mapping that will be returned by the buildInfo command and +# printed by the --version command-line option to mongod. +# Each mapped value is in turn a dict consisting of: +# key: +# value: +# inBuildInfo: : should it be included in buildInfo output +# inVersion: : should it be included in --version output +# The `value` field will be passed through env.subst, so you can use any SCons variables you +# want to define them. +def default_buildinfo_environment_data(compiler_path, extra_definitions): + data = ( + ( + "distmod", + extra_definitions["MONGO_DISTMOD"], + True, + True, + ), + ( + "distarch", + platform.machine().lower(), + True, + True, + ), + ( + "cc", + get_toolchain_ver("CC", compiler_path), + True, + False, + ), + ( + "ccflags", + extra_definitions["compile_variables"], + True, + False, + ), + ( + "cxx", + get_toolchain_ver("CXX", compiler_path), + True, + False, + ), + ( + "cxxflags", + extra_definitions["compile_variables"], + True, + False, + ), + ( + "linkflags", + extra_definitions["linkflags"], + True, + False, + ), + ( + "target_arch", + platform.machine().lower(), + True, + True, + ), + ( + "target_os", + get_running_os_name(), + True, + False, + ), + ( + "cppdefines", + extra_definitions["cpp_defines"], + True, + False, + ), + ) + return { + k: {"key": k, "value": v, "inBuildInfo": ibi, "inVersion": iv} for k, v, ibi, iv in data + } + + +# Render the MONGO_BUILDINFO_ENVIRONMENT_DATA dict into an initializer for a +# `std::vector`. +def fmt_build_info(data): + def fmt_bool(val): + return "true" if val else "false" + + def fmt_str(val): + return 'R"({0})"_sd'.format(val.replace("\\", r"\\")) + + def fmt_obj(obj): + return "{{{}, {}, {}, {}}}".format( + fmt_str(obj["key"]), + fmt_str(obj["value"]), + fmt_bool(obj["inBuildInfo"]), + fmt_bool(obj["inVersion"]), + ) + + return ",\n".join([fmt_obj(obj) for _, obj in data.items()]) + + +def get_git_version(): + """Return the git version.""" + if not os.path.exists(".git") or not os.path.isdir(".git"): + return "nogitversion" + + version = open(".git/HEAD", "r").read().strip() + if not version.startswith("ref: "): + return version + version = version[5:] + git_ver = ".git/" + version + if not os.path.exists(git_ver): + return version + return open(git_ver, "r").read().strip() + + +logfile_path: str = "" +loglock = threading.Lock() + + +def log_check(message): + global loglock, logfile_path + with loglock: + with open(logfile_path, "a") as f: + f.write(message + "\n") + + +def generate_config_header( + compiler_path, compiler_args, env_vars, logpath, additional_inputs, extra_definitions={} +) -> Dict[str, str]: + global logfile_path + logfile_path = logpath + + log_check("") + extra_definitions_dict = json.loads(extra_definitions) + buildInfoInitializer = fmt_build_info( + default_buildinfo_environment_data(compiler_path, extra_definitions_dict) + ) + + # This generates a numeric representation of the version string so that + # you can easily compare versions of MongoDB without having to parse + # the version string. + # + # Examples: + # 5.1.1-123 => ['5', '1', '1', '123', None, None] => [5, 1, 2, -100] + # 5.1.1-rc2 => ['5', '1', '1', 'rc2', 'rc', '2'] => [5, 1, 1, -23] + # 5.1.1-rc2-123 => ['5', '1', '1', 'rc2-123', 'rc', '2'] => [5, 1, 1, -23] + # 5.1.0-alpha-123 => ['5', '1', '0', 'alpha-123', 'alpha', ''] => [5, 1, 0, -50] + # 5.1.0-alpha1-123 => ['5', '1', '0', 'alpha1-123', 'alpha', '1'] => [5, 1, 0, -49] + # 5.1.1 => ['5', '1', '1', '', None, None] => [5, 1, 1, 0] + + version_parts = [ + x + for x in re.match( + r"^(\d+)\.(\d+)\.(\d+)-?((?:(rc|alpha)(\d?))?.*)?", + extra_definitions_dict["MONGO_VERSION"], + ).groups() + ] + version_extra = version_parts[3] if version_parts[3] else "" + if version_parts[4] == "rc": + version_parts[3] = int(version_parts[5]) + -25 + elif version_parts[4] == "alpha": + if version_parts[5] == "": + version_parts[3] = -50 + else: + version_parts[3] = int(version_parts[5]) + -50 + elif version_parts[3]: + version_parts[2] = int(version_parts[2]) + 1 + version_parts[3] = -100 + else: + version_parts[3] = 0 + version_parts = [int(x) for x in version_parts[:4]] + + modules = ["enterprise"] if "build_enterprise_enabled" in extra_definitions_dict else [] + module_list = ",\n".join(['"{0}"_sd'.format(x) for x in modules]) + + replacements = { + "@mongo_version@": extra_definitions_dict["MONGO_VERSION"], + "@mongo_version_major@": str(version_parts[0]), + "@mongo_version_minor@": str(version_parts[1]), + "@mongo_version_patch@": str(version_parts[2]), + "@mongo_version_extra@": str(version_parts[3]), + "@mongo_version_extra_str@": version_extra, + "@mongo_git_hash@": get_git_version(), + "@buildinfo_js_engine@": extra_definitions_dict["js_engine_ver"], + "@buildinfo_allocator@": extra_definitions_dict["MONGO_ALLOCATOR"], + "@buildinfo_modules@": module_list, + "@buildinfo_environment_data@": buildInfoInitializer, + } + + return replacements