From 6982155e4261966c00403a6b1e9d676110b4e289 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 19 Nov 2024 11:35:47 -0500 Subject: [PATCH] SERVER-97283 fix ninja compiledb (#29306) GitOrigin-RevId: c571ee716edb731ab2c0dbb7ab83d4ba56dd82fa --- .gitignore | 2 +- SConstruct | 30 +++++++++++++++++++++--- etc/scons/mongodbtoolchain_v4_clang.vars | 20 ++++++++++------ etc/scons/mongodbtoolchain_v4_gcc.vars | 16 +++++++++---- site_scons/mongo/ninja_bazel_build.py | 3 ++- site_scons/site_tools/integrate_bazel.py | 13 ++++++++-- 6 files changed, 65 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index ee3e365054f..1afb4326771 100644 --- a/.gitignore +++ b/.gitignore @@ -280,7 +280,7 @@ buildifier .bazelrc.gitinfo .bazelrc.workstation .bazelrc.xcode -.bazel_info_for_ninja.txt +*.bazel_info_for_ninja.txt .ninja_last_command_line_targets.txt bazel/coverity/analysis/BUILD.bazel .bazel_include_info.json diff --git a/SConstruct b/SConstruct index 5b8ace76759..86ed70d31d3 100644 --- a/SConstruct +++ b/SConstruct @@ -744,6 +744,31 @@ add_option( help="Name a toolchain root for use with toolchain selection Variables files in etc/scons", ) +if mongo_toolchain_execroot: + bin_dir = os.path.join(mongo_toolchain_execroot, "external/mongo_toolchain/v4/bin") + gcc_path = os.path.dirname( + os.path.realpath(os.path.join(bin_dir, os.readlink(os.path.join(bin_dir, "g++")))) + ) + clang_path = os.path.dirname( + os.path.realpath(os.path.join(bin_dir, os.readlink(os.path.join(bin_dir, "clang++")))) + ) +else: + gcc_path = "" + clang_path = "" + +add_option( + "bazel-toolchain-clang", + default=clang_path, + help="used in Variables files to help find the real bazel toolchain location.", +) + +add_option( + "bazel-toolchain-gcc", + default=gcc_path, + help="used in Variables files to help find the real bazel toolchain location.", +) + + add_option( "msvc-debugging-format", choices=["codeview", "pdb"], @@ -6615,9 +6640,8 @@ elif env.GetOption("build-mongot"): # __NINJA_NO is ninja callback to scons signal, in that case we care about # scons only targets not thin targets. -if env.get("__NINJA_NO") != "1": - env.Tool("integrate_bazel") -else: +env.Tool("integrate_bazel") +if env.get("__NINJA_NO") == "1": env.LoadBazelBuilders() def noop(*args, **kwargs): diff --git a/etc/scons/mongodbtoolchain_v4_clang.vars b/etc/scons/mongodbtoolchain_v4_clang.vars index b67f2872900..3715754bca0 100644 --- a/etc/scons/mongodbtoolchain_v4_clang.vars +++ b/etc/scons/mongodbtoolchain_v4_clang.vars @@ -56,17 +56,23 @@ if not ARGUMENTS.get('CC') and not ARGUMENTS.get("CXX") and platform.machine() ! DWP = os.path.join(local_toolchain_bindir, 'dwp') READELF = os.path.join(local_toolchain_bindir, 'readelf') GDB = os.path.join(local_toolchain_bindir, 'gdb') - + + bazel_toolchain_bin = SCons.Script.Main.GetOption('bazel-toolchain-clang') + if bazel_toolchain_bin: + cxx_realpath = os.path.join(bazel_toolchain_bin, "clang++") + else: + cxx_realpath = CXX + try: - AR = subprocess.check_output([CXX, '-print-prog-name=ar']).decode('utf-8').strip() - AS = subprocess.check_output([CXX, '-print-prog-name=as']).decode('utf-8').strip() - OBJCOPY = subprocess.check_output([CXX, '-print-prog-name=objcopy']).decode('utf-8').strip() - LLVM_SYMBOLIZER = subprocess.check_output([CXX, '-print-prog-name=llvm-symbolizer']).decode('utf-8').strip() + AR = subprocess.check_output([cxx_realpath, '-print-prog-name=ar']).decode('utf-8').strip() + AS = subprocess.check_output([cxx_realpath, '-print-prog-name=as']).decode('utf-8').strip() + OBJCOPY = subprocess.check_output([cxx_realpath, '-print-prog-name=objcopy']).decode('utf-8').strip() + LLVM_SYMBOLIZER = subprocess.check_output([cxx_realpath, '-print-prog-name=llvm-symbolizer']).decode('utf-8').strip() except subprocess.CalledProcessError as e: - print("Failed while invoking toolchain binary " + CXX + ": " + str(e.output)) + print("Failed while invoking toolchain binary " + cxx_realpath + ": " + str(e.output)) SCons.Script.Exit(-1) except OSError as e: - print("Failed to invoke toolchain binary " + CXX + ": " + str(e)) + print("Failed to invoke toolchain binary " + cxx_realpath + ": " + str(e)) SCons.Script.Exit(-1) DWARF_VERSION=4 diff --git a/etc/scons/mongodbtoolchain_v4_gcc.vars b/etc/scons/mongodbtoolchain_v4_gcc.vars index f3e1f76c17d..380049a68b2 100644 --- a/etc/scons/mongodbtoolchain_v4_gcc.vars +++ b/etc/scons/mongodbtoolchain_v4_gcc.vars @@ -56,15 +56,21 @@ if not ARGUMENTS.get('CC') and not ARGUMENTS.get("CXX"): READELF = os.path.join(local_toolchain_bindir, 'readelf') GDB = os.path.join(local_toolchain_bindir, 'gdb') + bazel_toolchain_bin = SCons.Script.Main.GetOption('bazel-toolchain-gcc') + if bazel_toolchain_bin: + cxx_realpath = os.path.join(bazel_toolchain_bin, "g++") + else: + cxx_realpath = CXX + try: - AR = subprocess.check_output([CXX, '-print-prog-name=ar']).decode('utf-8').strip() - AS = subprocess.check_output([CXX, '-print-prog-name=as']).decode('utf-8').strip() - OBJCOPY = subprocess.check_output([CXX, '-print-prog-name=objcopy']).decode('utf-8').strip() + AR = subprocess.check_output([cxx_realpath, '-print-prog-name=ar']).decode('utf-8').strip() + AS = subprocess.check_output([cxx_realpath, '-print-prog-name=as']).decode('utf-8').strip() + OBJCOPY = subprocess.check_output([cxx_realpath, '-print-prog-name=objcopy']).decode('utf-8').strip() except subprocess.CalledProcessError as e: - print('Failed while invoking toolchain binary ' + CXX + ': ' + str(e.output)) + print('Failed while invoking toolchain binary ' + cxx_realpath + ': ' + str(e.output)) SCons.Script.Exit(-1) except OSError as e: - print('Failed to invoke toolchain binary ' + CXX + ': ' + str(e)) + print('Failed to invoke toolchain binary ' + cxx_realpath + ': ' + str(e)) SCons.Script.Exit(-1) DWARF_VERSION=5 diff --git a/site_scons/mongo/ninja_bazel_build.py b/site_scons/mongo/ninja_bazel_build.py index ebfb88915b5..0de548d86eb 100644 --- a/site_scons/mongo/ninja_bazel_build.py +++ b/site_scons/mongo/ninja_bazel_build.py @@ -47,7 +47,8 @@ except OSError as exc: # the specific ninja file ninja_build_info = dict() try: - bazel_info_file = ".bazel_info_for_ninja.txt" + ninja_prefix = args.ninja_file.split(".")[0] + bazel_info_file = f".{ninja_prefix}.bazel_info_for_ninja.txt" with open(bazel_info_file) as f: ninja_build_info = json.load(f) except OSError as exc: diff --git a/site_scons/site_tools/integrate_bazel.py b/site_scons/site_tools/integrate_bazel.py index 9c23be0a93a..00e5db98f39 100644 --- a/site_scons/site_tools/integrate_bazel.py +++ b/site_scons/site_tools/integrate_bazel.py @@ -441,7 +441,10 @@ def bazel_build_thread_func(env, log_dir: str, verbose: bool, ninja_generate: bo os.remove(file) extra_args += ["--build_tag_filters=scons_link_lists"] - bazel_cmd = Globals.bazel_base_build_command + extra_args + ["//src/..."] + if SCons.Script.BUILD_TARGETS == ["compiledb"]: + bazel_cmd = Globals.bazel_base_build_command + extra_args + ["//:compiledb"] + else: + bazel_cmd = Globals.bazel_base_build_command + extra_args + ["//src/..."] if ninja_generate: print("Generating bazel link deps...") @@ -599,7 +602,7 @@ def generate_bazel_info_for_ninja(env: SCons.Environment.Environment) -> None: "CXX": env.get("CXX", ""), "USE_NATIVE_TOOLCHAIN": os.environ.get("USE_NATIVE_TOOLCHAIN"), } - with open(".bazel_info_for_ninja.txt", "w") as f: + with open(f".{env.subst('$NINJA_PREFIX')}.bazel_info_for_ninja.txt", "w") as f: json.dump(ninja_bazel_build_json, f) # we also store the outputs in the env (the passed env is intended to be @@ -616,6 +619,9 @@ def generate_bazel_info_for_ninja(env: SCons.Environment.Environment) -> None: ninja_bazel_outs += [bazel_t["bazel_output"]] ninja_bazel_ins += env.NinjaGetInputs(env.File(scons_t)) + if platform.system() == "Linux" and not os.environ.get("USE_NATIVE_TOOLCHAIN"): + ninja_bazel_outs += [env.get("CC"), env.get("CXX")] + # This is to be used directly by ninja later during generation of the ninja file env["NINJA_BAZEL_OUTPUTS"] = ninja_bazel_outs env["NINJA_BAZEL_INPUTS"] = ninja_bazel_ins @@ -1260,6 +1266,9 @@ def generate(env: SCons.Environment.Environment) -> None: "\\", "/" ) + if env.get("__NINJA_NO") == "1": + return + # ThinTarget builder is a special bazel target and should not be prefixed with Bazel in the builder # name to exclude it from the other BazelBuilder's. This builder excludes any normal builder # mechanisms like scanners or emitters and functions as a pass through for targets which exist