mongodb/site_scons/mongo/ninja_bazel_build.py

import argparse
import glob
import json
import os
import shutil
import subprocess
import sys
import tempfile

parser = argparse.ArgumentParser(description="Ninja Bazel builder.")

parser.add_argument("--ninja-file", type=str, help="The ninja file in use", default="build.ninja")
parser.add_argument("--verbose", action="store_true", help="Turn on verbose mode")
parser.add_argument(
    "--integration-debug",
    action="store_true",
    help="Turn on extra debug output about the ninja-bazel integration",
)

args = parser.parse_args()

# This corresponds to BAZEL_INTEGRATION_DEBUG=1 from SCons command line
if args.integration_debug:

    def print_debug(msg):
        print("[BAZEL_INTEGRATION_DEBUG] " + msg)
else:

    def print_debug(msg):
        pass


# our ninja python module intercepts the command lines and
# prints out the targets everytime ninja is executed
ninja_command_line_targets = []
try:
    ninja_last_cmd_file = ".ninja_last_command_line_targets.txt"
    with open(ninja_last_cmd_file) as f:
        ninja_command_line_targets = [target.strip() for target in f.readlines() if target.strip()]
except OSError as exc:
    print(
        f"Failed to open {ninja_last_cmd_file}, this is expected to be generated on ninja execution by the mongo-ninja-python module."
    )
    raise exc


# Our ninja generation process generates all the build info related to
# the specific ninja file
ninja_build_info = dict()
try:
    ninja_prefix = args.ninja_file.split(".")[0]
    bazel_info_file = f".{ninja_prefix}.bazel_info_for_ninja.txt"
    with open(bazel_info_file) as f:
        ninja_build_info = json.load(f)
except OSError as exc:
    print(
        f"Failed to open {bazel_info_file}, this is expected to be generated by scons during ninja generation."
    )
    raise exc


# flip the targets map for optimized use later
bazel_out_to_bazel_target = dict()
for bazel_t in ninja_build_info["targets"].values():
    bazel_out_to_bazel_target[bazel_t["bazel_output"]] = bazel_t["bazel_target"]

# run ninja and get the deps from the passed command line targets so we can check if any deps are bazel targets
ninja_inputs_cmd = ["ninja", "-f", args.ninja_file, "-t", "inputs"] + ninja_command_line_targets
print_debug(f"NINJA GET INPUTS CMD: {' '.join(ninja_inputs_cmd)}")

ninja_proc = subprocess.run(ninja_inputs_cmd, capture_output=True, text=True, check=True)
deps = [dep.replace("\\", "/") for dep in ninja_proc.stdout.split("\n") if dep]
print_debug(f"COMMAND LINE DEPS:{os.linesep}{os.linesep.join(deps)}")
os.unlink(ninja_last_cmd_file)

# isolate just the raw output files for the list intersection
bazel_outputs = [bazel_t["bazel_output"] for bazel_t in ninja_build_info["targets"].values()]
print_debug(f"BAZEL OUTPUTS:{os.linesep}{os.linesep.join(bazel_outputs)}")

# now out of possible bazel outputs find which are deps of the requested command line targets
outputs_to_build = list(set(deps).intersection(bazel_outputs))
print_debug(f"BAZEL OUTPUTS TO BUILD: {outputs_to_build}")

# convert from outputs (raw files) to bazel targets (bazel labels i.e //src/db/mongo:target)
targets_to_build = [bazel_out_to_bazel_target[out] for out in outputs_to_build]

if (
    not targets_to_build
    and "compiledb" not in ninja_command_line_targets
    and "compile_commands.json" not in ninja_command_line_targets
):
    print(
        "WARNING: Did not resolve any bazel specific targets to build, this might not be correct."
    )

list_files = glob.glob("bazel-out/**/*.gen_source_list", recursive=True)
gen_source_targets = []
for list_file in list_files:
    with open(list_file) as f:
        gen_source_targets.append(f.read().strip())
targets_to_build += gen_source_targets

# ninja will automatically create directories for any outputs, but in this case
# bazel will be creating a symlink for the bazel-out dir to its cache. We don't want
# ninja to interfere so delete the dir if it was not a link (made by bazel)
if sys.platform == "win32":
    if os.path.exists("bazel-out"):
        try:
            os.readlink("bazel-out")
        except OSError:
            shutil.rmtree("bazel-out")

else:
    if not os.path.islink("bazel-out"):
        shutil.rmtree("bazel-out")

env_flags = os.environ.get("BAZEL_FLAGS", [])
if env_flags:
    print(f"Using shell env BAZEL_FLAGS: {' '.join(env_flags)}")

if args.verbose:
    extra_args = []
else:
    extra_args = ["--output_filter=DONT_MATCH_ANYTHING"]

extra_args += env_flags

bazel_env = os.environ.copy()
if ninja_build_info.get("USE_NATIVE_TOOLCHAIN"):
    bazel_env["CC"] = ninja_build_info.get("CC")
    bazel_env["CXX"] = ninja_build_info.get("CXX")
    bazel_env["USE_NATIVE_TOOLCHAIN"] = "1"

with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
    tf_name = tf.name
    tpf = f"--target_pattern_file={tf_name}"
    extra_args += [tpf]
    bazel_cmd = " ".join(ninja_build_info["bazel_cmd"] + extra_args)
    sys.stderr.write(f"Running bazel command:\n{bazel_cmd} [{len(targets_to_build)} targets...]\n")
    tf.write("\n".join(targets_to_build))
    tf.close()
    bazel_proc = subprocess.run(
        ninja_build_info["bazel_cmd"] + extra_args,
        env=bazel_env,
    )

if bazel_proc.returncode != 0:
    print("Command that failed:")
    print(bazel_cmd)
    sys.exit(1)
else:
    os.remove(tf_name)
if (
    "compiledb" in ninja_command_line_targets
    or "compile_commands.json" in ninja_command_line_targets
):
    bazel_proc = subprocess.run(ninja_build_info["compiledb_cmd"], env=bazel_env)
    if bazel_proc.returncode != 0:
        print("Command that failed:")
        print(" ".join(ninja_build_info["compiledb_cmd"]))
        sys.exit(1)