diff --git a/SConstruct b/SConstruct index 78f34c326ee..c8762dc45c9 100644 --- a/SConstruct +++ b/SConstruct @@ -198,8 +198,9 @@ add_option( add_option( "bazel-includes-info", + action="append", help="write included headers in bazel label format to put files ([library].bazel_includes)", - default="", + default=[], ) add_option( diff --git a/buildscripts/convert_bazel_headers.py b/buildscripts/convert_bazel_headers.py index 113d961b765..1f68af446e9 100644 --- a/buildscripts/convert_bazel_headers.py +++ b/buildscripts/convert_bazel_headers.py @@ -8,61 +8,12 @@ import concurrent.futures import glob import traceback import shutil -from typing import Annotated +from typing import Annotated, List import typer -def main( - target_library: Annotated[str, typer.Option()], - silent: Annotated[bool, typer.Option()] = False, - skip_scons: Annotated[bool, typer.Option()] = False, -): - extra_args = [] - if os.name == "nt": - extra_args += [ - "CPPPATH=C:\sasl\include", - "LIBPATH=C:\sasl\lib", - ] - target_library = os.path.join( - os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib" - ) - - if platform.system() == "Darwin": - target_library = target_library[:-2] + "a" - - path = shutil.which("icecc") - if path is None: - extra_args += ["ICECC="] - - cmd = [ - sys.executable, - "buildscripts/scons.py", - "--build-profile=opt", - f"--bazel-includes-info={target_library}", - "--libdeps-linting=off", - "--ninja=disabled", - "compiledb", - ] + extra_args - - if not skip_scons: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - - while True: - line = p.stdout.readline() - if not line: - break - print(line.strip(), file=sys.stderr) - - _, _ = p.communicate() - - if p.returncode != 0: - print(f"SCons build failed, exit code {p.returncode}", file=sys.stderr) - sys.exit(1) - - with open("compile_commands.json") as f: - cc = json.load(f) - +def work(target_library: str, silent: bool, cpu_count: int, cc: List[str]): headers = set() original_headers = set() @@ -156,10 +107,6 @@ def main( if line.endswith("_gen.cpp"): line = line[:-4] sources.append(line) - if platform.system() == "Linux": - cpu_count = len(os.sched_getaffinity(0)) + 4 - else: - cpu_count = os.cpu_count() + 4 with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor: jobs = {executor.submit(get_headers, line.strip()): line.strip() for line in lines} @@ -282,29 +229,106 @@ def main( target_name = target_name[3:] local_bazel_path = os.path.dirname(target_library.replace("build/opt", "//src")) + ":" - print("mongo_cc_library(") - print(f' name = "{target_name}",') + bazel_target = "mongo_cc_library(\n" + bazel_target += f' name = "{target_name}",\n' if sources: - print(f" srcs = [") + bazel_target += " srcs = [\n" for src in sources: - print(f' "{src.replace(local_bazel_path, "")}",') - print(" ],") + bazel_target += f' "{src.replace(local_bazel_path, "")}",\n' + bazel_target += " ],\n" if minimal_headers: - print(" hdrs = [") + bazel_target += " hdrs = [\n" for header in sorted(minimal_headers): - print(f' "{header.replace(local_bazel_path, "")}",') - print(" ],") + bazel_target += f' "{header.replace(local_bazel_path, "")}",\n' + bazel_target += " ],\n" if header_deps: - print(" header_deps = [") + bazel_target += " header_deps = [\n" for dep in sorted(header_deps): - print(f' "{dep.strip().replace(local_bazel_path, "")}",') - print(" ],") + bazel_target += f' "{dep.strip().replace(local_bazel_path, "")}",\n' + bazel_target += " ],\n" if link_deps: - print(" deps = [") + bazel_target += " deps = [\n" for dep in sorted(link_deps): - print(f' "{dep.strip().replace(local_bazel_path, "")}",') - print(" ],") - print(")") + bazel_target += f' "{dep.strip().replace(local_bazel_path, "")}",\n' + bazel_target += " ],\n" + bazel_target += ")\n" + return bazel_target + + +def main( + target_libraries: Annotated[List[str], typer.Argument()], + silent: Annotated[bool, typer.Option()] = False, + skip_scons: Annotated[bool, typer.Option()] = False, +): + extra_args = [] + if os.name == "nt": + extra_args += [ + "CPPPATH=C:\sasl\include", + "LIBPATH=C:\sasl\lib", + ] + target_library = os.path.join( + os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib" + ) + + path = shutil.which("icecc") + if path is None: + extra_args += ["ICECC="] + + if os.name == "nt": + target_fmt = lambda target_library: os.path.join( + os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib" + ) + elif platform.system() == "Darwin": + target_fmt = lambda target_library: target_library[:-2] + "a" + else: + target_fmt = lambda x: None + + map(target_fmt, target_libraries) + + cmd = [ + sys.executable, + "buildscripts/scons.py", + "--build-profile=opt", + " ".join( + [f"--bazel-includes-info={target_library}" for target_library in target_libraries] + ), + "--libdeps-linting=off", + "--ninja=disabled", + "compiledb", + ] + extra_args + + if not skip_scons: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + while True: + line = p.stdout.readline() + if not line: + break + print(line.strip(), file=sys.stderr) + + _, _ = p.communicate() + + if p.returncode != 0: + print(f"SCons build failed, exit code {p.returncode}", file=sys.stderr) + sys.exit(1) + + with open("compile_commands.json") as f: + cc = json.load(f) + + if platform.system() == "Linux": + cpu_count = len(os.sched_getaffinity(0)) + 4 + else: + cpu_count = os.cpu_count() + 4 + + with concurrent.futures.ProcessPoolExecutor(max_workers=cpu_count) as executor: + jobs = { + executor.submit(work, target_library, silent, cpu_count, cc): target_library + for target_library in target_libraries + } + bazel_targets = [job.result() for job in concurrent.futures.as_completed(jobs)] + + print("====== Bazel Targets ======\n") + print("\n".join(bazel_targets)) if __name__ == "__main__": diff --git a/site_scons/site_tools/bazel_includes_info.py b/site_scons/site_tools/bazel_includes_info.py index 82bc59476df..02a6245a303 100644 --- a/site_scons/site_tools/bazel_includes_info.py +++ b/site_scons/site_tools/bazel_includes_info.py @@ -6,6 +6,7 @@ from pathlib import Path import SCons import libdeps_tool +from functools import partial def exists(env): @@ -154,13 +155,12 @@ def add_headers_from_gen_code(env, header_map): ] -def bazel_includes_emitter(target, source, env): - target_library = env.GetOption("bazel-includes-info").replace("\\", "/") +def bazel_includes_emitter(target_libraries, target, source, env): rel_target = os.path.relpath(str(target[0].abspath), start=env.Dir("#").abspath).replace( "\\", "/" ) - if rel_target == target_library: + if rel_target in target_libraries: objsuffix = ( env.subst("$OBJSUFFIX") if not env.TargetOSIs("linux") else env.subst("$SHOBJSUFFIX") ) @@ -236,6 +236,10 @@ def generate(env): add_headers_from_all_libraries(env, header_map) gen_header_map = {} add_headers_from_gen_code(env, gen_header_map) + target_libraries = { + target_library.split("=")[-1].replace("\\", "/") + for target_library in env.GetOption("bazel-includes-info")[0].split() + } bazel_include_info = { "header_map": header_map, @@ -250,5 +254,7 @@ def generate(env): for builder_name in ["SharedLibrary", "StaticLibrary", "Program"]: builder = env["BUILDERS"][builder_name] base_emitter = builder.emitter - new_emitter = SCons.Builder.ListEmitter([base_emitter, bazel_includes_emitter]) + new_emitter = SCons.Builder.ListEmitter( + [base_emitter, partial(bazel_includes_emitter, target_libraries)] + ) builder.emitter = new_emitter