mirror of
https://github.com/mongodb/mongo.git
synced 2024-11-24 00:17:37 +01:00
381 lines
14 KiB
Python
381 lines
14 KiB
Python
import concurrent.futures
|
|
import json
|
|
import os
|
|
import platform
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import traceback
|
|
from typing import Annotated, List
|
|
|
|
import typer
|
|
|
|
|
|
def work(target_library: str, silent: bool, cpu_count: int, cc: List[str]):
|
|
headers = set()
|
|
original_headers = set()
|
|
|
|
def get_headers(line):
|
|
nonlocal headers
|
|
try:
|
|
with open(target_library + ".bazel_headers") as f:
|
|
bazel_headers = [line.strip() for line in f.readlines()]
|
|
bazel_headers += [
|
|
"src/mongo/platform/basic.h",
|
|
"src/mongo/platform/windows_basic.h",
|
|
]
|
|
|
|
with open(target_library + ".env_vars") as f:
|
|
tmp_env_vars = json.load(f)
|
|
env_vars = {}
|
|
# subprocess requies only strings
|
|
for k, v in tmp_env_vars.items():
|
|
env_vars[str(k)] = str(v)
|
|
|
|
for command in cc:
|
|
cmd_output = command["output"].replace("\\", "/").strip("'").strip('"')
|
|
line_output = line.replace("\\", "/")
|
|
|
|
if cmd_output == line_output:
|
|
os.makedirs(os.path.dirname(line), exist_ok=True)
|
|
if os.name == "nt":
|
|
header_arg = " /showIncludes"
|
|
else:
|
|
header_arg = " -H"
|
|
|
|
if not silent:
|
|
print(f"compiling {line}")
|
|
|
|
p = subprocess.run(
|
|
shlex.split((command["command"].replace("\\", "/") + header_arg)),
|
|
env=env_vars,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if p.returncode != 0:
|
|
print(f"Error compiling, exitcode: {p.returncode}", file=sys.stderr)
|
|
print(f"STDOUT: {p.stdout}", file=sys.stderr)
|
|
print(f"STDERR: {p.stderr}", file=sys.stderr)
|
|
sys.exit(1)
|
|
if os.name == "nt":
|
|
for line in p.stdout.split("\n"):
|
|
line = (
|
|
line.replace("Note: including file:", "")
|
|
.strip(" ")
|
|
.replace("\\", "/")
|
|
)
|
|
|
|
if not line.startswith(os.getcwd().replace("\\", "/")):
|
|
continue
|
|
|
|
line = os.path.relpath(
|
|
line, start=os.getcwd().replace("\\", "/")
|
|
).replace("\\", "/")
|
|
if line not in bazel_headers:
|
|
if line.startswith("src/") or line.startswith("bazel-out/"):
|
|
original_headers.add(line)
|
|
line = "//" + line
|
|
line = ":".join(line.rsplit("/", 1))
|
|
|
|
headers.add(line)
|
|
else:
|
|
for line in p.stderr.split("\n"):
|
|
if ". src/" in line or ". bazel-out/" in line:
|
|
while line.startswith("."):
|
|
line = line[1:]
|
|
line = line.replace("\\", "/")
|
|
|
|
if line[1:] not in bazel_headers:
|
|
original_headers.add(line[1:])
|
|
line = "//" + line[1:]
|
|
line = ":".join(line.rsplit("/", 1))
|
|
|
|
headers.add(line)
|
|
except Exception as exc:
|
|
print(traceback.format_exc(), file=sys.stderr)
|
|
raise exc
|
|
|
|
sources = []
|
|
with open(target_library + ".obj_files") as f:
|
|
lines = f.readlines()
|
|
for line in lines:
|
|
line = line.strip()
|
|
line = line.replace("build/opt", "//src")
|
|
line = line[: line.find(".")] + ".cpp"
|
|
src_header = os.path.splitext(line[2:])[0] + ".h"
|
|
if os.path.exists(src_header):
|
|
src_header = "//" + ":".join(src_header.rsplit("/", 1))
|
|
headers.add(src_header)
|
|
line = ":".join(line.rsplit("/", 1))
|
|
if line.endswith("_gen.cpp"):
|
|
line = line[:-4]
|
|
sources.append(line)
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor:
|
|
jobs = {executor.submit(get_headers, line.strip()): line.strip() for line in lines}
|
|
for completed_job in concurrent.futures.as_completed(jobs):
|
|
if not silent:
|
|
print(f"finished {jobs[completed_job]}")
|
|
|
|
with open(".bazel_include_info.json") as f:
|
|
bazel_include_info = json.load(f)
|
|
|
|
header_map = bazel_include_info["header_map"]
|
|
gen_header_map = bazel_include_info["gen_header_map"]
|
|
bazel_exec = bazel_include_info["bazel_exec"]
|
|
bazel_config = bazel_include_info["config"]
|
|
|
|
global_headers = (
|
|
"src/mongo:config.h",
|
|
"src/mongo/config.h",
|
|
"src/mongo/platform/basic.h",
|
|
"src/mongo/platform/windows_basic.h",
|
|
)
|
|
|
|
reverse_header_map = {}
|
|
for k, v in header_map.items():
|
|
for hdr in v:
|
|
if not hdr or hdr.endswith(global_headers):
|
|
continue
|
|
bazel_header = "//" + hdr.replace("\\", "/")
|
|
bazel_header = ":".join(bazel_header.rsplit("/", 1))
|
|
if bazel_header.startswith("//src/third_party/SafeInt"):
|
|
reverse_header_map[bazel_header] = ["//src/third_party/SafeInt:headers"]
|
|
elif bazel_header.startswith("//src/third_party/immer"):
|
|
reverse_header_map[bazel_header] = ["//src/third_party/immer:headers"]
|
|
elif bazel_header in reverse_header_map:
|
|
if bazel_header.startswith("//src/third_party/"):
|
|
continue
|
|
reverse_header_map[bazel_header].append(k)
|
|
else:
|
|
reverse_header_map[bazel_header] = [k]
|
|
|
|
for k, v in gen_header_map.items():
|
|
for hdr in v:
|
|
if not hdr or hdr.endswith(global_headers):
|
|
continue
|
|
bazel_header = "//" + hdr.replace("\\", "/")
|
|
bazel_header = ":".join(bazel_header.rsplit("/", 1))
|
|
if bazel_header not in reverse_header_map:
|
|
reverse_header_map[bazel_header] = [k]
|
|
else:
|
|
reverse_header_map[bazel_header].append(k)
|
|
|
|
recommended_deps = set()
|
|
minimal_headers = set()
|
|
|
|
basename_sources = [os.path.splitext(src.rsplit(":", 1)[1])[0] for src in sources]
|
|
for header in headers:
|
|
header_basename = os.path.splitext(header.rsplit(":", 1)[1])[0]
|
|
if header_basename in basename_sources:
|
|
minimal_headers.add(header)
|
|
continue
|
|
|
|
if header in reverse_header_map:
|
|
found = False
|
|
for lib in reverse_header_map[header]:
|
|
recommended_deps.add(lib)
|
|
else:
|
|
if not header.endswith(global_headers):
|
|
minimal_headers.add(header)
|
|
|
|
deps_order_by_height = []
|
|
deps_queries = {}
|
|
|
|
with open(target_library + ".bazel_deps") as f:
|
|
original_deps = [line.strip() for line in f.readlines()]
|
|
|
|
for dep in recommended_deps | set(original_deps):
|
|
p = subprocess.run(
|
|
[bazel_exec, "cquery"]
|
|
+ bazel_config
|
|
+ [f'kind("extract_debuginfo|idl_generator|render_template", deps("@{dep}"))'],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
deps_queries[dep] = [
|
|
line.split(" ")[0] for line in p.stdout.splitlines() if line.startswith("//")
|
|
]
|
|
deps_order_by_height.append((dep, len(deps_queries[dep])))
|
|
|
|
deps_order_by_height.sort(key=lambda x: x[1])
|
|
|
|
deps_order_by_height = [dep[0] for dep in deps_order_by_height]
|
|
optimal_header_deps = set()
|
|
for header in headers:
|
|
if header in minimal_headers:
|
|
continue
|
|
|
|
path_header = "/".join(header.rsplit(":", 1))
|
|
path_header = path_header[2:]
|
|
for dep in deps_order_by_height:
|
|
if dep in header_map and path_header in header_map[dep]:
|
|
optimal_header_deps.add(dep)
|
|
break
|
|
found = False
|
|
for other_dep in deps_order_by_height:
|
|
if other_dep in gen_header_map:
|
|
continue
|
|
if dep in deps_queries[other_dep]:
|
|
optimal_header_deps.add(other_dep)
|
|
found = True
|
|
break
|
|
if found:
|
|
continue
|
|
if dep in gen_header_map:
|
|
minimal_headers.add(dep)
|
|
else:
|
|
raise Exception(
|
|
f"Should not happen, did not find way to add dep {dep} for {target_library}"
|
|
)
|
|
|
|
optimal_header_deps = list(optimal_header_deps)
|
|
|
|
working_deps = optimal_header_deps.copy()
|
|
for dep in optimal_header_deps:
|
|
if dep in working_deps:
|
|
for test_dep in optimal_header_deps:
|
|
if test_dep == dep:
|
|
continue
|
|
if test_dep in working_deps and test_dep in deps_queries[dep]:
|
|
working_deps.remove(test_dep)
|
|
|
|
link_deps = []
|
|
header_deps = []
|
|
for dep in sorted(list(set(list(working_deps) + list(set(original_deps))))):
|
|
if dep in original_deps:
|
|
link_deps.append(dep)
|
|
else:
|
|
header_deps.append(dep)
|
|
|
|
target_name = os.path.splitext(os.path.basename(target_library))[0]
|
|
if target_name.startswith("lib"):
|
|
target_name = target_name[3:]
|
|
|
|
bazel_target = f"{target_library}\n"
|
|
bazel_target += "=" * 50 + "\n"
|
|
local_bazel_path = os.path.dirname(target_library.replace("build/opt", "//src")) + ":"
|
|
bazel_target += "mongo_cc_library(\n"
|
|
bazel_target += f' name = "{target_name}",\n'
|
|
if sources:
|
|
bazel_target += " srcs = [\n"
|
|
for src in sorted([src.replace(local_bazel_path, "") for src in sources]):
|
|
bazel_target += f' "{src}",\n'
|
|
bazel_target += " ],\n"
|
|
if minimal_headers:
|
|
bazel_target += " hdrs = [\n"
|
|
for header in sorted([header.replace(local_bazel_path, "") for header in minimal_headers]):
|
|
bazel_target += f' "{header}",\n'
|
|
bazel_target += " ],\n"
|
|
if header_deps:
|
|
bazel_target += " header_deps = [\n"
|
|
for dep in sorted([dep.strip().replace(local_bazel_path, "") for dep in header_deps]):
|
|
bazel_target += f' "{dep}",\n'
|
|
bazel_target += " ],\n"
|
|
if link_deps:
|
|
bazel_target += " deps = [\n"
|
|
for dep in sorted([dep.strip().replace(local_bazel_path, "") for dep in link_deps]):
|
|
bazel_target += f' "{dep}",\n'
|
|
bazel_target += " ],\n"
|
|
bazel_target += ")\n"
|
|
return bazel_target
|
|
|
|
|
|
def main(
|
|
target_libraries: Annotated[List[str], typer.Argument()],
|
|
silent: Annotated[bool, typer.Option()] = False,
|
|
skip_scons: Annotated[bool, typer.Option()] = False,
|
|
debug_mode: Annotated[bool, typer.Option()] = False,
|
|
):
|
|
extra_args = []
|
|
if os.name == "nt":
|
|
extra_args += [
|
|
"CPPPATH=C:\sasl\include",
|
|
"LIBPATH=C:\sasl\lib",
|
|
]
|
|
target_library = os.path.join(
|
|
os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib"
|
|
)
|
|
|
|
path = shutil.which("icecc")
|
|
if path is None:
|
|
extra_args += ["ICECC="]
|
|
|
|
# Define separate functions instead of using lambdas
|
|
def target_fmt_nt(target_library: str) -> str:
|
|
return os.path.join(
|
|
os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib"
|
|
)
|
|
|
|
def target_fmt_darwin(target_library: str) -> str:
|
|
return target_library[:-2] + "a"
|
|
|
|
def target_fmt_default(x: str) -> None:
|
|
return None
|
|
|
|
if os.name == "nt":
|
|
target_fmt = target_fmt_nt
|
|
elif platform.system() == "Darwin":
|
|
target_fmt = target_fmt_darwin
|
|
else:
|
|
target_fmt = target_fmt_default
|
|
|
|
map(target_fmt, target_libraries)
|
|
|
|
cmd = [
|
|
sys.executable,
|
|
"buildscripts/scons.py",
|
|
"--build-profile=opt",
|
|
" ".join(
|
|
[f"--bazel-includes-info={target_library}" for target_library in target_libraries]
|
|
),
|
|
"--libdeps-linting=off",
|
|
"--ninja=disabled",
|
|
"compiledb",
|
|
] + extra_args
|
|
|
|
if not skip_scons:
|
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
|
|
while True:
|
|
line = p.stdout.readline()
|
|
if not line:
|
|
break
|
|
print(line.strip(), file=sys.stderr)
|
|
|
|
_, _ = p.communicate()
|
|
|
|
if p.returncode != 0:
|
|
print(f"SCons build failed, exit code {p.returncode}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
with open("compile_commands.json") as f:
|
|
cc = json.load(f)
|
|
if platform.system() == "Linux":
|
|
cpu_count = len(os.sched_getaffinity(0)) + 4
|
|
else:
|
|
cpu_count = os.cpu_count() + 4
|
|
|
|
# Process pool makes it harder to debug what is happening
|
|
# so for debug mode, we disabled process pool so things happen in order
|
|
# you can just print from the process.
|
|
if debug_mode:
|
|
bazel_targets = []
|
|
for target_library in target_libraries:
|
|
bazel_targets += [work(target_library, silent, cpu_count, cc)]
|
|
else:
|
|
with concurrent.futures.ProcessPoolExecutor(max_workers=cpu_count) as executor:
|
|
jobs = {
|
|
executor.submit(work, target_library, silent, cpu_count, cc): target_library
|
|
for target_library in target_libraries
|
|
}
|
|
bazel_targets = [job.result() for job in concurrent.futures.as_completed(jobs)]
|
|
|
|
print("====== Bazel Targets ======\n")
|
|
print("\n".join(bazel_targets))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
typer.run(main)
|