diff --git a/buildscripts/libdeps/gacli.py b/buildscripts/libdeps/gacli.py index 6e412a09537..474b72b9ce9 100755 --- a/buildscripts/libdeps/gacli.py +++ b/buildscripts/libdeps/gacli.py @@ -236,7 +236,14 @@ def setup_args_parser(): "--bazel-order", action="store_true", default=False, - help="Find candidate nodes for merging by searching the graph for nodes with only one node which depends on them.", + help="Print an optimal order of target conversion for the bazel conversion.", + ) + + parser.add_argument( + "--bazel-order-core", + action="store_true", + default=False, + help="Print an optimal order of target conversion for the bazel conversion focused just on the core binaries.", ) parser.add_argument( @@ -375,6 +382,9 @@ def main(): if args.bazel_order: analysis.append(libdeps_analyzer.BazelOrder(libdeps_graph)) + if args.bazel_order_core: + analysis.append(libdeps_analyzer.BazelOrderCore(libdeps_graph)) + for analyzer_args in args.critical_edges: analysis.append( libdeps_analyzer.CriticalEdges( diff --git a/buildscripts/libdeps/libdeps/analyzer.py b/buildscripts/libdeps/libdeps/analyzer.py index 002d8049444..31300c5769e 100644 --- a/buildscripts/libdeps/libdeps/analyzer.py +++ b/buildscripts/libdeps/libdeps/analyzer.py @@ -713,6 +713,54 @@ class BazelOrder(Analyzer): report[DependsReportTypes.BAZEL_ORDER.name] = self.run() +class BazelOrderCore(Analyzer): + """Generate a topo sort order to covnert bazel targets.""" + + def __init__(self, dependency_graph): + """Store graph and strip the nodes.""" + super().__init__(dependency_graph) + + @schema_check(schema_version=1) + def run(self): + """Let networkx generate the list for us.""" + order = [] + + mongod_graph = networkx.subgraph( + self._dependency_graph, + networkx.descendants(self._dependency_graph, "mongo/db/mongod") | {"mongo/db/mongod"}, + ) + mongos_graph = networkx.subgraph( + self._dependency_graph, + networkx.descendants(self._dependency_graph, "mongo/s/mongos") | {"mongo/s/mongos"}, + ) + mongo_graph = networkx.subgraph( + self._dependency_graph, + networkx.descendants(self._dependency_graph, "mongo/shell/mongo") + | {"mongo/shell/mongo"}, + ) + + for node in networkx.lexicographical_topological_sort( + networkx.compose_all( + [ + networkx.reverse_view(mongod_graph), + networkx.reverse_view(mongos_graph), + networkx.reverse_view(mongo_graph), + ] + ) + ): + node_type = self._dependents_graph.nodes()[node].get(NodeProps.bin_type.name, "Unknown") + if node_type != "ThinTarget": + order.append({"type": node_type, "name": node}) + return order + + def report(self, report): + """Add the symbol dependents list to the report.""" + + if DependsReportTypes.BAZEL_ORDER_CORE.name not in report: + report[DependsReportTypes.BAZEL_ORDER_CORE.name] = {} + report[DependsReportTypes.BAZEL_ORDER_CORE.name] = self.run() + + class Efficiency(Analyzer): """Find efficiency of each public dependency originating from each node in a given set.""" @@ -1049,6 +1097,29 @@ class GaJsonPrinter(GaPrinter): return json.dumps(self.serialize(results)) +def _bazel_order_print(results, report_name, title): + res = results[report_name] + builder_names = ("Bazel", "ThinTarget") + print(f"\n{title}:") + print(f"\tTotal targets: {len(res)}") + print( + f"\tTargets Converted: {len([node for node in res if node['type'].startswith(builder_names)])}" + ) + print( + f"\tTargets left to convert: {len([node for node in res if not node['type'].startswith(builder_names)])}" + ) + print("\tList of targets to convert (==== bars means independency groups):") + last_node = None + for num, node in enumerate(results[report_name]): + if last_node is None: + last_node = node["name"].lower() + if last_node != min(last_node, node["name"].lower()): + print("\t\t" + "=" * 60) + last_node = node["name"].lower() + target_type = "Bazel" if node["type"].startswith(builder_names) else "SCons" + print(f"\t\t{num}. {target_type}: {node['name']}") + + class GaPrettyPrinter(GaPrinter): """Printer for pretty console output.""" @@ -1160,26 +1231,15 @@ class GaPrettyPrinter(GaPrinter): print(f"\t{node}") if DependsReportTypes.BAZEL_ORDER.name in results: - res = results[DependsReportTypes.BAZEL_ORDER.name] - builder_names = ("Bazel", "ThinTarget") - print("\nBazel Target Conversion Order:") - print(f"\tTotal targets: {len(res)}") - print( - f"\tTargets Converted: {len([node for node in res if node['type'].startswith(builder_names)])}" + _bazel_order_print( + results, DependsReportTypes.BAZEL_ORDER.name, "Bazel Target Conversion Order" ) - print( - f"\tTargets left to convert: {len([node for node in res if not node['type'].startswith(builder_names)])}" + if DependsReportTypes.BAZEL_ORDER_CORE.name in results: + _bazel_order_print( + results, + DependsReportTypes.BAZEL_ORDER_CORE.name, + "Bazel Core Target Conversion Order", ) - print("\tList of targets to convert (==== bars means independency groups):") - last_node = None - for num, node in enumerate(results[DependsReportTypes.BAZEL_ORDER.name]): - if last_node is None: - last_node = node["name"].lower() - if last_node != min(last_node, node["name"].lower()): - print("\t\t" + "=" * 60) - last_node = node["name"].lower() - target_type = "Bazel" if node["type"].startswith(builder_names) else "SCons" - print(f"\t\t{num}. {target_type}: {node['name']}") if LinterTypes.EFFICIENCY_LINT.name in results: data = results[LinterTypes.EFFICIENCY_LINT.name] diff --git a/buildscripts/libdeps/libdeps/graph.py b/buildscripts/libdeps/libdeps/graph.py index ca207eb993f..68783d4ab3e 100755 --- a/buildscripts/libdeps/libdeps/graph.py +++ b/buildscripts/libdeps/libdeps/graph.py @@ -73,6 +73,7 @@ class DependsReportTypes(Enum): EFFICIENCY = auto() BAZEL_CONV_CANDIDATES = auto() BAZEL_ORDER = auto() + BAZEL_ORDER_CORE = auto() class LinterTypes(Enum): diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index c2f0792cadf..f602f795a52 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -887,6 +887,17 @@ tasks: content_type: text/plain display_name: Bazel Target Conversions + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/bazel_order_core.txt + remote_file: ${project}/${build_variant}/${revision}/artifacts/bazel_order_core.txt.${build_id}-${task_name}.${execution} + bucket: mciuploads + permissions: public-read + content_type: text/plain + display_name: Bazel Target Conversions Core Binaries + - command: s3.put params: aws_key: ${aws_key} diff --git a/evergreen/libdeps_run.sh b/evergreen/libdeps_run.sh index c84e322cba1..5ca77659e97 100755 --- a/evergreen/libdeps_run.sh +++ b/evergreen/libdeps_run.sh @@ -11,6 +11,7 @@ GRAPH_FILE=$(find build -name "libdeps.graphml") python buildscripts/libdeps/analyzer_unittests.py python buildscripts/libdeps/gacli.py --graph-file $GRAPH_FILE > results.txt python buildscripts/libdeps/gacli.py --graph-file $GRAPH_FILE --lint= --bazel-order > bazel_order.txt +python buildscripts/libdeps/gacli.py --graph-file $GRAPH_FILE --lint= --bazel-order-core > bazel_order_core.txt gzip $GRAPH_FILE mv $GRAPH_FILE.gz . targets_converted=$(grep "Targets Converted:" bazel_order.txt | cut -d":" -f 2)