From 2f4e03bbca3bf265eca475eb53a2f2dba3897ce2 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Mon, 14 Oct 2019 22:46:31 +0000 Subject: [PATCH] SERVER-43730 Small build system speed improvements These should speed up all SCons startup tasks for both vanilla SCons and Ninja generation --- SConstruct | 6 +++ jstests/SConscript | 16 ++++-- site_scons/libdeps.py | 18 ++++--- .../site_tools/auto_install_binaries.py | 54 +++++++++++++------ 4 files changed, 70 insertions(+), 24 deletions(-) diff --git a/SConstruct b/SConstruct index 398f2caf694..2113c959e3b 100644 --- a/SConstruct +++ b/SConstruct @@ -30,6 +30,12 @@ import mongo.generators as mongo_generators EnsurePythonVersion(3, 6) EnsureSConsVersion(3, 1, 1) +# Monkey patch SCons.FS.File.release_target_info to be a no-op. +# See https://github.com/SCons/scons/issues/3454 +def release_target_info_noop(self): + pass +SCons.Node.FS.File.release_target_info = release_target_info_noop + from buildscripts import utils from buildscripts import moduleconfig diff --git a/jstests/SConscript b/jstests/SConscript index c9aa97ef785..6c74052db81 100644 --- a/jstests/SConscript +++ b/jstests/SConscript @@ -1,5 +1,8 @@ # Includes the jstests in distribution tarballs generated by SCons +import os +from collections import defaultdict + Import("env") Import("get_option") @@ -8,10 +11,17 @@ env = env.Clone() if not get_option("install-mode") == "hygienic": Return() -for jstest in env.Glob("**/*.js"): +jstests = env.Glob("**/*.js") + +# Group by directory to avoid making a million calls to AutoInstall +jstests_by_dir = defaultdict(list) +for jstest in jstests: + jstests_by_dir[jstest.dir].append(jstest) + +for directory, files in jstests_by_dir.items(): env.AutoInstall( - target="$PREFIX_SHAREDIR/jstests/" + str(jstest.dir), - source=jstest, + target="$PREFIX_SHAREDIR/jstests/" + str(directory), + source=files, AIB_COMPONENT="jstests", AIB_ROLE="runtime", AIB_COMPONENTS_EXTRA=[ diff --git a/site_scons/libdeps.py b/site_scons/libdeps.py index a60b14fb937..b3dcfaae25c 100644 --- a/site_scons/libdeps.py +++ b/site_scons/libdeps.py @@ -282,9 +282,12 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit dependency(l, dependency_map[dependency.Private]) for l in env.get(libdeps_env_var + '_PRIVATE', []) if l) + lib_builder_prefix = lib_builder.get_prefix(env) + lib_builder_suffix = lib_builder.get_suffix(env) + for prereq in prereqs: - prereqWithIxes = SCons.Util.adjustixes(prereq.target_node, lib_builder.get_prefix(env), - lib_builder.get_suffix(env)) + prereqWithIxes = SCons.Util.adjustixes(prereq.target_node, lib_builder_prefix, + lib_builder_suffix) prereq.target_node = lib_node_factory(prereqWithIxes) for t in target: @@ -301,12 +304,15 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit visibility = dependent[1] dependent = dependent[0] - dependentWithIxes = SCons.Util.adjustixes(dependent, lib_builder.get_prefix(env), - lib_builder.get_suffix(env)) + dependentWithIxes = SCons.Util.adjustixes(dependent, lib_builder_prefix, + lib_builder_suffix) dependentNode = lib_node_factory(dependentWithIxes) __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])]) + prog_builder_prefix = prog_builder.get_prefix(env) + prog_builder_suffix = prog_builder.get_suffix(env) + if not ignore_progdeps: for dependent in env.get('PROGDEPS_DEPENDENTS', []): if dependent is None: @@ -318,8 +324,8 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit visibility = dependent[1] dependent = dependent[0] - dependentWithIxes = SCons.Util.adjustixes(dependent, prog_builder.get_prefix(env), - prog_builder.get_suffix(env)) + dependentWithIxes = SCons.Util.adjustixes(dependent, prog_builder_prefix, + prog_builder_suffix) dependentNode = prog_node_factory(dependentWithIxes) __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])]) diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py index a26838b8616..f2b8f26ea4e 100644 --- a/site_scons/site_tools/auto_install_binaries.py +++ b/site_scons/site_tools/auto_install_binaries.py @@ -107,6 +107,10 @@ RoleInfo = namedtuple( [ 'alias_name', 'alias', + 'components', + 'roles', + 'actions', + 'dependencies' ], ) @@ -225,6 +229,22 @@ def generate_alias(env, component, role, target="install"): role="" if env[ROLE_DECLARATIONS][role].silent else "-" + role, ) +def get_alias_map_entry(env, component, role): + c_entry = env[ALIAS_MAP][component] + try: + return c_entry[role] + except KeyError: + alias_name = generate_alias(env, component, role) + r_entry = RoleInfo( + alias_name=alias_name, + alias=[], + components=set(), + roles=set(), + actions=[], + dependencies=[] + ) + c_entry[role] = r_entry + return r_entry def get_package_name(env, component, role): """Return the package file name for the component and role combination.""" @@ -392,6 +412,9 @@ def archive_builder(source, target, env, for_signature): # will properly quote paths that have spaces in them on Posix # platforms and handle \ / on Windows. escape_func = env.get("ESCAPE", lambda x: x) + + # TODO: relpath is costly, and we do it for every file in the archive here. We should + # find a way to avoid the repeated relpath invocation, probably by bucketing by directory. relative_files = " ".join([ escape_func(os.path.relpath(path, common_ancestor)) for path in paths @@ -477,27 +500,30 @@ def auto_install(env, target, source, **kwargs): actions = env.Flatten(actions) for component, role in itertools.product(components, roles): - alias_name = generate_alias(env, component, role) - alias = env.Alias(alias_name, actions) - setattr(alias[0].attributes, COMPONENTS, components) - setattr(alias[0].attributes, ROLES, roles) + + entry = get_alias_map_entry(env, component, role) + entry.components.update(components) + entry.roles.update(roles) + entry.actions.extend(actions) # TODO: this hard codes behavior that should be done configurably if component != "common": - # We have to call env.Alias just in case the - # generated_alias does not already exist. - env.Depends(alias, env.Alias(generate_alias(env, "common", role))) - - env[ALIAS_MAP][component][role] = RoleInfo( - alias_name=alias_name, - alias=alias, - ) + dentry = get_alias_map_entry(env, "common", role) + entry.dependencies.append(dentry) return actions def finalize_install_dependencies(env): """Generates package aliases and wires install dependencies.""" + + for component, rolemap in env[ALIAS_MAP].items(): + for role, info in rolemap.items(): + info.alias.extend(env.Alias(info.alias_name, info.actions)) + setattr(info.alias[0].attributes, COMPONENTS, info.components) + setattr(info.alias[0].attributes, ROLES, info.roles) + env.Depends(info.alias, [d.alias for d in info.dependencies]) + common_rolemap = env[ALIAS_MAP].get("common") default_rolemap = env[ALIAS_MAP].get("default") @@ -515,10 +541,8 @@ def finalize_install_dependencies(env): for component, rolemap in env[ALIAS_MAP].items(): for role, info in rolemap.items(): - aliases = [info.alias] if common_rolemap and component != "common" and role in common_rolemap: env.Depends(info.alias, common_rolemap[role].alias) - aliases.extend(common_rolemap[role].alias) role_decl = env[ROLE_DECLARATIONS].get(role) for dependency in role_decl.dependencies: @@ -537,7 +561,7 @@ def finalize_install_dependencies(env): archive = env.__AibArchive( target="#{}.{}".format(pkg_name, pkg_suffix), - source=[make_archive_script] + aliases, + source=[make_archive_script] + info.alias, __AIB_ARCHIVE_TYPE=fmt, AIB_COMPONENT=component, AIB_ROLE=role,