SERVER-43730 Small build system speed improvements

These should speed up all SCons startup tasks for both
vanilla SCons and Ninja generation

SConstruct

@@ -30,6 +30,12 @@ import mongo.generators as mongo_generators
 EnsurePythonVersion(3, 6)
 EnsureSConsVersion(3, 1, 1)
 
+# Monkey patch SCons.FS.File.release_target_info to be a no-op.
+# See https://github.com/SCons/scons/issues/3454
+def release_target_info_noop(self):
+    pass
+SCons.Node.FS.File.release_target_info = release_target_info_noop
+
 from buildscripts import utils
 from buildscripts import moduleconfig
 

jstests/SConscript

@@ -1,5 +1,8 @@
 # Includes the jstests in distribution tarballs generated by SCons
 
+import os
+from collections import defaultdict
+
 Import("env")
 Import("get_option")
 
@@ -8,10 +11,17 @@ env = env.Clone()
 if not get_option("install-mode") == "hygienic":
     Return()
 
-for jstest in env.Glob("**/*.js"):
+jstests = env.Glob("**/*.js")
+
+# Group by directory to avoid making a million calls to AutoInstall
+jstests_by_dir = defaultdict(list)
+for jstest in jstests:
+    jstests_by_dir[jstest.dir].append(jstest)
+
+for directory, files in jstests_by_dir.items():
     env.AutoInstall(
-        target="$PREFIX_SHAREDIR/jstests/" + str(jstest.dir),
-        source=jstest,
+        target="$PREFIX_SHAREDIR/jstests/" + str(directory),
+        source=files,
         AIB_COMPONENT="jstests",
         AIB_ROLE="runtime",
         AIB_COMPONENTS_EXTRA=[

site_scons/libdeps.py

@@ -282,9 +282,12 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit
             dependency(l, dependency_map[dependency.Private])
             for l in env.get(libdeps_env_var + '_PRIVATE', []) if l)
 
+        lib_builder_prefix = lib_builder.get_prefix(env)
+        lib_builder_suffix = lib_builder.get_suffix(env)
+
         for prereq in prereqs:
-            prereqWithIxes = SCons.Util.adjustixes(prereq.target_node, lib_builder.get_prefix(env),
-                                                   lib_builder.get_suffix(env))
+            prereqWithIxes = SCons.Util.adjustixes(prereq.target_node, lib_builder_prefix,
+                                                   lib_builder_suffix)
             prereq.target_node = lib_node_factory(prereqWithIxes)
 
         for t in target:
@@ -301,12 +304,15 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit
                 visibility = dependent[1]
                 dependent = dependent[0]
 
-            dependentWithIxes = SCons.Util.adjustixes(dependent, lib_builder.get_prefix(env),
-                                                      lib_builder.get_suffix(env))
+            dependentWithIxes = SCons.Util.adjustixes(dependent, lib_builder_prefix,
+                                                      lib_builder_suffix)
             dependentNode = lib_node_factory(dependentWithIxes)
             __append_direct_libdeps(dependentNode,
                                     [dependency(target[0], dependency_map[visibility])])
 
+        prog_builder_prefix = prog_builder.get_prefix(env)
+        prog_builder_suffix = prog_builder.get_suffix(env)
+
         if not ignore_progdeps:
             for dependent in env.get('PROGDEPS_DEPENDENTS', []):
                 if dependent is None:
@@ -318,8 +324,8 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit
                     visibility = dependent[1]
                     dependent = dependent[0]
 
-                dependentWithIxes = SCons.Util.adjustixes(dependent, prog_builder.get_prefix(env),
-                                                          prog_builder.get_suffix(env))
+                dependentWithIxes = SCons.Util.adjustixes(dependent, prog_builder_prefix,
+                                                          prog_builder_suffix)
                 dependentNode = prog_node_factory(dependentWithIxes)
                 __append_direct_libdeps(dependentNode,
                                         [dependency(target[0], dependency_map[visibility])])

site_scons/site_tools/auto_install_binaries.py

@@ -107,6 +107,10 @@ RoleInfo = namedtuple(
     [
         'alias_name',
         'alias',
+        'components',
+        'roles',
+        'actions',
+        'dependencies'
     ],
 )
 
@@ -225,6 +229,22 @@ def generate_alias(env, component, role, target="install"):
         role="" if env[ROLE_DECLARATIONS][role].silent else "-" + role,
     )
 
+def get_alias_map_entry(env, component, role):
+    c_entry = env[ALIAS_MAP][component]
+    try:
+        return c_entry[role]
+    except KeyError:
+        alias_name = generate_alias(env, component, role)
+        r_entry = RoleInfo(
+            alias_name=alias_name,
+            alias=[],
+            components=set(),
+            roles=set(),
+            actions=[],
+            dependencies=[]
+        )
+        c_entry[role] = r_entry
+        return r_entry
 
 def get_package_name(env, component, role):
     """Return the package file name for the component and role combination."""
@@ -392,6 +412,9 @@ def archive_builder(source, target, env, for_signature):
     # will properly quote paths that have spaces in them on Posix
     # platforms and handle \ / on Windows.
     escape_func = env.get("ESCAPE", lambda x: x)
+
+    # TODO: relpath is costly, and we do it for every file in the archive here. We should
+    # find a way to avoid the repeated relpath invocation, probably by bucketing by directory.
     relative_files = " ".join([
         escape_func(os.path.relpath(path, common_ancestor))
         for path in paths
@@ -477,27 +500,30 @@ def auto_install(env, target, source, **kwargs):
 
     actions = env.Flatten(actions)
     for component, role in itertools.product(components, roles):
-        alias_name = generate_alias(env, component, role)
-        alias = env.Alias(alias_name, actions)
-        setattr(alias[0].attributes, COMPONENTS, components)
-        setattr(alias[0].attributes, ROLES, roles)
+
+        entry = get_alias_map_entry(env, component, role)
+        entry.components.update(components)
+        entry.roles.update(roles)
+        entry.actions.extend(actions)
 
         # TODO: this hard codes behavior that should be done configurably
         if component != "common":
-            # We have to call env.Alias just in case the
-            # generated_alias does not already exist.
-            env.Depends(alias, env.Alias(generate_alias(env, "common", role)))
-
-        env[ALIAS_MAP][component][role] = RoleInfo(
-            alias_name=alias_name,
-            alias=alias,
-        )
+            dentry = get_alias_map_entry(env, "common", role)
+            entry.dependencies.append(dentry)
 
     return actions
 
 
 def finalize_install_dependencies(env):
     """Generates package aliases and wires install dependencies."""
+
+    for component, rolemap in env[ALIAS_MAP].items():
+        for role, info in rolemap.items():
+            info.alias.extend(env.Alias(info.alias_name, info.actions))
+            setattr(info.alias[0].attributes, COMPONENTS, info.components)
+            setattr(info.alias[0].attributes, ROLES, info.roles)
+            env.Depends(info.alias, [d.alias for d in info.dependencies])
+
     common_rolemap = env[ALIAS_MAP].get("common")
     default_rolemap = env[ALIAS_MAP].get("default")
 
@@ -515,10 +541,8 @@ def finalize_install_dependencies(env):
     for component, rolemap in env[ALIAS_MAP].items():
         for role, info in rolemap.items():
 
-            aliases = [info.alias]
             if common_rolemap and component != "common" and role in common_rolemap:
                 env.Depends(info.alias, common_rolemap[role].alias)
-                aliases.extend(common_rolemap[role].alias)
 
             role_decl = env[ROLE_DECLARATIONS].get(role)
             for dependency in role_decl.dependencies:
@@ -537,7 +561,7 @@ def finalize_install_dependencies(env):
 
                 archive = env.__AibArchive(
                     target="#{}.{}".format(pkg_name, pkg_suffix),
-                    source=[make_archive_script] + aliases,
+                    source=[make_archive_script] + info.alias,
                     __AIB_ARCHIVE_TYPE=fmt,
                     AIB_COMPONENT=component,
                     AIB_ROLE=role,