SERVER-27380 Re-enable the thin archive tool

This refactors the thin_archive tool to use emitters and scanners
to note that when linking to a thin archive, you must also depend on
the children of that thin archive. Failing to do so is an error,
because a changed .o does not necessarily lead to a different .a,
which would subvert the SCons dependency mechanism.

This also includes a refactoring of the ABILINK tool to use a similar
mechanism, to achieve the opposite effect. For ABILINK, we want to
depend not on the actual .so, but on the hash of its abidw result. We
use emitters, actions, and scanners to produce an associated .abidw
file for each .so we build, and then update the scanner to depend on
the .abidw of our libraries, not the library itself. This allows us to
elide needless relinks.

SConstruct

@@ -1189,13 +1189,6 @@ env['BUILDERS']['LibraryObject'] = env['BUILDERS']['StaticObject']
 
 if link_model.startswith("dynamic"):
 
-    # Add in the abi linking tool if the user requested and it is
-    # supported on this platform.
-    if env.get('ABIDW'):
-        abilink = Tool('abilink')
-        if abilink.exists(env):
-            abilink(env)
-
     # Redirect the 'Library' target, which we always use instead of 'StaticLibrary' for things
     # that can be built in either mode, to point to SharedLibrary.
     env['BUILDERS']['Library'] = env['BUILDERS']['SharedLibrary']
@@ -1359,6 +1352,20 @@ if env['_LIBDEPS'] == '$_LIBDEPS_OBJS':
 
 libdeps.setup_environment(env, emitting_shared=(link_model.startswith("dynamic")))
 
+# Both the abidw tool and the thin archive tool must be loaded after
+# libdeps, so that the scanners they inject can see the library
+# dependencies added by libdeps.
+if link_model.startswith("dynamic"):
+    # Add in the abi linking tool if the user requested and it is
+    # supported on this platform.
+    if env.get('ABIDW'):
+        abilink = Tool('abilink')
+        if abilink.exists(env):
+            abilink(env)
+
+if env['_LIBDEPS'] == '$_LIBDEPS_LIBS':
+    env.Tool('thin_archive')
+
 if env.TargetOSIs('linux', 'freebsd', 'openbsd'):
     env['LINK_LIBGROUP_START'] = '-Wl,--start-group'
     env['LINK_LIBGROUP_END'] = '-Wl,--end-group'

site_scons/site_tools/abilink.py

@@ -15,6 +15,11 @@
 import SCons
 import subprocess
 
+# TODO: Make a SUFF variable for the suffix to write to
+# TODO: Prevent using abilink when -gsplit-dwarf is in play, since it doesn't work
+# TODO: Make a variable for the md5sum utility (allow any hasher)
+# TODO: Add an ABILINKCOM variable to the Action, so it can be silenced.
+
 def _detect(env):
     try:
         abidw = env['ABIDW']
@@ -26,53 +31,52 @@ def _detect(env):
 
     return env.WhereIs('abidw')
 
-def generate(env):
+def _add_emitter(builder):
+    base_emitter = builder.emitter
 
-    class AbilinkNode(SCons.Node.FS.File):
-        def __init__(self, name, directory, fs):
-            SCons.Node.FS.File.__init__(self, name, directory, fs)
+    def new_emitter(target, source, env):
+        new_targets = []
+        for t in target:
+            abidw = str(t) + ".abidw"
+            abidw = (t.builder.target_factory or env.File)(abidw)
+            new_targets.append(abidw)
+            setattr(t.attributes, "abidw", abidw)
+        targets = target + new_targets
+        return (targets, source)
 
-        def get_contents(self):
-            if not self.rexists():
-                return ''
+    new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter])
+    builder.emitter = new_emitter
 
-            fname = self.rfile().abspath
-            contents = None
+def _add_scanner(builder):
+    old_scanner = builder.target_scanner
+    path_function = old_scanner.path_function
 
-            try:
-                # TODO: If there were python bindings for libabigail, we
-                # could avoid the shell out (and probably be faster, as we
-                # could get exactly the information we want).
-                contents = subprocess.check_output([env.subst('$ABIDW'), fname])
-            except subprocess.CalledProcessError, e:
-                # ABIDW sometimes fails. In that case, log an error
-                # and fall back to the normal contents
-                print "WARNING: ABIDW failed for target %s, please file a bug report" % fname
-                try:
-                    contents = open(fname, "rb").read()
-                except EnvironmentError, e:
-                    if not e.filename:
-                        e.filename = fname
-                    raise
+    def new_scanner(node, env, path):
+        old_results = old_scanner(node, env, path)
+        new_results = []
+        for base in old_results:
+            abidw = getattr(env.Entry(base).attributes, "abidw", None)
+            new_results.append(abidw if abidw else base)
+        return new_results
 
-            return contents
+    builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function)
 
-        def get_content_hash(self):
-            return SCons.Util.MD5signature(self.get_contents())
-
-    env['ABIDW'] = _detect(env)
-
-    def ShlibNode(env, name, directory = None, create = 1):
-        return env.fs._lookup(env.subst(name), directory, AbilinkNode, create)
-
-    env.AddMethod(ShlibNode, 'ShlibNode')
-
-    def shlib_target_factory(arg):
-        return env.ShlibNode(arg)
-
-    env['BUILDERS']['SharedLibrary'].target_factory = shlib_target_factory
-    env['BUILDERS']['LoadableModule'].target_factory = shlib_target_factory
+def _add_action(builder):
+    actions = builder.action
+    builder.action = actions + SCons.Action.Action("$ABIDW $TARGET | md5sum > ${TARGET}.abidw")
 
 def exists(env):
     result = _detect(env) != None
     return result
+
+def generate(env):
+
+    if not exists(env):
+        return
+
+    builder = env['BUILDERS']['SharedLibrary']
+    _add_emitter(builder)
+    _add_action(builder)
+    _add_scanner(builder)
+    _add_scanner(env['BUILDERS']['Program'])
+    _add_scanner(env['BUILDERS']['LoadableModule'])

site_scons/site_tools/thin_archive.py

@@ -45,33 +45,36 @@ def exists(env):
 
     return bool(isgnu)
 
+def _add_emitter(builder):
+    base_emitter = builder.emitter
+
+    def new_emitter(target, source, env):
+        for t in target:
+            setattr(t.attributes, "thin_archive", True)
+        return (target, source)
+
+    new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter])
+    builder.emitter = new_emitter
+
+def _add_scanner(builder):
+    old_scanner = builder.target_scanner
+    path_function = old_scanner.path_function
+
+    def new_scanner(node, env, path):
+        old_results = old_scanner(node, env, path)
+        new_results = []
+        for base in old_results:
+            new_results.append(base)
+            if getattr(env.Entry(base).attributes, "thin_archive", None):
+                new_results.extend(base.children())
+        return new_results
+
+    builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function)
 
 def generate(env):
     if not exists(env):
         return
 
-    class ThinArchiveNode(SCons.Node.FS.File):
-        def __init__(self, name, directory, fs):
-            SCons.Node.FS.File.__init__(self, name, directory, fs)
-
-        def get_contents(self):
-            child_sigs = sorted([child.get_csig() for child in self.children()])
-            return ''.join(child_sigs)
-
-        def get_content_hash(self):
-            return SCons.Util.MD5signature(self.get_contents())
-
-
-    def _ThinArchiveNode(env, name, directory = None, create = 1):
-        return env.fs._lookup(env.subst(name), directory, ThinArchiveNode, create)
-
-    env.AddMethod(_ThinArchiveNode, 'ThinArchiveNode')
-
-    def archive_target_factory(arg):
-        return env.ThinArchiveNode(arg)
-
-    env['BUILDERS']['StaticLibrary'].target_factory = archive_target_factory
-
     env['ARFLAGS'] = SCons.Util.CLVar([arflag if arflag != "rc" else "rcsTD" for arflag in env['ARFLAGS']])
 
     def noop_action(env, target, source):
@@ -80,3 +83,10 @@ def generate(env):
     # Disable running ranlib, since we added 's' above
     env['RANLIBCOM'] = noop_action
     env['RANLIBCOMSTR'] = 'Skipping ranlib for thin archive $TARGET'
+
+    builder = env['BUILDERS']['StaticLibrary']
+    _add_emitter(builder)
+
+    _add_scanner(env['BUILDERS']['SharedLibrary'])
+    _add_scanner(env['BUILDERS']['LoadableModule'])
+    _add_scanner(env['BUILDERS']['Program'])