SERVER-49798 Added LIBDEPS_TYPEINFO for ubsan builds to add typeinfo dependencies.

SConstruct

@@ -1723,15 +1723,29 @@ if env['_LIBDEPS'] == '$_LIBDEPS_OBJS':
     # command but instead runs a function.
     env["BUILDERS"]["StaticLibrary"].action = SCons.Action.Action(write_uuid_to_file, "Generating placeholder library $TARGET")
 
+libdeps_typeinfo = False
+
 if get_option('build-tools') == 'next':
     import libdeps_next as libdeps
+
+    if (has_option('sanitize') and 'undefined' in get_option('sanitize')
+        and env.ToolchainIs('clang', 'gcc')
+        and (env.TargetOSIs('posix') and not env.TargetOSIs('darwin'))):
+
+        libdeps_typeinfo = True
+
+    libdeps.setup_environment(
+        env,
+        emitting_shared=(link_model.startswith("dynamic")),
+        linting=get_option('libdeps-linting'),
+        sanitize_typeinfo=libdeps_typeinfo)
 else:
     import libdeps
 
-libdeps.setup_environment(
-    env,
-    emitting_shared=(link_model.startswith("dynamic")),
-    linting=get_option('libdeps-linting'))
+    libdeps.setup_environment(
+        env,
+        emitting_shared=(link_model.startswith("dynamic")),
+        linting=get_option('libdeps-linting'))
 
 # Both the abidw tool and the thin archive tool must be loaded after
 # libdeps, so that the scanners they inject can see the library
@@ -4115,6 +4129,41 @@ if get_option('ninja') != 'disabled':
 
     env['NINJA_REGENERATE_DEPS'] = ninja_generate_deps
 
+    if libdeps_typeinfo and get_option('build-tools') == 'next':
+        # ninja will not handle the list action libdeps creates so in order for
+        # to build ubsan with ninja, we need to undo the list action and then
+        # create a special rule to handle the shlink typeinfo checks. If ninja is
+        # updated to handle list actions correctly, this whole section can go away.
+        base_action = env['BUILDERS']['SharedLibrary'].action
+        base_action.list[:] = base_action.list[:-2]
+
+        # Now we rewrite the command and set it up as a rule for ninja shlinks. We are
+        # cramming this all into a single command for ninja, so it is broken apart with
+        # commentation for each part.
+        env.NinjaRule(
+            "SHLINK",
+            libdeps.get_typeinfo_link_command().format(
+                ninjalink="$env$SHLINK @$out.rsp && ",
+                ldpath="",
+                target="${out}",
+                libdeps_tags="printenv",
+                tag='libdeps-cyclic-typeinfo'
+            ),
+            description="Linking $out",
+            deps=None,
+            pool="local_pool",
+            use_depfile=False,
+            use_response_file=True)
+
+        provider = env.NinjaGenResponseFileProvider(
+            "SHLINK",
+            "$SHLINK",
+            custom_env={
+                "TYPEINFO_TAGS":"'$LIBDEPS_TAGS'",
+                "LD_LIBRARY_PATH":"'$_LIBDEPS_LD_PATH'"})
+        env.NinjaRuleMapping("${SHLINKCOM}", provider)
+        env.NinjaRuleMapping(env["SHLINKCOM"], provider)
+
     # idlc.py has the ability to print it's implicit dependencies
     # while generating, Ninja can consume these prints using the
     # deps=msvc method.

site_scons/libdeps_next.py

@@ -59,7 +59,7 @@ import textwrap
 import SCons.Errors
 import SCons.Scanner
 import SCons.Util
-
+import SCons
 
 class Constants:
     Libdeps = "LIBDEPS"
@@ -67,6 +67,7 @@ class Constants:
     LibdepsDependents = "LIBDEPS_DEPENDENTS"
     LibdepsInterface ="LIBDEPS_INTERFACE"
     LibdepsPrivate = "LIBDEPS_PRIVATE"
+    LibdepsTypeinfo = "LIBDEPS_TYPEINFO"
     LibdepsTags = "LIBDEPS_TAGS"
     LibdepsTagExpansion = "LIBDEPS_TAG_EXPANSIONS"
     MissingLibdep = "MISSING_LIBDEP_"
@@ -76,7 +77,7 @@ class Constants:
     SysLibdepsPrivate = "SYSLIBDEPS_PRIVATE"
 
 class dependency:
-    Public, Private, Interface = list(range(3))
+    Public, Private, Interface, Typeinfo = list(range(4))
 
     def __init__(self, value, deptype, listed_name):
         self.target_node = value
@@ -564,10 +565,18 @@ dependency_visibility_honored = {
     dependency.Interface: dependency.Interface,
 }
 
+dependency_visibility_typeinfo = {
+    dependency.Public: dependency.Public,
+    dependency.Private: dependency.Private,
+    dependency.Interface: dependency.Interface,
+    dependency.Typeinfo: dependency.Private,
+}
+
 dep_type_to_env_var = {
     dependency.Public: Constants.Libdeps,
     dependency.Private: Constants.LibdepsPrivate,
     dependency.Interface: Constants.LibdepsInterface,
+    dependency.Typeinfo: Constants.LibdepsTypeinfo,
 }
 
 class DependencyCycleError(SCons.Errors.UserError):
@@ -954,8 +963,59 @@ def expand_libdeps_with_flags(source, target, env, for_signature):
 
     return libdeps_with_flags
 
+def get_typeinfo_link_command():
+    if LibdepLinter.skip_linting:
+        return "{ninjalink}"
+    else:
+        return (
+        # This command is has flexibility to be used by the ninja tool, but the ninja tool
+        # does not currently support list actions as the command is used below, so we can
+        # allow ninja to put its link command here in front simulating a list action. For
+        # a normal scons build ninjalink should be empty string.
+        # TODO: When the ninja tool supports multi rule type link actions this can go
+        # away and turn into functions actions.
+        # https://jira.mongodb.org/browse/SERVER-51435
+        # https://jira.mongodb.org/browse/SERVER-51436
+        "{ninjalink}"
 
-def setup_environment(env, emitting_shared=False, linting='on'):
+        # Dependencies must exist in a shared lib that was built as a
+        # dependent to the current node, so LD_LIBRARY_PATH should be set to
+        # include all the LIBDEPS directories.
+        + "UNDEF_SYMBOLS=$$({ldpath}ldd -r {target} "
+
+        # filter out only undefined symbols and then remove the sanitizer symbols because
+        # we are focused on the missing mongo db symbols.
+        + "| grep '^undefined symbol: ' | grep -v '__[a-z]*san' "
+
+        # Demangle the remaining symbols, and filter out mongo typeinfo symbols for the current library.
+        + "| c++filt | grep 'mongo::' | grep 'typeinfo for' | grep {target}) "
+
+        # Check if a exemption tag exists to see if we should skip this error, or if
+        # we are in libdeps printing mode, disregard its existence since we will always print
+        + "&& {} ".format("true" if LibdepLinter.print_linter_errors else " [ -z \"$$({libdeps_tags} | grep \"{tag}\")\" ]")
+
+        # output the missing symbols to stdout and use sed to insert tabs at the front of
+        # each new line for neatness.
+        + r"""&& echo '\n\tLibdepLinter:\n\t\tMissing typeinfo definitions:' """
+        + r"""&& echo "$$UNDEF_SYMBOLS\n" | sed 's/^/\t\t\t/g' 1>&2 """
+
+        # Fail the build if we found issues and are not in print mode.
+        + "&& return {} ".format(int(not LibdepLinter.print_linter_errors))
+
+        # The final checks will pass the build if the tag exists or no symbols were found.
+        + "|| {libdeps_tags} | grep -q \"{tag}\" || [ -z \"$$UNDEF_SYMBOLS\" ]")
+
+def get_libdeps_ld_path(source, target, env, for_signature):
+    result = ""
+    for libdep in env['_LIBDEPS_GET_LIBS'](source, target, env, for_signature):
+        if libdep:
+            result += os.path.dirname(str(libdep)) + ":"
+    if result:
+        result = result[:-1]
+
+    return [result]
+
+def setup_environment(env, emitting_shared=False, linting='on', sanitize_typeinfo=False):
     """Set up the given build environment to do LIBDEPS tracking."""
 
     LibdepLinter.skip_linting = linting == 'off'
@@ -974,6 +1034,34 @@ def setup_environment(env, emitting_shared=False, linting='on'):
     env[Constants.Libdeps] = SCons.Util.CLVar()
     env[Constants.SysLibdeps] = SCons.Util.CLVar()
 
+    if sanitize_typeinfo and not LibdepLinter.skip_linting:
+
+        # Some sanitizers, notably the vptr check of ubsan, can cause
+        # additional symbol dependencies to exist. Unfortunately,
+        # building with the sanitizers also requires that we not build
+        # with -z,defs, which means that we cannot make such undefined
+        # symbols errors at link time. We can however hack together
+        # something which looks for undefined typeinfo nodes in the
+        # mongo namespace using `ldd -r`.  See
+        # https://jira.mongodb.org/browse/SERVER-49798 for more
+        # details.
+        env["_LIBDEPS_LD_PATH"] = get_libdeps_ld_path
+
+        base_action = env['BUILDERS']['SharedLibrary'].action
+        if not isinstance(base_action, SCons.Action.ListAction):
+            base_action = SCons.Action.ListAction([base_action])
+
+        base_action.list.extend([
+            SCons.Action.Action(
+                get_typeinfo_link_command().format(
+                    ninjalink="",
+                    ldpath="LD_LIBRARY_PATH=$_LIBDEPS_LD_PATH ",
+                    target="${TARGET}",
+                    libdeps_tags="echo \"$LIBDEPS_TAGS\"",
+                    tag='libdeps-cyclic-typeinfo'),
+                None)
+        ])
+
     # We need a way for environments to alter just which libdeps
     # emitter they want, without altering the overall program or
     # library emitter which may have important effects. The
@@ -992,7 +1080,7 @@ def setup_environment(env, emitting_shared=False, linting='on'):
         LIBDEPS_SHAREMITTER=make_libdeps_emitter("SharedArchive", ignore_progdeps=True),
         SHAREMITTER=make_indirect_emitter("LIBDEPS_SHAREMITTER"),
         LIBDEPS_SHLIBEMITTER=make_libdeps_emitter(
-            "SharedLibrary", dependency_visibility_honored
+            "SharedLibrary", dependency_visibility_typeinfo if sanitize_typeinfo else dependency_visibility_honored
         ),
         SHLIBEMITTER=make_indirect_emitter("LIBDEPS_SHLIBEMITTER"),
         LIBDEPS_PROGEMITTER=make_libdeps_emitter(

site_scons/site_tools/next/ninja.py

@@ -942,13 +942,13 @@ def get_command_env(env):
             # doesn't make builds on paths with spaces (Ninja and SCons issues)
             # nor expanding response file paths with spaces (Ninja issue) work.
             value = value.replace(r' ', r'$ ')
-            command_env += "{}='{}' ".format(key, value)
+            command_env += "export {}='{}';".format(key, value)
 
     env["NINJA_ENV_VAR_CACHE"] = command_env
     return command_env
 
 
-def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False):
+def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}):
     """Generate a response file command provider for rule name."""
 
     # If win32 using the environment with a response file command will cause
@@ -996,6 +996,11 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False):
         variables[rule] = cmd
         if use_command_env:
             variables["env"] = get_command_env(env)
+
+            for key, value in custom_env.items():
+                variables["env"] += env.subst(
+                    f"export {key}={value};", target=targets, source=sources, executor=executor
+                ) + " "
         return rule, variables, [tool_command]
 
     return get_response_file_command
@@ -1176,7 +1181,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule):
     __NINJA_RULE_MAPPING[pre_subst_string] = rule
 
 
-def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False):
+def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False):
     """Allows specification of Ninja rules from inside SCons files."""
     rule_obj = {
         "command": command,
@@ -1192,6 +1197,10 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non
     if pool is not None:
         rule_obj["pool"] = pool
 
+    if use_response_file:
+        rule_obj["rspfile"] = "$out.rsp"
+        rule_obj["rspfile_content"] = "$rspc"
+
     env[NINJA_RULES][rule] = rule_obj
 
 

src/mongo/db/SConscript

@@ -607,6 +607,9 @@ env.Library(
         'commands/test_commands_enabled',
         'service_context',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/commands',
+    ]
 )
 
 env.Library(
@@ -781,6 +784,9 @@ env.Library(
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
 )
 
 env.Library(

src/mongo/db/catalog/SConscript

@@ -167,6 +167,9 @@ env.Library(
         '$BUILD_DIR/mongo/base',
         'index_catalog_entry',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
 )
 
 env.Library(
@@ -206,6 +209,9 @@ env.Library(
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ]
 )
 
 env.Library(
@@ -454,6 +460,9 @@ env.Library(
         '$BUILD_DIR/mongo/db/service_context',
         '$BUILD_DIR/mongo/db/update_index_data',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/index/index_access_methods'
+    ]
 )
 
 env.Library(

src/mongo/db/commands/SConscript

@@ -45,6 +45,9 @@ env.Library(
         '$BUILD_DIR/mongo/util/processinfo',
         'server_status_core',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/rpc/rpc',
+    ]
 )
 
 env.Library(

src/mongo/db/concurrency/SConscript

@@ -67,6 +67,9 @@ env.Library(
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/base',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
 )
 
 env.Benchmark(

src/mongo/db/ftdc/SConscript

@@ -76,6 +76,9 @@ env.Library(
         '$BUILD_DIR/mongo/db/auth/auth',
         '$BUILD_DIR/mongo/db/auth/authprivilege',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/rpc/rpc',
+    ]
 )
 
 env.Library(

src/mongo/db/index/SConscript

@@ -108,12 +108,15 @@ env.Library(
 
 env.Library(
     target='index_access_method_factory',
-        source=[
-            'index_access_method_factory.cpp',
-        ],
-        LIBDEPS=[
-            '$BUILD_DIR/mongo/base',
-        ],
+    source=[
+        'index_access_method_factory.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+    ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
 )
 
 serveronlyEnv = env.Clone()
@@ -143,6 +146,9 @@ serveronlyEnv.Library(
         '$BUILD_DIR/mongo/idl/server_parameter',
         'skipped_record_tracker',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/matcher/expressions',
+    ]
 )
 
 env.Library(
@@ -161,7 +167,7 @@ env.Library(
         '$BUILD_DIR/mongo/base',
         'index_access_method',
         'key_generator',
-    ],
+    ]
 )
 
 env.Library(

src/mongo/db/matcher/SConscript

@@ -94,6 +94,9 @@ env.Library(
         '$BUILD_DIR/mongo/scripting/scripting_server',
         'expressions',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/index/index_access_methods'
+    ]
 )
 
 env.CppUnitTest(

src/mongo/db/query/SConscript

@@ -249,27 +249,30 @@ env.Library(
 )
 
 env.Library(
-     target="sort_pattern",
-     source=[
-         "sort_pattern.cpp",
-     ],
-     LIBDEPS=[
-         '$BUILD_DIR/mongo/base',
-         '$BUILD_DIR/mongo/db/exec/document_value/document_value',
-         '$BUILD_DIR/mongo/db/pipeline/expression_context',
-     ],
+    target="sort_pattern",
+    source=[
+        "sort_pattern.cpp",
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/db/exec/document_value/document_value',
+        '$BUILD_DIR/mongo/db/pipeline/expression_context',
+    ],
  )
 
 env.Library(
-     target="plan_yield_policy",
-     source=[
-         "plan_yield_policy.cpp",
-     ],
-     LIBDEPS=[
-         '$BUILD_DIR/mongo/base',
-         '$BUILD_DIR/mongo/util/elapsed_tracker',
-         '$BUILD_DIR/mongo/util/fail_point',
-     ],
+    target="plan_yield_policy",
+    source=[
+        "plan_yield_policy.cpp",
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/util/elapsed_tracker',
+        '$BUILD_DIR/mongo/util/fail_point',
+    ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
  )
 
 env.CppUnitTest(

src/mongo/db/repl/SConscript

@@ -77,6 +77,9 @@ env.Library(
         '$BUILD_DIR/mongo/client/authentication',
         '$BUILD_DIR/mongo/db/auth/authorization_manager_global',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/client/clientdriver_minimal',
+    ]
 )
 
 env.Library(

src/mongo/db/stats/SConscript

@@ -85,6 +85,9 @@ env.Library(
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/idl/server_parameter',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
 )
 
 env.Library(

src/mongo/db/storage/SConscript

@@ -242,6 +242,9 @@ env.Library(
         'index_entry_comparison',
         'key_string',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context_test_fixture',
+    ]
 )
 
 env.Library(
@@ -268,6 +271,9 @@ env.Library(
         '$BUILD_DIR/mongo/db/storage/storage_options',
         '$BUILD_DIR/mongo/unittest/unittest',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context_test_fixture',
+    ]
 )
 
 
@@ -281,6 +287,9 @@ env.Library(
         '$BUILD_DIR/mongo/db/service_context',
         '$BUILD_DIR/mongo/unittest/unittest',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context_test_fixture',
+    ]
 )
 
 env.Library(
@@ -350,6 +359,10 @@ env.Library(
         '$BUILD_DIR/mongo/util/fail_point',
         'recovery_unit_base',
     ],
+    LIBDEPS_TAGS=[
+        'libdeps-cyclic-typeinfo'
+    ],
+
 )
 
 env.Library(
@@ -440,6 +453,9 @@ env.Library(
         '$BUILD_DIR/mongo/util/fail_point',
         'flow_control_parameters',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
+    ]
 )
 
 env.CppUnitTest(

src/mongo/db/storage/devnull/SConscript

@@ -15,6 +15,9 @@ env.Library(
         '$BUILD_DIR/mongo/db/storage/oplog_hack',
         '$BUILD_DIR/mongo/db/storage/recovery_unit_base',
     ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
 )
 
 env.Library(

src/mongo/executor/SConscript

@@ -144,7 +144,10 @@ env.Library(
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
-    ]
+    ],
+    LIBDEPS_TYPEINFO=[
+        '$BUILD_DIR/mongo/db/service_context',
+    ],
 )
 
 env.Library(
@@ -232,6 +235,9 @@ env.Library(
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
+    ],
+    LIBDEPS_TYPEINFO=[
+        'network_interface',
     ]
 )