SERVER-76740 SERVER-69843 added pretty printer tests framework

SConstruct

@@ -917,6 +917,7 @@ def variable_tools_converter(val):
         "mongo_integrationtest",
         "mongo_unittest",
         "mongo_libfuzzer",
+        "mongo_pretty_printer_tests",
         "textfile",
     ]
 
@@ -1641,6 +1642,8 @@ envDict = dict(
     # TODO: Move unittests.txt to $BUILD_DIR, but that requires
     # changes to MCI.
     UNITTEST_LIST='$BUILD_ROOT/unittests.txt',
+    PRETTY_PRINTER_TEST_ALIAS='install-pretty-printer-tests',
+    PRETTY_PRINTER_TEST_LIST='$BUILD_DIR/pretty_printer_tests.txt',
     LIBFUZZER_TEST_ALIAS='install-fuzzertests',
     LIBFUZZER_TEST_LIST='$BUILD_ROOT/libfuzzer_tests.txt',
     INTEGRATION_TEST_ALIAS='install-integration-tests',
@@ -6021,6 +6024,14 @@ env.AddPackageNameAlias(
     name="mh-debugsymbols",
 )
 
+env.AutoInstall(
+    target='$PREFIX',
+    source='$PRETTY_PRINTER_TEST_LIST',
+    AIB_ROLE='runtime',
+    AIB_COMPONENT='pretty-printer-tests',
+    AIB_COMPONENTS_EXTRA=['dist-test'],
+)
+
 env['RPATH_ESCAPED_DOLLAR_ORIGIN'] = '\\$$$$ORIGIN'
 
 

buildscripts/gdb/mongo.py

@@ -159,6 +159,36 @@ def get_thread_id():
     raise ValueError("Failed to find thread id in {}".format(thread_info))
 
 
+MAIN_GLOBAL_BLOCK = None
+
+
+def lookup_type(gdb_type_str: str) -> gdb.Type:
+    """
+    Try to find the type object from string.
+
+    GDB says it searches the global blocks, however this appear not to be the
+    case or at least it doesn't search all global blocks, sometimes it required
+    to get the global block based off the current frame.
+    """
+    global MAIN_GLOBAL_BLOCK  # pylint: disable=global-statement
+
+    exceptions = []
+    try:
+        return gdb.lookup_type(gdb_type_str)
+    except Exception as exc:
+        exceptions.append(exc)
+
+    if MAIN_GLOBAL_BLOCK is None:
+        MAIN_GLOBAL_BLOCK = gdb.lookup_symbol("main")[0].symtab.global_block()
+
+    try:
+        return gdb.lookup_type(gdb_type_str, MAIN_GLOBAL_BLOCK)
+    except Exception as exc:
+        exceptions.append(exc)
+
+    raise gdb.error("Failed to get type, tried:\n%s" % '\n'.join([str(exc) for exc in exceptions]))
+
+
 def get_current_thread_name():
     """Return the name of the current GDB thread."""
     fallback_name = '"%s"' % (gdb.selected_thread().name or '')
@@ -217,7 +247,7 @@ def get_wt_session(recovery_unit, recovery_unit_impl_type):
     if not wt_session_handle.dereference().address:
         return None
     wt_session = wt_session_handle.dereference().cast(
-        gdb.lookup_type("mongo::WiredTigerSession"))["_session"]
+        lookup_type("mongo::WiredTigerSession"))["_session"]
     return wt_session
 
 
@@ -230,13 +260,13 @@ def get_decorations(obj):
     TODO: De-duplicate the logic between here and DecorablePrinter. This code was copied from there.
     """
     type_name = str(obj.type).replace("class", "").replace(" ", "")
-    decorable = obj.cast(gdb.lookup_type("mongo::Decorable<{}>".format(type_name)))
+    decorable = obj.cast(lookup_type("mongo::Decorable<{}>".format(type_name)))
     decl_vector = decorable["_decorations"]["_registry"]["_decorationInfo"]
     start = decl_vector["_M_impl"]["_M_start"]
     finish = decl_vector["_M_impl"]["_M_finish"]
 
     decorable_t = decorable.type.template_argument(0)
-    decinfo_t = gdb.lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format(
+    decinfo_t = lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format(
         str(decorable_t).replace("class", "").strip()))
     count = int((int(finish) - int(start)) / decinfo_t.sizeof)
 
@@ -255,7 +285,7 @@ def get_decorations(obj):
             type_name = type_name[0:len(type_name) - 1]
         type_name = type_name.rstrip()
         try:
-            type_t = gdb.lookup_type(type_name)
+            type_t = lookup_type(type_name)
             obj = decoration_data[dindex].cast(type_t)
             yield (type_name, obj)
         except Exception as err:
@@ -501,7 +531,7 @@ class DumpMongoDSessionCatalog(gdb.Command):
             val = get_boost_optional(txn_part_observable_state['txnResourceStash'])
             if val:
                 locker_addr = get_unique_ptr(val["_locker"])
-                locker_obj = locker_addr.dereference().cast(gdb.lookup_type("mongo::LockerImpl"))
+                locker_obj = locker_addr.dereference().cast(lookup_type("mongo::LockerImpl"))
                 print('txnResourceStash._locker', "@", locker_addr)
                 print("txnResourceStash._locker._id", "=", locker_obj["_id"])
             else:
@@ -645,7 +675,7 @@ class MongoDBDumpRecoveryUnits(gdb.Command):
                 recovery_unit_handle = get_unique_ptr(operation_context["_recoveryUnit"])
                 # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit"
                 recovery_unit = recovery_unit_handle.dereference().cast(
-                    gdb.lookup_type(recovery_unit_impl_type))
+                    lookup_type(recovery_unit_impl_type))
 
             output_doc["recoveryUnit"] = hex(recovery_unit_handle) if recovery_unit else "0x0"
             wt_session = get_wt_session(recovery_unit, recovery_unit_impl_type)
@@ -690,7 +720,7 @@ class MongoDBDumpRecoveryUnits(gdb.Command):
                 recovery_unit_handle = get_unique_ptr(txn_resource_stash["_recoveryUnit"])
                 # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit"
                 recovery_unit = recovery_unit_handle.dereference().cast(
-                    gdb.lookup_type(recovery_unit_impl_type))
+                    lookup_type(recovery_unit_impl_type))
 
         output_doc["recoveryUnit"] = hex(recovery_unit_handle) if recovery_unit else "0x0"
         wt_session = get_wt_session(recovery_unit, recovery_unit_impl_type)

buildscripts/gdb/mongo_lock.py

@@ -9,7 +9,7 @@ import gdb.printing
 
 if not gdb:
     sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent))
-    from buildscripts.gdb.mongo import get_current_thread_name, get_thread_id, RegisterMongoCommand
+    from buildscripts.gdb.mongo import get_current_thread_name, get_thread_id, lookup_type, RegisterMongoCommand
 
 if sys.version_info[0] < 3:
     raise gdb.GdbError(
@@ -323,7 +323,7 @@ def find_lock_manager_holders(graph, thread_dict, show):
     (_, lock_waiter_lwpid, _) = gdb.selected_thread().ptid
     lock_waiter = thread_dict[lock_waiter_lwpid]
 
-    locker_ptr_type = gdb.lookup_type("mongo::LockerImpl").pointer()
+    locker_ptr_type = lookup_type("mongo::LockerImpl").pointer()
 
     lock_head = gdb.parse_and_eval(
         "mongo::LockManager::get((mongo::ServiceContext*) mongo::getGlobalServiceContext())->_getBucket(resId)->findOrInsert(resId)"

buildscripts/gdb/mongo_printers.py

@@ -15,7 +15,7 @@ if ROOT_PATH not in sys.path:
 from src.third_party.immer.dist.tools.gdb_pretty_printers.printers import ListIter as ImmerListIter  # pylint: disable=wrong-import-position
 
 if not gdb:
-    from buildscripts.gdb.mongo import get_boost_optional
+    from buildscripts.gdb.mongo import get_boost_optional, lookup_type
     from buildscripts.gdb.optimizer_printers import register_abt_printers
 
 try:
@@ -142,7 +142,7 @@ class BSONObjPrinter(object):
     def __init__(self, val):
         """Initialize BSONObjPrinter."""
         self.val = val
-        self.ptr = self.val['_objdata'].cast(gdb.lookup_type('void').pointer())
+        self.ptr = self.val['_objdata'].cast(lookup_type('void').pointer())
         self.is_valid = False
 
         # Handle the endianness of the BSON object size, which is represented as a 32-bit integer
@@ -301,7 +301,7 @@ class RecordIdPrinter(object):
             holder = holder_ptr.dereference()
             str_len = int(holder["_capacity"])
             # Start of data is immediately after pointer for holder
-            start_ptr = (holder_ptr + 1).dereference().cast(gdb.lookup_type("char")).address
+            start_ptr = (holder_ptr + 1).dereference().cast(lookup_type("char")).address
             raw_bytes = [int(start_ptr[i]) for i in range(0, str_len)]
             hex_bytes = [hex(b & 0xFF)[2:].zfill(2) for b in raw_bytes]
             return "RecordId big string %d hex bytes @ %s: %s" % (str_len, holder_ptr + 1,
@@ -322,7 +322,7 @@ class DecorablePrinter(object):
         self.start = decl_vector["_M_impl"]["_M_start"]
         finish = decl_vector["_M_impl"]["_M_finish"]
         decorable_t = val.type.template_argument(0)
-        decinfo_t = gdb.lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format(
+        decinfo_t = lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format(
             str(decorable_t).replace("class", "").strip()))
         self.count = int((int(finish) - int(self.start)) / decinfo_t.sizeof)
 
@@ -357,7 +357,7 @@ class DecorablePrinter(object):
             type_name = type_name.rstrip()
 
             # Cast the raw char[] into the actual object that is stored there.
-            type_t = gdb.lookup_type(type_name)
+            type_t = lookup_type(type_name)
             obj = decoration_data[dindex].cast(type_t)
 
             yield ('key', "%d:%s:%s" % (index, obj.address, type_name))
@@ -791,7 +791,7 @@ def make_inverse_enum_dict(enum_type_name):
     For example, if the enum type is 'mongo::sbe::vm::Builtin' with an element 'regexMatch', the
     dictionary will contain 'regexMatch' value and not 'mongo::sbe::vm::Builtin::regexMatch'.
     """
-    enum_dict = gdb.types.make_enum_dict(gdb.lookup_type(enum_type_name))
+    enum_dict = gdb.types.make_enum_dict(lookup_type(enum_type_name))
     enum_inverse_dic = dict()
     for key, value in enum_dict.items():
         enum_inverse_dic[int(value)] = key.split('::')[-1]  # take last element
@@ -838,11 +838,11 @@ class SbeCodeFragmentPrinter(object):
         # either use an inline buffer or an allocated one. The choice of storage is decoded in the
         # last bit of the 'metadata_' field.
         storage = self.val['_instrs']['storage_']
-        meta = storage['metadata_'].cast(gdb.lookup_type('size_t'))
+        meta = storage['metadata_'].cast(lookup_type('size_t'))
         self.is_inlined = (meta % 2 == 0)
         self.size = (meta >> 1)
         self.pdata = \
-            storage['data_']['inlined']['inlined_data'].cast(gdb.lookup_type('uint8_t').pointer()) \
+            storage['data_']['inlined']['inlined_data'].cast(lookup_type('uint8_t').pointer()) \
             if self.is_inlined \
             else storage['data_']['allocated']['allocated_data']
 
@@ -866,17 +866,17 @@ class SbeCodeFragmentPrinter(object):
         yield 'instrs total size', self.size
 
         # Sizes for types we'll use when parsing the insructions stream.
-        int_size = gdb.lookup_type('int').sizeof
-        ptr_size = gdb.lookup_type('void').pointer().sizeof
-        tag_size = gdb.lookup_type('mongo::sbe::value::TypeTags').sizeof
-        value_size = gdb.lookup_type('mongo::sbe::value::Value').sizeof
-        uint8_size = gdb.lookup_type('uint8_t').sizeof
-        uint32_size = gdb.lookup_type('uint32_t').sizeof
-        uint64_size = gdb.lookup_type('uint64_t').sizeof
-        builtin_size = gdb.lookup_type('mongo::sbe::vm::Builtin').sizeof
-        time_unit_size = gdb.lookup_type('mongo::TimeUnit').sizeof
-        timezone_size = gdb.lookup_type('mongo::TimeZone').sizeof
-        day_of_week_size = gdb.lookup_type('mongo::DayOfWeek').sizeof
+        int_size = lookup_type('int').sizeof
+        ptr_size = lookup_type('void').pointer().sizeof
+        tag_size = lookup_type('mongo::sbe::value::TypeTags').sizeof
+        value_size = lookup_type('mongo::sbe::value::Value').sizeof
+        uint8_size = lookup_type('uint8_t').sizeof
+        uint32_size = lookup_type('uint32_t').sizeof
+        uint64_size = lookup_type('uint64_t').sizeof
+        builtin_size = lookup_type('mongo::sbe::vm::Builtin').sizeof
+        time_unit_size = lookup_type('mongo::TimeUnit').sizeof
+        timezone_size = lookup_type('mongo::TimeZone').sizeof
+        day_of_week_size = lookup_type('mongo::DayOfWeek').sizeof
 
         cur_op = self.pdata
         end_op = self.pdata + self.size
@@ -921,9 +921,9 @@ class SbeCodeFragmentPrinter(object):
                 cur_op += uint32_size
             elif op_name in ['function', 'functionSmall']:
                 arity_size = \
-                    gdb.lookup_type('mongo::sbe::vm::ArityType').sizeof \
+                    lookup_type('mongo::sbe::vm::ArityType').sizeof \
                     if op_name == 'function' \
-                    else gdb.lookup_type('mongo::sbe::vm::SmallArityType').sizeof
+                    else lookup_type('mongo::sbe::vm::SmallArityType').sizeof
                 builtin_id = read_as_integer(cur_op, builtin_size)
                 args = 'builtin: ' + self.builtins_lookup.get(builtin_id, "unknown")
                 args += ' arity: ' + str(read_as_integer(cur_op + builtin_size, arity_size))

buildscripts/gdb/optimizer_printers.py

@@ -8,7 +8,7 @@ import gdb.printing
 
 if not gdb:
     sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent))
-    from buildscripts.gdb.mongo import get_boost_optional
+    from buildscripts.gdb.mongo import get_boost_optional, lookup_type
 
 
 def eval_print_fn(val, print_fn):
@@ -893,7 +893,7 @@ class PolyValuePrinter(object):
     def to_string(self):
         dynamic_type = self.get_dynamic_type()
         try:
-            dynamic_type = gdb.lookup_type(dynamic_type).strip_typedefs()
+            dynamic_type = lookup_type(dynamic_type).strip_typedefs()
         except gdb.error:
             return "Unknown PolyValue tag: {}, did you add a new one?".format(self.tag)
         # GDB automatically formats types with children, remove the extra characters to get the
@@ -979,7 +979,7 @@ class ABTPrinter(PolyValuePrinter):
     def get_bound_projections(node):
         # Casts the input node to an ExpressionBinder and returns the set of bound projection names.
         pp = PolyValuePrinter(ABTPrinter.abt_type_set, ABTPrinter.abt_namespace, node)
-        dynamic_type = gdb.lookup_type(pp.get_dynamic_type()).strip_typedefs()
+        dynamic_type = lookup_type(pp.get_dynamic_type()).strip_typedefs()
         binder = pp.cast_control_block(dynamic_type)
         return Vector(binder["_names"])
 
@@ -1111,7 +1111,7 @@ def register_abt_printers(pp):
     # stale.
     try:
         # ABT printer.
-        abt_type = gdb.lookup_type("mongo::optimizer::ABT").strip_typedefs()
+        abt_type = lookup_type("mongo::optimizer::ABT").strip_typedefs()
         pp.add('ABT', abt_type.name, False, ABTPrinter)
 
         abt_ref_type = abt_type.name + "::Reference"

buildscripts/gdb/wt_dump_table.py

@@ -1,6 +1,13 @@
 import gdb
 import bson
+import sys
+import os
 from pprint import pprint
+from pathlib import Path
+
+if not gdb:
+    sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent))
+    from buildscripts.gdb.mongo import lookup_type
 
 DEBUGGING = False
 '''
@@ -21,7 +28,7 @@ Some behaviors/limitations:
 
 
 def dump_pages_for_table(ident):
-    conn_impl_type = gdb.lookup_type("WT_CONNECTION_IMPL")
+    conn_impl_type = lookup_type("WT_CONNECTION_IMPL")
     if not conn_impl_type:
         print('WT_CONNECTION_IMPL type not found. Try invoking this function from a different \
 thread and frame.')
@@ -104,7 +111,7 @@ def get_data_handle(conn, handle_name):
 
 
 def get_btree_handle(dhandle):
-    btree = gdb.lookup_type('WT_BTREE').pointer()
+    btree = lookup_type('WT_BTREE').pointer()
     return dhandle['handle'].reinterpret_cast(btree).dereference()
 
 

buildscripts/resmokeconfig/suites/pretty-printer-tests.yml

@@ -0,0 +1,7 @@
+test_kind: pretty_printer_test
+
+selector:
+  root: build/install/dist-test/pretty_printer_tests.txt
+
+executor:
+  config: {}

buildscripts/resmokelib/selector.py

@@ -604,6 +604,39 @@ class _CppTestSelector(_Selector):
         return _Selector.select(self, selector_config)
 
 
+class _PrettyPrinterTestSelectorConfig(_SelectorConfig):
+    """_SelectorConfig subclass for pretty-printer-tests."""
+
+    def __init__(self, root=config.DEFAULT_INTEGRATION_TEST_LIST, roots=None, include_files=None,
+                 exclude_files=None):
+        """Initialize _PrettyPrinterTestSelectorConfig."""
+        if roots:
+            # The 'roots' argument is only present when tests are specified on the command line
+            # and in that case they take precedence over the tests in the root file.
+            _SelectorConfig.__init__(self, roots=roots, include_files=include_files,
+                                     exclude_files=exclude_files)
+        else:
+            _SelectorConfig.__init__(self, root=root, include_files=include_files,
+                                     exclude_files=exclude_files)
+
+
+class _PrettyPrinterTestSelector(_Selector):
+    """_Selector subclass for pretty-printer-tests."""
+
+    def __init__(self, test_file_explorer):
+        """Initialize _PrettyPrinterTestSelector."""
+        _Selector.__init__(self, test_file_explorer)
+
+    def select(self, selector_config):
+        """Return selected tests."""
+        if selector_config.roots:
+            # Tests have been specified on the command line. We use them without additional
+            # filtering.
+            test_list = _TestList(self._test_file_explorer, selector_config.roots)
+            return test_list.get_tests()
+        return _Selector.select(self, selector_config)
+
+
 class _DbTestSelectorConfig(_SelectorConfig):
     """_Selector config subclass for db_test tests."""
 
@@ -715,6 +748,7 @@ _DEFAULT_TEST_FILE_EXPLORER = TestFileExplorer()
 _SELECTOR_REGISTRY = {
     "cpp_integration_test": (_CppTestSelectorConfig, _CppTestSelector),
     "cpp_unit_test": (_CppTestSelectorConfig, _CppTestSelector),
+    "pretty_printer_test": (_PrettyPrinterTestSelectorConfig, _PrettyPrinterTestSelector),
     "benchmark_test": (_CppTestSelectorConfig, _CppTestSelector),
     "sdam_json_test": (_FileBasedSelectorConfig, _Selector),
     "server_selection_json_test": (_FileBasedSelectorConfig, _Selector),

buildscripts/resmokelib/testing/testcases/pretty_printer_testcase.py

@@ -0,0 +1,25 @@
+"""The unittest.TestCase for pretty printer tests."""
+import os
+
+from buildscripts.resmokelib import config
+from buildscripts.resmokelib import core
+from buildscripts.resmokelib import utils
+from buildscripts.resmokelib.testing.testcases import interface
+
+
+class PrettyPrinterTestCase(interface.ProcessTestCase):
+    """A pretty printer test to execute."""
+
+    REGISTERED_NAME = "pretty_printer_test"
+
+    def __init__(self, logger, program_executable, program_options=None):
+        """Initialize the PrettyPrinterTestCase with the executable to run."""
+
+        interface.ProcessTestCase.__init__(self, logger, "pretty printer test", program_executable)
+
+        self.program_executable = program_executable
+        self.program_options = utils.default_if_none(program_options, {}).copy()
+
+    def _make_process(self):
+        return core.programs.make_process(self.logger, [self.program_executable],
+                                          **self.program_options)

etc/evergreen_yml_components/definitions.yml

@@ -535,7 +535,11 @@ functions:
     params:
       binary: bash
       args:
-      - "src/evergreen/functions/binaries_extract.sh"
+      - "src/evergreen/run_python_script.sh"
+      - "evergreen/functions/binaries_extract.py"
+      - "--tarball=mongo-binaries.tgz"
+      - "--extraction-command=${decompress}"
+      - "--change-dir=${extraction_change_dir}"
 
   "get version expansions": &get_version_expansions
     command: s3.get
@@ -2982,6 +2986,45 @@ tasks:
       suite: unittests
       install_dir: build/install/bin
 
+## pretty_printer ##
+- <<: *task_template
+  name: run_pretty_printer_tests
+  tags: []
+  commands:
+  - func: "git get project and add git tag"
+  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
+  - func: "set up venv"
+  - func: "upload pip requirements"
+  - func: "configure evergreen api credentials"
+  - func: "do setup"
+    vars:
+      extraction_change_dir: build/install/
+  - command: s3.get
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      remote_file: ${mongo_debugsymbols}
+      bucket: mciuploads
+      local_file: src/mongo-debugsymbols.tgz
+      optional: true
+  - command: subprocess.exec
+    params:
+      binary: bash
+      args:
+      - "src/evergreen/run_python_script.sh"
+      - "evergreen/functions/binaries_extract.py"
+      - "--tarball=mongo-debugsymbols.tgz"
+      - "--extraction-command=${decompress}"
+      - "--change-dir=build/install/"
+    optional: true
+  - func: "run tests"
+    vars:
+      suite: pretty-printer-tests
+      install_dir: build/install/dist-test/bin
+
+
 ## run_unittests with UndoDB live-record ##
 #- name: run_unittests_with_recording
 #  depends_on:

etc/evergreen_yml_components/variants/compile_static_analysis.yml

@@ -79,6 +79,7 @@ buildvariants:
     - name: compile_test_and_package_parallel_dbtest_stream_TG
     - name: compile_integration_and_test_parallel_stream_TG
     - name: generate_buildid_to_debug_symbols_mapping
+    - name: run_pretty_printer_tests
     - name: server_discovery_and_monitoring_json_test_TG
       distros:
       - rhel80-large
@@ -224,6 +225,7 @@ buildvariants:
     - name: resmoke_validation_tests
     - name: server_discovery_and_monitoring_json_test_TG
     - name: server_selection_json_test_TG
+    - name: run_pretty_printer_tests
 
 - <<: *linux-arm64-dynamic-compile-params
   name: &amazon-linux2-arm64-crypt-compile amazon-linux2-arm64-crypt-compile

etc/evergreen_yml_components/variants/sanitizer.yml

@@ -165,6 +165,7 @@ buildvariants:
   - name: compile_integration_and_test_parallel_stream_TG
     distros:
     - rhel80-large
+  - name: run_pretty_printer_tests
   - name: .aggregation !.feature_flag_guarded
   - name: .auth
   - name: audit

evergreen/functions/binaries_extract.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+#
+# Copyright 2020 MongoDB Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+import argparse
+import subprocess
+import os
+import sys
+import pathlib
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('--change-dir', type=str, action='store',
+                    help="The directory to change into to perform the extraction.")
+parser.add_argument('--extraction-command', type=str, action='store',
+                    help="The command to use for the extraction.")
+parser.add_argument('--tarball', type=str, action='store',
+                    help="The tarball to perform the extraction on.")
+
+args = parser.parse_args()
+
+if args.change_dir:
+    working_dir = pathlib.Path(args.change_dir).as_posix()
+    tarball = pathlib.Path(args.tarball).resolve().as_posix()
+    print(f"Switching to {working_dir} to perform the extraction in.")
+    os.makedirs(working_dir, exist_ok=True)
+else:
+    working_dir = None
+    tarball = pathlib.Path(args.tarball).as_posix()
+
+if sys.platform == 'win32':
+    proc = subprocess.run(['C:/cygwin/bin/cygpath.exe', '-w', os.environ['SHELL']], text=True,
+                          capture_output=True)
+    bash = pathlib.Path(proc.stdout.strip())
+    cmd = [bash.as_posix(), '-c', f"{args.extraction_command} {tarball}"]
+else:
+    cmd = [os.environ['SHELL'], '-c', f"{args.extraction_command} {tarball}"]
+
+print(f"Extracting: {' '.join(cmd)}")
+proc = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                      cwd=working_dir)
+
+print(proc.stdout)
+sys.exit(proc.returncode)

evergreen/functions/binaries_extract.sh

@@ -1,7 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/../prelude.sh"
-
-cd src
-
-set -o errexit
-${decompress} mongo-binaries.tgz

evergreen/run_python_script.sh

@@ -8,4 +8,5 @@ set -o verbose
 cd src
 
 activate_venv
-$python $@
+echo $python $@
+$python "$@"

site_scons/site_tools/auto_install_binaries.py

@@ -387,7 +387,7 @@ def auto_install_pseudobuilder(env, target, source, **kwargs):
 
         new_installed_files = env.Install(target=target_for_source, source=s)
         setattr(s.attributes, INSTALLED_FILES, new_installed_files)
-
+        setattr(new_installed_files[0].attributes, 'AIB_INSTALL_FROM', s)
         installed_files.extend(new_installed_files)
 
     entry.files.update(installed_files)

site_scons/site_tools/mongo_pretty_printer_tests.py

@@ -0,0 +1,208 @@
+# Copyright 2020 MongoDB Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+"""Pseudo-builders for building and registering tests for pretty printers."""
+import subprocess
+import os
+import sys
+
+import SCons
+from SCons.Script import Chmod
+
+not_building_already_warned = False
+
+
+def print_warning(message: str):
+    global not_building_already_warned
+    if not not_building_already_warned:
+        not_building_already_warned = True
+        print(message)
+
+
+def exists(env):
+    return True
+
+
+def build_pretty_printer_test(env, target, **kwargs):
+    if not isinstance(target, list):
+        target = [target]
+
+    if env.GetOption('ninja') != 'disabled':
+        print_warning("Can't build pretty printer tests with ninja enabled.")
+        return []
+
+    gdb_bin = None
+    if env.get('GDB'):
+        gdb_bin = env.get('GDB')
+    elif env.ToolchainIs('gcc', 'clang'):
+        # Always prefer v4 gdb, otherwise try anything in the path
+        gdb_bin = env.WhereIs('gdb', ['/opt/mongodbtoolchain/v4/bin']) or env.WhereIs('gdb')
+
+    if gdb_bin is None:
+        print_warning("Can't find gdb, not building pretty printer tests.")
+        return []
+
+    test_component = {"dist-test"}
+
+    if "AIB_COMPONENTS_EXTRA" in kwargs:
+        kwargs["AIB_COMPONENTS_EXTRA"] = set(kwargs["AIB_COMPONENTS_EXTRA"]).union(test_component)
+    else:
+        kwargs["AIB_COMPONENTS_EXTRA"] = list(test_component)
+
+    # GDB has a built in python interpreter, but it may have a python binary on the system which
+    # we can use to check package requirements.
+    python_bin = None
+    result = subprocess.run([gdb_bin, '--configuration'], capture_output=True, text=True)
+    if result.returncode == 0:
+        for line in result.stdout.splitlines():
+            if line.strip().startswith('--with-python='):
+                python_root = line.strip()[len('--with-python=') - 1:]
+                if python_root.endswith(' (relocatable)'):
+                    python_root = python_root[:-len(' (relocatable)')]
+                python_bin = os.path.join(python_root, 'bin/python3')
+    if not python_bin:
+        print(
+            f"Failed to find gdb's python from gdb '--configuration', defaulting to {sys.executable}"
+        )
+        python_bin = sys.executable
+
+    test_program = kwargs.get("TEST_PROGRAM", ['$DESTDIR/$PREFIX/bin/mongod'])
+    if isinstance(test_program, list):
+        test_program = test_program[0]
+    test_args = kwargs.get('TEST_ARGS', [])
+    gdb_test_script = env.File(target[0]).srcnode().abspath
+
+    if not gdb_test_script:
+        env.FatalError(
+            f"{target[0]}: You must supply a gdb python script to use in the pretty printer test.")
+
+    with open(gdb_test_script) as test_script:
+        verify_reqs_file = env.File('#site_scons/mongo/pip_requirements.py')
+
+        gen_test_script = env.Textfile(
+            target=os.path.basename(gdb_test_script),
+            source=verify_reqs_file.get_contents().decode('utf-8').split('\n') + [
+                "import os,subprocess,sys",
+                "cmd = 'python -c \"import os,sys;print(os.linesep.join(sys.path).strip())\"'",
+                "paths = subprocess.check_output(cmd,shell=True).decode('utf-8').split()",
+                "sys.path.extend(paths)",
+                "symbols_loaded = False",
+                "try:",
+                "    if gdb.objfiles()[0].lookup_global_symbol('main') is not None:",
+                "        symbols_loaded = True",
+                "except Exception:",
+                "    pass",
+                "if not symbols_loaded:",
+                r"    gdb.write('Could not find main symbol, debug info may not be loaded.\n')",
+                r"    gdb.write('TEST FAILED -- No Symbols.\\\n')",
+                "    gdb.execute('quit 1', to_string=True)",
+                "else:",
+                r"    gdb.write('Symbols loaded.\n')",
+                "gdb.execute('set confirm off')",
+                "gdb.execute('source .gdbinit')",
+                "try:",
+                "    verify_requirements('etc/pip/components/core.req', executable=f'@python_executable@')",
+                "except MissingRequirements as ex:",
+                "    print(ex)",
+                "    print('continuing testing anyways!')",
+            ] + [line.rstrip() for line in test_script.readlines()])
+
+        gen_test_script_install = env.AutoInstall(
+            target='$PREFIX_BINDIR',
+            source=gen_test_script,
+            AIB_ROLE='runtime',
+            AIB_COMPONENT='pretty-printer-tests',
+            AIB_COMPONENTS_EXTRA=kwargs["AIB_COMPONENTS_EXTRA"],
+        )
+
+    pretty_printer_test_launcher = env.Substfile(
+        target=f'pretty_printer_test_launcher_{target[0]}',
+        source='#/src/mongo/util/pretty_printer_test_launcher.py.in', SUBST_DICT={
+            '@VERBOSE@':
+                str(env.Verbose()),
+            '@pretty_printer_test_py@':
+                gen_test_script_install[0].path,
+            '@gdb_path@':
+                gdb_bin,
+            '@pretty_printer_test_program@':
+                env.File(test_program).path,
+            '@test_args@':
+                '["' + '", "'.join([env.subst(arg, target=target) for arg in test_args]) + '"]',
+        }, AIB_ROLE='runtime', AIB_COMPONENT='pretty-printer-tests',
+        AIB_COMPONENTS_EXTRA=kwargs["AIB_COMPONENTS_EXTRA"])
+    env.Depends(
+        pretty_printer_test_launcher[0],
+        [
+            test_program,
+            gen_test_script_install,
+        ],
+    )
+    env.AddPostAction(pretty_printer_test_launcher[0],
+                      Chmod(pretty_printer_test_launcher[0], 'ugo+x'))
+
+    pretty_printer_test_launcher_install = env.AutoInstall(
+        target='$PREFIX_BINDIR',
+        source=pretty_printer_test_launcher,
+        AIB_ROLE='runtime',
+        AIB_COMPONENT='pretty-printer-tests',
+        AIB_COMPONENTS_EXTRA=kwargs["AIB_COMPONENTS_EXTRA"],
+    )
+
+    def new_scanner(node, env, path=()):
+        source_binary = getattr(
+            env.File(env.get('TEST_PROGRAM')).attributes, 'AIB_INSTALL_FROM', None)
+        if source_binary:
+            debug_files = getattr(env.File(source_binary).attributes, 'separate_debug_files', None)
+            if debug_files:
+                if debug_files:
+                    installed_debug_files = getattr(
+                        env.File(debug_files[0]).attributes, 'AIB_INSTALLED_FILES', None)
+                    if installed_debug_files:
+                        if env.Verbose():
+                            print(
+                                f"Found and installing pretty_printer_test {node} test_program {env.File(env.get('TEST_PROGRAM'))} debug file {installed_debug_files[0]}"
+                            )
+                        return installed_debug_files
+        if env.Verbose():
+            print(f"Did not find separate debug files for pretty_printer_test {node}")
+        return []
+
+    scanner = SCons.Scanner.Scanner(function=new_scanner)
+
+    run_test = env.Command(target='+' + os.path.splitext(os.path.basename(gdb_test_script))[0],
+                           source=pretty_printer_test_launcher_install, action=str(
+                               pretty_printer_test_launcher_install[0]), TEST_PROGRAM=test_program,
+                           target_scanner=scanner)
+    env.Pseudo(run_test)
+    env.Alias('+' + os.path.splitext(os.path.basename(gdb_test_script))[0], run_test)
+    env.Depends(pretty_printer_test_launcher_install, [gen_test_script_install, test_program])
+
+    env.RegisterTest('$PRETTY_PRINTER_TEST_LIST', pretty_printer_test_launcher_install[0])
+    env.Alias("$PRETTY_PRINTER_TEST_ALIAS", pretty_printer_test_launcher_install[0])
+    env.Alias('+pretty-printer-tests', run_test)
+    return run_test
+
+
+def generate(env):
+    env.TestList("$PRETTY_PRINTER_TEST_LIST", source=[])
+    env.AddMethod(build_pretty_printer_test, "PrettyPrinterTest")
+    alias = env.Alias("$PRETTY_PRINTER_TEST_ALIAS", "$PRETTY_PRINTER_TEST_LIST")
+    env.Alias('+pretty-printer-tests', alias)

src/mongo/db/concurrency/SConscript

@@ -1,4 +1,5 @@
 # -*- mode: python -*-
+import sys
 
 Import("env")
 
@@ -105,3 +106,5 @@ env.CppUnitTest(
         'lock_manager',
     ],
 )
+
+env.PrettyPrinterTest(target="lock_gdb_test.py")

src/mongo/db/concurrency/lock_gdb_test.py

@@ -0,0 +1,15 @@
+"""Script to be invoked by GDB for testing lock manager pretty printer.
+"""
+
+import gdb
+import traceback
+
+try:
+    gdb.execute('break main')
+    gdb.execute('run')
+    gdb_type = lookup_type('mongo::LockManager')
+    assert gdb_type is not None, 'Failed to lookup type mongo::LockManager'
+    gdb.write('TEST PASSED\n')
+except Exception as err:
+    gdb.write('TEST FAILED -- {!s}\n'.format(traceback.format_exc()))
+    gdb.execute('quit 1', to_string=True)

src/mongo/util/SConscript

@@ -936,95 +936,17 @@ env.Benchmark(
     ],
 )
 
-if env.ToolchainIs('gcc', 'clang') and env.GetOption('ninja') == 'disabled':
-    # Always prefer v4 gdb, otherwise try anything in the path
-    gdb_bin = env.WhereIs('gdb', ['/opt/mongodbtoolchain/v4/bin']) or env.WhereIs('gdb')
-    if not gdb_bin:
-        Return()
-
-    # GDB has a built in python interpreter, but it may have a python binary on the system which
-    # we can use to check package requirements.
-    python_bin = None
-    result = subprocess.run([gdb_bin, '--configuration'], capture_output=True, text=True)
-    if result.returncode == 0:
-        for line in result.stdout.splitlines():
-            if line.strip().startswith('--with-python='):
-                python_root = line.strip()[len('--with-python=') - 1:]
-                if python_root.endswith(' (relocatable)'):
-                    python_root = python_root[:-len(' (relocatable)')]
-                python_bin = os.path.join(python_root, 'bin/python3')
-    if not python_bin:
-        python_bin = sys.executable
-
-    pretty_printer_test_program = env.Program(target='pretty_printer_test_program', source=[
+pretty_printer_test_program = env.Program(
+    target='pretty_printer_test_program',
+    source=[
         'pretty_printer_test_program.cpp',
-    ], LIBDEPS=[
+    ],
+    LIBDEPS=[
         '$BUILD_DIR/mongo/base',
-    ], AIB_COMPONENT='pretty-printer-test', AIB_COMPONENTS_EXTRA=[
-        'unittests',
-        'tests',
-    ])
-    pretty_printer_test_program_installed = env.GetAutoInstalledFiles(
-        pretty_printer_test_program[0])
+    ],
+    AIB_COMPONENT='pretty-printer-test',
+    AIB_COMPONENTS_EXTRA=['dist-test'],
+)
+pretty_printer_test_program_installed = env.GetAutoInstalledFiles(pretty_printer_test_program[0])
 
-    verify_reqs_file = env.File('#site_scons/mongo/pip_requirements.py')
-    pretty_printer_test = env.Substfile(
-        target='pretty_printer_test.py', source='pretty_printer_test.py.in', SUBST_DICT={
-            '@verify_requirements@':
-                verify_reqs_file.get_contents().decode('utf-8').replace('\\', '\\\\'),
-            '@python_executable@':
-                python_bin
-        })
-    env.Depends(pretty_printer_test, verify_reqs_file)
-    pretty_printer_test_installed = env.AutoInstall(
-        target='$PREFIX_BINDIR',
-        source=pretty_printer_test,
-        AIB_ROLE='runtime',
-        AIB_COMPONENT='pretty-printer-test',
-        AIB_COMPONENTS_EXTRA=[
-            'unittests',
-            'tests',
-        ],
-    )
-
-    pretty_printer_test_launcher = env.Substfile(
-        target='pretty_printer_test_launcher.py',
-        source='pretty_printer_test_launcher.py.in',
-        SUBST_DICT={
-            '@pretty_printer_test_py@': pretty_printer_test_installed[0].path,
-            '@gdb_path@': gdb_bin,
-            '@pretty_printer_test_program@': pretty_printer_test_program_installed[0].path,
-        },
-        AIB_ROLE='runtime',
-        AIB_COMPONENT='pretty-printer-test',
-        AIB_COMPONENTS_EXTRA=[
-            'unittests',
-            'tests',
-        ],
-    )
-    env.Depends(
-        pretty_printer_test_launcher[0],
-        [
-            pretty_printer_test_program,
-            pretty_printer_test,
-        ],
-    )
-    env.AddPostAction(pretty_printer_test_launcher[0],
-                      Chmod(pretty_printer_test_launcher[0], 'ugo+x'))
-
-    pretty_printer_test_launcher_install = env.AutoInstall(
-        target='$PREFIX_BINDIR',
-        source=pretty_printer_test_launcher,
-        AIB_ROLE='runtime',
-        AIB_COMPONENT='pretty-printer-test',
-        AIB_COMPONENTS_EXTRA=[
-            'unittests',
-            'tests',
-        ],
-    )
-    env.Depends(pretty_printer_test_launcher_install,
-                [pretty_printer_test_installed, pretty_printer_test_program_installed])
-    test_env = env.Clone()
-    test_env['ENV'] = os.environ.copy()
-    test_env.RegisterTest('$UNITTEST_LIST', pretty_printer_test_launcher_install[0])
-    test_env.Alias("$UNITTEST_ALIAS", pretty_printer_test_launcher_install[0])
+env.PrettyPrinterTest('pretty_printer_test.py', TEST_PROGRAM=pretty_printer_test_program_installed)

src/mongo/util/pretty_printer_test.py

@@ -3,15 +3,6 @@
 
 import gdb
 import re
-import sys
-
-# Here we substitute in the pip_requirements verification code, so this test
-# can remain a self contained test with little complexity. This is essentially an
-# import we are generating into the file instead to avoid and complex importer logic.
-# Start of pip verification code
-@verify_requirements@
-# End of pip verification code
-
 
 expected_patterns = [
     r'Decorable<MyDecorable\> with 3 elems',
@@ -23,11 +14,13 @@ up_pattern = r'std::unique_ptr<int\> = \{get\(\) \= 0x[0-9a-fA-F]+\}'
 set_pattern = r'std::[__debug::]*set with 4 elements'
 static_member_pattern = '128'
 
+
 def search(pattern, s):
     match = re.search(pattern, s)
     assert match is not None, 'Did not find {!s} in {!s}'.format(pattern, s)
     return match
 
+
 def test_decorable():
     gdb.execute('run')
     gdb.execute('frame function main')
@@ -35,25 +28,13 @@ def test_decorable():
     for pattern in expected_patterns:
         search(pattern, d1_str)
 
-    search(up_pattern,  gdb.execute('print up', to_string=True))
+    search(up_pattern, gdb.execute('print up', to_string=True))
     search(set_pattern, gdb.execute('print set_type', to_string=True))
     search(static_member_pattern, gdb.execute('print testClass::static_member', to_string=True))
 
 
-def configure_gdb():
-    import os,subprocess,sys
-    cmd = 'python -c "import os,sys;print(os.linesep.join(sys.path).strip())"'
-    paths = subprocess.check_output(cmd,shell=True).decode("utf-8").split()
-    sys.path.extend(paths)
-    gdb.execute('source .gdbinit')
-    try:
-        verify_requirements('etc/pip/components/core.req', executable=f"@python_executable@")
-    except MissingRequirements as ex:
-        print(ex)
-
 if __name__ == '__main__':
     try:
-        configure_gdb()
         test_decorable()
         gdb.write('TEST PASSED\n')
     except Exception as err:

src/mongo/util/pretty_printer_test_launcher.py.in

@@ -7,23 +7,34 @@ interpolated by scons.
 
 import subprocess
 import os
-
+import shlex
+verbose = @VERBOSE@
+test_script = r'@pretty_printer_test_py@'
+test_args = @test_args@
 gdb_path = '@gdb_path@'
 args = [
     gdb_path,
     '-nx',
     '-batch',
     '-ex',
-    r'source @pretty_printer_test_py@',
+    f'source {test_script}',
     '-args',
     r'@pretty_printer_test_program@',
-]
-print(f"Pretty printer test running command:\n{' '.join(args)}")
+] + test_args
+
+
+if verbose:
+    print(f"Pretty printer test running command:\n{shlex.join(args)}")
 # We assume we are running from project root, and require modules buried in the src directory
 python_env = os.environ.copy()
 python_env['PYTHONPATH'] = os.getcwd() + os.pathsep + python_env.get('PYTHONPATH', "")
 
 proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
                       env=python_env)
-print(proc.stdout)
+
+if verbose:
+    out = '\n'.join([f'[{os.path.basename(test_script)}] {line}' for line in proc.stdout.split('\n')])
+    print(out)
+else:
+    print(f'[{os.path.basename(test_script)}] {"passed" if proc.returncode == 0 else "failed"}')
 exit(proc.returncode)