From 1450624c71fb35ab61b36e87c7348395e659c395 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 19 May 2023 16:13:26 +0000 Subject: [PATCH] SERVER-76740 SERVER-69843 added pretty printer tests framework --- SConstruct | 11 + buildscripts/gdb/mongo.py | 44 +++- buildscripts/gdb/mongo_lock.py | 4 +- buildscripts/gdb/mongo_printers.py | 42 ++-- buildscripts/gdb/optimizer_printers.py | 8 +- buildscripts/gdb/wt_dump_table.py | 11 +- .../suites/pretty-printer-tests.yml | 7 + buildscripts/resmokelib/selector.py | 34 +++ .../testcases/pretty_printer_testcase.py | 25 +++ etc/evergreen_yml_components/definitions.yml | 45 +++- .../variants/compile_static_analysis.yml | 2 + .../variants/sanitizer.yml | 1 + evergreen/functions/binaries_extract.py | 64 ++++++ evergreen/functions/binaries_extract.sh | 7 - evergreen/run_python_script.sh | 3 +- .../site_tools/auto_install_binaries.py | 2 +- .../site_tools/mongo_pretty_printer_tests.py | 208 ++++++++++++++++++ src/mongo/db/concurrency/SConscript | 3 + src/mongo/db/concurrency/lock_gdb_test.py | 15 ++ src/mongo/util/SConscript | 100 +-------- ...nter_test.py.in => pretty_printer_test.py} | 25 +-- .../util/pretty_printer_test_launcher.py.in | 21 +- 22 files changed, 520 insertions(+), 162 deletions(-) create mode 100644 buildscripts/resmokeconfig/suites/pretty-printer-tests.yml create mode 100644 buildscripts/resmokelib/testing/testcases/pretty_printer_testcase.py create mode 100644 evergreen/functions/binaries_extract.py delete mode 100755 evergreen/functions/binaries_extract.sh create mode 100644 site_scons/site_tools/mongo_pretty_printer_tests.py create mode 100644 src/mongo/db/concurrency/lock_gdb_test.py rename src/mongo/util/{pretty_printer_test.py.in => pretty_printer_test.py} (58%) diff --git a/SConstruct b/SConstruct index f9ab97fd905..716f98ed50c 100644 --- a/SConstruct +++ b/SConstruct @@ -917,6 +917,7 @@ def variable_tools_converter(val): "mongo_integrationtest", "mongo_unittest", "mongo_libfuzzer", + "mongo_pretty_printer_tests", "textfile", ] @@ -1641,6 +1642,8 @@ envDict = dict( # TODO: Move unittests.txt to $BUILD_DIR, but that requires # changes to MCI. UNITTEST_LIST='$BUILD_ROOT/unittests.txt', + PRETTY_PRINTER_TEST_ALIAS='install-pretty-printer-tests', + PRETTY_PRINTER_TEST_LIST='$BUILD_DIR/pretty_printer_tests.txt', LIBFUZZER_TEST_ALIAS='install-fuzzertests', LIBFUZZER_TEST_LIST='$BUILD_ROOT/libfuzzer_tests.txt', INTEGRATION_TEST_ALIAS='install-integration-tests', @@ -6021,6 +6024,14 @@ env.AddPackageNameAlias( name="mh-debugsymbols", ) +env.AutoInstall( + target='$PREFIX', + source='$PRETTY_PRINTER_TEST_LIST', + AIB_ROLE='runtime', + AIB_COMPONENT='pretty-printer-tests', + AIB_COMPONENTS_EXTRA=['dist-test'], +) + env['RPATH_ESCAPED_DOLLAR_ORIGIN'] = '\\$$$$ORIGIN' diff --git a/buildscripts/gdb/mongo.py b/buildscripts/gdb/mongo.py index 8f20f1caa85..fe1a5d443d0 100644 --- a/buildscripts/gdb/mongo.py +++ b/buildscripts/gdb/mongo.py @@ -159,6 +159,36 @@ def get_thread_id(): raise ValueError("Failed to find thread id in {}".format(thread_info)) +MAIN_GLOBAL_BLOCK = None + + +def lookup_type(gdb_type_str: str) -> gdb.Type: + """ + Try to find the type object from string. + + GDB says it searches the global blocks, however this appear not to be the + case or at least it doesn't search all global blocks, sometimes it required + to get the global block based off the current frame. + """ + global MAIN_GLOBAL_BLOCK # pylint: disable=global-statement + + exceptions = [] + try: + return gdb.lookup_type(gdb_type_str) + except Exception as exc: + exceptions.append(exc) + + if MAIN_GLOBAL_BLOCK is None: + MAIN_GLOBAL_BLOCK = gdb.lookup_symbol("main")[0].symtab.global_block() + + try: + return gdb.lookup_type(gdb_type_str, MAIN_GLOBAL_BLOCK) + except Exception as exc: + exceptions.append(exc) + + raise gdb.error("Failed to get type, tried:\n%s" % '\n'.join([str(exc) for exc in exceptions])) + + def get_current_thread_name(): """Return the name of the current GDB thread.""" fallback_name = '"%s"' % (gdb.selected_thread().name or '') @@ -217,7 +247,7 @@ def get_wt_session(recovery_unit, recovery_unit_impl_type): if not wt_session_handle.dereference().address: return None wt_session = wt_session_handle.dereference().cast( - gdb.lookup_type("mongo::WiredTigerSession"))["_session"] + lookup_type("mongo::WiredTigerSession"))["_session"] return wt_session @@ -230,13 +260,13 @@ def get_decorations(obj): TODO: De-duplicate the logic between here and DecorablePrinter. This code was copied from there. """ type_name = str(obj.type).replace("class", "").replace(" ", "") - decorable = obj.cast(gdb.lookup_type("mongo::Decorable<{}>".format(type_name))) + decorable = obj.cast(lookup_type("mongo::Decorable<{}>".format(type_name))) decl_vector = decorable["_decorations"]["_registry"]["_decorationInfo"] start = decl_vector["_M_impl"]["_M_start"] finish = decl_vector["_M_impl"]["_M_finish"] decorable_t = decorable.type.template_argument(0) - decinfo_t = gdb.lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format( + decinfo_t = lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format( str(decorable_t).replace("class", "").strip())) count = int((int(finish) - int(start)) / decinfo_t.sizeof) @@ -255,7 +285,7 @@ def get_decorations(obj): type_name = type_name[0:len(type_name) - 1] type_name = type_name.rstrip() try: - type_t = gdb.lookup_type(type_name) + type_t = lookup_type(type_name) obj = decoration_data[dindex].cast(type_t) yield (type_name, obj) except Exception as err: @@ -501,7 +531,7 @@ class DumpMongoDSessionCatalog(gdb.Command): val = get_boost_optional(txn_part_observable_state['txnResourceStash']) if val: locker_addr = get_unique_ptr(val["_locker"]) - locker_obj = locker_addr.dereference().cast(gdb.lookup_type("mongo::LockerImpl")) + locker_obj = locker_addr.dereference().cast(lookup_type("mongo::LockerImpl")) print('txnResourceStash._locker', "@", locker_addr) print("txnResourceStash._locker._id", "=", locker_obj["_id"]) else: @@ -645,7 +675,7 @@ class MongoDBDumpRecoveryUnits(gdb.Command): recovery_unit_handle = get_unique_ptr(operation_context["_recoveryUnit"]) # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit" recovery_unit = recovery_unit_handle.dereference().cast( - gdb.lookup_type(recovery_unit_impl_type)) + lookup_type(recovery_unit_impl_type)) output_doc["recoveryUnit"] = hex(recovery_unit_handle) if recovery_unit else "0x0" wt_session = get_wt_session(recovery_unit, recovery_unit_impl_type) @@ -690,7 +720,7 @@ class MongoDBDumpRecoveryUnits(gdb.Command): recovery_unit_handle = get_unique_ptr(txn_resource_stash["_recoveryUnit"]) # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit" recovery_unit = recovery_unit_handle.dereference().cast( - gdb.lookup_type(recovery_unit_impl_type)) + lookup_type(recovery_unit_impl_type)) output_doc["recoveryUnit"] = hex(recovery_unit_handle) if recovery_unit else "0x0" wt_session = get_wt_session(recovery_unit, recovery_unit_impl_type) diff --git a/buildscripts/gdb/mongo_lock.py b/buildscripts/gdb/mongo_lock.py index a7f7e271926..38dcaa8b0f9 100644 --- a/buildscripts/gdb/mongo_lock.py +++ b/buildscripts/gdb/mongo_lock.py @@ -9,7 +9,7 @@ import gdb.printing if not gdb: sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent)) - from buildscripts.gdb.mongo import get_current_thread_name, get_thread_id, RegisterMongoCommand + from buildscripts.gdb.mongo import get_current_thread_name, get_thread_id, lookup_type, RegisterMongoCommand if sys.version_info[0] < 3: raise gdb.GdbError( @@ -323,7 +323,7 @@ def find_lock_manager_holders(graph, thread_dict, show): (_, lock_waiter_lwpid, _) = gdb.selected_thread().ptid lock_waiter = thread_dict[lock_waiter_lwpid] - locker_ptr_type = gdb.lookup_type("mongo::LockerImpl").pointer() + locker_ptr_type = lookup_type("mongo::LockerImpl").pointer() lock_head = gdb.parse_and_eval( "mongo::LockManager::get((mongo::ServiceContext*) mongo::getGlobalServiceContext())->_getBucket(resId)->findOrInsert(resId)" diff --git a/buildscripts/gdb/mongo_printers.py b/buildscripts/gdb/mongo_printers.py index 95780dd50dc..31ce5fd12c1 100644 --- a/buildscripts/gdb/mongo_printers.py +++ b/buildscripts/gdb/mongo_printers.py @@ -15,7 +15,7 @@ if ROOT_PATH not in sys.path: from src.third_party.immer.dist.tools.gdb_pretty_printers.printers import ListIter as ImmerListIter # pylint: disable=wrong-import-position if not gdb: - from buildscripts.gdb.mongo import get_boost_optional + from buildscripts.gdb.mongo import get_boost_optional, lookup_type from buildscripts.gdb.optimizer_printers import register_abt_printers try: @@ -142,7 +142,7 @@ class BSONObjPrinter(object): def __init__(self, val): """Initialize BSONObjPrinter.""" self.val = val - self.ptr = self.val['_objdata'].cast(gdb.lookup_type('void').pointer()) + self.ptr = self.val['_objdata'].cast(lookup_type('void').pointer()) self.is_valid = False # Handle the endianness of the BSON object size, which is represented as a 32-bit integer @@ -301,7 +301,7 @@ class RecordIdPrinter(object): holder = holder_ptr.dereference() str_len = int(holder["_capacity"]) # Start of data is immediately after pointer for holder - start_ptr = (holder_ptr + 1).dereference().cast(gdb.lookup_type("char")).address + start_ptr = (holder_ptr + 1).dereference().cast(lookup_type("char")).address raw_bytes = [int(start_ptr[i]) for i in range(0, str_len)] hex_bytes = [hex(b & 0xFF)[2:].zfill(2) for b in raw_bytes] return "RecordId big string %d hex bytes @ %s: %s" % (str_len, holder_ptr + 1, @@ -322,7 +322,7 @@ class DecorablePrinter(object): self.start = decl_vector["_M_impl"]["_M_start"] finish = decl_vector["_M_impl"]["_M_finish"] decorable_t = val.type.template_argument(0) - decinfo_t = gdb.lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format( + decinfo_t = lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format( str(decorable_t).replace("class", "").strip())) self.count = int((int(finish) - int(self.start)) / decinfo_t.sizeof) @@ -357,7 +357,7 @@ class DecorablePrinter(object): type_name = type_name.rstrip() # Cast the raw char[] into the actual object that is stored there. - type_t = gdb.lookup_type(type_name) + type_t = lookup_type(type_name) obj = decoration_data[dindex].cast(type_t) yield ('key', "%d:%s:%s" % (index, obj.address, type_name)) @@ -791,7 +791,7 @@ def make_inverse_enum_dict(enum_type_name): For example, if the enum type is 'mongo::sbe::vm::Builtin' with an element 'regexMatch', the dictionary will contain 'regexMatch' value and not 'mongo::sbe::vm::Builtin::regexMatch'. """ - enum_dict = gdb.types.make_enum_dict(gdb.lookup_type(enum_type_name)) + enum_dict = gdb.types.make_enum_dict(lookup_type(enum_type_name)) enum_inverse_dic = dict() for key, value in enum_dict.items(): enum_inverse_dic[int(value)] = key.split('::')[-1] # take last element @@ -838,11 +838,11 @@ class SbeCodeFragmentPrinter(object): # either use an inline buffer or an allocated one. The choice of storage is decoded in the # last bit of the 'metadata_' field. storage = self.val['_instrs']['storage_'] - meta = storage['metadata_'].cast(gdb.lookup_type('size_t')) + meta = storage['metadata_'].cast(lookup_type('size_t')) self.is_inlined = (meta % 2 == 0) self.size = (meta >> 1) self.pdata = \ - storage['data_']['inlined']['inlined_data'].cast(gdb.lookup_type('uint8_t').pointer()) \ + storage['data_']['inlined']['inlined_data'].cast(lookup_type('uint8_t').pointer()) \ if self.is_inlined \ else storage['data_']['allocated']['allocated_data'] @@ -866,17 +866,17 @@ class SbeCodeFragmentPrinter(object): yield 'instrs total size', self.size # Sizes for types we'll use when parsing the insructions stream. - int_size = gdb.lookup_type('int').sizeof - ptr_size = gdb.lookup_type('void').pointer().sizeof - tag_size = gdb.lookup_type('mongo::sbe::value::TypeTags').sizeof - value_size = gdb.lookup_type('mongo::sbe::value::Value').sizeof - uint8_size = gdb.lookup_type('uint8_t').sizeof - uint32_size = gdb.lookup_type('uint32_t').sizeof - uint64_size = gdb.lookup_type('uint64_t').sizeof - builtin_size = gdb.lookup_type('mongo::sbe::vm::Builtin').sizeof - time_unit_size = gdb.lookup_type('mongo::TimeUnit').sizeof - timezone_size = gdb.lookup_type('mongo::TimeZone').sizeof - day_of_week_size = gdb.lookup_type('mongo::DayOfWeek').sizeof + int_size = lookup_type('int').sizeof + ptr_size = lookup_type('void').pointer().sizeof + tag_size = lookup_type('mongo::sbe::value::TypeTags').sizeof + value_size = lookup_type('mongo::sbe::value::Value').sizeof + uint8_size = lookup_type('uint8_t').sizeof + uint32_size = lookup_type('uint32_t').sizeof + uint64_size = lookup_type('uint64_t').sizeof + builtin_size = lookup_type('mongo::sbe::vm::Builtin').sizeof + time_unit_size = lookup_type('mongo::TimeUnit').sizeof + timezone_size = lookup_type('mongo::TimeZone').sizeof + day_of_week_size = lookup_type('mongo::DayOfWeek').sizeof cur_op = self.pdata end_op = self.pdata + self.size @@ -921,9 +921,9 @@ class SbeCodeFragmentPrinter(object): cur_op += uint32_size elif op_name in ['function', 'functionSmall']: arity_size = \ - gdb.lookup_type('mongo::sbe::vm::ArityType').sizeof \ + lookup_type('mongo::sbe::vm::ArityType').sizeof \ if op_name == 'function' \ - else gdb.lookup_type('mongo::sbe::vm::SmallArityType').sizeof + else lookup_type('mongo::sbe::vm::SmallArityType').sizeof builtin_id = read_as_integer(cur_op, builtin_size) args = 'builtin: ' + self.builtins_lookup.get(builtin_id, "unknown") args += ' arity: ' + str(read_as_integer(cur_op + builtin_size, arity_size)) diff --git a/buildscripts/gdb/optimizer_printers.py b/buildscripts/gdb/optimizer_printers.py index 756b7580e35..047f993ecd6 100644 --- a/buildscripts/gdb/optimizer_printers.py +++ b/buildscripts/gdb/optimizer_printers.py @@ -8,7 +8,7 @@ import gdb.printing if not gdb: sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent)) - from buildscripts.gdb.mongo import get_boost_optional + from buildscripts.gdb.mongo import get_boost_optional, lookup_type def eval_print_fn(val, print_fn): @@ -893,7 +893,7 @@ class PolyValuePrinter(object): def to_string(self): dynamic_type = self.get_dynamic_type() try: - dynamic_type = gdb.lookup_type(dynamic_type).strip_typedefs() + dynamic_type = lookup_type(dynamic_type).strip_typedefs() except gdb.error: return "Unknown PolyValue tag: {}, did you add a new one?".format(self.tag) # GDB automatically formats types with children, remove the extra characters to get the @@ -979,7 +979,7 @@ class ABTPrinter(PolyValuePrinter): def get_bound_projections(node): # Casts the input node to an ExpressionBinder and returns the set of bound projection names. pp = PolyValuePrinter(ABTPrinter.abt_type_set, ABTPrinter.abt_namespace, node) - dynamic_type = gdb.lookup_type(pp.get_dynamic_type()).strip_typedefs() + dynamic_type = lookup_type(pp.get_dynamic_type()).strip_typedefs() binder = pp.cast_control_block(dynamic_type) return Vector(binder["_names"]) @@ -1111,7 +1111,7 @@ def register_abt_printers(pp): # stale. try: # ABT printer. - abt_type = gdb.lookup_type("mongo::optimizer::ABT").strip_typedefs() + abt_type = lookup_type("mongo::optimizer::ABT").strip_typedefs() pp.add('ABT', abt_type.name, False, ABTPrinter) abt_ref_type = abt_type.name + "::Reference" diff --git a/buildscripts/gdb/wt_dump_table.py b/buildscripts/gdb/wt_dump_table.py index 699cb3381e4..1add011e3e9 100644 --- a/buildscripts/gdb/wt_dump_table.py +++ b/buildscripts/gdb/wt_dump_table.py @@ -1,6 +1,13 @@ import gdb import bson +import sys +import os from pprint import pprint +from pathlib import Path + +if not gdb: + sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent)) + from buildscripts.gdb.mongo import lookup_type DEBUGGING = False ''' @@ -21,7 +28,7 @@ Some behaviors/limitations: def dump_pages_for_table(ident): - conn_impl_type = gdb.lookup_type("WT_CONNECTION_IMPL") + conn_impl_type = lookup_type("WT_CONNECTION_IMPL") if not conn_impl_type: print('WT_CONNECTION_IMPL type not found. Try invoking this function from a different \ thread and frame.') @@ -104,7 +111,7 @@ def get_data_handle(conn, handle_name): def get_btree_handle(dhandle): - btree = gdb.lookup_type('WT_BTREE').pointer() + btree = lookup_type('WT_BTREE').pointer() return dhandle['handle'].reinterpret_cast(btree).dereference() diff --git a/buildscripts/resmokeconfig/suites/pretty-printer-tests.yml b/buildscripts/resmokeconfig/suites/pretty-printer-tests.yml new file mode 100644 index 00000000000..6d2a2d2c0ce --- /dev/null +++ b/buildscripts/resmokeconfig/suites/pretty-printer-tests.yml @@ -0,0 +1,7 @@ +test_kind: pretty_printer_test + +selector: + root: build/install/dist-test/pretty_printer_tests.txt + +executor: + config: {} diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index b26ac4a374b..9d347259e6d 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -604,6 +604,39 @@ class _CppTestSelector(_Selector): return _Selector.select(self, selector_config) +class _PrettyPrinterTestSelectorConfig(_SelectorConfig): + """_SelectorConfig subclass for pretty-printer-tests.""" + + def __init__(self, root=config.DEFAULT_INTEGRATION_TEST_LIST, roots=None, include_files=None, + exclude_files=None): + """Initialize _PrettyPrinterTestSelectorConfig.""" + if roots: + # The 'roots' argument is only present when tests are specified on the command line + # and in that case they take precedence over the tests in the root file. + _SelectorConfig.__init__(self, roots=roots, include_files=include_files, + exclude_files=exclude_files) + else: + _SelectorConfig.__init__(self, root=root, include_files=include_files, + exclude_files=exclude_files) + + +class _PrettyPrinterTestSelector(_Selector): + """_Selector subclass for pretty-printer-tests.""" + + def __init__(self, test_file_explorer): + """Initialize _PrettyPrinterTestSelector.""" + _Selector.__init__(self, test_file_explorer) + + def select(self, selector_config): + """Return selected tests.""" + if selector_config.roots: + # Tests have been specified on the command line. We use them without additional + # filtering. + test_list = _TestList(self._test_file_explorer, selector_config.roots) + return test_list.get_tests() + return _Selector.select(self, selector_config) + + class _DbTestSelectorConfig(_SelectorConfig): """_Selector config subclass for db_test tests.""" @@ -715,6 +748,7 @@ _DEFAULT_TEST_FILE_EXPLORER = TestFileExplorer() _SELECTOR_REGISTRY = { "cpp_integration_test": (_CppTestSelectorConfig, _CppTestSelector), "cpp_unit_test": (_CppTestSelectorConfig, _CppTestSelector), + "pretty_printer_test": (_PrettyPrinterTestSelectorConfig, _PrettyPrinterTestSelector), "benchmark_test": (_CppTestSelectorConfig, _CppTestSelector), "sdam_json_test": (_FileBasedSelectorConfig, _Selector), "server_selection_json_test": (_FileBasedSelectorConfig, _Selector), diff --git a/buildscripts/resmokelib/testing/testcases/pretty_printer_testcase.py b/buildscripts/resmokelib/testing/testcases/pretty_printer_testcase.py new file mode 100644 index 00000000000..15cb9cff96d --- /dev/null +++ b/buildscripts/resmokelib/testing/testcases/pretty_printer_testcase.py @@ -0,0 +1,25 @@ +"""The unittest.TestCase for pretty printer tests.""" +import os + +from buildscripts.resmokelib import config +from buildscripts.resmokelib import core +from buildscripts.resmokelib import utils +from buildscripts.resmokelib.testing.testcases import interface + + +class PrettyPrinterTestCase(interface.ProcessTestCase): + """A pretty printer test to execute.""" + + REGISTERED_NAME = "pretty_printer_test" + + def __init__(self, logger, program_executable, program_options=None): + """Initialize the PrettyPrinterTestCase with the executable to run.""" + + interface.ProcessTestCase.__init__(self, logger, "pretty printer test", program_executable) + + self.program_executable = program_executable + self.program_options = utils.default_if_none(program_options, {}).copy() + + def _make_process(self): + return core.programs.make_process(self.logger, [self.program_executable], + **self.program_options) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index d1b763bf16c..7efc33bb1d6 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -535,7 +535,11 @@ functions: params: binary: bash args: - - "src/evergreen/functions/binaries_extract.sh" + - "src/evergreen/run_python_script.sh" + - "evergreen/functions/binaries_extract.py" + - "--tarball=mongo-binaries.tgz" + - "--extraction-command=${decompress}" + - "--change-dir=${extraction_change_dir}" "get version expansions": &get_version_expansions command: s3.get @@ -2982,6 +2986,45 @@ tasks: suite: unittests install_dir: build/install/bin +## pretty_printer ## +- <<: *task_template + name: run_pretty_printer_tests + tags: [] + commands: + - func: "git get project and add git tag" + - *f_expansions_write + - *kill_processes + - *cleanup_environment + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - func: "do setup" + vars: + extraction_change_dir: build/install/ + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${mongo_debugsymbols} + bucket: mciuploads + local_file: src/mongo-debugsymbols.tgz + optional: true + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/run_python_script.sh" + - "evergreen/functions/binaries_extract.py" + - "--tarball=mongo-debugsymbols.tgz" + - "--extraction-command=${decompress}" + - "--change-dir=build/install/" + optional: true + - func: "run tests" + vars: + suite: pretty-printer-tests + install_dir: build/install/dist-test/bin + + ## run_unittests with UndoDB live-record ## #- name: run_unittests_with_recording # depends_on: diff --git a/etc/evergreen_yml_components/variants/compile_static_analysis.yml b/etc/evergreen_yml_components/variants/compile_static_analysis.yml index 3ab4426b699..3994e8002f3 100644 --- a/etc/evergreen_yml_components/variants/compile_static_analysis.yml +++ b/etc/evergreen_yml_components/variants/compile_static_analysis.yml @@ -79,6 +79,7 @@ buildvariants: - name: compile_test_and_package_parallel_dbtest_stream_TG - name: compile_integration_and_test_parallel_stream_TG - name: generate_buildid_to_debug_symbols_mapping + - name: run_pretty_printer_tests - name: server_discovery_and_monitoring_json_test_TG distros: - rhel80-large @@ -224,6 +225,7 @@ buildvariants: - name: resmoke_validation_tests - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG + - name: run_pretty_printer_tests - <<: *linux-arm64-dynamic-compile-params name: &amazon-linux2-arm64-crypt-compile amazon-linux2-arm64-crypt-compile diff --git a/etc/evergreen_yml_components/variants/sanitizer.yml b/etc/evergreen_yml_components/variants/sanitizer.yml index 13d3681ddec..c971c045283 100644 --- a/etc/evergreen_yml_components/variants/sanitizer.yml +++ b/etc/evergreen_yml_components/variants/sanitizer.yml @@ -165,6 +165,7 @@ buildvariants: - name: compile_integration_and_test_parallel_stream_TG distros: - rhel80-large + - name: run_pretty_printer_tests - name: .aggregation !.feature_flag_guarded - name: .auth - name: audit diff --git a/evergreen/functions/binaries_extract.py b/evergreen/functions/binaries_extract.py new file mode 100644 index 00000000000..11cbd6c4975 --- /dev/null +++ b/evergreen/functions/binaries_extract.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# +# Copyright 2020 MongoDB Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import argparse +import subprocess +import os +import sys +import pathlib + +parser = argparse.ArgumentParser() + +parser.add_argument('--change-dir', type=str, action='store', + help="The directory to change into to perform the extraction.") +parser.add_argument('--extraction-command', type=str, action='store', + help="The command to use for the extraction.") +parser.add_argument('--tarball', type=str, action='store', + help="The tarball to perform the extraction on.") + +args = parser.parse_args() + +if args.change_dir: + working_dir = pathlib.Path(args.change_dir).as_posix() + tarball = pathlib.Path(args.tarball).resolve().as_posix() + print(f"Switching to {working_dir} to perform the extraction in.") + os.makedirs(working_dir, exist_ok=True) +else: + working_dir = None + tarball = pathlib.Path(args.tarball).as_posix() + +if sys.platform == 'win32': + proc = subprocess.run(['C:/cygwin/bin/cygpath.exe', '-w', os.environ['SHELL']], text=True, + capture_output=True) + bash = pathlib.Path(proc.stdout.strip()) + cmd = [bash.as_posix(), '-c', f"{args.extraction_command} {tarball}"] +else: + cmd = [os.environ['SHELL'], '-c', f"{args.extraction_command} {tarball}"] + +print(f"Extracting: {' '.join(cmd)}") +proc = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd=working_dir) + +print(proc.stdout) +sys.exit(proc.returncode) diff --git a/evergreen/functions/binaries_extract.sh b/evergreen/functions/binaries_extract.sh deleted file mode 100755 index ec7b8dd3f8f..00000000000 --- a/evergreen/functions/binaries_extract.sh +++ /dev/null @@ -1,7 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/../prelude.sh" - -cd src - -set -o errexit -${decompress} mongo-binaries.tgz diff --git a/evergreen/run_python_script.sh b/evergreen/run_python_script.sh index 35181dec3c1..746c229e9d8 100644 --- a/evergreen/run_python_script.sh +++ b/evergreen/run_python_script.sh @@ -8,4 +8,5 @@ set -o verbose cd src activate_venv -$python $@ +echo $python $@ +$python "$@" diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py index 7ec9810e2d2..5fdce083dc2 100644 --- a/site_scons/site_tools/auto_install_binaries.py +++ b/site_scons/site_tools/auto_install_binaries.py @@ -387,7 +387,7 @@ def auto_install_pseudobuilder(env, target, source, **kwargs): new_installed_files = env.Install(target=target_for_source, source=s) setattr(s.attributes, INSTALLED_FILES, new_installed_files) - + setattr(new_installed_files[0].attributes, 'AIB_INSTALL_FROM', s) installed_files.extend(new_installed_files) entry.files.update(installed_files) diff --git a/site_scons/site_tools/mongo_pretty_printer_tests.py b/site_scons/site_tools/mongo_pretty_printer_tests.py new file mode 100644 index 00000000000..5de2f3a325c --- /dev/null +++ b/site_scons/site_tools/mongo_pretty_printer_tests.py @@ -0,0 +1,208 @@ +# Copyright 2020 MongoDB Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +"""Pseudo-builders for building and registering tests for pretty printers.""" +import subprocess +import os +import sys + +import SCons +from SCons.Script import Chmod + +not_building_already_warned = False + + +def print_warning(message: str): + global not_building_already_warned + if not not_building_already_warned: + not_building_already_warned = True + print(message) + + +def exists(env): + return True + + +def build_pretty_printer_test(env, target, **kwargs): + if not isinstance(target, list): + target = [target] + + if env.GetOption('ninja') != 'disabled': + print_warning("Can't build pretty printer tests with ninja enabled.") + return [] + + gdb_bin = None + if env.get('GDB'): + gdb_bin = env.get('GDB') + elif env.ToolchainIs('gcc', 'clang'): + # Always prefer v4 gdb, otherwise try anything in the path + gdb_bin = env.WhereIs('gdb', ['/opt/mongodbtoolchain/v4/bin']) or env.WhereIs('gdb') + + if gdb_bin is None: + print_warning("Can't find gdb, not building pretty printer tests.") + return [] + + test_component = {"dist-test"} + + if "AIB_COMPONENTS_EXTRA" in kwargs: + kwargs["AIB_COMPONENTS_EXTRA"] = set(kwargs["AIB_COMPONENTS_EXTRA"]).union(test_component) + else: + kwargs["AIB_COMPONENTS_EXTRA"] = list(test_component) + + # GDB has a built in python interpreter, but it may have a python binary on the system which + # we can use to check package requirements. + python_bin = None + result = subprocess.run([gdb_bin, '--configuration'], capture_output=True, text=True) + if result.returncode == 0: + for line in result.stdout.splitlines(): + if line.strip().startswith('--with-python='): + python_root = line.strip()[len('--with-python=') - 1:] + if python_root.endswith(' (relocatable)'): + python_root = python_root[:-len(' (relocatable)')] + python_bin = os.path.join(python_root, 'bin/python3') + if not python_bin: + print( + f"Failed to find gdb's python from gdb '--configuration', defaulting to {sys.executable}" + ) + python_bin = sys.executable + + test_program = kwargs.get("TEST_PROGRAM", ['$DESTDIR/$PREFIX/bin/mongod']) + if isinstance(test_program, list): + test_program = test_program[0] + test_args = kwargs.get('TEST_ARGS', []) + gdb_test_script = env.File(target[0]).srcnode().abspath + + if not gdb_test_script: + env.FatalError( + f"{target[0]}: You must supply a gdb python script to use in the pretty printer test.") + + with open(gdb_test_script) as test_script: + verify_reqs_file = env.File('#site_scons/mongo/pip_requirements.py') + + gen_test_script = env.Textfile( + target=os.path.basename(gdb_test_script), + source=verify_reqs_file.get_contents().decode('utf-8').split('\n') + [ + "import os,subprocess,sys", + "cmd = 'python -c \"import os,sys;print(os.linesep.join(sys.path).strip())\"'", + "paths = subprocess.check_output(cmd,shell=True).decode('utf-8').split()", + "sys.path.extend(paths)", + "symbols_loaded = False", + "try:", + " if gdb.objfiles()[0].lookup_global_symbol('main') is not None:", + " symbols_loaded = True", + "except Exception:", + " pass", + "if not symbols_loaded:", + r" gdb.write('Could not find main symbol, debug info may not be loaded.\n')", + r" gdb.write('TEST FAILED -- No Symbols.\\\n')", + " gdb.execute('quit 1', to_string=True)", + "else:", + r" gdb.write('Symbols loaded.\n')", + "gdb.execute('set confirm off')", + "gdb.execute('source .gdbinit')", + "try:", + " verify_requirements('etc/pip/components/core.req', executable=f'@python_executable@')", + "except MissingRequirements as ex:", + " print(ex)", + " print('continuing testing anyways!')", + ] + [line.rstrip() for line in test_script.readlines()]) + + gen_test_script_install = env.AutoInstall( + target='$PREFIX_BINDIR', + source=gen_test_script, + AIB_ROLE='runtime', + AIB_COMPONENT='pretty-printer-tests', + AIB_COMPONENTS_EXTRA=kwargs["AIB_COMPONENTS_EXTRA"], + ) + + pretty_printer_test_launcher = env.Substfile( + target=f'pretty_printer_test_launcher_{target[0]}', + source='#/src/mongo/util/pretty_printer_test_launcher.py.in', SUBST_DICT={ + '@VERBOSE@': + str(env.Verbose()), + '@pretty_printer_test_py@': + gen_test_script_install[0].path, + '@gdb_path@': + gdb_bin, + '@pretty_printer_test_program@': + env.File(test_program).path, + '@test_args@': + '["' + '", "'.join([env.subst(arg, target=target) for arg in test_args]) + '"]', + }, AIB_ROLE='runtime', AIB_COMPONENT='pretty-printer-tests', + AIB_COMPONENTS_EXTRA=kwargs["AIB_COMPONENTS_EXTRA"]) + env.Depends( + pretty_printer_test_launcher[0], + [ + test_program, + gen_test_script_install, + ], + ) + env.AddPostAction(pretty_printer_test_launcher[0], + Chmod(pretty_printer_test_launcher[0], 'ugo+x')) + + pretty_printer_test_launcher_install = env.AutoInstall( + target='$PREFIX_BINDIR', + source=pretty_printer_test_launcher, + AIB_ROLE='runtime', + AIB_COMPONENT='pretty-printer-tests', + AIB_COMPONENTS_EXTRA=kwargs["AIB_COMPONENTS_EXTRA"], + ) + + def new_scanner(node, env, path=()): + source_binary = getattr( + env.File(env.get('TEST_PROGRAM')).attributes, 'AIB_INSTALL_FROM', None) + if source_binary: + debug_files = getattr(env.File(source_binary).attributes, 'separate_debug_files', None) + if debug_files: + if debug_files: + installed_debug_files = getattr( + env.File(debug_files[0]).attributes, 'AIB_INSTALLED_FILES', None) + if installed_debug_files: + if env.Verbose(): + print( + f"Found and installing pretty_printer_test {node} test_program {env.File(env.get('TEST_PROGRAM'))} debug file {installed_debug_files[0]}" + ) + return installed_debug_files + if env.Verbose(): + print(f"Did not find separate debug files for pretty_printer_test {node}") + return [] + + scanner = SCons.Scanner.Scanner(function=new_scanner) + + run_test = env.Command(target='+' + os.path.splitext(os.path.basename(gdb_test_script))[0], + source=pretty_printer_test_launcher_install, action=str( + pretty_printer_test_launcher_install[0]), TEST_PROGRAM=test_program, + target_scanner=scanner) + env.Pseudo(run_test) + env.Alias('+' + os.path.splitext(os.path.basename(gdb_test_script))[0], run_test) + env.Depends(pretty_printer_test_launcher_install, [gen_test_script_install, test_program]) + + env.RegisterTest('$PRETTY_PRINTER_TEST_LIST', pretty_printer_test_launcher_install[0]) + env.Alias("$PRETTY_PRINTER_TEST_ALIAS", pretty_printer_test_launcher_install[0]) + env.Alias('+pretty-printer-tests', run_test) + return run_test + + +def generate(env): + env.TestList("$PRETTY_PRINTER_TEST_LIST", source=[]) + env.AddMethod(build_pretty_printer_test, "PrettyPrinterTest") + alias = env.Alias("$PRETTY_PRINTER_TEST_ALIAS", "$PRETTY_PRINTER_TEST_LIST") + env.Alias('+pretty-printer-tests', alias) diff --git a/src/mongo/db/concurrency/SConscript b/src/mongo/db/concurrency/SConscript index 872b28b8047..961696fa8e7 100644 --- a/src/mongo/db/concurrency/SConscript +++ b/src/mongo/db/concurrency/SConscript @@ -1,4 +1,5 @@ # -*- mode: python -*- +import sys Import("env") @@ -105,3 +106,5 @@ env.CppUnitTest( 'lock_manager', ], ) + +env.PrettyPrinterTest(target="lock_gdb_test.py") diff --git a/src/mongo/db/concurrency/lock_gdb_test.py b/src/mongo/db/concurrency/lock_gdb_test.py new file mode 100644 index 00000000000..b9b1d8dc89f --- /dev/null +++ b/src/mongo/db/concurrency/lock_gdb_test.py @@ -0,0 +1,15 @@ +"""Script to be invoked by GDB for testing lock manager pretty printer. +""" + +import gdb +import traceback + +try: + gdb.execute('break main') + gdb.execute('run') + gdb_type = lookup_type('mongo::LockManager') + assert gdb_type is not None, 'Failed to lookup type mongo::LockManager' + gdb.write('TEST PASSED\n') +except Exception as err: + gdb.write('TEST FAILED -- {!s}\n'.format(traceback.format_exc())) + gdb.execute('quit 1', to_string=True) diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index 992990e9fbc..8c5f4a0e63a 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -936,95 +936,17 @@ env.Benchmark( ], ) -if env.ToolchainIs('gcc', 'clang') and env.GetOption('ninja') == 'disabled': - # Always prefer v4 gdb, otherwise try anything in the path - gdb_bin = env.WhereIs('gdb', ['/opt/mongodbtoolchain/v4/bin']) or env.WhereIs('gdb') - if not gdb_bin: - Return() - - # GDB has a built in python interpreter, but it may have a python binary on the system which - # we can use to check package requirements. - python_bin = None - result = subprocess.run([gdb_bin, '--configuration'], capture_output=True, text=True) - if result.returncode == 0: - for line in result.stdout.splitlines(): - if line.strip().startswith('--with-python='): - python_root = line.strip()[len('--with-python=') - 1:] - if python_root.endswith(' (relocatable)'): - python_root = python_root[:-len(' (relocatable)')] - python_bin = os.path.join(python_root, 'bin/python3') - if not python_bin: - python_bin = sys.executable - - pretty_printer_test_program = env.Program(target='pretty_printer_test_program', source=[ +pretty_printer_test_program = env.Program( + target='pretty_printer_test_program', + source=[ 'pretty_printer_test_program.cpp', - ], LIBDEPS=[ + ], + LIBDEPS=[ '$BUILD_DIR/mongo/base', - ], AIB_COMPONENT='pretty-printer-test', AIB_COMPONENTS_EXTRA=[ - 'unittests', - 'tests', - ]) - pretty_printer_test_program_installed = env.GetAutoInstalledFiles( - pretty_printer_test_program[0]) + ], + AIB_COMPONENT='pretty-printer-test', + AIB_COMPONENTS_EXTRA=['dist-test'], +) +pretty_printer_test_program_installed = env.GetAutoInstalledFiles(pretty_printer_test_program[0]) - verify_reqs_file = env.File('#site_scons/mongo/pip_requirements.py') - pretty_printer_test = env.Substfile( - target='pretty_printer_test.py', source='pretty_printer_test.py.in', SUBST_DICT={ - '@verify_requirements@': - verify_reqs_file.get_contents().decode('utf-8').replace('\\', '\\\\'), - '@python_executable@': - python_bin - }) - env.Depends(pretty_printer_test, verify_reqs_file) - pretty_printer_test_installed = env.AutoInstall( - target='$PREFIX_BINDIR', - source=pretty_printer_test, - AIB_ROLE='runtime', - AIB_COMPONENT='pretty-printer-test', - AIB_COMPONENTS_EXTRA=[ - 'unittests', - 'tests', - ], - ) - - pretty_printer_test_launcher = env.Substfile( - target='pretty_printer_test_launcher.py', - source='pretty_printer_test_launcher.py.in', - SUBST_DICT={ - '@pretty_printer_test_py@': pretty_printer_test_installed[0].path, - '@gdb_path@': gdb_bin, - '@pretty_printer_test_program@': pretty_printer_test_program_installed[0].path, - }, - AIB_ROLE='runtime', - AIB_COMPONENT='pretty-printer-test', - AIB_COMPONENTS_EXTRA=[ - 'unittests', - 'tests', - ], - ) - env.Depends( - pretty_printer_test_launcher[0], - [ - pretty_printer_test_program, - pretty_printer_test, - ], - ) - env.AddPostAction(pretty_printer_test_launcher[0], - Chmod(pretty_printer_test_launcher[0], 'ugo+x')) - - pretty_printer_test_launcher_install = env.AutoInstall( - target='$PREFIX_BINDIR', - source=pretty_printer_test_launcher, - AIB_ROLE='runtime', - AIB_COMPONENT='pretty-printer-test', - AIB_COMPONENTS_EXTRA=[ - 'unittests', - 'tests', - ], - ) - env.Depends(pretty_printer_test_launcher_install, - [pretty_printer_test_installed, pretty_printer_test_program_installed]) - test_env = env.Clone() - test_env['ENV'] = os.environ.copy() - test_env.RegisterTest('$UNITTEST_LIST', pretty_printer_test_launcher_install[0]) - test_env.Alias("$UNITTEST_ALIAS", pretty_printer_test_launcher_install[0]) +env.PrettyPrinterTest('pretty_printer_test.py', TEST_PROGRAM=pretty_printer_test_program_installed) diff --git a/src/mongo/util/pretty_printer_test.py.in b/src/mongo/util/pretty_printer_test.py similarity index 58% rename from src/mongo/util/pretty_printer_test.py.in rename to src/mongo/util/pretty_printer_test.py index c54406ced98..735a700011f 100644 --- a/src/mongo/util/pretty_printer_test.py.in +++ b/src/mongo/util/pretty_printer_test.py @@ -3,15 +3,6 @@ import gdb import re -import sys - -# Here we substitute in the pip_requirements verification code, so this test -# can remain a self contained test with little complexity. This is essentially an -# import we are generating into the file instead to avoid and complex importer logic. -# Start of pip verification code -@verify_requirements@ -# End of pip verification code - expected_patterns = [ r'Decorable with 3 elems', @@ -23,11 +14,13 @@ up_pattern = r'std::unique_ptr = \{get\(\) \= 0x[0-9a-fA-F]+\}' set_pattern = r'std::[__debug::]*set with 4 elements' static_member_pattern = '128' + def search(pattern, s): match = re.search(pattern, s) assert match is not None, 'Did not find {!s} in {!s}'.format(pattern, s) return match + def test_decorable(): gdb.execute('run') gdb.execute('frame function main') @@ -35,25 +28,13 @@ def test_decorable(): for pattern in expected_patterns: search(pattern, d1_str) - search(up_pattern, gdb.execute('print up', to_string=True)) + search(up_pattern, gdb.execute('print up', to_string=True)) search(set_pattern, gdb.execute('print set_type', to_string=True)) search(static_member_pattern, gdb.execute('print testClass::static_member', to_string=True)) -def configure_gdb(): - import os,subprocess,sys - cmd = 'python -c "import os,sys;print(os.linesep.join(sys.path).strip())"' - paths = subprocess.check_output(cmd,shell=True).decode("utf-8").split() - sys.path.extend(paths) - gdb.execute('source .gdbinit') - try: - verify_requirements('etc/pip/components/core.req', executable=f"@python_executable@") - except MissingRequirements as ex: - print(ex) - if __name__ == '__main__': try: - configure_gdb() test_decorable() gdb.write('TEST PASSED\n') except Exception as err: diff --git a/src/mongo/util/pretty_printer_test_launcher.py.in b/src/mongo/util/pretty_printer_test_launcher.py.in index fc70a831e08..77f117fea61 100644 --- a/src/mongo/util/pretty_printer_test_launcher.py.in +++ b/src/mongo/util/pretty_printer_test_launcher.py.in @@ -7,23 +7,34 @@ interpolated by scons. import subprocess import os - +import shlex +verbose = @VERBOSE@ +test_script = r'@pretty_printer_test_py@' +test_args = @test_args@ gdb_path = '@gdb_path@' args = [ gdb_path, '-nx', '-batch', '-ex', - r'source @pretty_printer_test_py@', + f'source {test_script}', '-args', r'@pretty_printer_test_program@', -] -print(f"Pretty printer test running command:\n{' '.join(args)}") +] + test_args + + +if verbose: + print(f"Pretty printer test running command:\n{shlex.join(args)}") # We assume we are running from project root, and require modules buried in the src directory python_env = os.environ.copy() python_env['PYTHONPATH'] = os.getcwd() + os.pathsep + python_env.get('PYTHONPATH', "") proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=python_env) -print(proc.stdout) + +if verbose: + out = '\n'.join([f'[{os.path.basename(test_script)}] {line}' for line in proc.stdout.split('\n')]) + print(out) +else: + print(f'[{os.path.basename(test_script)}] {"passed" if proc.returncode == 0 else "failed"}') exit(proc.returncode)