diff --git a/buildscripts/combine_reports.py b/buildscripts/combine_reports.py new file mode 100755 index 00000000000..94f5494a0c0 --- /dev/null +++ b/buildscripts/combine_reports.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +""" +Combines JSON report files used in Evergreen +""" + +from __future__ import absolute_import +from __future__ import print_function + +import json +import os +import sys +from optparse import OptionParser + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from buildscripts.resmokelib.testing import report + + +def read_json_file(json_file): + with open(json_file) as json_data: + return json.load(json_data) + + +def report_exit(combined_test_report): + """The exit code of this script is based on the following: + 0: All tests have status "pass", or only non-dynamic tests have status "silentfail". + 31: At least one test has status "fail" or "timeout". + 32: At least one dynamic test has status "silentfail", + but no tests have status "fail" or "timeout". + Note: A test can be considered dynamic if its name contains a ":" character.""" + + ret = 0 + for test in combined_test_report.test_infos: + if test.status in ["fail", "timeout"]: + return 31 + if test.dynamic and test.status == "silentfail": + ret = 32 + return ret + + +def main(): + + usage = "usage: %prog [options] report1.json report2.json ..." + parser = OptionParser(description=__doc__, usage=usage) + parser.add_option("-o", "--output-file", + dest="outfile", + default="-", + help="If '-', then the combined report file is written to stdout." + " Any other value is treated as the output file name. By default," + " output is written to stdout.") + + (options, args) = parser.parse_args() + + if not args: + sys.exit("No report files were specified") + + report_files = args + test_reports = [] + for report_file in report_files: + report_file_json = read_json_file(report_file) + test_reports.append(report.TestReport.from_dict(report_file_json)) + + combined_test_report = report.TestReport.combine(*test_reports) + combined_report = combined_test_report.as_dict() + if options.outfile != "-": + with open(options.outfile, "w") as fstream: + json.dump(combined_report, fstream) + else: + print(json.dumps(combined_report)) + + sys.exit(report_exit(combined_test_report)) + + +if __name__ == "__main__": + main() diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index 18e3edb63e3..8afc50ac68f 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -308,6 +308,27 @@ class TestReport(unittest.TestResult): "failures": self.num_failed + self.num_errored + self.num_interrupted, } + @classmethod + def from_dict(cls, report_dict): + """ + Returns the test report instance copied from a dict (generated in as_dict). + + Used when combining reports instances. + """ + + report = cls(logging.loggers.EXECUTOR_LOGGER) + for result in report_dict["results"]: + # By convention, dynamic tests are named ":". + is_dynamic = ":" in result["test_file"] + test_info = _TestInfo(result["test_file"], is_dynamic) + test_info.url_endpoint = result.get("url") + test_info.status = result["status"] + test_info.return_code = result["exit_code"] + test_info.start_time = result["start"] + test_info.end_time = result["end"] + report.test_infos.append(test_info) + return report + def reset(self): """ Resets the test report back to its initial state.