From aa9a359ca2663195b0f04eef46109c28c4ff74d3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 05:35:39 +0200 Subject: [PATCH] gh-108388: Split test_multiprocessing_spawn (#108396) Split test_multiprocessing_fork, test_multiprocessing_forkserver and test_multiprocessing_spawn into test packages. Each package is made of 4 sub-tests: processes, threads, manager and misc. It allows running more tests in parallel and so reduce the total test duration. --- Lib/test/_test_multiprocessing.py | 10 +++++++++- Lib/test/libregrtest/runtest.py | 3 +++ .../__init__.py} | 11 ++++------- Lib/test/test_multiprocessing_fork/test_manager.py | 7 +++++++ Lib/test/test_multiprocessing_fork/test_misc.py | 7 +++++++ Lib/test/test_multiprocessing_fork/test_processes.py | 7 +++++++ Lib/test/test_multiprocessing_fork/test_threads.py | 7 +++++++ .../__init__.py} | 11 ++++------- .../test_multiprocessing_forkserver/test_manager.py | 7 +++++++ .../test_multiprocessing_forkserver/test_misc.py | 7 +++++++ .../test_processes.py | 7 +++++++ .../test_multiprocessing_forkserver/test_threads.py | 7 +++++++ Lib/test/test_multiprocessing_spawn.py | 12 ------------ Lib/test/test_multiprocessing_spawn/__init__.py | 9 +++++++++ Lib/test/test_multiprocessing_spawn/test_manager.py | 7 +++++++ Lib/test/test_multiprocessing_spawn/test_misc.py | 7 +++++++ .../test_multiprocessing_spawn/test_processes.py | 7 +++++++ Lib/test/test_multiprocessing_spawn/test_threads.py | 7 +++++++ .../2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst | 4 ++++ 19 files changed, 117 insertions(+), 27 deletions(-) rename Lib/test/{test_multiprocessing_fork.py => test_multiprocessing_fork/__init__.py} (66%) create mode 100644 Lib/test/test_multiprocessing_fork/test_manager.py create mode 100644 Lib/test/test_multiprocessing_fork/test_misc.py create mode 100644 Lib/test/test_multiprocessing_fork/test_processes.py create mode 100644 Lib/test/test_multiprocessing_fork/test_threads.py rename Lib/test/{test_multiprocessing_forkserver.py => test_multiprocessing_forkserver/__init__.py} (58%) create mode 100644 Lib/test/test_multiprocessing_forkserver/test_manager.py create mode 100644 Lib/test/test_multiprocessing_forkserver/test_misc.py create mode 100644 Lib/test/test_multiprocessing_forkserver/test_processes.py create mode 100644 Lib/test/test_multiprocessing_forkserver/test_threads.py delete mode 100644 Lib/test/test_multiprocessing_spawn.py create mode 100644 Lib/test/test_multiprocessing_spawn/__init__.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_manager.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_misc.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_processes.py create mode 100644 Lib/test/test_multiprocessing_spawn/test_threads.py create mode 100644 Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a826e31f5ee..343e9dcf769 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6126,7 +6126,8 @@ class ThreadsMixin(BaseMixin): # Functions used to create test cases from the base ones in this module # -def install_tests_in_module_dict(remote_globs, start_method): +def install_tests_in_module_dict(remote_globs, start_method, + only_type=None, exclude_types=False): __module__ = remote_globs['__name__'] local_globs = globals() ALL_TYPES = {'processes', 'threads', 'manager'} @@ -6139,6 +6140,10 @@ def install_tests_in_module_dict(remote_globs, start_method): continue assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES for type_ in base.ALLOWED_TYPES: + if only_type and type_ != only_type: + continue + if exclude_types: + continue newname = 'With' + type_.capitalize() + name[1:] Mixin = local_globs[type_.capitalize() + 'Mixin'] class Temp(base, Mixin, unittest.TestCase): @@ -6149,6 +6154,9 @@ def install_tests_in_module_dict(remote_globs, start_method): Temp.__module__ = __module__ remote_globs[newname] = Temp elif issubclass(base, unittest.TestCase): + if only_type: + continue + class Temp(base, object): pass Temp.__name__ = Temp.__qualname__ = name diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e927079da47..cddb37c87aa 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -132,6 +132,9 @@ PROGRESS_MIN_TIME = 30.0 # seconds SPLITTESTDIRS = { "test_asyncio", + "test_multiprocessing_fork", + "test_multiprocessing_forkserver", + "test_multiprocessing_spawn", } # Storage of uncollectable objects diff --git a/Lib/test/test_multiprocessing_fork.py b/Lib/test/test_multiprocessing_fork/__init__.py similarity index 66% rename from Lib/test/test_multiprocessing_fork.py rename to Lib/test/test_multiprocessing_fork/__init__.py index 5000edb7c5c..aa1fff50b28 100644 --- a/Lib/test/test_multiprocessing_fork.py +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -1,7 +1,6 @@ -import unittest -import test._test_multiprocessing - +import os.path import sys +import unittest from test import support if support.PGO: @@ -13,7 +12,5 @@ if sys.platform == "win32": if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") -test._test_multiprocessing.install_tests_in_module_dict(globals(), 'fork') - -if __name__ == '__main__': - unittest.main() +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_fork/test_manager.py b/Lib/test/test_multiprocessing_fork/test_manager.py new file mode 100644 index 00000000000..9efbb83bbb7 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_misc.py b/Lib/test/test_multiprocessing_fork/test_misc.py new file mode 100644 index 00000000000..891a494020c --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_processes.py b/Lib/test/test_multiprocessing_fork/test_processes.py new file mode 100644 index 00000000000..e64e9afc010 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_processes.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="processes") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_threads.py b/Lib/test/test_multiprocessing_fork/test_threads.py new file mode 100644 index 00000000000..1670e34cb17 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver.py b/Lib/test/test_multiprocessing_forkserver/__init__.py similarity index 58% rename from Lib/test/test_multiprocessing_forkserver.py rename to Lib/test/test_multiprocessing_forkserver/__init__.py index 6ad5faf9e8a..d91715a344d 100644 --- a/Lib/test/test_multiprocessing_forkserver.py +++ b/Lib/test/test_multiprocessing_forkserver/__init__.py @@ -1,7 +1,6 @@ -import unittest -import test._test_multiprocessing - +import os.path import sys +import unittest from test import support if support.PGO: @@ -10,7 +9,5 @@ if support.PGO: if sys.platform == "win32": raise unittest.SkipTest("forkserver is not available on Windows") -test._test_multiprocessing.install_tests_in_module_dict(globals(), 'forkserver') - -if __name__ == '__main__': - unittest.main() +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_forkserver/test_manager.py b/Lib/test/test_multiprocessing_forkserver/test_manager.py new file mode 100644 index 00000000000..14f8f10dfb4 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_misc.py b/Lib/test/test_multiprocessing_forkserver/test_misc.py new file mode 100644 index 00000000000..9cae1b50f71 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_processes.py b/Lib/test/test_multiprocessing_forkserver/test_processes.py new file mode 100644 index 00000000000..360967cf1ae --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_processes.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="processes") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_threads.py b/Lib/test/test_multiprocessing_forkserver/test_threads.py new file mode 100644 index 00000000000..719c752aa05 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn.py b/Lib/test/test_multiprocessing_spawn.py deleted file mode 100644 index 6558952308f..00000000000 --- a/Lib/test/test_multiprocessing_spawn.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest -import test._test_multiprocessing - -from test import support - -if support.PGO: - raise unittest.SkipTest("test is not helpful for PGO") - -test._test_multiprocessing.install_tests_in_module_dict(globals(), 'spawn') - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/__init__.py b/Lib/test/test_multiprocessing_spawn/__init__.py new file mode 100644 index 00000000000..3fd0f9b3906 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/__init__.py @@ -0,0 +1,9 @@ +import os.path +import unittest +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_spawn/test_manager.py b/Lib/test/test_multiprocessing_spawn/test_manager.py new file mode 100644 index 00000000000..b40bea0bf61 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_misc.py b/Lib/test/test_multiprocessing_spawn/test_misc.py new file mode 100644 index 00000000000..32f37c5cc81 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_processes.py b/Lib/test/test_multiprocessing_spawn/test_processes.py new file mode 100644 index 00000000000..af764b0d848 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_processes.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="processes") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_threads.py b/Lib/test/test_multiprocessing_spawn/test_threads.py new file mode 100644 index 00000000000..c1257749b9c --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst b/Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst new file mode 100644 index 00000000000..8cf77b1cc18 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-24-04-23-35.gh-issue-108388.mr0MeE.rst @@ -0,0 +1,4 @@ +Split test_multiprocessing_fork, test_multiprocessing_forkserver and +test_multiprocessing_spawn into test packages. Each package is made of 4 +sub-tests: processes, threads, manager and misc. It allows running more tests +in parallel and so reduce the total test duration. Patch by Victor Stinner.