Skip to content

Commit e5f10a7

Browse files
colesburycolesbury
authored
gh-127933: Add option to run regression tests in parallel (gh-128003)
This adds a new command line argument, `--parallel-threads` to the regression test runner to allow it to run individual tests in multiple threads in parallel in order to find multithreading bugs. Some tests pass when run with `--parallel-threads`, but there's still more work before the entire suite passes.
1 parent 285c1c4 commit e5f10a7

12 files changed

Lines changed: 150 additions & 3 deletions

File tree

‎Doc/library/test.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -792,6 +792,11 @@ The :mod:`test.support` module defines the following functions:
792792
Decorator for invoking :func:`check_impl_detail` on *guards*. If that
793793
returns ``False``, then uses *msg* as the reason for skipping the test.
794794

795+
.. decorator:: thread_unsafe(reason=None)
796+
797+
Decorator for marking tests as thread-unsafe. This test always runs in one
798+
thread even when invoked with ``--parallel-threads``.
799+
795800

796801
.. decorator:: no_tracing
797802

‎Lib/test/libregrtest/cmdline.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ def __init__(self, **kwargs) -> None:
160160
self.print_slow = False
161161
self.random_seed = None
162162
self.use_mp = None
163+
self.parallel_threads = None
163164
self.forever = False
164165
self.header = False
165166
self.failfast = False
@@ -316,6 +317,10 @@ def _create_parser():
316317
'a single process, ignore -jN option, '
317318
'and failed tests are also rerun sequentially '
318319
'in the same process')
320+
group.add_argument('--parallel-threads', metavar='PARALLEL_THREADS',
321+
type=int,
322+
help='run copies of each test in PARALLEL_THREADS at '
323+
'once')
319324
group.add_argument('-T', '--coverage', action='store_true',
320325
dest='trace',
321326
help='turn on code coverage tracing using the trace '

‎Lib/test/libregrtest/main.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,8 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
142142
else:
143143
self.random_seed = ns.random_seed
144144

145+
self.parallel_threads = ns.parallel_threads
146+
145147
# tests
146148
self.first_runtests: RunTests | None = None
147149

@@ -506,6 +508,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests:
506508
python_cmd=self.python_cmd,
507509
randomize=self.randomize,
508510
random_seed=self.random_seed,
511+
parallel_threads=self.parallel_threads,
509512
)
510513

511514
def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:

‎Lib/test/libregrtest/parallel_case.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
"""Run a test case multiple times in parallel threads."""
2+
3+
import copy
4+
import functools
5+
import threading
6+
import unittest
7+
8+
from unittest import TestCase
9+
10+
11+
class ParallelTestCase(TestCase):
12+
def __init__(self, test_case: TestCase, num_threads: int):
13+
self.test_case = test_case
14+
self.num_threads = num_threads
15+
self._testMethodName = test_case._testMethodName
16+
self._testMethodDoc = test_case._testMethodDoc
17+
18+
def __str__(self):
19+
return f"{str(self.test_case)} [threads={self.num_threads}]"
20+
21+
def run_worker(self, test_case: TestCase, result: unittest.TestResult,
22+
barrier: threading.Barrier):
23+
barrier.wait()
24+
test_case.run(result)
25+
26+
def run(self, result=None):
27+
if result is None:
28+
result = test_case.defaultTestResult()
29+
startTestRun = getattr(result, 'startTestRun', None)
30+
stopTestRun = getattr(result, 'stopTestRun', None)
31+
if startTestRun is not None:
32+
startTestRun()
33+
else:
34+
stopTestRun = None
35+
36+
# Called at the beginning of each test. See TestCase.run.
37+
result.startTest(self)
38+
39+
cases = [copy.copy(self.test_case) for _ in range(self.num_threads)]
40+
results = [unittest.TestResult() for _ in range(self.num_threads)]
41+
42+
barrier = threading.Barrier(self.num_threads)
43+
threads = []
44+
for i, (case, r) in enumerate(zip(cases, results)):
45+
thread = threading.Thread(target=self.run_worker,
46+
args=(case, r, barrier),
47+
name=f"{str(self.test_case)}-{i}",
48+
daemon=True)
49+
threads.append(thread)
50+
51+
for thread in threads:
52+
thread.start()
53+
54+
for threads in threads:
55+
threads.join()
56+
57+
# Aggregate test results
58+
if all(r.wasSuccessful() for r in results):
59+
result.addSuccess(self)
60+
61+
# Note: We can't call result.addError, result.addFailure, etc. because
62+
# we no longer have the original exception, just the string format.
63+
for r in results:
64+
if len(r.errors) > 0 or len(r.failures) > 0:
65+
result._mirrorOutput = True
66+
result.errors.extend(r.errors)
67+
result.failures.extend(r.failures)
68+
result.skipped.extend(r.skipped)
69+
result.expectedFailures.extend(r.expectedFailures)
70+
result.unexpectedSuccesses.extend(r.unexpectedSuccesses)
71+
result.collectedDurations.extend(r.collectedDurations)
72+
73+
if any(r.shouldStop for r in results):
74+
result.stop()
75+
76+
# Test has finished running
77+
result.stopTest(self)
78+
if stopTestRun is not None:
79+
stopTestRun()

‎Lib/test/libregrtest/runtests.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ class RunTests:
100100
python_cmd: tuple[str, ...] | None
101101
randomize: bool
102102
random_seed: int | str
103+
parallel_threads: int | None
103104

104105
def copy(self, **override) -> 'RunTests':
105106
state = dataclasses.asdict(self)
@@ -184,6 +185,8 @@ def bisect_cmd_args(self) -> list[str]:
184185
args.extend(("--python", cmd))
185186
if self.randomize:
186187
args.append(f"--randomize")
188+
if self.parallel_threads:
189+
args.append(f"--parallel-threads={self.parallel_threads}")
187190
args.append(f"--randseed={self.random_seed}")
188191
return args
189192

‎Lib/test/libregrtest/single.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from .save_env import saved_test_environment
1818
from .setup import setup_tests
1919
from .testresult import get_test_runner
20+
from .parallel_case import ParallelTestCase
2021
from .utils import (
2122
TestName,
2223
clear_caches, remove_testfn, abs_module_name, print_warning)
@@ -27,14 +28,17 @@
2728
PROGRESS_MIN_TIME = 30.0 # seconds
2829

2930

30-
def run_unittest(test_mod):
31+
def run_unittest(test_mod, runtests: RunTests):
3132
loader = unittest.TestLoader()
3233
tests = loader.loadTestsFromModule(test_mod)
34+
3335
for error in loader.errors:
3436
print(error, file=sys.stderr)
3537
if loader.errors:
3638
raise Exception("errors while loading tests")
3739
_filter_suite(tests, match_test)
40+
if runtests.parallel_threads:
41+
_parallelize_tests(tests, runtests.parallel_threads)
3842
return _run_suite(tests)
3943

4044
def _filter_suite(suite, pred):
@@ -49,6 +53,28 @@ def _filter_suite(suite, pred):
4953
newtests.append(test)
5054
suite._tests = newtests
5155

56+
def _parallelize_tests(suite, parallel_threads: int):
57+
def is_thread_unsafe(test):
58+
test_method = getattr(test, test._testMethodName)
59+
instance = test_method.__self__
60+
return (getattr(test_method, "__unittest_thread_unsafe__", False) or
61+
getattr(instance, "__unittest_thread_unsafe__", False))
62+
63+
newtests: list[object] = []
64+
for test in suite._tests:
65+
if isinstance(test, unittest.TestSuite):
66+
_parallelize_tests(test, parallel_threads)
67+
newtests.append(test)
68+
continue
69+
70+
if is_thread_unsafe(test):
71+
# Don't parallelize thread-unsafe tests
72+
newtests.append(test)
73+
continue
74+
75+
newtests.append(ParallelTestCase(test, parallel_threads))
76+
suite._tests = newtests
77+
5278
def _run_suite(suite):
5379
"""Run tests from a unittest.TestSuite-derived class."""
5480
runner = get_test_runner(sys.stdout,
@@ -133,7 +159,7 @@ def _load_run_test(result: TestResult, runtests: RunTests) -> None:
133159
raise Exception(f"Module {test_name} defines test_main() which "
134160
f"is no longer supported by regrtest")
135161
def test_func():
136-
return run_unittest(test_mod)
162+
return run_unittest(test_mod, runtests)
137163

138164
try:
139165
regrtest_runner(result, test_func, runtests)

‎Lib/test/support/__init__.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
"anticipate_failure", "load_package_tests", "detect_api_mismatch",
4141
"check__all__", "skip_if_buggy_ucrt_strfptime",
4242
"check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer",
43-
"requires_limited_api", "requires_specialization",
43+
"requires_limited_api", "requires_specialization", "thread_unsafe",
4444
# sys
4545
"MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi",
4646
"is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval",
@@ -382,6 +382,21 @@ def wrapper(*args, **kw):
382382
return decorator
383383

384384

385+
def thread_unsafe(reason):
386+
"""Mark a test as not thread safe. When the test runner is run with
387+
--parallel-threads=N, the test will be run in a single thread."""
388+
def decorator(test_item):
389+
test_item.__unittest_thread_unsafe__ = True
390+
# the reason is not currently used
391+
test_item.__unittest_thread_unsafe__why__ = reason
392+
return test_item
393+
if isinstance(reason, types.FunctionType):
394+
test_item = reason
395+
reason = ''
396+
return decorator(test_item)
397+
return decorator
398+
399+
385400
def skip_if_buildbot(reason=None):
386401
"""Decorator raising SkipTest if running on a buildbot."""
387402
import getpass

‎Lib/test/test_class.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"Test the functionality of Python classes implementing operators."
22

33
import unittest
4+
from test import support
45
from test.support import cpython_only, import_helper, script_helper, skip_emscripten_stack_overflow
56

67
testmeths = [
@@ -134,6 +135,7 @@ def __%s__(self, *args):
134135
AllTests = type("AllTests", (object,), d)
135136
del d, statictests, method, method_template
136137

138+
@support.thread_unsafe("callLst is shared between threads")
137139
class ClassTests(unittest.TestCase):
138140
def setUp(self):
139141
callLst[:] = []

‎Lib/test/test_descr.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1103,6 +1103,7 @@ class MyFrozenSet(frozenset):
11031103
with self.assertRaises(TypeError):
11041104
frozenset().__class__ = MyFrozenSet
11051105

1106+
@support.thread_unsafe
11061107
def test_slots(self):
11071108
# Testing __slots__...
11081109
class C0(object):
@@ -5485,6 +5486,7 @@ def __repr__(self):
54855486
{pickle.dumps, pickle._dumps},
54865487
{pickle.loads, pickle._loads}))
54875488

5489+
@support.thread_unsafe
54885490
def test_pickle_slots(self):
54895491
# Tests pickling of classes with __slots__.
54905492

@@ -5552,6 +5554,7 @@ class E(C):
55525554
y = pickle_copier.copy(x)
55535555
self._assert_is_copy(x, y)
55545556

5557+
@support.thread_unsafe
55555558
def test_reduce_copying(self):
55565559
# Tests pickling and copying new-style classes and objects.
55575560
global C1

‎Lib/test/test_operator.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -666,6 +666,7 @@ class COperatorTestCase(OperatorTestCase, unittest.TestCase):
666666
module = c_operator
667667

668668

669+
@support.thread_unsafe("swaps global operator module")
669670
class OperatorPickleTestCase:
670671
def copy(self, obj, proto):
671672
with support.swap_item(sys.modules, 'operator', self.module):

0 commit comments

Comments
 (0)