Skip to content

[3.11] gh-108822: Backport libregrtest changes from the main branch #108820

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 27 commits into from
Sep 3, 2023
Merged

[3.11] gh-108822: Backport libregrtest changes from the main branch #108820

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
a4e5e7b
Revert "[3.11] gh-101634: regrtest reports decoding error as failed t…
vstinner Sep 2, 2023
39bf879
Revert "[3.11] bpo-46523: fix tests rerun when `setUp[Class|Module]` …
vstinner Sep 2, 2023
9a03f9c
Revert "gh-95027: Fix regrtest stdout encoding on Windows (GH-98492)"
vstinner Sep 2, 2023
57c050e
Revert "[3.11] gh-94026: Buffer regrtest worker stdout in temporary f…
vstinner Sep 2, 2023
f8e8df3
Revert "Run Tools/scripts/reindent.py (GH-94225)"
vstinner Sep 2, 2023
b95c315
Revert "gh-94052: Don't re-run failed tests with --python option (GH-…
vstinner Sep 2, 2023
9689029
Revert "[3.11] gh-84461: Fix Emscripten umask and permission issues (…
vstinner Sep 2, 2023
9583edd
gh-93353: regrtest checks for leaked temporary files (#93776)
vstinner Jun 14, 2022
7f3f0b3
gh-93353: Fix regrtest for -jN with N >= 2 (GH-93813)
vstinner Jun 14, 2022
9007571
gh-93353: regrtest supports checking tmp files with -j2 (#93909)
vstinner Jun 16, 2022
cb9c17d
gh-84461: Fix Emscripten umask and permission issues (GH-94002)
tiran Jun 19, 2022
70d8b30
gh-94052: Don't re-run failed tests with --python option (#94054)
tiran Jun 21, 2022
5c87ac3
Run Tools/scripts/reindent.py (#94225)
vstinner Jun 26, 2022
24d94ca
gh-94026: Buffer regrtest worker stdout in temporary file (GH-94253)
tiran Jun 29, 2022
693a035
gh-96465: Clear fractions hash lru_cache under refleak testing (GH-96…
zware Sep 8, 2022
73334eb
gh-95027: Fix regrtest stdout encoding on Windows (#98492)
vstinner Oct 21, 2022
2cc75b6
gh-98903: Test suite fails with exit code 4 if no tests ran (#98904)
vstinner Nov 2, 2022
c5bfed5
gh-100086: Add build info to test.libregrtest (#100093)
vstinner Dec 8, 2022
14ef8cf
bpo-46523: fix tests rerun when `setUp[Class|Module]` fails (#30895)
sobolevn Apr 7, 2023
ec875d5
gh-82054: allow test runner to split test_asyncio to execute in paral…
zitterbewegung Apr 30, 2023
5d764de
Display the sanitizer config in the regrtest header. (#105301)
gpshead Jun 6, 2023
f848b2b
gh-101634: regrtest reports decoding error as failed test (#106169)
vstinner Jun 28, 2023
15425da
gh-108223: test.pythoninfo and libregrtest log Py_NOGIL (#108238)
vstinner Aug 21, 2023
deb932f
gh-90791: test.pythoninfo logs ASAN_OPTIONS env var (#108289)
vstinner Aug 22, 2023
30120ff
gh-108388: regrtest splits test_asyncio package (#108393)
vstinner Aug 24, 2023
b882af4
regrtest computes statistics (#108793)
vstinner Sep 2, 2023
b8104bf
gh-108822: Add Changelog entry for regrtest statistics (#108821)
vstinner Sep 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
gh-98903: Test suite fails with exit code 4 if no tests ran (#98904) 
The Python test suite now fails wit exit code 4 if no tests ran. It
should help detecting typos in test names and test methods.

* Add "EXITCODE_" constants to Lib/test/libregrtest/main.py.
* Fix a typo: "NO TEST RUN" becomes "NO TESTS RAN"

(cherry picked from commit c76db37)
  • Loading branch information
@vstinner
vstinner committed Sep 2, 2023
commit 2cc75b63a4d9720226af2aa73bf1ff13a64d16d8
22 changes: 16 additions & 6 deletions Lib/test/libregrtest/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
# Must be smaller than buildbot "1200 seconds without output" limit.
EXIT_TIMEOUT = 120.0

EXITCODE_BAD_TEST = 2
EXITCODE_INTERRUPTED = 130
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4


class Regrtest:
"""Execute a test suite.
Expand Down Expand Up @@ -493,15 +498,18 @@ def display_header(self):
print("== encodings: locale=%s, FS=%s"
% (locale.getencoding(), sys.getfilesystemencoding()))

def no_tests_run(self):
return not any((self.good, self.bad, self.skipped, self.interrupted,
self.environment_changed))

def get_tests_result(self):
result = []
if self.bad:
result.append("FAILURE")
elif self.ns.fail_env_changed and self.environment_changed:
result.append("ENV CHANGED")
elif not any((self.good, self.bad, self.skipped, self.interrupted,
self.environment_changed)):
result.append("NO TEST RUN")
elif self.no_tests_run():
result.append("NO TESTS RAN")

if self.interrupted:
result.append("INTERRUPTED")
Expand Down Expand Up @@ -750,11 +758,13 @@ def _main(self, tests, kwargs):
self.save_xml_result()

if self.bad:
sys.exit(2)
sys.exit(EXITCODE_BAD_TEST)
if self.interrupted:
sys.exit(130)
sys.exit(EXITCODE_INTERRUPTED)
if self.ns.fail_env_changed and self.environment_changed:
sys.exit(3)
sys.exit(EXITCODE_ENV_CHANGED)
if self.no_tests_run():
sys.exit(EXITCODE_NO_TESTS_RAN)
sys.exit(0)


Expand Down
54 changes: 34 additions & 20 deletions Lib/test/test_regrtest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'

EXITCODE_BAD_TEST = 2
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4
EXITCODE_INTERRUPTED = 130

TEST_INTERRUPTED = textwrap.dedent("""
from signal import SIGINT, raise_signal
try:
Expand Down Expand Up @@ -499,7 +504,7 @@ def list_regex(line_format, tests):
result.append('INTERRUPTED')
if not any((good, result, failed, interrupted, skipped,
env_changed, fail_env_changed)):
result.append("NO TEST RUN")
result.append("NO TESTS RAN")
elif not result:
result.append('SUCCESS')
result = ', '.join(result)
Expand Down Expand Up @@ -709,7 +714,7 @@ def test_failing(self):
test_failing = self.create_test('failing', code=code)
tests = [test_ok, test_failing]

output = self.run_tests(*tests, exitcode=2)
output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, tests, failed=test_failing)

def test_resources(self):
Expand Down Expand Up @@ -750,13 +755,14 @@ def test_random(self):
test = self.create_test('random', code)

# first run to get the output with the random seed
output = self.run_tests('-r', test)
output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN)
randseed = self.parse_random_seed(output)
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
test_random = int(match.group(1))

# try to reproduce with the random seed
output = self.run_tests('-r', '--randseed=%s' % randseed, test)
output = self.run_tests('-r', '--randseed=%s' % randseed, test,
exitcode=EXITCODE_NO_TESTS_RAN)
randseed2 = self.parse_random_seed(output)
self.assertEqual(randseed2, randseed)

Expand Down Expand Up @@ -815,7 +821,7 @@ def test_fromfile(self):
def test_interrupted(self):
code = TEST_INTERRUPTED
test = self.create_test('sigint', code=code)
output = self.run_tests(test, exitcode=130)
output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED)
self.check_executed_tests(output, test, omitted=test,
interrupted=True)

Expand All @@ -840,7 +846,7 @@ def test_slowest_interrupted(self):
args = ("--slowest", "-j2", test)
else:
args = ("--slowest", test)
output = self.run_tests(*args, exitcode=130)
output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED)
self.check_executed_tests(output, test,
omitted=test, interrupted=True)

Expand Down Expand Up @@ -880,7 +886,7 @@ def test_run(self):
builtins.__dict__['RUN'] = 1
""")
test = self.create_test('forever', code=code)
output = self.run_tests('--forever', test, exitcode=2)
output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [test]*3, failed=test)

def check_leak(self, code, what):
Expand All @@ -889,7 +895,7 @@ def check_leak(self, code, what):
filename = 'reflog.txt'
self.addCleanup(os_helper.unlink, filename)
output = self.run_tests('--huntrleaks', '3:3:', test,
exitcode=2,
exitcode=EXITCODE_BAD_TEST,
stderr=subprocess.STDOUT)
self.check_executed_tests(output, [test], failed=test)

Expand Down Expand Up @@ -971,7 +977,7 @@ def test_crashed(self):
crash_test = self.create_test(name="crash", code=code)

tests = [crash_test]
output = self.run_tests("-j2", *tests, exitcode=2)
output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, tests, failed=crash_test,
randomize=True)

Expand Down Expand Up @@ -1071,7 +1077,8 @@ def test_env_changed(self):
self.check_executed_tests(output, [testname], env_changed=testname)

# fail with --fail-env-changed
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname], env_changed=testname,
fail_env_changed=True)

Expand All @@ -1090,7 +1097,7 @@ def test_fail_always(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=2)
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [testname],
failed=testname, rerun={testname: "test_fail_always"})

Expand Down Expand Up @@ -1125,7 +1132,8 @@ def test_bug(self):
""")
testname = self.create_test(code=code)

output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0)
output = self.run_tests(testname, "-m", "nosuchtest",
exitcode=EXITCODE_NO_TESTS_RAN)
self.check_executed_tests(output, [testname], no_test_ran=testname)

def test_no_tests_ran_skip(self):
Expand All @@ -1138,7 +1146,7 @@ def test_skipped(self):
""")
testname = self.create_test(code=code)

output = self.run_tests(testname, exitcode=0)
output = self.run_tests(testname)
self.check_executed_tests(output, [testname])

def test_no_tests_ran_multiple_tests_nonexistent(self):
Expand All @@ -1152,7 +1160,8 @@ def test_bug(self):
testname = self.create_test(code=code)
testname2 = self.create_test(code=code)

output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0)
output = self.run_tests(testname, testname2, "-m", "nosuchtest",
exitcode=EXITCODE_NO_TESTS_RAN)
self.check_executed_tests(output, [testname, testname2],
no_test_ran=[testname, testname2])

Expand Down Expand Up @@ -1200,7 +1209,8 @@ def test_garbage(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("--fail-env-changed", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand All @@ -1226,7 +1236,8 @@ def test_sleep(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2)
output = self.run_tests("-j2", "--timeout=1.0", testname,
exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [testname],
failed=testname)
self.assertRegex(output,
Expand Down Expand Up @@ -1258,7 +1269,8 @@ def test_unraisable_exc(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", "-v", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand Down Expand Up @@ -1289,7 +1301,8 @@ def test_threading_excepthook(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", "-v", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand Down Expand Up @@ -1330,7 +1343,7 @@ def test_print_warning(self):
for option in ("-v", "-W"):
with self.subTest(option=option):
cmd = ["--fail-env-changed", option, testname]
output = self.run_tests(*cmd, exitcode=3)
output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand Down Expand Up @@ -1375,7 +1388,8 @@ def test_leak_tmp_file(self):
""")
testnames = [self.create_test(code=code) for _ in range(3)]

output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3)
output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, testnames,
env_changed=testnames,
fail_env_changed=True,
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The Python test suite now fails wit exit code 4 if no tests ran. It should
help detecting typos in test names and test methods.