Skip to content

gh-151626: Fix tests that fail when PYTHONPYCACHEPREFIX is set #151952

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zangjiucheng wants to merge 1 commit into
base: main
Choose a base branch
from
Open

gh-151626: Fix tests that fail when PYTHONPYCACHEPREFIX is set #151952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 45 additions & 20 deletions Lib/test/test_compileall.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,24 @@ def temporary_pycache_prefix(self):
finally:
sys.pycache_prefix = old_prefix

@contextlib.contextmanager
def no_pycache_prefix(self):
"""Ignore any ambient pycache prefix for the duration of the test.

Some tests assume bytecode is written next to the source in a
__pycache__ directory. When the test suite is run with
PYTHONPYCACHEPREFIX set, neutralize it both in this process (used by
cache_from_source) and in any spawned subprocesses.
"""
old_prefix = sys.pycache_prefix
sys.pycache_prefix = None
try:
with os_helper.EnvironmentVarGuard() as env:
env.unset('PYTHONPYCACHEPREFIX')
yield
finally:
sys.pycache_prefix = old_prefix

def _get_run_args(self, args):
return [*support.optim_args_from_interpreter_flags(),
'-S', '-m', 'compileall',
Expand Down Expand Up @@ -650,15 +668,16 @@ def test_legacy_paths(self):
def test_multiple_runs(self):
# Bug 8527 reported that multiple calls produced empty
# __pycache__/__pycache__ directories.
self.assertRunOK('-q', self.pkgdir)
# Verify the __pycache__ directory contents.
self.assertTrue(os.path.exists(self.pkgdir_cachedir))
cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__')
self.assertFalse(os.path.exists(cachecachedir))
# Call compileall again.
self.assertRunOK('-q', self.pkgdir)
self.assertTrue(os.path.exists(self.pkgdir_cachedir))
self.assertFalse(os.path.exists(cachecachedir))
with self.no_pycache_prefix():
self.assertRunOK('-q', self.pkgdir)
# Verify the __pycache__ directory contents.
self.assertTrue(os.path.exists(self.pkgdir_cachedir))
cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__')
self.assertFalse(os.path.exists(cachecachedir))
# Call compileall again.
self.assertRunOK('-q', self.pkgdir)
self.assertTrue(os.path.exists(self.pkgdir_cachedir))
self.assertFalse(os.path.exists(cachecachedir))

@without_source_date_epoch # timestamp invalidation test
def test_force(self):
Expand Down Expand Up @@ -731,10 +750,13 @@ def test_symlink_loop(self):
script_helper.make_pkg(pkg)
os.symlink('.', os.path.join(pkg, 'evil'))
os.symlink('.', os.path.join(pkg, 'evil2'))
self.assertRunOK('-q', self.pkgdir)
self.assertCompiled(os.path.join(
self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
))
# This relies on the __pycache__ layout (shared across the symlinked
# paths), so neutralize any ambient PYTHONPYCACHEPREFIX.
with self.no_pycache_prefix():
self.assertRunOK('-q', self.pkgdir)
self.assertCompiled(os.path.join(
self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
))

def test_quiet(self):
noisy = self.assertRunOK(self.pkgdir)
Expand Down Expand Up @@ -821,13 +843,16 @@ def test_include_on_stdin(self):
f2 = script_helper.make_script(self.pkgdir, 'f2', '')
f3 = script_helper.make_script(self.pkgdir, 'f3', '')
f4 = script_helper.make_script(self.pkgdir, 'f4', '')
p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-']))
p.stdin.write((f3+os.linesep).encode('ascii'))
script_helper.kill_python(p)
self.assertNotCompiled(f1)
self.assertNotCompiled(f2)
self.assertCompiled(f3)
self.assertNotCompiled(f4)
# spawn_python() runs with -E, ignoring PYTHONPYCACHEPREFIX, so make
# cache_from_source() in this process agree by neutralizing it too.
with self.no_pycache_prefix():
p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-']))
p.stdin.write((f3+os.linesep).encode('ascii'))
script_helper.kill_python(p)
self.assertNotCompiled(f1)
self.assertNotCompiled(f2)
self.assertCompiled(f3)
self.assertNotCompiled(f4)

def test_compiles_as_much_as_possible(self):
bingfn = script_helper.make_script(self.pkgdir, 'bing', 'syntax(error')
Expand Down
6 changes: 6 additions & 0 deletions Lib/test/test_import/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1669,6 +1669,11 @@ def _clean(self):
unlink(self.source)

def setUp(self):
# These tests assume bytecode is written next to the source in a
# local __pycache__ directory, so neutralize any pycache prefix (e.g.
# when the test suite is run with PYTHONPYCACHEPREFIX set).
self._orig_pycache_prefix = sys.pycache_prefix
sys.pycache_prefix = None
self.source = TESTFN + '.py'
self._clean()
with open(self.source, 'w', encoding='utf-8') as fp:
Expand All @@ -1680,6 +1685,7 @@ def tearDown(self):
assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]'
del sys.path[0]
self._clean()
sys.pycache_prefix = self._orig_pycache_prefix

@skip_if_dont_write_bytecode
def test_import_pyc_path(self):
Expand Down
11 changes: 11 additions & 0 deletions Lib/test/test_importlib/test_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,17 @@ class PEP3147Tests:

tag = sys.implementation.cache_tag

def setUp(self):
# Most of these tests assume the default (unset) pycache prefix, so
# clear it for the duration of the test (e.g. when the test suite is
# run with PYTHONPYCACHEPREFIX set). Tests that need a specific prefix
# set their own via util.temporary_pycache_prefix().
self._orig_pycache_prefix = sys.pycache_prefix
sys.pycache_prefix = None

def tearDown(self):
sys.pycache_prefix = self._orig_pycache_prefix

@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag not be None')
def test_cache_from_source(self):
Expand Down
25 changes: 22 additions & 3 deletions Lib/test/test_inspect/test_inspect.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import functools
import gc
import importlib
import importlib.util
import inspect
import io
import linecache
Expand Down Expand Up @@ -6576,12 +6577,18 @@ def test_details_option_with_package(self):
args = support.optim_args_from_interpreter_flags()
rc, out, err = assert_python_ok(*args, '-m', 'inspect',
module_name, '--details')
# assert_python_ok() runs the subprocess in isolated mode (-I), which
# ignores PYTHONPYCACHEPREFIX, so compute the expected cached path the
# same way (i.e. without any pycache prefix) to stay independent of the
# environment the test suite is run in.
with support.swap_attr(sys, 'pycache_prefix', None):
cached = importlib.util.cache_from_source(module.__spec__.origin)
# Full rendering check on the expected output
expected_lines = [
f"Target: {module.__name__}", # No aliasing
f"Origin: {module.__spec__.origin}",
f"Source: {module.__file__}",
f"Cached: {module.__spec__.cached}", # None is still displayed
f"Cached: {cached}", # None is still displayed
f"Loader: {_clean_object_ids(repr(module.__spec__.loader))}",
f"Submodule search paths: {module.__path__}",
"",
Expand Down Expand Up @@ -6619,13 +6626,19 @@ def test_details_option_with_data_target(self):
args = support.optim_args_from_interpreter_flags()
rc, out, err = assert_python_ok(*args, '-m', 'inspect',
cli_target, '--details')
# assert_python_ok() runs the subprocess in isolated mode (-I), which
# ignores PYTHONPYCACHEPREFIX, so compute the expected cached path the
# same way (i.e. without any pycache prefix) to stay independent of the
# environment the test suite is run in.
with support.swap_attr(sys, 'pycache_prefix', None):
cached = importlib.util.cache_from_source(module.__spec__.origin)
# Full rendering check on the expected output
# The error is only informational when reading source details
expected_lines = [
f"Target: {cli_target}", # No aliasing
f"Origin: {module.__spec__.origin}",
f"Source: {module.__file__}",
f"Cached: {module.__spec__.cached}", # None is still displayed
f"Cached: {cached}", # None is still displayed
self.NO_SOURCE_TARGET_ERROR,
"",
]
Expand All @@ -6644,12 +6657,18 @@ def test_details_option_with_aliased_target(self):
args = support.optim_args_from_interpreter_flags()
rc, out, err = assert_python_ok(*args, '-m', 'inspect',
cli_target, '--details')
# assert_python_ok() runs the subprocess in isolated mode (-I), which
# ignores PYTHONPYCACHEPREFIX, so compute the expected cached path the
# same way (i.e. without any pycache prefix) to stay independent of the
# environment the test suite is run in.
with support.swap_attr(sys, 'pycache_prefix', None):
cached = importlib.util.cache_from_source(module.__spec__.origin)
# Full rendering check on the expected output
expected_lines = [
f'Target: {defining_target} (looked up as "{cli_target}")',
f"Origin: {module.__spec__.origin}",
f"Source: {module.__file__}",
f"Cached: {module.__spec__.cached}", # None is still displayed
f"Cached: {cached}", # None is still displayed
f"Line: {inspect.findsource(target)[1]}",
"",
]
Expand Down
41 changes: 25 additions & 16 deletions Lib/test/test_py_compile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,21 +186,24 @@ def test_source_date_epoch(self):
def test_double_dot_no_clobber(self):
# http://bugs.python.org/issue22966
# py_compile foo.bar.py -> __pycache__/foo.cpython-34.pyc
weird_path = os.path.join(self.directory, 'foo.bar.py')
cache_path = importlib.util.cache_from_source(weird_path)
pyc_path = weird_path + 'c'
head, tail = os.path.split(cache_path)
penultimate_tail = os.path.basename(head)
self.assertEqual(
os.path.join(penultimate_tail, tail),
os.path.join(
'__pycache__',
'foo.bar.{}.pyc'.format(sys.implementation.cache_tag)))
with open(weird_path, 'w') as file:
file.write('x = 123\n')
py_compile.compile(weird_path)
self.assertTrue(os.path.exists(cache_path))
self.assertFalse(os.path.exists(pyc_path))
# This test asserts the default __pycache__ layout, so neutralize any
# pycache prefix (e.g. when run with PYTHONPYCACHEPREFIX set).
with support.swap_attr(sys, 'pycache_prefix', None):
weird_path = os.path.join(self.directory, 'foo.bar.py')
cache_path = importlib.util.cache_from_source(weird_path)
pyc_path = weird_path + 'c'
head, tail = os.path.split(cache_path)
penultimate_tail = os.path.basename(head)
self.assertEqual(
os.path.join(penultimate_tail, tail),
os.path.join(
'__pycache__',
'foo.bar.{}.pyc'.format(sys.implementation.cache_tag)))
with open(weird_path, 'w') as file:
file.write('x = 123\n')
py_compile.compile(weird_path)
self.assertTrue(os.path.exists(cache_path))
self.assertFalse(os.path.exists(pyc_path))

@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag is not None')
Expand Down Expand Up @@ -307,7 +310,13 @@ def test_with_files(self):
self.assertEqual(rc, 0)
self.assertEqual(stdout, b'')
self.assertEqual(stderr, b'')
self.assertTrue(os.path.exists(self.cache_path))
# pycompilecmd() runs the interpreter in isolated mode (-I), which
# ignores PYTHONPYCACHEPREFIX, so the bytecode is written next to the
# source. Compute the expected cache path the same way.
with support.swap_attr(sys, 'pycache_prefix', None):
cache_path = importlib.util.cache_from_source(
self.source_path, optimization='' if __debug__ else 1)
self.assertTrue(os.path.exists(cache_path))

def test_bad_syntax(self):
bad_syntax = os.path.join(os.path.dirname(__file__),
Expand Down
5 changes: 5 additions & 0 deletions Misc/NEWS.d/next/Tests/2026-06-22-19-45-00.gh-issue-151626.K9pZ2x.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix several tests in :mod:`test.test_inspect`, :mod:`test.test_import`,
:mod:`test.test_importlib`, :mod:`test.test_py_compile` and
:mod:`test.test_compileall` that failed when the test suite was run with
:envvar:`PYTHONPYCACHEPREFIX` set. These tests now neutralize the pycache
prefix where they assume the default ``__pycache__`` bytecode layout.
Loading