diff --git a/Lib/importlib/_abc.py b/Lib/importlib/_abc.py index 693b4661126..2598036dac4 100644 --- a/Lib/importlib/_abc.py +++ b/Lib/importlib/_abc.py @@ -19,21 +19,3 @@ def create_module(self, spec): # We don't define exec_module() here since that would break # hasattr checks we do to support backward compatibility. - - def load_module(self, fullname): - """Return the loaded module. - - The module must be added to sys.modules and have import-related - attributes set properly. The fullname is a str. - - ImportError is raised on failure. - - This method is deprecated in favor of loader.exec_module(). If - exec_module() exists then it is used to provide a backwards-compatible - functionality for this method. - - """ - if not hasattr(self, 'exec_module'): - raise ImportError - # Warning implemented in _load_module_shim(). - return _bootstrap._load_module_shim(self, fullname) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9d911e1dcab..75bc901e238 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -424,6 +424,64 @@ def __exit__(self, *args, **kwargs): self._lock.release() +def _get_module_chain(name): + """Return the chain of dotted-name prefixes from root to leaf. + + For example: 'a.b.c' -> ['a', 'a.b', 'a.b.c'] + """ + parts = name.split('.') + return ['.'.join(parts[:i+1]) for i in range(len(parts))] + + +class _HierarchicalLockManager: + """Manages acquisition of multiple module locks in hierarchical order. + + This prevents deadlocks by ensuring all threads acquire locks in the + same order (parent modules before child modules). + """ + + def __init__(self, name): + self._name = name + self._module_chain = _get_module_chain(name) + self._locks = [] + + def __enter__(self): + try: + for module_name in self._module_chain: + # Only acquire lock if module is not already fully loaded + module = sys.modules.get(module_name) + if (module is None or + getattr(getattr(module, "__spec__", None), + "_initializing", False)): + lock = _get_module_lock(module_name) + try: + lock.acquire() + except _DeadlockError: + if module_name == self._name: + raise + # The parent is being initialised by a thread that + # is (transitively) waiting on a lock we hold. + # Apply the same policy as _lock_unlock_module(): + # accept a partially-initialised parent for circular + # imports rather than failing the whole chain. + continue + self._locks.append((module_name, lock)) + except: + # __exit__ is not called when __enter__ raises (e.g. _DeadlockError + # on the leaf lock, or KeyboardInterrupt), so release whatever we + # already hold to avoid permanently leaking held module locks. + for module_name, lock in reversed(self._locks): + lock.release() + self._locks.clear() + raise + return self + + def __exit__(self, *args, **kwargs): + for module_name, lock in reversed(self._locks): + lock.release() + self._locks.clear() + + # The following two functions are for consumption by Python/import.c. def _get_module_lock(name): @@ -521,24 +579,6 @@ def _requires_frozen_wrapper(self, fullname): return _requires_frozen_wrapper -# Typically used by loader classes as a method replacement. -def _load_module_shim(self, fullname): - """Load the specified module into sys.modules and return it. - - This method is deprecated. Use loader.exec_module() instead. - - """ - msg = ("the load_module() method is deprecated and slated for removal in " - "Python 3.15; use exec_module() instead") - _warnings.warn(msg, DeprecationWarning) - spec = spec_from_loader(fullname, self) - if fullname in sys.modules: - module = sys.modules[fullname] - _exec(spec, module) - return sys.modules[fullname] - else: - return _load(spec) - # Module specifications ####################################################### def _module_repr(module): @@ -583,8 +623,7 @@ class ModuleSpec: `has_location` indicates that a spec's "origin" reflects a location. When this is True, `__file__` attribute of the module is set. - `cached` is the location of the cached bytecode file, if any. It - corresponds to the `__cached__` attribute. + `cached` is the location of the cached bytecode file, if any. `submodule_search_locations` is the sequence of path entries to search when importing submodules. If set, is_package should be @@ -717,10 +756,6 @@ def _spec_from_module(module, loader=None, origin=None): origin = getattr(loader, '_ORIGIN', None) if not origin and location is not None: origin = location - try: - cached = module.__cached__ - except AttributeError: - cached = None try: submodule_search_locations = list(module.__path__) except AttributeError: @@ -728,7 +763,7 @@ def _spec_from_module(module, loader=None, origin=None): spec = ModuleSpec(name, loader, origin=origin) spec._set_fileattr = False if location is None else (origin == location) - spec.cached = cached + spec.cached = None spec.submodule_search_locations = submodule_search_locations return spec @@ -763,8 +798,7 @@ def _init_module_attrs(spec, module, *, override=False): # should also be None for consistency. While a bit of a hack, # this is the best place to ensure this consistency. # - # See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module - # and bpo-32305 + # See bpo-32305 module.__file__ = None try: module.__loader__ = loader @@ -789,7 +823,7 @@ def _init_module_attrs(spec, module, *, override=False): module.__path__ = spec.submodule_search_locations except AttributeError: pass - # __file__/__cached__ + # __file__ if spec.has_location: if override or getattr(module, '__file__', None) is None: try: @@ -797,12 +831,6 @@ def _init_module_attrs(spec, module, *, override=False): except AttributeError: pass - if override or getattr(module, '__cached__', None) is None: - if spec.cached is not None: - try: - module.__cached__ = spec.cached - except AttributeError: - pass return module @@ -844,7 +872,7 @@ def _module_repr_from_spec(spec): return f'' -# Used by importlib.reload() and _load_module_shim(). +# Used by importlib.reload(). def _exec(spec, module): """Execute the spec's specified module in an existing module's namespace.""" name = spec.name @@ -860,13 +888,7 @@ def _exec(spec, module): _init_module_attrs(spec, module, override=True) else: _init_module_attrs(spec, module, override=True) - if not hasattr(spec.loader, 'exec_module'): - msg = (f"{_object_name(spec.loader)}.exec_module() not found; " - "falling back to load_module()") - _warnings.warn(msg, ImportWarning) - spec.loader.load_module(name) - else: - spec.loader.exec_module(module) + spec.loader.exec_module(module) finally: # Update the order of insertion into sys.modules for module # clean-up at shutdown. @@ -874,53 +896,8 @@ def _exec(spec, module): sys.modules[spec.name] = module return module - -def _load_backward_compatible(spec): - # It is assumed that all callers have been warned about using load_module() - # appropriately before calling this function. - try: - spec.loader.load_module(spec.name) - except: - if spec.name in sys.modules: - module = sys.modules.pop(spec.name) - sys.modules[spec.name] = module - raise - # The module must be in sys.modules at this point! - # Move it to the end of sys.modules. - module = sys.modules.pop(spec.name) - sys.modules[spec.name] = module - if getattr(module, '__loader__', None) is None: - try: - module.__loader__ = spec.loader - except AttributeError: - pass - if getattr(module, '__package__', None) is None: - try: - # Since module.__path__ may not line up with - # spec.submodule_search_paths, we can't necessarily rely - # on spec.parent here. - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = spec.name.rpartition('.')[0] - except AttributeError: - pass - if getattr(module, '__spec__', None) is None: - try: - module.__spec__ = spec - except AttributeError: - pass - return module - def _load_unlocked(spec): # A helper for direct use by the import system. - if spec.loader is not None: - # Not a namespace package. - if not hasattr(spec.loader, 'exec_module'): - msg = (f"{_object_name(spec.loader)}.exec_module() not found; " - "falling back to load_module()") - _warnings.warn(msg, ImportWarning) - return _load_backward_compatible(spec) - module = module_from_spec(spec) # This must be done before putting the module in sys.modules @@ -954,8 +931,7 @@ def _load_unlocked(spec): return module -# A method used during testing of _load_unlocked() and by -# _load_module_shim(). +# A method used during testing of _load_unlocked(). def _load(spec): """Return a new module object, loaded by the spec's loader. @@ -1020,8 +996,6 @@ def is_package(cls, fullname): """Return False as built-in modules are never packages.""" return False - load_module = classmethod(_load_module_shim) - class FrozenImporter: @@ -1178,25 +1152,6 @@ def exec_module(module): code = _call_with_frames_removed(_imp.get_frozen_object, name) exec(code, module.__dict__) - @classmethod - def load_module(cls, fullname): - """Load a frozen module. - - This method is deprecated. Use exec_module() instead. - - """ - # Warning about deprecation implemented in _load_module_shim(). - module = _load_module_shim(cls, fullname) - info = _imp.find_frozen(fullname) - assert info is not None - _, ispkg, origname = info - module.__origname__ = origname - vars(module).pop('__file__', None) - if ispkg: - module.__path__ = [] - cls._fix_up_module(module) - return module - @classmethod @_requires_frozen def get_code(cls, fullname): @@ -1305,8 +1260,16 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -def _find_and_load_unlocked(name, import_): +def _find_and_load_unlocked(name, import_, *, lazy_submodule=False): path = None + sys.audit( + "import", + name, + path, + getattr(sys, "path", None), + getattr(sys, "meta_path", None), + getattr(sys, "path_hooks", None) + ) parent = name.rpartition('.')[0] parent_spec = None if parent: @@ -1320,6 +1283,8 @@ def _find_and_load_unlocked(name, import_): try: path = parent_module.__path__ except AttributeError: + if lazy_submodule: + return None msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ @@ -1332,6 +1297,8 @@ def _find_and_load_unlocked(name, import_): child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: + if lazy_submodule: + return None raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name) else: if parent_spec: @@ -1351,13 +1318,19 @@ def _find_and_load_unlocked(name, import_): except AttributeError: msg = f"Cannot set an attribute on {parent!r} for child module {child!r}" _warnings.warn(msg, ImportWarning) + # Set attributes to lazy submodules on the module. + try: + _imp._set_lazy_attributes(module, name) + except Exception as e: + msg = f"Cannot set lazy attributes on {name!r}: {e!r}" + _warnings.warn(msg, ImportWarning) return module _NEEDS_LOADING = object() -def _find_and_load(name, import_): +def _find_and_load(name, import_, *, lazy_submodule=False): """Find and load the module.""" # Optimization: we avoid unneeded module locking if the module @@ -1365,10 +1338,17 @@ def _find_and_load(name, import_): module = sys.modules.get(name, _NEEDS_LOADING) if (module is _NEEDS_LOADING or getattr(getattr(module, "__spec__", None), "_initializing", False)): - with _ModuleLockManager(name): + + if '.' in name: + lock_manager = _HierarchicalLockManager(name) + else: + lock_manager = _ModuleLockManager(name) + + with lock_manager: module = sys.modules.get(name, _NEEDS_LOADING) if module is _NEEDS_LOADING: - return _find_and_load_unlocked(name, import_) + return _find_and_load_unlocked( + name, import_, lazy_submodule=lazy_submodule) # Optimization: only call _bootstrap._lock_unlock_module() if # module.__spec__._initializing is True. @@ -1382,7 +1362,7 @@ def _find_and_load(name, import_): # to preserve normal semantics: the caller gets the exception from # the actual import failure rather than a synthetic error. if sys.modules.get(name) is not module: - return _find_and_load(name, import_) + return _find_and_load(name, import_, lazy_submodule=lazy_submodule) if module is None: message = f'import of {name} halted; None in sys.modules' @@ -1391,6 +1371,10 @@ def _find_and_load(name, import_): return module +def _find_and_load_lazy_submodule(name, import_): + return _find_and_load(name, import_, lazy_submodule=True) + + def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 6a828ae75ed..a1cb729efb7 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -236,7 +236,7 @@ def _write_atomic(path, data, mode=0o666): # Deprecated. DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES -def cache_from_source(path, debug_override=None, *, optimization=None): +def cache_from_source(path, *, optimization=None): """Given the path to a .py file, return the path to its .pyc file. The .py file does not need to exist; this simply returns the path to the @@ -247,20 +247,9 @@ def cache_from_source(path, debug_override=None, *, optimization=None): of the argument is taken and verified to be alphanumeric (else ValueError is raised). - The debug_override parameter is deprecated. If debug_override is not None, - a True value is the same as setting 'optimization' to the empty string - while a False value is equivalent to setting 'optimization' to '1'. - If sys.implementation.cache_tag is None then NotImplementedError is raised. """ - if debug_override is not None: - _warnings.warn('the debug_override parameter is deprecated; use ' - "'optimization' instead", DeprecationWarning) - if optimization is not None: - message = 'debug_override or optimization must be set to None' - raise TypeError(message) - optimization = '' if debug_override else 1 path = _os.fspath(path) head, tail = _path_split(path) base, sep, rest = tail.rpartition('.') @@ -548,8 +537,7 @@ def decode_source(source_bytes): import tokenize # To avoid bootstrap issues. source_bytes_readline = _io.BytesIO(source_bytes).readline encoding = tokenize.detect_encoding(source_bytes_readline) - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - return newline_decoder.decode(source_bytes.decode(encoding[0])) + return _io.TextIOWrapper(_io.BytesIO(source_bytes), encoding=encoding[0], newline=None).read() # Module specifications ####################################################### @@ -758,11 +746,6 @@ def exec_module(self, module): 'get_code() returns None') _bootstrap._call_with_frames_removed(exec, code, module.__dict__) - def load_module(self, fullname): - """This method is deprecated.""" - # Warning implemented in _load_module_shim(). - return _bootstrap._load_module_shim(self, fullname) - class SourceLoader(_LoaderBasics): @@ -815,13 +798,14 @@ def get_source(self, fullname): name=fullname) from exc return decode_source(source_bytes) - def source_to_code(self, data, path, *, _optimize=-1): + def source_to_code(self, data, path, fullname=None, *, _optimize=-1): """Return the code object compiled from source. The 'data' argument can be any object type that compile() supports. """ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True, optimize=_optimize) + dont_inherit=True, optimize=_optimize, + module=fullname) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -890,7 +874,7 @@ def get_code(self, fullname): source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) - code_object = self.source_to_code(source_bytes, source_path) + code_object = self.source_to_code(source_bytes, source_path, fullname) _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): @@ -927,18 +911,6 @@ def __eq__(self, other): def __hash__(self): return hash(self.name) ^ hash(self.path) - @_check_name - def load_module(self, fullname): - """Load a module from a file. - - This method is deprecated. Use exec_module() instead. - - """ - # The only reason for this method is for the name check. - # Issue #14857: Avoid the zero-argument form of super so the implementation - # of that form can be updated without breaking the frozen module. - return super(FileLoader, self).load_module(fullname) - @_check_name def get_filename(self, fullname): """Return the path to the source file as found by the finder.""" @@ -1082,12 +1054,18 @@ def get_filename(self, fullname): return self.path -class _NamespacePath: - """Represents a namespace package's path. It uses the module name - to find its parent module, and from there it looks up the parent's - __path__. When this changes, the module's own path is recomputed, - using path_finder. For top-level modules, the parent module's path - is sys.path.""" +class NamespacePath: + """Represents a namespace package's path. + + It uses the module *name* to find its parent module, and from there it looks + up the parent's __path__. When this changes, the module's own path is + recomputed, using *path_finder*. The initial value is set to *path*. + + For top-level modules, the parent module's path is sys.path. + + *path_finder* should be a callable with the same signature as + MetaPathFinder.find_spec((fullname, path, target=None) -> spec). + """ # When invalidate_caches() is called, this epoch is incremented # https://bugs.python.org/issue45703 @@ -1112,7 +1090,15 @@ def _find_parent_path_names(self): def _get_parent_path(self): parent_module_name, path_attr_name = self._find_parent_path_names() - return getattr(sys.modules[parent_module_name], path_attr_name) + try: + module = sys.modules[parent_module_name] + except KeyError as e: + raise ModuleNotFoundError( + f"{parent_module_name!r} must be imported before finding {self._name!r}.", + name=parent_module_name, + ) from e + else: + return getattr(module, path_attr_name) def _recalculate(self): # If the parent's path has changed, recalculate _path @@ -1141,7 +1127,7 @@ def __len__(self): return len(self._recalculate()) def __repr__(self): - return f'_NamespacePath({self._path!r})' + return f'NamespacePath({self._path!r})' def __contains__(self, item): return item in self._recalculate() @@ -1150,12 +1136,16 @@ def append(self, item): self._path.append(item) +# For backwards-compatibility for anyone desperate enough to get at the class back in the day. +_NamespacePath = NamespacePath + + # This class is actually exposed publicly in a namespace package's __loader__ # attribute, so it should be available through a non-private name. # https://github.com/python/cpython/issues/92054 class NamespaceLoader: def __init__(self, name, path, path_finder): - self._path = _NamespacePath(name, path, path_finder) + self._path = NamespacePath(name, path, path_finder) def is_package(self, fullname): return True @@ -1164,7 +1154,7 @@ def get_source(self, fullname): return '' def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) + return compile('', '', 'exec', dont_inherit=True, module=fullname) def create_module(self, spec): """Use default semantics for module creation.""" @@ -1172,18 +1162,6 @@ def create_module(self, spec): def exec_module(self, module): pass - def load_module(self, fullname): - """Load a namespace module. - - This method is deprecated. Use exec_module() instead. - - """ - # The import system never calls this method. - _bootstrap._verbose_message('namespace module loaded with path {!r}', - self._path) - # Warning implemented in _load_module_shim(). - return _bootstrap._load_module_shim(self, fullname) - def get_resource_reader(self, module): from importlib.readers import NamespaceReader return NamespaceReader(self._path) @@ -1210,9 +1188,9 @@ def invalidate_caches(): del sys.path_importer_cache[name] elif hasattr(finder, 'invalidate_caches'): finder.invalidate_caches() - # Also invalidate the caches of _NamespacePaths + # Also invalidate the caches of NamespacePaths # https://bugs.python.org/issue45703 - _NamespacePath._epoch += 1 + NamespacePath._epoch += 1 from importlib.metadata import MetadataPathFinder MetadataPathFinder.invalidate_caches() @@ -1298,13 +1276,30 @@ def find_spec(cls, fullname, path=None, target=None): # We found at least one namespace path. Return a spec which # can create the namespace package. spec.origin = None - spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) + spec.submodule_search_locations = NamespacePath(fullname, namespace_path, cls._get_spec) return spec else: return None else: return spec + @classmethod + def discover(cls, parent=None): + if parent is None: + path = sys.path + elif parent.submodule_search_locations is None: + raise ValueError(f'{parent} is not a package module') + else: + path = parent.submodule_search_locations + + for entry in set(path): + if not isinstance(entry, str): + continue + if (finder := cls._path_importer_cache(entry)) is None: + continue + if discover := getattr(finder, 'discover', None): + yield from discover(parent) + @staticmethod def find_distributions(*args, **kwargs): """ @@ -1454,6 +1449,37 @@ def path_hook_for_FileFinder(path): return path_hook_for_FileFinder + def _find_children(self): + with _os.scandir(self.path) as scan_iterator: + while True: + try: + entry = next(scan_iterator) + if entry.name == _PYCACHE: + continue + # packages + if entry.is_dir() and '.' not in entry.name: + yield entry.name + # files + if entry.is_file(): + yield from { + entry.name.removesuffix(suffix) + for suffix, _ in self._loaders + if entry.name.endswith(suffix) + } + except OSError: + pass # ignore exceptions from next(scan_iterator) and os.DirEntry + except StopIteration: + break + + def discover(self, parent=None): + if parent and parent.submodule_search_locations is None: + raise ValueError(f'{parent} is not a package module') + + module_prefix = f'{parent.name}.' if parent else '' + for child_name in self._find_children(): + if spec := self.find_spec(module_prefix + child_name): + yield spec + def __repr__(self): return f'FileFinder({self.path!r})' @@ -1525,7 +1551,6 @@ def _fix_up_module(ns, name, pathname, cpathname=None): ns['__spec__'] = spec ns['__loader__'] = loader ns['__file__'] = pathname - ns['__cached__'] = cpathname except Exception: # Not important enough to report. pass diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 1e47495f65f..9ca127ad9c7 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -45,6 +45,16 @@ def invalidate_caches(self): This method is used by importlib.invalidate_caches(). """ + def discover(self, parent=None): + """An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, MetaPathFinder.discover will search + for top-level modules. + + Returns an iterable of possible specs. + """ + return () + + _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.PathFinder, machinery.WindowsRegistryFinder) @@ -58,6 +68,15 @@ def invalidate_caches(self): This method is used by PathFinder.invalidate_caches(). """ + def discover(self, parent=None): + """An optional method which searches for possible specs with given + *parent* module spec. If *parent* is *None*, PathEntryFinder.discover + will search for top-level modules. + + Returns an iterable of possible specs. + """ + return () + _register(PathEntryFinder, machinery.FileFinder) @@ -108,7 +127,7 @@ def get_code(self, fullname): source = self.get_source(fullname) if source is None: return None - return self.source_to_code(source) + return self.source_to_code(source, '', fullname) @abc.abstractmethod def get_source(self, fullname): @@ -120,15 +139,14 @@ def get_source(self, fullname): raise ImportError @staticmethod - def source_to_code(data, path=''): + def source_to_code(data, path='', fullname=None): """Compile 'data' into a code object. The 'data' argument can be anything that compile() can handle. The'path' argument should be where the data was retrieved (when applicable).""" - return compile(data, path, 'exec', dont_inherit=True) + return compile(data, path, 'exec', dont_inherit=True, module=fullname) exec_module = _bootstrap_external._LoaderBasics.exec_module - load_module = _bootstrap_external._LoaderBasics.load_module _register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.NamespaceLoader) @@ -163,9 +181,8 @@ def get_code(self, fullname): try: path = self.get_filename(fullname) except ImportError: - return self.source_to_code(source) - else: - return self.source_to_code(source, path) + path = '' + return self.source_to_code(source, path, fullname) _register( ExecutionLoader, diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 63d726445c3..023f77d750f 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -16,6 +16,7 @@ from ._bootstrap_external import ExtensionFileLoader from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader +from ._bootstrap_external import NamespacePath def all_suffixes(): diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 8ce62dd864f..32f4b7d2d6e 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,40 +1,50 @@ +""" +APIs exposing metadata from third-party Python packages. + +This codebase is shared between importlib.metadata in the stdlib +and importlib_metadata in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" + from __future__ import annotations -import os -import re import abc -import sys -import json +import collections import email -import types -import inspect -import pathlib -import zipfile -import operator -import textwrap -import warnings import functools import itertools +import operator +import os +import pathlib import posixpath -import collections - -from . import _meta -from ._collections import FreezableDefaultDict, Pair -from ._functools import method_cache, pass_none -from ._itertools import always_iterable, unique_everseen -from ._meta import PackageMetadata, SimplePath - +import re +import sys +import textwrap +import types +from collections.abc import Iterable, Mapping from contextlib import suppress from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast +from typing import Any + +from . import _meta +from ._collections import FreezableDefaultDict, Pair +from ._context import ExceptionTrap +from ._functools import method_cache, noop, pass_none, passthrough +from ._itertools import always_iterable, bucket, unique_everseen +from ._meta import PackageMetadata, SimplePath +from ._typing import md_none __all__ = [ 'Distribution', 'DistributionFinder', 'PackageMetadata', 'PackageNotFoundError', + 'PackagePath', + 'MetadataNotFound', + 'SimplePath', 'distribution', 'distributions', 'entry_points', @@ -53,11 +63,15 @@ def __str__(self) -> str: return f"No package metadata was found for {self.name}" @property - def name(self) -> str: # type: ignore[override] + def name(self) -> str: # type: ignore[override] # make readonly (name,) = self.args return name +class MetadataNotFound(FileNotFoundError): + """No metadata file is present in the distribution.""" + + class Sectioned: """ A simple entry point config parser for performance @@ -123,6 +137,12 @@ def valid(line: str): return line and not line.startswith('#') +class _EntryPointMatch(types.SimpleNamespace): + module: str + attr: str + extras: str + + class EntryPoint: """An entry point as defined by Python packaging conventions. @@ -138,6 +158,30 @@ class EntryPoint: 'attr' >>> ep.extras ['extra1', 'extra2'] + + If the value package or module are not valid identifiers, a + ValueError is raised on access. + + >>> EntryPoint(name=None, group=None, value='invalid-name').module + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + >>> EntryPoint(name=None, group=None, value='invalid-name').attr + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + >>> EntryPoint(name=None, group=None, value='invalid-name').extras + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + + The same thing happens on construction. + + >>> EntryPoint(name=None, group=None, value='invalid-name') + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + """ pattern = re.compile( @@ -165,38 +209,44 @@ class EntryPoint: value: str group: str - dist: Optional[Distribution] = None + dist: Distribution | None = None def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) + self.module def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = cast(Match, self.pattern.match(self.value)) - module = import_module(match.group('module')) - attrs = filter(None, (match.group('attr') or '').split('.')) + module = import_module(self.module) + attrs = filter(None, (self.attr or '').split('.')) return functools.reduce(getattr, attrs, module) @property def module(self) -> str: - match = self.pattern.match(self.value) - assert match is not None - return match.group('module') + return self._match.module @property def attr(self) -> str: - match = self.pattern.match(self.value) - assert match is not None - return match.group('attr') + return self._match.attr @property - def extras(self) -> List[str]: + def extras(self) -> list[str]: + return re.findall(r'\w+', self._match.extras or '') + + @functools.cached_property + def _match(self) -> _EntryPointMatch: match = self.pattern.match(self.value) - assert match is not None - return re.findall(r'\w+', match.group('extras') or '') + if not match: + raise ValueError( + 'Invalid object reference. ' + 'See https://packaging.python.org' + '/en/latest/specifications/entry-points/#data-model', + self.value, + ) + return _EntryPointMatch(**match.groupdict()) def _for(self, dist): vars(self).update(dist=dist) @@ -222,9 +272,26 @@ def matches(self, **params): >>> ep.matches(attr='bong') True """ + self._disallow_dist(params) attrs = (getattr(self, param) for param in params) return all(map(operator.eq, params.values(), attrs)) + @staticmethod + def _disallow_dist(params): + """ + Querying by dist is not allowed (dist objects are not comparable). + >>> EntryPoint(name='fan', value='fav', group='fag').matches(dist='foo') + Traceback (most recent call last): + ... + ValueError: "dist" is not suitable for matching... + """ + if "dist" in params: + raise ValueError( + '"dist" is not suitable for matching. ' + "Instead, use Distribution.entry_points.select() on a " + "located distribution." + ) + def _key(self): return self.name, self.value, self.group @@ -254,7 +321,7 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] + def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] # Work with str instead of int """ Get the EntryPoint in self matching name. """ @@ -278,14 +345,14 @@ def select(self, **params) -> EntryPoints: return EntryPoints(ep for ep in self if ep.matches(**params)) @property - def names(self) -> Set[str]: + def names(self) -> set[str]: """ Return the set of all names of all entry points. """ return {ep.name for ep in self} @property - def groups(self) -> Set[str]: + def groups(self) -> set[str]: """ Return the set of all groups of all entry points. """ @@ -306,11 +373,11 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - hash: Optional[FileHash] + hash: FileHash | None size: int dist: Distribution - def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] + def read_text(self, encoding: str = 'utf-8') -> str: return self.locate().read_text(encoding=encoding) def read_binary(self) -> bytes: @@ -329,27 +396,7 @@ def __repr__(self) -> str: return f'' -class DeprecatedNonAbstract: - # Required until Python 3.14 - def __new__(cls, *args, **kwargs): - all_names = { - name for subclass in inspect.getmro(cls) for name in vars(subclass) - } - abstract = { - name - for name in all_names - if getattr(getattr(cls, name), '__isabstractmethod__', False) - } - if abstract: - warnings.warn( - f"Unimplemented abstract methods {abstract}", - DeprecationWarning, - stacklevel=2, - ) - return super().__new__(cls) - - -class Distribution(DeprecatedNonAbstract): +class Distribution(metaclass=abc.ABCMeta): """ An abstract Python distribution package. @@ -361,7 +408,7 @@ class Distribution(DeprecatedNonAbstract): """ @abc.abstractmethod - def read_text(self, filename) -> Optional[str]: + def read_text(self, filename) -> str | None: """Attempt to load metadata file given by the name. Python distribution metadata is organized by blobs of text @@ -388,6 +435,17 @@ def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ Given a path to a file in this distribution, return a SimplePath to it. + + This method is used by callers of ``Distribution.files()`` to + locate files within the distribution. If it's possible for a + Distribution to represent files in the distribution as + ``SimplePath`` objects, it should implement this method + to resolve such objects. + + Some Distribution providers may elect not to resolve SimplePath + objects within the distribution by raising a + NotImplementedError, but consumers of such a Distribution would + be unable to invoke ``Distribution.files()``. """ @classmethod @@ -404,13 +462,13 @@ def from_name(cls, name: str) -> Distribution: if not name: raise ValueError("A distribution name is required.") try: - return next(iter(cls.discover(name=name))) + return next(iter(cls._prefer_valid(cls.discover(name=name)))) except StopIteration: - raise PackageNotFoundError(name) + raise PackageNotFoundError(name) from None @classmethod def discover( - cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + cls, *, context: DistributionFinder.Context | None = None, **kwargs ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. @@ -428,6 +486,21 @@ def discover( resolver(context) for resolver in cls._discover_resolvers() ) + @staticmethod + def _prefer_valid(dists: Iterable[Distribution]) -> Iterable[Distribution]: + """ + Prefer (move to the front) distributions that have metadata. + + Ref python/importlib_resources#489. + """ + + has_metadata = ExceptionTrap(MetadataNotFound).passes( + operator.attrgetter('metadata') + ) + + buckets = bucket(dists, has_metadata) + return itertools.chain(buckets[True], buckets[False]) + @staticmethod def at(path: str | os.PathLike[str]) -> Distribution: """Return a Distribution for the indicated metadata path. @@ -455,11 +528,11 @@ def metadata(self) -> _meta.PackageMetadata: Custom providers may provide the METADATA file or override this property. + + :raises MetadataNotFound: If no metadata file is present. """ - # deferred for performance (python/cpython#109829) - from . import _adapters - opt_text = ( + text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') # This last clause is here to support old egg-info files. Its @@ -467,9 +540,21 @@ def metadata(self) -> _meta.PackageMetadata: # (which points to the egg-info file) attribute unchanged. or self.read_text('') ) - text = cast(str, opt_text) + return self._assemble_message(self._ensure_metadata_present(text)) + + @staticmethod + def _assemble_message(text: str) -> _meta.PackageMetadata: + # deferred for performance (python/cpython#109829) + from . import _adapters + return _adapters.Message(email.message_from_string(text)) + def _ensure_metadata_present(self, text: str | None) -> str: + if text is not None: + return text + + raise MetadataNotFound('No package metadata was found.') + @property def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" @@ -496,7 +581,7 @@ def entry_points(self) -> EntryPoints: return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property - def files(self) -> Optional[List[PackagePath]]: + def files(self) -> list[PackagePath] | None: """Files in this distribution. :return: List of PackagePath for this distribution or None @@ -589,7 +674,7 @@ def _read_files_egginfo_sources(self): return text and map('"{}"'.format, text.splitlines()) @property - def requires(self) -> Optional[List[str]]: + def requires(self) -> list[str] | None: """Generated requirements specified for this Distribution""" reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() return reqs and list(reqs) @@ -645,6 +730,9 @@ def origin(self): return self._load_json('direct_url.json') def _load_json(self, filename): + # Deferred for performance (python/importlib_metadata#503) + import json + return pass_none(json.loads)( self.read_text(filename), object_hook=lambda data: types.SimpleNamespace(**data), @@ -692,7 +780,7 @@ def __init__(self, **kwargs): vars(self).update(kwargs) @property - def path(self) -> List[str]: + def path(self) -> list[str]: """ The sequence of directory path that a distribution finder should search. @@ -713,6 +801,20 @@ def find_distributions(self, context=Context()) -> Iterable[Distribution]: """ +@passthrough +def _clear_after_fork(cached): + """Ensure ``func`` clears cached state after ``fork`` when supported. + + ``FastPath`` caches zip-backed ``pathlib.Path`` objects that retain a + reference to the parent's open ``ZipFile`` handle. Re-using a cached + instance in a forked child can therefore resurrect invalid file pointers + and trigger ``BadZipFile``/``OSError`` failures (python/importlib_metadata#520). + Registering ``cache_clear`` with ``os.register_at_fork`` keeps each process + on its own cache. + """ + getattr(os, 'register_at_fork', noop)(after_in_child=cached.cache_clear) + + class FastPath: """ Micro-optimized class for searching a root for children. @@ -729,7 +831,8 @@ class FastPath: True """ - @functools.lru_cache() # type: ignore + @_clear_after_fork # type: ignore[misc] + @functools.lru_cache() def __new__(cls, root): return super().__new__(cls) @@ -747,6 +850,9 @@ def children(self): return [] def zip_children(self): + # deferred for performance (python/importlib_metadata#502) + import zipfile + zip_path = zipfile.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath @@ -841,7 +947,7 @@ class Prepared: normalized = None legacy_normalized = None - def __init__(self, name: Optional[str]): + def __init__(self, name: str | None): self.name = name if name is None: return @@ -852,8 +958,15 @@ def __init__(self, name: Optional[str]): def normalize(name): """ PEP 503 normalization plus dashes as underscores. + + Specifically avoids ``re.sub`` as prescribed for performance + benefits (see python/cpython#143658). """ - return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') + value = name.lower().replace("-", "_").replace(".", "_") + # Condense repeats + while "__" in value: + value = value.replace("__", "_") + return value @staticmethod def legacy_normalize(name): @@ -904,7 +1017,7 @@ def __init__(self, path: SimplePath) -> None: """ self._path = path - def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: + def read_text(self, filename: str | os.PathLike[str]) -> str | None: with suppress( FileNotFoundError, IsADirectoryError, @@ -973,6 +1086,7 @@ def metadata(distribution_name: str) -> _meta.PackageMetadata: :param distribution_name: The name of the distribution package to query. :return: A PackageMetadata containing the parsed metadata. + :raises MetadataNotFound: If no metadata file is present in the distribution. """ return Distribution.from_name(distribution_name).metadata @@ -1011,7 +1125,7 @@ def entry_points(**params) -> EntryPoints: return EntryPoints(eps).select(**params) -def files(distribution_name: str) -> Optional[List[PackagePath]]: +def files(distribution_name: str) -> list[PackagePath] | None: """Return a list of files for the named package. :param distribution_name: The name of the distribution package to query. @@ -1020,7 +1134,7 @@ def files(distribution_name: str) -> Optional[List[PackagePath]]: return distribution(distribution_name).files -def requires(distribution_name: str) -> Optional[List[str]]: +def requires(distribution_name: str) -> list[str] | None: """ Return a list of requirements for the named package. @@ -1030,7 +1144,7 @@ def requires(distribution_name: str) -> Optional[List[str]]: return distribution(distribution_name).requires -def packages_distributions() -> Mapping[str, List[str]]: +def packages_distributions() -> Mapping[str, list[str]]: """ Return a mapping of top-level packages to their distributions. @@ -1051,7 +1165,7 @@ def _top_level_declared(dist): return (dist.read_text('top_level.txt') or '').split() -def _topmost(name: PackagePath) -> Optional[str]: +def _topmost(name: PackagePath) -> str | None: """ Return the top-most parent as long as there is a parent. """ @@ -1077,11 +1191,10 @@ def _get_toplevel_name(name: PackagePath) -> str: >>> _get_toplevel_name(PackagePath('foo.dist-info')) 'foo.dist-info' """ - return _topmost(name) or ( - # python/typeshed#10328 - inspect.getmodulename(name) # type: ignore - or str(name) - ) + # Defer import of inspect for performance (python/cpython#118761) + import inspect + + return _topmost(name) or inspect.getmodulename(name) or str(name) def _top_level_inferred(dist): diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index 59116880895..dede395d79a 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -1,22 +1,59 @@ -import functools -import warnings +import email.message +import email.policy import re import textwrap -import email.message from ._text import FoldedCase -# Do not remove prior to 2024-01-01 or Python 3.14 -_warn = functools.partial( - warnings.warn, - "Implicit None on return values is deprecated and will raise KeyErrors.", - DeprecationWarning, - stacklevel=2, -) +class RawPolicy(email.policy.EmailPolicy): + def fold(self, name, value): + folded = self.linesep.join( + textwrap + .indent(value, prefix=' ' * 8, predicate=lambda line: True) + .lstrip() + .splitlines() + ) + return f'{name}: {folded}{self.linesep}' class Message(email.message.Message): + r""" + Specialized Message subclass to handle metadata naturally. + + Reads values that may have newlines in them and converts the + payload to the Description. + + >>> msg_text = textwrap.dedent(''' + ... Name: Foo + ... Version: 3.0 + ... License: blah + ... de-blah + ... + ... First line of description. + ... Second line of description. + ... + ... Fourth line! + ... ''').lstrip().replace('', '') + >>> msg = Message(email.message_from_string(msg_text)) + >>> msg['Description'] + 'First line of description.\nSecond line of description.\n\nFourth line!\n' + + Message should render even if values contain newlines. + + >>> print(msg) + Name: Foo + Version: 3.0 + License: blah + de-blah + Description: First line of description. + Second line of description. + + Fourth line! + + + """ + multiple_use_keys = set( map( FoldedCase, @@ -52,26 +89,36 @@ def __iter__(self): def __getitem__(self, item): """ - Warn users that a ``KeyError`` can be expected when a - missing key is supplied. Ref python/importlib_metadata#371. + Override parent behavior to typical dict behavior. + + ``email.message.Message`` will emit None values for missing + keys. Typical mappings, including this ``Message``, will raise + a key error for missing keys. + + Ref python/importlib_metadata#371. """ res = super().__getitem__(item) if res is None: - _warn() + raise KeyError(item) return res def _repair_headers(self): def redent(value): "Correct for RFC822 indentation" - if not value or '\n' not in value: + indent = ' ' * 8 + if not value or '\n' + indent not in value: return value - return textwrap.dedent(' ' * 8 + value) + return textwrap.dedent(indent + value) headers = [(key, redent(value)) for key, value in vars(self)['_headers']] if self._payload: headers.append(('Description', self.get_payload())) + self.set_payload('') return headers + def as_string(self): + return super().as_string(policy=RawPolicy()) + @property def json(self): """ diff --git a/Lib/importlib/metadata/_collections.py b/Lib/importlib/metadata/_collections.py index cf0954e1a30..fc5045d36be 100644 --- a/Lib/importlib/metadata/_collections.py +++ b/Lib/importlib/metadata/_collections.py @@ -1,4 +1,5 @@ import collections +import typing # from jaraco.collections 3.3 @@ -24,7 +25,10 @@ def freeze(self): self._frozen = lambda key: self.default_factory() -class Pair(collections.namedtuple('Pair', 'name value')): +class Pair(typing.NamedTuple): + name: str + value: str + @classmethod def parse(cls, text): return cls(*map(str.strip, text.split("=", 1))) diff --git a/Lib/importlib/metadata/_context.py b/Lib/importlib/metadata/_context.py new file mode 100644 index 00000000000..2635b164ce8 --- /dev/null +++ b/Lib/importlib/metadata/_context.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import functools +import operator + + +# from jaraco.context 6.1 +class ExceptionTrap: + """ + A context manager that will catch certain exceptions and provide an + indication they occurred. + + >>> with ExceptionTrap() as trap: + ... raise Exception() + >>> bool(trap) + True + + >>> with ExceptionTrap() as trap: + ... pass + >>> bool(trap) + False + + >>> with ExceptionTrap(ValueError) as trap: + ... raise ValueError("1 + 1 is not 3") + >>> bool(trap) + True + >>> trap.value + ValueError('1 + 1 is not 3') + >>> trap.tb + + + >>> with ExceptionTrap(ValueError) as trap: + ... raise Exception() + Traceback (most recent call last): + ... + Exception + + >>> bool(trap) + False + """ + + exc_info = None, None, None + + def __init__(self, exceptions=(Exception,)): + self.exceptions = exceptions + + def __enter__(self): + return self + + @property + def type(self): + return self.exc_info[0] + + @property + def value(self): + return self.exc_info[1] + + @property + def tb(self): + return self.exc_info[2] + + def __exit__(self, *exc_info): + type = exc_info[0] + matches = type and issubclass(type, self.exceptions) + if matches: + self.exc_info = exc_info + return matches + + def __bool__(self): + return bool(self.type) + + def raises(self, func, *, _test=bool): + """ + Wrap func and replace the result with the truth + value of the trap (True if an exception occurred). + + First, give the decorator an alias to support Python 3.8 + Syntax. + + >>> raises = ExceptionTrap(ValueError).raises + + Now decorate a function that always fails. + + >>> @raises + ... def fail(): + ... raise ValueError('failed') + >>> fail() + True + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + with ExceptionTrap(self.exceptions) as trap: + func(*args, **kwargs) + return _test(trap) + + return wrapper + + def passes(self, func): + """ + Wrap func and replace the result with the truth + value of the trap (True if no exception). + + First, give the decorator an alias to support Python 3.8 + Syntax. + + >>> passes = ExceptionTrap(ValueError).passes + + Now decorate a function that always fails. + + >>> @passes + ... def fail(): + ... raise ValueError('failed') + + >>> fail() + False + """ + return self.raises(func, _test=operator.not_) diff --git a/Lib/importlib/metadata/_functools.py b/Lib/importlib/metadata/_functools.py index 71f66bd03cb..c159b46e489 100644 --- a/Lib/importlib/metadata/_functools.py +++ b/Lib/importlib/metadata/_functools.py @@ -1,5 +1,7 @@ -import types import functools +import types +from collections.abc import Callable +from typing import TypeVar # from jaraco.functools 3.3 @@ -102,3 +104,33 @@ def wrapper(param, *args, **kwargs): return func(param, *args, **kwargs) return wrapper + + +# From jaraco.functools 4.4 +def noop(*args, **kwargs): + """ + A no-operation function that does nothing. + + >>> noop(1, 2, three=3) + """ + + +_T = TypeVar('_T') + + +# From jaraco.functools 4.4 +def passthrough(func: Callable[..., object]) -> Callable[[_T], _T]: + """ + Wrap the function to always return the first parameter. + + >>> passthrough(print)('3') + 3 + '3' + """ + + @functools.wraps(func) + def wrapper(first: _T, *args, **kwargs) -> _T: + func(first, *args, **kwargs) + return first + + return wrapper # type: ignore[return-value] diff --git a/Lib/importlib/metadata/_itertools.py b/Lib/importlib/metadata/_itertools.py index d4ca9b9140e..79d37198ce7 100644 --- a/Lib/importlib/metadata/_itertools.py +++ b/Lib/importlib/metadata/_itertools.py @@ -1,3 +1,4 @@ +from collections import defaultdict, deque from itertools import filterfalse @@ -71,3 +72,100 @@ def always_iterable(obj, base_type=(str, bytes)): return iter(obj) except TypeError: return iter((obj,)) + + +# Copied from more_itertools 10.3 +class bucket: + """Wrap *iterable* and return an object that buckets the iterable into + child iterables based on a *key* function. + + >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] + >>> s = bucket(iterable, key=lambda x: x[0]) # Bucket by 1st character + >>> sorted(list(s)) # Get the keys + ['a', 'b', 'c'] + >>> a_iterable = s['a'] + >>> next(a_iterable) + 'a1' + >>> next(a_iterable) + 'a2' + >>> list(s['b']) + ['b1', 'b2', 'b3'] + + The original iterable will be advanced and its items will be cached until + they are used by the child iterables. This may require significant storage. + + By default, attempting to select a bucket to which no items belong will + exhaust the iterable and cache all values. + If you specify a *validator* function, selected buckets will instead be + checked against it. + + >>> from itertools import count + >>> it = count(1, 2) # Infinite sequence of odd numbers + >>> key = lambda x: x % 10 # Bucket by last digit + >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only + >>> s = bucket(it, key=key, validator=validator) + >>> 2 in s + False + >>> list(s[2]) + [] + + """ + + def __init__(self, iterable, key, validator=None): + self._it = iter(iterable) + self._key = key + self._cache = defaultdict(deque) + self._validator = validator or (lambda x: True) + + def __contains__(self, value): + if not self._validator(value): + return False + + try: + item = next(self[value]) + except StopIteration: + return False + else: + self._cache[value].appendleft(item) + + return True + + def _get_values(self, value): + """ + Helper to yield items from the parent iterator that match *value*. + Items that don't match are stored in the local cache as they + are encountered. + """ + while True: + # If we've cached some items that match the target value, emit + # the first one and evict it from the cache. + if self._cache[value]: + yield self._cache[value].popleft() + # Otherwise we need to advance the parent iterator to search for + # a matching item, caching the rest. + else: + while True: + try: + item = next(self._it) + except StopIteration: + return + item_value = self._key(item) + if item_value == value: + yield item + break + elif self._validator(item_value): + self._cache[item_value].append(item) + + def __iter__(self): + for item in self._it: + item_value = self._key(item) + if self._validator(item_value): + self._cache[item_value].append(item) + + yield from self._cache.keys() + + def __getitem__(self, value): + if not self._validator(value): + return iter(()) + + return self._get_values(value) diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index 1927d0f624d..0c20eff3da7 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,9 +1,13 @@ from __future__ import annotations import os -from typing import Protocol -from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload - +from collections.abc import Iterator +from typing import ( + Any, + Protocol, + TypeVar, + overload, +) _T = TypeVar("_T") @@ -20,25 +24,25 @@ def __iter__(self) -> Iterator[str]: ... # pragma: no cover @overload def get( self, name: str, failobj: None = None - ) -> Optional[str]: ... # pragma: no cover + ) -> str | None: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover + def get(self, name: str, failobj: _T) -> str | _T: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload def get_all( self, name: str, failobj: None = None - ) -> Optional[List[Any]]: ... # pragma: no cover + ) -> list[Any] | None: ... # pragma: no cover @overload - def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: + def get_all(self, name: str, failobj: _T) -> list[Any] | _T: """ Return all values associated with a possibly multi-valued key. """ @property - def json(self) -> Dict[str, Union[str, List[str]]]: + def json(self) -> dict[str, str | list[str]]: """ A JSON-compatible form of the metadata. """ @@ -50,11 +54,11 @@ class SimplePath(Protocol): """ def joinpath( - self, other: Union[str, os.PathLike[str]] + self, other: str | os.PathLike[str] ) -> SimplePath: ... # pragma: no cover def __truediv__( - self, other: Union[str, os.PathLike[str]] + self, other: str | os.PathLike[str] ) -> SimplePath: ... # pragma: no cover @property diff --git a/Lib/importlib/metadata/_typing.py b/Lib/importlib/metadata/_typing.py new file mode 100644 index 00000000000..32b1d2b98ac --- /dev/null +++ b/Lib/importlib/metadata/_typing.py @@ -0,0 +1,15 @@ +import functools +import typing + +from ._meta import PackageMetadata + +md_none = functools.partial(typing.cast, PackageMetadata) +""" +Suppress type errors for optional metadata. + +Although Distribution.metadata can return None when metadata is corrupt +and thus None, allow callers to assume it's not None and crash if +that's the case. + +# python/importlib_metadata#493 +""" diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index 723c9f9eb33..27d6c7f8930 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -8,12 +8,11 @@ """ from ._common import ( + Anchor, + Package, as_file, files, - Package, - Anchor, ) - from ._functional import ( contents, is_resource, @@ -23,10 +22,8 @@ read_binary, read_text, ) - from .abc import ResourceReader - __all__ = [ 'Package', 'Anchor', diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index 4e9014c45a0..6f87d77492f 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -1,54 +1,20 @@ -import os -import pathlib -import tempfile -import functools import contextlib -import types +import functools import importlib import inspect -import warnings import itertools +import os +import pathlib +import tempfile +import types +from typing import Optional, cast -from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -Package = Union[types.ModuleType, str] +Package = types.ModuleType | str Anchor = Package -def package_to_anchor(func): - """ - Replace 'package' parameter as 'anchor' and warn about the change. - - Other errors should fall through. - - >>> files('a', 'b') - Traceback (most recent call last): - TypeError: files() takes from 0 to 1 positional arguments but 2 were given - - Remove this compatibility in Python 3.14. - """ - undefined = object() - - @functools.wraps(func) - def wrapper(anchor=undefined, package=undefined): - if package is not undefined: - if anchor is not undefined: - return func(anchor, package) - warnings.warn( - "First parameter to files is renamed to 'anchor'", - DeprecationWarning, - stacklevel=2, - ) - return func(package) - elif anchor is undefined: - return func() - return func(anchor) - - return wrapper - - -@package_to_anchor def files(anchor: Optional[Anchor] = None) -> Traversable: """ Get a Traversable resource for an anchor. @@ -105,6 +71,19 @@ def is_wrapper(frame_info): return next(callers).frame +def _assert_spec(package: types.ModuleType) -> None: + """ + Provide a nicer error message when package is ``__main__`` + and its ``__spec__`` is ``None`` + (https://docs.python.org/3/reference/import.html#main-spec). + """ + if package.__spec__ is None: + raise TypeError( + f"Cannot access resources for '{package.__name__}' " + "as it does not appear to correspond to an importable module (its __spec__ is None)." + ) + + def from_package(package: types.ModuleType): """ Return a Traversable object for the given package. @@ -113,6 +92,7 @@ def from_package(package: types.ModuleType): # deferred for performance (python/cpython#109829) from ._adapters import wrap_spec + _assert_spec(package) spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py index f59416f2dd6..b08a5c6efe2 100644 --- a/Lib/importlib/resources/_functional.py +++ b/Lib/importlib/resources/_functional.py @@ -2,8 +2,8 @@ import warnings -from ._common import files, as_file - +from ._common import as_file, files +from .abc import TraversalError _MISSING = object() @@ -42,7 +42,10 @@ def is_resource(anchor, *path_names): Otherwise returns ``False``. """ - return _get_resource(anchor, path_names).is_file() + try: + return _get_resource(anchor, path_names).is_file() + except TraversalError: + return False def contents(anchor, *path_names): diff --git a/Lib/importlib/resources/abc.py b/Lib/importlib/resources/abc.py index 6750a7aaf14..0b5fdee80e8 100644 --- a/Lib/importlib/resources/abc.py +++ b/Lib/importlib/resources/abc.py @@ -1,14 +1,22 @@ import abc -import io import itertools import os import pathlib -from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional -from typing import runtime_checkable, Protocol -from typing import Union - - -StrPath = Union[str, os.PathLike[str]] +from collections.abc import Iterable, Iterator +from typing import ( + Any, + BinaryIO, + Literal, + NoReturn, + Optional, + Protocol, + Text, + TextIO, + overload, + runtime_checkable, +) + +StrPath = str | os.PathLike[str] __all__ = ["ResourceReader", "Traversable", "TraversableResources"] @@ -82,11 +90,13 @@ def read_bytes(self) -> bytes: with self.open('rb') as strm: return strm.read() - def read_text(self, encoding: Optional[str] = None) -> str: + def read_text( + self, encoding: Optional[str] = None, errors: Optional[str] = None + ) -> str: """ Read contents of self as text """ - with self.open(encoding=encoding) as strm: + with self.open(encoding=encoding, errors=errors) as strm: return strm.read() @abc.abstractmethod @@ -132,8 +142,14 @@ def __truediv__(self, child: StrPath) -> "Traversable": """ return self.joinpath(child) + @overload + def open(self, mode: Literal['r'] = 'r', *args: Any, **kwargs: Any) -> TextIO: ... + + @overload + def open(self, mode: Literal['rb'], *args: Any, **kwargs: Any) -> BinaryIO: ... + @abc.abstractmethod - def open(self, mode='r', *args, **kwargs): + def open(self, mode: str = 'r', *args: Any, **kwargs: Any) -> TextIO | BinaryIO: """ mode may be 'r' or 'rb' to open as text or binary. Return a handle suitable for reading (same as pathlib.Path.open). @@ -160,7 +176,7 @@ class TraversableResources(ResourceReader): def files(self) -> "Traversable": """Return a Traversable object for the loaded package.""" - def open_resource(self, resource: StrPath) -> io.BufferedReader: + def open_resource(self, resource: StrPath) -> BinaryIO: return self.files().joinpath(resource).open('rb') def resource_path(self, resource: Any) -> NoReturn: diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index 70fc7e2b9c0..5d0ae46d672 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -3,15 +3,14 @@ import collections import contextlib import itertools -import pathlib import operator +import pathlib import re import warnings import zipfile from collections.abc import Iterator from . import abc - from ._itertools import only diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 2e75299b13a..5e182d12607 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -5,7 +5,7 @@ import abc import io import itertools -from typing import BinaryIO, List +from typing import BinaryIO from .abc import Traversable, TraversableResources @@ -24,14 +24,14 @@ def package(self) -> str: """ @abc.abstractmethod - def children(self) -> List['SimpleReader']: + def children(self) -> list['SimpleReader']: """ Obtain an iterable of SimpleReader for available child containers (e.g. directories). """ @abc.abstractmethod - def resources(self) -> List[str]: + def resources(self) -> list[str]: """ Obtain available named resources for this virtual package. """ diff --git a/Lib/inspect.py b/Lib/inspect.py index cf61ae943b1..92664f46d35 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -153,9 +153,9 @@ import itertools import linecache import os -import re +lazy import re import sys -import tokenize +lazy import tokenize import token import types import functools @@ -163,9 +163,9 @@ from keyword import iskeyword from operator import attrgetter from collections import namedtuple, OrderedDict -from weakref import ref as make_weakref +from _weakref import ref as make_weakref -# Create constants for the compiler flags in Include/code.h +# Create constants for the compiler flags in Include/cpython/code.h # We try to get them from dis to avoid duplication mod_dict = globals() for k, v in dis.COMPILER_FLAG_NAMES.items(): @@ -421,7 +421,6 @@ def iscode(object): co_freevars tuple of names of free variables co_posonlyargcount number of positional only arguments co_kwonlyargcount number of keyword only arguments (not including ** arg) - co_lnotab encoded mapping of line numbers to bytecode indices co_name name with which this code object was defined co_names tuple of names other than arguments and function locals co_nlocals number of local variables @@ -712,8 +711,8 @@ def _findclass(func): return None return cls -def _finddoc(obj): - if isclass(obj): +def _finddoc(obj, *, search_in_class=True): + if search_in_class and isclass(obj): for base in obj.__mro__: if base is not object: try: @@ -753,6 +752,12 @@ def _finddoc(obj): cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None + # Should be tested before ismethoddescriptor() + elif isinstance(obj, functools.cached_property): + name = obj.attrname + cls = _findclass(obj.func) + if cls is None or getattr(cls, name) is not obj: + return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): name = obj.__name__ cls = obj.__objclass__ @@ -773,19 +778,37 @@ def _finddoc(obj): return doc return None -def getdoc(object): +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def getdoc(object, *, fallback_to_class_doc=True, inherit_class_doc=True): """Get the documentation string for an object. All tabs are expanded to spaces. To clean up docstrings that are indented to line up with blocks of code, any whitespace than can be uniformly removed from the second line onwards is removed.""" - try: - doc = object.__doc__ - except AttributeError: - return None + if fallback_to_class_doc: + try: + doc = object.__doc__ + except AttributeError: + return None + else: + doc = _getowndoc(object) if doc is None: try: - doc = _finddoc(object) + doc = _finddoc(object, search_in_class=inherit_class_doc) except (AttributeError, TypeError): return None if not isinstance(doc, str): @@ -1236,11 +1259,10 @@ def getargs(co): FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') -def getfullargspec(func): +def getfullargspec(func, *, annotation_format=Format.VALUE): """Get the names and default values of a callable object's parameters. - A tuple of seven things is returned: - (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). + A FullArgSpec namedtuple is returned, which has the following attributes: 'args' is a list of the parameter names. 'varargs' and 'varkw' are the names of the * and ** parameters or None. 'defaults' is an n-tuple of the default values of the last n parameters. @@ -1248,6 +1270,9 @@ def getfullargspec(func): 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping parameter names to annotations. + The *annotation_format* parameter controls the format of the annotations. + See the annotationlib documentation for details. + Notable differences from inspect.signature(): - the "self" parameter is always reported, even for bound methods - wrapper chains defined by __wrapped__ *not* unwrapped automatically @@ -1273,7 +1298,8 @@ def getfullargspec(func): follow_wrapper_chains=False, skip_bound_arg=False, sigcls=Signature, - eval_str=False) + eval_str=False, + annotation_format=annotation_format) except Exception as ex: # Most of the times 'signature' will raise ValueError. # But, it can also raise AttributeError, and, maybe something @@ -1615,7 +1641,6 @@ def getframeinfo(frame, context=1): def getlineno(frame): """Get the line number from a frame object, allowing for optimization.""" - # FrameType.f_lineno is now a descriptor that grovels co_lnotab return frame.f_lineno _FrameInfo = namedtuple('_FrameInfo', ('frame',) + Traceback._fields) @@ -1709,7 +1734,8 @@ def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): class_dict = dunder_dict['__dict__'] if not (type(class_dict) is types.GetSetDescriptorType and class_dict.__name__ == "__dict__" and - class_dict.__objclass__ is entry): + (class_dict.__objclass__ is object or + class_dict.__objclass__ is entry)): return class_dict return _sentinel @@ -1793,13 +1819,7 @@ def getgeneratorstate(generator): GEN_SUSPENDED: Currently suspended at a yield expression. GEN_CLOSED: Execution has completed. """ - if generator.gi_running: - return GEN_RUNNING - if generator.gi_suspended: - return GEN_SUSPENDED - if generator.gi_frame is None: - return GEN_CLOSED - return GEN_CREATED + return generator.gi_state def getgeneratorlocals(generator): @@ -1835,13 +1855,7 @@ def getcoroutinestate(coroutine): CORO_SUSPENDED: Currently suspended at an await expression. CORO_CLOSED: Execution has completed. """ - if coroutine.cr_running: - return CORO_RUNNING - if coroutine.cr_suspended: - return CORO_SUSPENDED - if coroutine.cr_frame is None: - return CORO_CLOSED - return CORO_CREATED + return coroutine.cr_state def getcoroutinelocals(coroutine): @@ -1874,13 +1888,7 @@ def getasyncgenstate(agen): AGEN_SUSPENDED: Currently suspended at a yield expression. AGEN_CLOSED: Execution has completed. """ - if agen.ag_running: - return AGEN_RUNNING - if agen.ag_suspended: - return AGEN_SUSPENDED - if agen.ag_frame is None: - return AGEN_CLOSED - return AGEN_CREATED + return agen.ag_state def getasyncgenlocals(agen): @@ -2121,7 +2129,6 @@ def _signature_strip_non_python_syntax(signature): current_parameter = 0 OP = token.OP - ERRORTOKEN = token.ERRORTOKEN # token stream always starts with ENCODING token, skip it t = next(token_stream) @@ -2198,7 +2205,8 @@ def wrap_value(s): except NameError: raise ValueError - if isinstance(value, (str, int, float, bytes, bool, type(None))): + if isinstance(value, (str, int, float, bytes, bool, type(None), + sentinel)): return ast.Constant(value) raise ValueError @@ -3356,22 +3364,113 @@ class BufferFlags(enum.IntFlag): WRITE = 0x200 +def _get_details_for_cli(module, nominal_target, resolved_target): + # Determine if the given module name is an alias for another module, + # or if it is reexporting a name that is actually defined elsewhere + resolved_module = getmodule(resolved_target) + if resolved_module is not None and resolved_module is not module: + # Referenced target indicates it was defined somewhere else, + # so report the details of that module rather than the lookup module + module = resolved_module + reported_module_name = module.__name__ + # Ensure the reported source file reflects the actual defining location + try: + source_file = getsourcefile(resolved_target) + except Exception: + try: + source_file = getsourcefile(module) + except Exception: + source_file = None + # Determine if the nominal target location is its defining location + if resolved_target is module: + reported_target = reported_module_name + else: + reported_qualname = getattr(resolved_target, "__qualname__", None) + if not reported_qualname: + reported_qualname = nominal_target.partition(":")[2] + reported_target = f"{reported_module_name}:{reported_qualname}" + # Special case for looking up functions in frozen modules + if source_file == f"": + source_file = module.__file__ + # Populate the actual details to be reported + details = { + "target": reported_target, + "origin": module.__spec__.origin, + "cached": module.__spec__.cached, + "source": source_file, + } + if reported_target != nominal_target: + details["alias"] = nominal_target + error = None + if not source_file: + if module.__name__ in sys.builtin_module_names: + error = "No source code available for builtin module" + else: + error = "No source code available for defining module" + if resolved_target is module: + details["loader"] = repr(module.__spec__.loader) + if hasattr(module, '__path__'): + details["submodule_paths"] = str(module.__path__) + elif source_file: + try: + __, lineno = findsource(resolved_target) + except Exception: + error = "Failed to retrieve source code for given target" + else: + details["lineno"] = lineno + if error: + details["error"] = error + return details + +def _render_details_for_cli(details): + resolved_target = details["target"] + alias = details.get("alias") + if alias: + rendered_target = f'{resolved_target} (looked up as "{alias}")' + else: + rendered_target = resolved_target + lines = [ + f'Target: {rendered_target}', + f'Origin: {details["origin"]}', + f'Source: {details["source"]}', + f'Cached: {details["cached"]}', + ] + loader = details.get("loader") + if loader: + lines.append(f'Loader: {loader}') + submodule_paths = details.get("submodule_paths") + if submodule_paths: + lines.append(f'Submodule search paths: {submodule_paths}') + else: + error = details.get("error") + if error: + # The error is only informational when retrieving object details + lines.append(error) + else: + lines.append(f'Line: {details["lineno"]}') + + lines.append("") + return "\n".join(lines) + + def _main(): """ Logic for inspecting an object given at command line """ import argparse import importlib - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument( 'object', help="The object to be analysed. " - "It supports the 'module:qualname' syntax") + "It supports the `module:qualname` syntax") parser.add_argument( '-d', '--details', action='store_true', help='Display info about the module rather than its source code') args = parser.parse_args() + # We don't use `pkgutil.resolve_name` here because we want to obtain + # references to both the module *and* the fully resolved target object target = args.object mod_name, has_attrs, attrs = target.partition(":") try: @@ -3389,29 +3488,16 @@ def _main(): for part in parts: obj = getattr(obj, part) - if module.__name__ in sys.builtin_module_names: - print("Can't get info for builtin modules.", file=sys.stderr) - sys.exit(1) - + details = _get_details_for_cli(module, target, obj) if args.details: - print('Target: {}'.format(target)) - print('Origin: {}'.format(getsourcefile(module))) - print('Cached: {}'.format(module.__cached__)) - if obj is module: - print('Loader: {}'.format(repr(module.__loader__))) - if hasattr(module, '__path__'): - print('Submodule search path: {}'.format(module.__path__)) - else: - try: - __, lineno = findsource(obj) - except Exception: - pass - else: - print('Line: {}'.format(lineno)) - - print('\n') + print(_render_details_for_cli(details)) else: - print(getsource(obj)) + # Attempt to render target source details + error = details.get("error") + if error: + sys.exit(error) + else: + print(getsource(obj)) if __name__ == "__main__": diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 4c7eac0b7eb..e8a3d176ad6 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -4,6 +4,7 @@ import importlib.machinery import importlib.util import os +import py_compile import shutil import sys import textwrap @@ -49,20 +50,31 @@ def forget(modname): # combinations of PEP 3147/488 and legacy pyc files. unlink(source + 'c') for opt in ('', 1, 2): - unlink(importlib.util.cache_from_source(source, optimization=opt)) + try: + unlink(importlib.util.cache_from_source(source, optimization=opt)) + except NotImplementedError: + pass -def make_legacy_pyc(source): +def make_legacy_pyc(source, allow_compile=False): """Move a PEP 3147/488 pyc file to its legacy pyc location. :param source: The file system path to the source file. The source file - does not need to exist, however the PEP 3147/488 pyc file must exist. + does not need to exist, however the PEP 3147/488 pyc file must exist or + allow_compile must be set. + :param allow_compile: If True, uses py_compile to create a .pyc if it does + not exist. This should be passed as True if cache_tag may be None. :return: The file system path to the legacy pyc file. """ - pyc_file = importlib.util.cache_from_source(source) assert source.endswith('.py') legacy_pyc = source + 'c' - shutil.move(pyc_file, legacy_pyc) + try: + pyc_file = importlib.util.cache_from_source(source) + shutil.move(pyc_file, legacy_pyc) + except FileNotFoundError, NotImplementedError: + if not allow_compile: + raise + py_compile.compile(source, legacy_pyc, doraise=True) return legacy_pyc @@ -313,7 +325,7 @@ def ready_to_import(name=None, source=""): sys.modules.pop(name, None) -def ensure_lazy_imports(imported_module, modules_to_block): +def ensure_lazy_imports(imported_module, modules_to_block, *, additional_code=None): """Test that when imported_module is imported, none of the modules in modules_to_block are imported as a side effect.""" modules_to_block = frozenset(modules_to_block) @@ -331,6 +343,16 @@ def ensure_lazy_imports(imported_module, modules_to_block): raise AssertionError(f'unexpectedly imported after importing {imported_module}: {{after}}') """ ) + if additional_code: + script += additional_code + script += textwrap.dedent( + f""" + if unexpected := modules_to_block & sys.modules.keys(): + after = ", ".join(unexpected) + raise AssertionError(f'unexpectedly imported after additional code: {{after}}') + """ + ) + from .script_helper import assert_python_ok assert_python_ok("-S", "-c", script) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index daf6060940e..12d34dedfe1 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -1,6 +1,7 @@ import collections.abc import contextlib import errno +import functools import logging import os import re @@ -433,7 +434,10 @@ def _rmtree_inner(path): file=sys.__stderr__) mode = 0 if stat.S_ISDIR(mode): - _waitfor(_rmtree_inner, fullname, waitall=True) + # Do not follow junctions, which os.lstat() reports + # as directories. + if not os.path.isjunction(fullname): + _waitfor(_rmtree_inner, fullname, waitall=True) _force_run(fullname, os.rmdir, fullname) else: _force_run(fullname, os.unlink, fullname) @@ -806,6 +810,48 @@ def __exit__(self, *ignore_exc): os.environ = self._environ +def without_source_date_epoch(fxn): + """Runs function with SOURCE_DATE_EPOCH unset.""" + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + with EnvironmentVarGuard() as env: + env.unset('SOURCE_DATE_EPOCH') + return fxn(*args, **kwargs) + return wrapper + + +_MISSING = sentinel("MISSING") + +def with_source_date_epoch(fxn=_MISSING, *, epoch=123456789): + """Runs function with SOURCE_DATE_EPOCH set to *epoch*.""" + if fxn is _MISSING: + return functools.partial(with_source_date_epoch, epoch=epoch) + + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + with EnvironmentVarGuard() as env: + env['SOURCE_DATE_EPOCH'] = str(epoch) + return fxn(*args, **kwargs) + return wrapper + + +# Run tests with SOURCE_DATE_EPOCH set or unset explicitly. +class SourceDateEpochTestMeta(type(unittest.TestCase)): + def __new__(mcls, name, bases, dct, *, source_date_epoch): + cls = super().__new__(mcls, name, bases, dct) + + for attr in dir(cls): + if attr.startswith('test_'): + meth = getattr(cls, attr) + if source_date_epoch: + wrapper = with_source_date_epoch(meth) + else: + wrapper = without_source_date_epoch(meth) + setattr(cls, attr, wrapper) + + return cls + + try: if support.MS_WINDOWS: import ctypes diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3233a72750b..5d8e764b569 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -15,6 +15,7 @@ import os import py_compile import random +import re import shutil import stat import subprocess @@ -23,6 +24,7 @@ import threading import time import types +import warnings import unittest from unittest import mock import _imp @@ -52,7 +54,7 @@ TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper from test.support import threading_helper -from test.test_importlib.util import uncache +from test.test_importlib.util import uncache, temporary_pycache_prefix from types import ModuleType try: import _testsinglephase @@ -64,8 +66,10 @@ _testmultiphase = None try: import _interpreters + import concurrent.interpreters except ModuleNotFoundError: _interpreters = None + concurrent = None try: import _testinternalcapi except ImportError: @@ -73,7 +77,7 @@ skip_if_dont_write_bytecode = unittest.skipIf( - sys.dont_write_bytecode, + sys.dont_write_bytecode or sys.implementation.cache_tag is None, "test meaningful only when writing bytecode") @@ -363,11 +367,11 @@ def test_import_raises_ModuleNotFoundError(self): def test_import_null_byte_in_name_raises_ModuleNotFoundError(self): # gh-150633: module names containing null bytes should not # lead to duplicates in sys.modules - before = set(sys.modules.keys()) + before = set(sys.modules) with self.assertRaises(ModuleNotFoundError): __import__('zipimport\x00junk') - self.assertEqual(set(sys.modules.keys()), before) + self.assertEqual(set(sys.modules), before) def test_from_import_missing_module_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): @@ -422,7 +426,6 @@ def test_from_import_missing_attr_path_is_canonical(self): self.assertIsNotNone(cm.exception) def test_from_import_star_invalid_type(self): - import re with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("__all__ = [b'invalid_type']") @@ -511,7 +514,7 @@ def test_module_with_large_stack(self, module='longlist'): try: # Compile & remove .py file; we only need .pyc. # Bytecode must be relocated from the PEP 3147 bytecode-only location. - py_compile.compile(filename) + make_legacy_pyc(filename, allow_compile=True) finally: unlink(filename) @@ -521,7 +524,6 @@ def test_module_with_large_stack(self, module='longlist'): namespace = {} try: - make_legacy_pyc(filename) # This used to crash. exec('import ' + module, None, namespace) finally: @@ -786,7 +788,7 @@ def test_dll_dependency_import(self): pydname = importlib.util.find_spec("_sqlite3").origin depname = os.path.join( os.path.dirname(pydname), - "sqlite3{}.dll".format("_d" if "_d" in pydname else "")) + "sqlite3{}.dll".format("_d" if "_d" in os.path.basename(pydname) else "")) with os_helper.temp_dir() as tmp: tmp2 = os.path.join(tmp, "DLLs") @@ -1261,6 +1263,44 @@ class Spec2: origin = "a\x00b" _imp.create_dynamic(Spec2()) + def test_create_builtin(self): + class Spec: + pass + spec = Spec() + + spec.name = "sys" + self.assertIs(_imp.create_builtin(spec), sys) + + spec.name = None + with self.assertRaisesRegex(TypeError, 'name must be string, not NoneType'): + _imp.create_builtin(spec) + + # gh-142029 + spec.name = "nonexistent_lib" + with self.assertRaises(ModuleNotFoundError): + _imp.create_builtin(spec) + + # gh-142029 + spec.name = "" + with self.assertRaisesRegex(ValueError, 'name must not be empty'): + _imp.create_builtin(spec) + + def test_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + unload('test.test_import.data.syntax_warnings') + with (os_helper.temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + import test.test_import.data.syntax_warnings + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = test.test_import.data.syntax_warnings.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -1369,7 +1409,10 @@ def func(): """ dir_name = os.path.abspath(TESTFN) file_name = os.path.join(dir_name, module_name) + os.extsep + "py" - compiled_name = importlib.util.cache_from_source(file_name) + try: + compiled_name = importlib.util.cache_from_source(file_name) + except NotImplementedError: + compiled_name = None def setUp(self): self.sys_path = sys.path[:] @@ -1387,7 +1430,8 @@ def tearDown(self): else: unload(self.module_name) unlink(self.file_name) - unlink(self.compiled_name) + if self.compiled_name: + unlink(self.compiled_name) rmtree(self.dir_name) def import_module(self): @@ -1406,6 +1450,8 @@ def test_basics(self): self.assertEqual(mod.code_filename, self.file_name) self.assertEqual(mod.func_filename, self.file_name) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag is not None') def test_incorrect_code_name(self): py_compile.compile(self.file_name, dfile="another_module.py") mod = self.import_module() @@ -1415,9 +1461,9 @@ def test_incorrect_code_name(self): def test_module_without_source(self): target = "another_module.py" - py_compile.compile(self.file_name, dfile=target) + pyc_file = self.file_name + 'c' + py_compile.compile(self.file_name, pyc_file, dfile=target) os.remove(self.file_name) - pyc_file = make_legacy_pyc(self.file_name) importlib.invalidate_caches() mod = self.import_module() self.assertEqual(mod.module_filename, pyc_file) @@ -1425,8 +1471,9 @@ def test_module_without_source(self): self.assertEqual(mod.func_filename, target) def test_foreign_code(self): - py_compile.compile(self.file_name) - with open(self.compiled_name, "rb") as f: + compiled_name = self.compiled_name or (self.file_name + 'c') + py_compile.compile(self.file_name, compiled_name) + with open(compiled_name, "rb") as f: header = f.read(16) code = marshal.load(f) constants = list(code.co_consts) @@ -1434,9 +1481,11 @@ def test_foreign_code(self): pos = constants.index(1000) constants[pos] = foreign_code code = code.replace(co_consts=tuple(constants)) - with open(self.compiled_name, "wb") as f: + with open(compiled_name, "wb") as f: f.write(header) marshal.dump(code, f) + if not self.compiled_name: + os.remove(self.file_name) mod = self.import_module() self.assertEqual(mod.constant.co_filename, foreign_code.co_filename) @@ -1621,6 +1670,11 @@ def _clean(self): unlink(self.source) def setUp(self): + # These tests assume bytecode is written next to the source in a + # local __pycache__ directory, so neutralize any pycache prefix (e.g. + # when the test suite is run with PYTHONPYCACHEPREFIX set). + self._orig_pycache_prefix = sys.pycache_prefix + sys.pycache_prefix = None self.source = TESTFN + '.py' self._clean() with open(self.source, 'w', encoding='utf-8') as fp: @@ -1632,6 +1686,7 @@ def tearDown(self): assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]' del sys.path[0] self._clean() + sys.pycache_prefix = self._orig_pycache_prefix @skip_if_dont_write_bytecode def test_import_pyc_path(self): @@ -1691,78 +1746,6 @@ def test_missing_source_legacy(self): finally: os.remove(pyc_file) - def test___cached__(self): - # Modules now also have an __cached__ that points to the pyc file. - m = __import__(TESTFN) - pyc_file = importlib.util.cache_from_source(TESTFN + '.py') - self.assertEqual(m.__cached__, os.path.join(os.getcwd(), pyc_file)) - - @skip_if_dont_write_bytecode - def test___cached___legacy_pyc(self): - # Like test___cached__() except that for backward compatibility, - # when the pyc file lives where the py file would have been (and named - # without the tag), it is importable. The __cached__ of the imported - # module is the pyc location. - __import__(TESTFN) - # pyc_file gets removed in _clean() via tearDown(). - pyc_file = make_legacy_pyc(self.source) - os.remove(self.source) - unload(TESTFN) - importlib.invalidate_caches() - m = __import__(TESTFN) - self.assertEqual(m.__cached__, - os.path.join(os.getcwd(), os.path.relpath(pyc_file))) - - @skip_if_dont_write_bytecode - def test_package___cached__(self): - # Like test___cached__ but for packages. - def cleanup(): - rmtree('pep3147') - unload('pep3147.foo') - unload('pep3147') - os.mkdir('pep3147') - self.addCleanup(cleanup) - # Touch the __init__.py - with open(os.path.join('pep3147', '__init__.py'), 'wb'): - pass - with open(os.path.join('pep3147', 'foo.py'), 'wb'): - pass - importlib.invalidate_caches() - m = __import__('pep3147.foo') - init_pyc = importlib.util.cache_from_source( - os.path.join('pep3147', '__init__.py')) - self.assertEqual(m.__cached__, os.path.join(os.getcwd(), init_pyc)) - foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py')) - self.assertEqual(sys.modules['pep3147.foo'].__cached__, - os.path.join(os.getcwd(), foo_pyc)) - - def test_package___cached___from_pyc(self): - # Like test___cached__ but ensuring __cached__ when imported from a - # PEP 3147 pyc file. - def cleanup(): - rmtree('pep3147') - unload('pep3147.foo') - unload('pep3147') - os.mkdir('pep3147') - self.addCleanup(cleanup) - # Touch the __init__.py - with open(os.path.join('pep3147', '__init__.py'), 'wb'): - pass - with open(os.path.join('pep3147', 'foo.py'), 'wb'): - pass - importlib.invalidate_caches() - m = __import__('pep3147.foo') - unload('pep3147.foo') - unload('pep3147') - importlib.invalidate_caches() - m = __import__('pep3147.foo') - init_pyc = importlib.util.cache_from_source( - os.path.join('pep3147', '__init__.py')) - self.assertEqual(m.__cached__, os.path.join(os.getcwd(), init_pyc)) - foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py')) - self.assertEqual(sys.modules['pep3147.foo'].__cached__, - os.path.join(os.getcwd(), foo_pyc)) - def test_recompute_pyc_same_second(self): # Even when the source file doesn't change timestamp, a change in # source size is enough to trigger recomputation of the pyc file. @@ -2031,10 +2014,6 @@ def test_import_bug(self): # away from the traceback. self.create_module("foo", "") importlib = sys.modules['_frozen_importlib_external'] - if 'load_module' in vars(importlib.SourceLoader): - old_exec_module = importlib.SourceLoader.exec_module - else: - old_exec_module = None try: def exec_module(*args): 1/0 @@ -2047,10 +2026,7 @@ def exec_module(*args): self.fail("ZeroDivisionError should have been raised") self.assert_traceback(tb, [__file__, '>> @suppress(KeyError) - ... def key_error(): - ... {}[''] - >>> key_error() - """ diff --git a/Lib/test/test_importlib/_path.py b/Lib/test/test_importlib/_path.py deleted file mode 100644 index 71a704389b9..00000000000 --- a/Lib/test/test_importlib/_path.py +++ /dev/null @@ -1,109 +0,0 @@ -# from jaraco.path 3.5 - -import functools -import pathlib -from typing import Dict, Union - -try: - from typing import Protocol, runtime_checkable -except ImportError: # pragma: no cover - # Python 3.7 - from typing_extensions import Protocol, runtime_checkable # type: ignore - - -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore - - -@runtime_checkable -class TreeMaker(Protocol): - def __truediv__(self, *args, **kwargs): - ... # pragma: no cover - - def mkdir(self, **kwargs): - ... # pragma: no cover - - def write_text(self, content, **kwargs): - ... # pragma: no cover - - def write_bytes(self, content): - ... # pragma: no cover - - -def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: - return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore - - -def build( - spec: FilesSpec, - prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore -): - """ - Build a set of files/directories, as described by the spec. - - Each key represents a pathname, and the value represents - the content. Content may be a nested directory. - - >>> spec = { - ... 'README.txt': "A README file", - ... "foo": { - ... "__init__.py": "", - ... "bar": { - ... "__init__.py": "", - ... }, - ... "baz.py": "# Some code", - ... } - ... } - >>> target = getfixture('tmp_path') - >>> build(spec, target) - >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') - '# Some code' - """ - for name, contents in spec.items(): - create(contents, _ensure_tree_maker(prefix) / name) - - -@functools.singledispatch -def create(content: Union[str, bytes, FilesSpec], path): - path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore - - -@create.register -def _(content: bytes, path): - path.write_bytes(content) - - -@create.register -def _(content: str, path): - path.write_text(content, encoding='utf-8') - - -@create.register -def _(content: str, path): - path.write_text(content, encoding='utf-8') - - -class Recording: - """ - A TreeMaker object that records everything that would be written. - - >>> r = Recording() - >>> build({'foo': {'foo1.txt': 'yes'}, 'bar.txt': 'abc'}, r) - >>> r.record - ['foo/foo1.txt', 'bar.txt'] - """ - - def __init__(self, loc=pathlib.PurePosixPath(), record=None): - self.loc = loc - self.record = record if record is not None else [] - - def __truediv__(self, other): - return Recording(self.loc / other, self.record) - - def write_text(self, content, **kwargs): - self.record.append(str(self.loc)) - - write_bytes = write_text - - def mkdir(self, **kwargs): - return diff --git a/Lib/test/test_importlib/abc.py b/Lib/test/test_importlib/abc.py index 5d4b9587676..0da98e0c99e 100644 --- a/Lib/test/test_importlib/abc.py +++ b/Lib/test/test_importlib/abc.py @@ -40,22 +40,6 @@ def test_failure(self): class LoaderTests(metaclass=abc.ABCMeta): - @abc.abstractmethod - def test_module(self): - """A module should load without issue. - - After the loader returns the module should be in sys.modules. - - Attributes to verify: - - * __file__ - * __loader__ - * __name__ - * No __path__ - - """ - pass - @abc.abstractmethod def test_package(self): """Loading a package should work. diff --git a/Lib/test/test_importlib/builtin/test_loader.py b/Lib/test/test_importlib/builtin/test_loader.py index 7e9d1b1960f..2a5c2a8f39e 100644 --- a/Lib/test/test_importlib/builtin/test_loader.py +++ b/Lib/test/test_importlib/builtin/test_loader.py @@ -7,70 +7,6 @@ import unittest import warnings -@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') -class LoaderTests(abc.LoaderTests): - - """Test load_module() for built-in modules.""" - - def setUp(self): - self.verification = {'__name__': 'errno', '__package__': '', - '__loader__': self.machinery.BuiltinImporter} - - def verify(self, module): - """Verify that the module matches against what it should have.""" - self.assertIsInstance(module, types.ModuleType) - for attr, value in self.verification.items(): - self.assertEqual(getattr(module, attr), value) - self.assertIn(module.__name__, sys.modules) - - def load_module(self, name): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.machinery.BuiltinImporter.load_module(name) - - def test_module(self): - # Common case. - with util.uncache(util.BUILTINS.good_name): - module = self.load_module(util.BUILTINS.good_name) - self.verify(module) - - # Built-in modules cannot be a package. - test_package = test_lacking_parent = None - - # No way to force an import failure. - test_state_after_failure = None - - def test_module_reuse(self): - # Test that the same module is used in a reload. - with util.uncache(util.BUILTINS.good_name): - module1 = self.load_module(util.BUILTINS.good_name) - module2 = self.load_module(util.BUILTINS.good_name) - self.assertIs(module1, module2) - - def test_unloadable(self): - name = 'dssdsdfff' - assert name not in sys.builtin_module_names - with self.assertRaises(ImportError) as cm: - self.load_module(name) - self.assertEqual(cm.exception.name, name) - - def test_already_imported(self): - # Using the name of a module already imported but not a built-in should - # still fail. - module_name = 'builtin_reload_test' - assert module_name not in sys.builtin_module_names - with util.uncache(module_name): - module = types.ModuleType(module_name) - sys.modules[module_name] = module - with self.assertRaises(ImportError) as cm: - self.load_module(module_name) - self.assertEqual(cm.exception.name, module_name) - - -(Frozen_LoaderTests, - Source_LoaderTests - ) = util.test_both(LoaderTests, machinery=machinery) - @unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') class InspectLoaderTests: diff --git a/Lib/test/test_importlib/data/__init__.py b/Lib/test/test_importlib/data/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl deleted file mode 100644 index 641ab07f7aa..00000000000 Binary files a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl and /dev/null differ diff --git a/Lib/test/test_importlib/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/data/example-21.12-py3.6.egg deleted file mode 100644 index cdb298a19b0..00000000000 Binary files a/Lib/test/test_importlib/data/example-21.12-py3.6.egg and /dev/null differ diff --git a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl deleted file mode 100644 index 5ca93657f81..00000000000 Binary files a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/test/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file deleted file mode 100644 index eaf36c1dacc..00000000000 Binary files a/Lib/test/test_importlib/data01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file deleted file mode 100644 index eaf36c1dacc..00000000000 Binary files a/Lib/test/test_importlib/data01/subdirectory/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file deleted file mode 100644 index 2cb772295ef..00000000000 Binary files a/Lib/test/test_importlib/data01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index cdc8884d668..e33c0465654 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,10 +1,11 @@ -from test.support import is_apple_mobile +from test.support import is_apple_mobile, Py_GIL_DISABLED from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') import unittest import sys +import sysconfig class FinderTests(abc.FinderTests): @@ -59,6 +60,28 @@ def test_module(self): def test_failure(self): self.assertIsNone(self.find_spec('asdfjkl;')) + def test_abi3_extension_suffixes(self): + suffixes = self.machinery.EXTENSION_SUFFIXES + platform = sysconfig.get_config_var("SOABI_PLATFORM") + if 'win32' in sys.platform: + # Either "_d.pyd" or ".pyd" must be in suffixes + self.assertTrue({"_d.pyd", ".pyd"}.intersection(suffixes)) + elif 'cygwin' in sys.platform: + pass + else: + if Py_GIL_DISABLED: + self.assertNotIn(".abi3.so", suffixes) + else: + self.assertIn(".abi3.so", suffixes) + self.assertIn(".abi3t.so", suffixes) + + if platform: + if Py_GIL_DISABLED: + self.assertNotIn(f".abi3-{platform}.so", suffixes) + else: + self.assertIn(f".abi3-{platform}.so", suffixes) + self.assertIn(f".abi3t-{platform}.so", suffixes) + (Frozen_FinderTests, Source_FinderTests diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 0dd21e079eb..15d5a7d4752 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -35,11 +35,6 @@ def setUp(self): self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) - def load_module(self, fullname): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.loader.load_module(fullname) - def test_equality(self): other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) @@ -48,25 +43,6 @@ def test_inequality(self): other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) - def test_load_module_API(self): - # Test the default argument for load_module(). - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.loader.load_module() - self.loader.load_module(None) - with self.assertRaises(ImportError): - self.load_module('XXX') - - def test_module(self): - with util.uncache(util.EXTENSIONS.name): - module = self.load_module(util.EXTENSIONS.name) - for attr, value in [('__name__', util.EXTENSIONS.name), - ('__file__', util.EXTENSIONS.file_path), - ('__package__', '')]: - self.assertEqual(getattr(module, attr), value) - self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, self.LoaderClass) - # No extension module as __init__ available for testing. test_package = None @@ -76,18 +52,6 @@ def test_module(self): # No easy way to trigger a failure after a successful import. test_state_after_failure = None - def test_unloadable(self): - name = 'asdfjkl;' - with self.assertRaises(ImportError) as cm: - self.load_module(name) - self.assertEqual(cm.exception.name, name) - - def test_module_reuse(self): - with util.uncache(util.EXTENSIONS.name): - module1 = self.load_module(util.EXTENSIONS.name) - module2 = self.load_module(util.EXTENSIONS.name) - self.assertIs(module1, module2) - def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: @@ -126,11 +90,6 @@ def setUp(self): self.loader = self.LoaderClass(self.name, self.spec.origin) - def load_module(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.loader.load_module(self.name) - def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin @@ -140,19 +99,6 @@ def load_module_by_name(self, fullname): loader.exec_module(module) return module - def test_module(self): - # Test loading an extension module. - with util.uncache(self.name): - module = self.load_module() - for attr, value in [('__name__', self.name), - ('__file__', self.spec.origin), - ('__package__', '')]: - self.assertEqual(getattr(module, attr), value) - with self.assertRaises(AttributeError): - module.__path__ - self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, self.LoaderClass) - # No extension module as __init__ available for testing. test_package = None @@ -213,12 +159,6 @@ def setUp(self): assert self.spec self.loader = self.LoaderClass(self.name, self.spec.origin) - def load_module(self): - # Load the module from the test extension. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.loader.load_module(self.name) - def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin @@ -237,60 +177,6 @@ def load_module_by_name(self, fullname): # Handling failure on reload is the up to the module. test_state_after_failure = None - def test_module(self): - # Test loading an extension module. - with util.uncache(self.name): - module = self.load_module() - for attr, value in [('__name__', self.name), - ('__file__', self.spec.origin), - ('__package__', '')]: - self.assertEqual(getattr(module, attr), value) - with self.assertRaises(AttributeError): - module.__path__ - self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, self.LoaderClass) - - def test_functionality(self): - # Test basic functionality of stuff defined in an extension module. - with util.uncache(self.name): - module = self.load_module() - self.assertIsInstance(module, types.ModuleType) - ex = module.Example() - self.assertEqual(ex.demo('abcd'), 'abcd') - self.assertEqual(ex.demo(), None) - with self.assertRaises(AttributeError): - ex.abc - ex.abc = 0 - self.assertEqual(ex.abc, 0) - self.assertEqual(module.foo(9, 9), 18) - self.assertIsInstance(module.Str(), str) - self.assertEqual(module.Str(1) + '23', '123') - with self.assertRaises(module.error): - raise module.error() - self.assertEqual(module.int_const, 1969) - self.assertEqual(module.str_const, 'something different') - - def test_reload(self): - # Test that reload didn't re-set the module's attributes. - with util.uncache(self.name): - module = self.load_module() - ex_class = module.Example - importlib.reload(module) - self.assertIs(ex_class, module.Example) - - def test_try_registration(self): - # Assert that the PyState_{Find,Add,Remove}Module C API doesn't work. - with util.uncache(self.name): - module = self.load_module() - with self.subTest('PyState_FindModule'): - self.assertEqual(module.call_state_registration_func(0), None) - with self.subTest('PyState_AddModule'): - with self.assertRaises(SystemError): - module.call_state_registration_func(1) - with self.subTest('PyState_RemoveModule'): - with self.assertRaises(SystemError): - module.call_state_registration_func(2) - def test_load_submodule(self): # Test loading a simulated submodule. module = self.load_module_by_name('pkg.' + self.name) diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py deleted file mode 100644 index 73e5da2ba92..00000000000 --- a/Lib/test/test_importlib/fixtures.py +++ /dev/null @@ -1,382 +0,0 @@ -import os -import sys -import copy -import shutil -import pathlib -import tempfile -import textwrap -import functools -import contextlib - -from test.support.os_helper import FS_NONASCII -from test.support import requires_zlib - -from . import _path -from ._path import FilesSpec - - -try: - from importlib import resources # type: ignore - - getattr(resources, 'files') - getattr(resources, 'as_file') -except (ImportError, AttributeError): - import importlib_resources as resources # type: ignore - - -@contextlib.contextmanager -def tempdir(): - tmpdir = tempfile.mkdtemp() - try: - yield pathlib.Path(tmpdir) - finally: - shutil.rmtree(tmpdir) - - -@contextlib.contextmanager -def save_cwd(): - orig = os.getcwd() - try: - yield - finally: - os.chdir(orig) - - -@contextlib.contextmanager -def tempdir_as_cwd(): - with tempdir() as tmp: - with save_cwd(): - os.chdir(str(tmp)) - yield tmp - - -@contextlib.contextmanager -def install_finder(finder): - sys.meta_path.append(finder) - try: - yield - finally: - sys.meta_path.remove(finder) - - -class Fixtures: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - -class SiteDir(Fixtures): - def setUp(self): - super().setUp() - self.site_dir = self.fixtures.enter_context(tempdir()) - - -class OnSysPath(Fixtures): - @staticmethod - @contextlib.contextmanager - def add_sys_path(dir): - sys.path[:0] = [str(dir)] - try: - yield - finally: - sys.path.remove(str(dir)) - - def setUp(self): - super().setUp() - self.fixtures.enter_context(self.add_sys_path(self.site_dir)) - - -class DistInfoPkg(OnSysPath, SiteDir): - files: FilesSpec = { - "distinfo_pkg-1.0.0.dist-info": { - "METADATA": """ - Name: distinfo-pkg - Author: Steven Ma - Version: 1.0.0 - Requires-Dist: wheel >= 1.0 - Requires-Dist: pytest; extra == 'test' - Keywords: sample package - - Once upon a time - There was a distinfo pkg - """, - "RECORD": "mod.py,sha256=abc,20\n", - "entry_points.txt": """ - [entries] - main = mod:main - ns:sub = mod:main - """, - }, - "mod.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) - - def make_uppercase(self): - """ - Rewrite metadata with everything uppercase. - """ - shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") - files = copy.deepcopy(DistInfoPkg.files) - info = files["distinfo_pkg-1.0.0.dist-info"] - info["METADATA"] = info["METADATA"].upper() - build_files(files, self.site_dir) - - -class DistInfoPkgWithDot(OnSysPath, SiteDir): - files: FilesSpec = { - "pkg_dot-1.0.0.dist-info": { - "METADATA": """ - Name: pkg.dot - Version: 1.0.0 - """, - }, - } - - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDot.files, self.site_dir) - - -class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir): - files: FilesSpec = { - "pkg.dot-1.0.0.dist-info": { - "METADATA": """ - Name: pkg.dot - Version: 1.0.0 - """, - }, - "pkg.lot.egg-info": { - "METADATA": """ - Name: pkg.lot - Version: 1.0.0 - """, - }, - } - - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDotLegacy.files, self.site_dir) - - -class DistInfoPkgOffPath(SiteDir): - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) - - -class EggInfoPkg(OnSysPath, SiteDir): - files: FilesSpec = { - "egginfo_pkg.egg-info": { - "PKG-INFO": """ - Name: egginfo-pkg - Author: Steven Ma - License: Unknown - Version: 1.0.0 - Classifier: Intended Audience :: Developers - Classifier: Topic :: Software Development :: Libraries - Keywords: sample package - Description: Once upon a time - There was an egginfo package - """, - "SOURCES.txt": """ - mod.py - egginfo_pkg.egg-info/top_level.txt - """, - "entry_points.txt": """ - [entries] - main = mod:main - """, - "requires.txt": """ - wheel >= 1.0; python_version >= "2.7" - [test] - pytest - """, - "top_level.txt": "mod\n", - }, - "mod.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkg.files, prefix=self.site_dir) - - -class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteDir): - files: FilesSpec = { - "egg_with_module_pkg.egg-info": { - "PKG-INFO": "Name: egg_with_module-pkg", - # SOURCES.txt is made from the source archive, and contains files - # (setup.py) that are not present after installation. - "SOURCES.txt": """ - egg_with_module.py - setup.py - egg_with_module_pkg.egg-info/PKG-INFO - egg_with_module_pkg.egg-info/SOURCES.txt - egg_with_module_pkg.egg-info/top_level.txt - """, - # installed-files.txt is written by pip, and is a strictly more - # accurate source than SOURCES.txt as to the installed contents of - # the package. - "installed-files.txt": """ - ../egg_with_module.py - PKG-INFO - SOURCES.txt - top_level.txt - """, - # missing top_level.txt (to trigger fallback to installed-files.txt) - }, - "egg_with_module.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoToplevel.files, prefix=self.site_dir) - - -class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteDir): - files: FilesSpec = { - "egg_with_no_modules_pkg.egg-info": { - "PKG-INFO": "Name: egg_with_no_modules-pkg", - # SOURCES.txt is made from the source archive, and contains files - # (setup.py) that are not present after installation. - "SOURCES.txt": """ - setup.py - egg_with_no_modules_pkg.egg-info/PKG-INFO - egg_with_no_modules_pkg.egg-info/SOURCES.txt - egg_with_no_modules_pkg.egg-info/top_level.txt - """, - # installed-files.txt is written by pip, and is a strictly more - # accurate source than SOURCES.txt as to the installed contents of - # the package. - "installed-files.txt": """ - PKG-INFO - SOURCES.txt - top_level.txt - """, - # top_level.txt correctly reflects that no modules are installed - "top_level.txt": b"\n", - }, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoModules.files, prefix=self.site_dir) - - -class EggInfoPkgSourcesFallback(OnSysPath, SiteDir): - files: FilesSpec = { - "sources_fallback_pkg.egg-info": { - "PKG-INFO": "Name: sources_fallback-pkg", - # SOURCES.txt is made from the source archive, and contains files - # (setup.py) that are not present after installation. - "SOURCES.txt": """ - sources_fallback.py - setup.py - sources_fallback_pkg.egg-info/PKG-INFO - sources_fallback_pkg.egg-info/SOURCES.txt - """, - # missing installed-files.txt (i.e. not installed by pip) and - # missing top_level.txt (to trigger fallback to SOURCES.txt) - }, - "sources_fallback.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkgSourcesFallback.files, prefix=self.site_dir) - - -class EggInfoFile(OnSysPath, SiteDir): - files: FilesSpec = { - "egginfo_file.egg-info": """ - Metadata-Version: 1.0 - Name: egginfo_file - Version: 0.1 - Summary: An example package - Home-page: www.example.com - Author: Eric Haffa-Vee - Author-email: eric@example.coms - License: UNKNOWN - Description: UNKNOWN - Platform: UNKNOWN - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoFile.files, prefix=self.site_dir) - - -# dedent all text strings before writing -orig = _path.create.registry[str] -_path.create.register(str, lambda content, path: orig(DALS(content), path)) - - -build_files = _path.build - - -def build_record(file_defs): - return ''.join(f'{name},,\n' for name in record_names(file_defs)) - - -def record_names(file_defs): - recording = _path.Recording() - _path.build(file_defs, recording) - return recording.record - - -class FileBuilder: - def unicode_filename(self): - return FS_NONASCII or self.skip("File system does not support non-ascii.") - - -def DALS(str): - "Dedent and left-strip" - return textwrap.dedent(str).lstrip() - - -@requires_zlib() -class ZipFixtures: - root = 'test.test_importlib.data' - - def _fixture_on_path(self, filename): - pkg_file = resources.files(self.root).joinpath(filename) - file = self.resources.enter_context(resources.as_file(pkg_file)) - assert file.name.startswith('example'), file.name - sys.path.insert(0, str(file)) - self.resources.callback(sys.path.pop, 0) - - def setUp(self): - # Add self.zip_name to the front of sys.path. - self.resources = contextlib.ExitStack() - self.addCleanup(self.resources.close) - - -def parameterize(*args_set): - """Run test method with a series of parameters.""" - - def wrapper(func): - @functools.wraps(func) - def _inner(self): - for args in args_set: - with self.subTest(**args): - func(self, **args) - - return _inner - - return wrapper diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index d6ad590b3d4..ea1e29a4e5f 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -27,13 +27,6 @@ def exec_module(module): raise ImportError('I cannot be loaded!') -class BadLoaderFinder: - @classmethod - def load_module(cls, fullname): - if fullname == SUBMOD_NAME: - raise ImportError('I cannot be loaded!') - - class APITest: """Test API-specific details for __import__ (e.g. raising the right @@ -93,45 +86,6 @@ def test_blocked_fromlist(self): self.assertEqual(cm.exception.name, SUBMOD_NAME) -class OldAPITests(APITest): - bad_finder_loader = BadLoaderFinder - - def test_raises_ModuleNotFoundError(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_raises_ModuleNotFoundError() - - def test_name_requires_rparition(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_name_requires_rparition() - - def test_negative_level(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_negative_level() - - def test_nonexistent_fromlist_entry(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_nonexistent_fromlist_entry() - - def test_fromlist_load_error_propagates(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_fromlist_load_error_propagates - - def test_blocked_fromlist(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_blocked_fromlist() - - -(Frozen_OldAPITests, - Source_OldAPITests - ) = util.test_both(OldAPITests, __import__=util.__import__) - - class SpecAPITests(APITest): bad_finder_loader = BadSpecFinderLoader diff --git a/Lib/test/test_importlib/import_/test_helpers.py b/Lib/test/test_importlib/import_/test_helpers.py index 550f88d1d7a..7587276a41e 100644 --- a/Lib/test/test_importlib/import_/test_helpers.py +++ b/Lib/test/test_importlib/import_/test_helpers.py @@ -19,8 +19,7 @@ def test_no_loader_but_spec(self): ns = {"__spec__": spec} _bootstrap_external._fix_up_module(ns, name, path) - expected = {"__spec__": spec, "__loader__": loader, "__file__": path, - "__cached__": None} + expected = {"__spec__": spec, "__loader__": loader, "__file__": path} self.assertEqual(ns, expected) def test_no_loader_no_spec_but_sourceless(self): @@ -29,7 +28,7 @@ def test_no_loader_no_spec_but_sourceless(self): ns = {} _bootstrap_external._fix_up_module(ns, name, path, path) - expected = {"__file__": path, "__cached__": path} + expected = {"__file__": path} for key, val in expected.items(): with self.subTest(f"{key}: {val}"): @@ -51,7 +50,7 @@ def test_no_loader_no_spec_but_source(self): ns = {} _bootstrap_external._fix_up_module(ns, name, path) - expected = {"__file__": path, "__cached__": None} + expected = {"__file__": path} for key, val in expected.items(): with self.subTest(f"{key}: {val}"): diff --git a/Lib/test/test_importlib/metadata/__init__.py b/Lib/test/test_importlib/metadata/__init__.py index 8b137891791..e69de29bb2d 100644 --- a/Lib/test/test_importlib/metadata/__init__.py +++ b/Lib/test/test_importlib/metadata/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py index b3cfb9cd549..e63d889f96b 100644 --- a/Lib/test/test_importlib/metadata/_path.py +++ b/Lib/test/test_importlib/metadata/_path.py @@ -1,9 +1,14 @@ -# from jaraco.path 3.7 +# from jaraco.path 3.7.2 + +from __future__ import annotations import functools import pathlib -from typing import Dict, Protocol, Union -from typing import runtime_checkable +from collections.abc import Mapping +from typing import TYPE_CHECKING, Protocol, Union, runtime_checkable + +if TYPE_CHECKING: + from typing_extensions import Self class Symlink(str): @@ -12,29 +17,25 @@ class Symlink(str): """ -FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] # type: ignore +FilesSpec = Mapping[str, Union[str, bytes, Symlink, 'FilesSpec']] @runtime_checkable class TreeMaker(Protocol): - def __truediv__(self, *args, **kwargs): ... # pragma: no cover - - def mkdir(self, **kwargs): ... # pragma: no cover - - def write_text(self, content, **kwargs): ... # pragma: no cover - - def write_bytes(self, content): ... # pragma: no cover - - def symlink_to(self, target): ... # pragma: no cover + def __truediv__(self, other, /) -> Self: ... + def mkdir(self, *, exist_ok) -> object: ... + def write_text(self, content, /, *, encoding) -> object: ... + def write_bytes(self, content, /) -> object: ... + def symlink_to(self, target, /) -> object: ... -def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: - return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore +def _ensure_tree_maker(obj: str | TreeMaker) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) def build( spec: FilesSpec, - prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore + prefix: str | TreeMaker = pathlib.Path(), ): """ Build a set of files/directories, as described by the spec. @@ -66,23 +67,24 @@ def build( @functools.singledispatch -def create(content: Union[str, bytes, FilesSpec], path): +def create(content: str | bytes | FilesSpec, path: TreeMaker) -> None: path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + # Mypy only looks at the signature of the main singledispatch method. So it must contain the complete Union + build(content, prefix=path) # type: ignore[arg-type] # python/mypy#11727 @create.register -def _(content: bytes, path): +def _(content: bytes, path: TreeMaker) -> None: path.write_bytes(content) @create.register -def _(content: str, path): +def _(content: str, path: TreeMaker) -> None: path.write_text(content, encoding='utf-8') @create.register -def _(content: Symlink, path): +def _(content: Symlink, path: TreeMaker) -> None: path.symlink_to(content) diff --git a/Lib/test/test_importlib/metadata/data/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py index 8b137891791..e69de29bb2d 100644 --- a/Lib/test/test_importlib/metadata/data/__init__.py +++ b/Lib/test/test_importlib/metadata/data/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py index 826b1b3259b..3283697d418 100644 --- a/Lib/test/test_importlib/metadata/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -1,11 +1,12 @@ -import sys +import contextlib import copy +import functools import json -import shutil import pathlib +import shutil +import sys import textwrap -import functools -import contextlib +from importlib import resources from test.support import import_helper from test.support import os_helper @@ -15,15 +16,6 @@ from ._path import FilesSpec -try: - from importlib import resources # type: ignore - - getattr(resources, 'files') - getattr(resources, 'as_file') -except (ImportError, AttributeError): - import importlib_resources as resources # type: ignore - - @contextlib.contextmanager def tmp_path(): """ diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py index 2256e0c502e..5449f048449 100644 --- a/Lib/test/test_importlib/metadata/test_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -1,14 +1,12 @@ +import importlib import re import textwrap import unittest -import warnings -import importlib -import contextlib -from . import fixtures from importlib.metadata import ( Distribution, PackageNotFoundError, + Prepared, distribution, entry_points, files, @@ -17,12 +15,7 @@ version, ) - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx +from . import fixtures class APITests( @@ -153,13 +146,13 @@ def test_metadata_for_this_package(self): classifiers = md.get_all('Classifier') assert 'Topic :: Software Development :: Libraries' in classifiers - def test_missing_key_legacy(self): + def test_missing_key(self): """ - Requesting a missing key will still return None, but warn. + Requesting a missing key raises KeyError. """ md = metadata('distinfo-pkg') - with suppress_known_deprecation(): - assert md['does-not-exist'] is None + with self.assertRaises(KeyError): + md['does-not-exist'] def test_get_key(self): """ @@ -321,3 +314,34 @@ class InvalidateCache(unittest.TestCase): def test_invalidate_cache(self): # No externally observable behavior, but ensures test coverage... importlib.invalidate_caches() + + +class PreparedTests(unittest.TestCase): + @fixtures.parameterize( + # Simple + dict(input='sample', expected='sample'), + # Mixed case + dict(input='Sample', expected='sample'), + dict(input='SAMPLE', expected='sample'), + dict(input='SaMpLe', expected='sample'), + # Separator conversions + dict(input='sample-pkg', expected='sample_pkg'), + dict(input='sample.pkg', expected='sample_pkg'), + dict(input='sample_pkg', expected='sample_pkg'), + # Multiple separators + dict(input='sample---pkg', expected='sample_pkg'), + dict(input='sample___pkg', expected='sample_pkg'), + dict(input='sample...pkg', expected='sample_pkg'), + # Mixed separators + dict(input='sample-._pkg', expected='sample_pkg'), + dict(input='sample_.-pkg', expected='sample_pkg'), + # Complex + dict(input='Sample__Pkg-name.foo', expected='sample_pkg_name_foo'), + dict(input='Sample__Pkg.name__foo', expected='sample_pkg_name_foo'), + # Uppercase with separators + dict(input='SAMPLE-PKG', expected='sample_pkg'), + dict(input='Sample.Pkg', expected='sample_pkg'), + dict(input='SAMPLE_PKG', expected='sample_pkg'), + ) + def test_normalize(self, input, expected): + self.assertEqual(Prepared.normalize(input), expected) diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py index e4218076f8c..aae052160d9 100644 --- a/Lib/test/test_importlib/metadata/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -1,23 +1,18 @@ -import re +import importlib import pickle +import re import unittest -import warnings -import importlib -import importlib.metadata -import contextlib -from test.support import os_helper try: import pyfakefs.fake_filesystem_unittest as ffs except ImportError: from .stubs import fake_filesystem_unittest as ffs +from test.support import os_helper -from . import fixtures -from ._context import suppress -from ._path import Symlink from importlib.metadata import ( Distribution, EntryPoint, + MetadataNotFound, PackageNotFoundError, _unique, distributions, @@ -27,12 +22,8 @@ version, ) - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx +from . import fixtures +from ._path import Symlink class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): @@ -59,9 +50,6 @@ def test_package_not_found_mentions_metadata(self): assert "metadata" in str(ctx.exception) - # expected to fail until ABC is enforced - @suppress(AssertionError) - @suppress_known_deprecation() def test_abc_enforced(self): with self.assertRaises(TypeError): type('DistributionSubclass', (Distribution,), {})() @@ -146,6 +134,43 @@ def test_unique_distributions(self): assert len(after) == len(before) +class InvalidMetadataTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def make_pkg(name, files=dict(METADATA="VERSION: 1.0")): + """ + Create metadata for a dist-info package with name and files. + """ + return { + f'{name}.dist-info': files, + } + + def test_valid_dists_preferred(self): + """ + Dists with metadata should be preferred when discovered by name. + + Ref python/importlib_metadata#489. + """ + # create three dists with the valid one in the middle (lexicographically) + # such that on most file systems, the valid one is never naturally first. + fixtures.build_files(self.make_pkg('foo-4.0', files={}), self.site_dir) + fixtures.build_files(self.make_pkg('foo-4.1'), self.site_dir) + fixtures.build_files(self.make_pkg('foo-4.2', files={}), self.site_dir) + dist = Distribution.from_name('foo') + assert dist.version == "1.0" + + def test_missing_metadata(self): + """ + Dists with a missing metadata file should raise ``MetadataNotFound``. + + Ref python/importlib_metadata#493 and python/cpython#143387. + """ + fixtures.build_files(self.make_pkg('foo-4.3', files={}), self.site_dir) + with self.assertRaises(MetadataNotFound): + Distribution.from_name('foo').metadata + with self.assertRaises(MetadataNotFound): + metadata('foo') + + class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod def pkg_with_non_ascii_description(site_dir): diff --git a/Lib/test/test_importlib/metadata/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py index 276f6288c91..97168549667 100644 --- a/Lib/test/test_importlib/metadata/test_zip.py +++ b/Lib/test/test_importlib/metadata/test_zip.py @@ -1,8 +1,12 @@ +import multiprocessing +import os import sys import unittest -from . import fixtures +from test.support import requires_fork, warnings_helper + from importlib.metadata import ( + FastPath, PackageNotFoundError, distribution, distributions, @@ -11,6 +15,8 @@ version, ) +from . import fixtures + class TestZip(fixtures.ZipFixtures, unittest.TestCase): def setUp(self): @@ -46,6 +52,39 @@ def test_one_distribution(self): dists = list(distributions(path=sys.path[:1])) assert len(dists) == 1 + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @requires_fork() + @unittest.skipUnless( + hasattr(os, 'register_at_fork') + and 'fork' in multiprocessing.get_all_start_methods(), + 'requires fork-based multiprocessing support', + ) + def test_fastpath_cache_cleared_in_forked_child(self): + zip_path = sys.path[0] + + FastPath(zip_path) + assert FastPath.__new__.cache_info().currsize >= 1 + + ctx = multiprocessing.get_context('fork') + parent_conn, child_conn = ctx.Pipe() + + def child(conn, root): + try: + before = FastPath.__new__.cache_info().currsize + FastPath(root) + after = FastPath.__new__.cache_info().currsize + conn.send((before, after)) + finally: + conn.close() + + proc = ctx.Process(target=child, args=(child_conn, zip_path)) + proc.start() + child_conn.close() + cache_sizes = parent_conn.recv() + proc.join() + + self.assertEqual(cache_sizes, (0, 1)) + class TestEgg(TestZip): def setUp(self): diff --git a/Lib/test/test_importlib/namespace_pkgs/foo/README.md b/Lib/test/test_importlib/namespace_pkgs/foo/README.md new file mode 100644 index 00000000000..9f6bc74941d --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/foo/README.md @@ -0,0 +1,2 @@ +This directory should not be a package, but should share a name with an +unrelated subpackage. diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py index 8b137891791..e69de29bb2d 100644 --- a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py +++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/namespacedata01/binary.file b/Lib/test/test_importlib/namespacedata01/binary.file deleted file mode 100644 index eaf36c1dacc..00000000000 Binary files a/Lib/test/test_importlib/namespacedata01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/namespacedata01/utf-16.file b/Lib/test/test_importlib/namespacedata01/utf-16.file deleted file mode 100644 index 2cb772295ef..00000000000 Binary files a/Lib/test/test_importlib/namespacedata01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/__init__.py b/Lib/test/test_importlib/resources/__init__.py index 8b137891791..e69de29bb2d 100644 --- a/Lib/test/test_importlib/resources/__init__.py +++ b/Lib/test/test_importlib/resources/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index b144628cb73..3720af7c508 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -1,10 +1,6 @@ -import pathlib import functools - -from typing import Dict, Union -from typing import runtime_checkable -from typing import Protocol - +import pathlib +from typing import Protocol, Union, runtime_checkable #### # from jaraco.path 3.7.1 @@ -16,7 +12,7 @@ class Symlink(str): """ -FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] +FilesSpec = dict[str, Union[str, bytes, Symlink, 'FilesSpec']] @runtime_checkable @@ -32,13 +28,13 @@ def write_bytes(self, content): ... # pragma: no cover def symlink_to(self, target): ... # pragma: no cover -def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: +def _ensure_tree_maker(obj: str | TreeMaker) -> TreeMaker: return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] def build( spec: FilesSpec, - prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] + prefix: str | TreeMaker = pathlib.Path(), # type: ignore[assignment] ): """ Build a set of files/directories, as described by the spec. @@ -70,7 +66,7 @@ def build( @functools.singledispatch -def create(content: Union[str, bytes, FilesSpec], path): +def create(content: str | bytes | FilesSpec, path): path.mkdir(exist_ok=True) build(content, prefix=path) # type: ignore[arg-type] diff --git a/Lib/test/test_importlib/resources/data01/__init__.py b/Lib/test/test_importlib/resources/data01/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data01/binary.file b/Lib/test/test_importlib/resources/data01/binary.file deleted file mode 100644 index eaf36c1dacc..00000000000 Binary files a/Lib/test/test_importlib/resources/data01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py b/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file deleted file mode 100644 index eaf36c1dacc..00000000000 Binary files a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-16.file b/Lib/test/test_importlib/resources/data01/utf-16.file deleted file mode 100644 index 2cb772295ef..00000000000 Binary files a/Lib/test/test_importlib/resources/data01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-8.file b/Lib/test/test_importlib/resources/data01/utf-8.file deleted file mode 100644 index 1c0132ad90a..00000000000 --- a/Lib/test/test_importlib/resources/data01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/data02/__init__.py b/Lib/test/test_importlib/resources/data02/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/__init__.py b/Lib/test/test_importlib/resources/data02/one/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/resource1.txt b/Lib/test/test_importlib/resources/data02/one/resource1.txt deleted file mode 100644 index 61a813e4017..00000000000 --- a/Lib/test/test_importlib/resources/data02/one/resource1.txt +++ /dev/null @@ -1 +0,0 @@ -one resource diff --git a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt deleted file mode 100644 index 48f587a2d0a..00000000000 --- a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt +++ /dev/null @@ -1 +0,0 @@ -a resource \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/data02/two/__init__.py b/Lib/test/test_importlib/resources/data02/two/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data02/two/resource2.txt b/Lib/test/test_importlib/resources/data02/two/resource2.txt deleted file mode 100644 index a80ce46ea36..00000000000 --- a/Lib/test/test_importlib/resources/data02/two/resource2.txt +++ /dev/null @@ -1 +0,0 @@ -two resource diff --git a/Lib/test/test_importlib/resources/data03/__init__.py b/Lib/test/test_importlib/resources/data03/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/resource1.txt b/Lib/test/test_importlib/resources/data03/namespace/resource1.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/namespacedata01/binary.file b/Lib/test/test_importlib/resources/namespacedata01/binary.file deleted file mode 100644 index eaf36c1dacc..00000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file b/Lib/test/test_importlib/resources/namespacedata01/utf-16.file deleted file mode 100644 index 2cb772295ef..00000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file b/Lib/test/test_importlib/resources/namespacedata01/utf-8.file deleted file mode 100644 index 1c0132ad90a..00000000000 --- a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/test_compatibilty_files.py b/Lib/test/test_importlib/resources/test_compatibilty_files.py index bcf608d9e2c..2fd39bedf25 100644 --- a/Lib/test/test_importlib/resources/test_compatibilty_files.py +++ b/Lib/test/test_importlib/resources/test_compatibilty_files.py @@ -1,8 +1,6 @@ +import importlib.resources as resources import io import unittest - -from importlib import resources - from importlib.resources._adapters import ( CompatibilityFiles, wrap_spec, @@ -26,51 +24,46 @@ def files(self): return resources.files(self.package) def test_spec_path_iter(self): - self.assertEqual( - sorted(path.name for path in self.files.iterdir()), - ['a', 'b', 'c'], - ) + assert sorted(path.name for path in self.files.iterdir()) == ['a', 'b', 'c'] def test_child_path_iter(self): - self.assertEqual(list((self.files / 'a').iterdir()), []) + assert list((self.files / 'a').iterdir()) == [] def test_orphan_path_iter(self): - self.assertEqual(list((self.files / 'a' / 'a').iterdir()), []) - self.assertEqual(list((self.files / 'a' / 'a' / 'a').iterdir()), []) + assert list((self.files / 'a' / 'a').iterdir()) == [] + assert list((self.files / 'a' / 'a' / 'a').iterdir()) == [] def test_spec_path_is(self): - self.assertFalse(self.files.is_file()) - self.assertFalse(self.files.is_dir()) + assert not self.files.is_file() + assert not self.files.is_dir() def test_child_path_is(self): - self.assertTrue((self.files / 'a').is_file()) - self.assertFalse((self.files / 'a').is_dir()) + assert (self.files / 'a').is_file() + assert not (self.files / 'a').is_dir() def test_orphan_path_is(self): - self.assertFalse((self.files / 'a' / 'a').is_file()) - self.assertFalse((self.files / 'a' / 'a').is_dir()) - self.assertFalse((self.files / 'a' / 'a' / 'a').is_file()) - self.assertFalse((self.files / 'a' / 'a' / 'a').is_dir()) + assert not (self.files / 'a' / 'a').is_file() + assert not (self.files / 'a' / 'a').is_dir() + assert not (self.files / 'a' / 'a' / 'a').is_file() + assert not (self.files / 'a' / 'a' / 'a').is_dir() def test_spec_path_name(self): - self.assertEqual(self.files.name, 'testingpackage') + assert self.files.name == 'testingpackage' def test_child_path_name(self): - self.assertEqual((self.files / 'a').name, 'a') + assert (self.files / 'a').name == 'a' def test_orphan_path_name(self): - self.assertEqual((self.files / 'a' / 'b').name, 'b') - self.assertEqual((self.files / 'a' / 'b' / 'c').name, 'c') + assert (self.files / 'a' / 'b').name == 'b' + assert (self.files / 'a' / 'b' / 'c').name == 'c' def test_spec_path_open(self): - self.assertEqual(self.files.read_bytes(), b'Hello, world!') - self.assertEqual(self.files.read_text(encoding='utf-8'), 'Hello, world!') + assert self.files.read_bytes() == b'Hello, world!' + assert self.files.read_text(encoding='utf-8') == 'Hello, world!' def test_child_path_open(self): - self.assertEqual((self.files / 'a').read_bytes(), b'Hello, world!') - self.assertEqual( - (self.files / 'a').read_text(encoding='utf-8'), 'Hello, world!' - ) + assert (self.files / 'a').read_bytes() == b'Hello, world!' + assert (self.files / 'a').read_text(encoding='utf-8') == 'Hello, world!' def test_orphan_path_open(self): with self.assertRaises(FileNotFoundError): @@ -88,7 +81,7 @@ def test_orphan_path_invalid(self): def test_wrap_spec(self): spec = wrap_spec(self.package) - self.assertIsInstance(spec.loader.get_resource_reader(None), CompatibilityFiles) + assert isinstance(spec.loader.get_resource_reader(None), CompatibilityFiles) class CompatibilityFilesNoReaderTests(unittest.TestCase): @@ -101,4 +94,4 @@ def files(self): return resources.files(self.package) def test_spec_path_joinpath(self): - self.assertIsInstance(self.files / 'a', CompatibilityFiles.OrphanPath) + assert isinstance(self.files / 'a', CompatibilityFiles.OrphanPath) diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 4e4e0e9c337..bdc158d85a2 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,5 +1,5 @@ +import importlib.resources as resources import unittest -from importlib import resources from . import util diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 640f90fc0dd..a7fc6bc35c5 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -1,12 +1,12 @@ -import unittest import contextlib +import importlib.resources as resources import pathlib +import unittest +from importlib.resources import abc +from importlib.resources.abc import ResourceReader, TraversableResources from test.support import os_helper -from importlib import resources -from importlib.resources import abc -from importlib.resources.abc import TraversableResources, ResourceReader from . import util diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 3ce44999f98..c922d32cedc 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,15 +1,16 @@ +import contextlib +import importlib +import importlib.resources as resources import pathlib import py_compile import textwrap import unittest import warnings -import importlib -import contextlib - -from importlib import resources from importlib.resources.abc import Traversable + +from test.support import import_helper, os_helper + from . import util -from test.support import os_helper, import_helper @contextlib.contextmanager @@ -36,15 +37,7 @@ def test_traversable(self): def test_joinpath_with_multiple_args(self): files = resources.files(self.data) binfile = files.joinpath('subdirectory', 'binary.file') - self.assertTrue(binfile.is_file()) - - def test_old_parameter(self): - """ - Files used to take a 'package' parameter. Make sure anyone - passing by name is still supported. - """ - with suppress_known_deprecation(): - resources.files(package=self.data) + assert binfile.is_file() class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): @@ -70,7 +63,7 @@ def test_non_paths_in_dunder_path(self): to cause the ``PathEntryFinder`` to be called when searching for packages. In that case, resources should still be loadable. """ - import namespacedata01 + import namespacedata01 # type: ignore[import-not-found] namespacedata01.__path__.append( '__editable__.sample_namespace-1.0.finder.__path_hook__' @@ -161,7 +154,9 @@ def _compile_importlib(self): sources = pathlib.Path(resources.__file__).parent for source_path in sources.glob('**/*.py'): - c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix( + '.pyc' + ) py_compile.compile(source_path, c_path) self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py index e8d25fa4d9f..9cec6af9a5d 100644 --- a/Lib/test/test_importlib/resources/test_functional.py +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -1,17 +1,12 @@ -import unittest -import os import importlib +import importlib.resources as resources +import os +import unittest from test.support import warnings_helper -from importlib import resources - from . import util -# Since the functional API forwards to Traversable, we only test -# filesystem resources here -- not zip files, namespace packages etc. -# We do test for two kinds of Anchor, though. - class StringAnchorMixin: anchor01 = 'data01' @@ -28,7 +23,7 @@ def anchor02(self): return importlib.import_module('data02') -class FunctionalAPIBase(util.DiskSetup): +class FunctionalAPIBase: def setUp(self): super().setUp() self.load_fixture('data02') @@ -43,83 +38,90 @@ def _gen_resourcetxt_path_parts(self): with self.subTest(path_parts=path_parts): yield path_parts + @staticmethod + def remove_utf16_bom(string): + """Remove an architecture-specific UTF-16 BOM prefix when present. + + Some platforms surface UTF-16 BOM bytes as escaped text when the + fixture is intentionally decoded as UTF-8 with ``errors='backslashreplace'``. + Strip that prefix so assertions validate content consistently.""" + for bom in ('\\xff\\xfe', '\\xfe\\xff', '\ufeff'): + if string.startswith(bom): + return string[len(bom) :] + return string + def test_read_text(self): - self.assertEqual( - resources.read_text(self.anchor01, 'utf-8.file'), - 'Hello, UTF-8 world!\n', + assert ( + resources.read_text(self.anchor01, 'utf-8.file') == 'Hello, UTF-8 world!\n' ) - self.assertEqual( + assert ( resources.read_text( self.anchor02, 'subdirectory', 'subsubdir', 'resource.txt', encoding='utf-8', - ), - 'a resource', + ) + == 'a resource' ) for path_parts in self._gen_resourcetxt_path_parts(): - self.assertEqual( + assert ( resources.read_text( self.anchor02, *path_parts, encoding='utf-8', - ), - 'a resource', + ) + == 'a resource' ) # Use generic OSError, since e.g. attempting to read a directory can # fail with PermissionError rather than IsADirectoryError with self.assertRaises(OSError): resources.read_text(self.anchor01) - with self.assertRaises(OSError): + with self.assertRaises((OSError, resources.abc.TraversalError)): resources.read_text(self.anchor01, 'no-such-file') with self.assertRaises(UnicodeDecodeError): resources.read_text(self.anchor01, 'utf-16.file') - self.assertEqual( + assert ( resources.read_text( self.anchor01, 'binary.file', encoding='latin1', - ), - '\x00\x01\x02\x03', + ) + == '\x00\x01\x02\x03' ) - self.assertEndsWith( # ignore the BOM + assert self.remove_utf16_bom( resources.read_text( self.anchor01, 'utf-16.file', errors='backslashreplace', ), - 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( - errors='backslashreplace', - ), + ) == 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', ) def test_read_binary(self): - self.assertEqual( - resources.read_binary(self.anchor01, 'utf-8.file'), - b'Hello, UTF-8 world!\n', + assert ( + resources.read_binary(self.anchor01, 'utf-8.file') + == b'Hello, UTF-8 world!\n' ) for path_parts in self._gen_resourcetxt_path_parts(): - self.assertEqual( - resources.read_binary(self.anchor02, *path_parts), - b'a resource', - ) + assert resources.read_binary(self.anchor02, *path_parts) == b'a resource' def test_open_text(self): with resources.open_text(self.anchor01, 'utf-8.file') as f: - self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + assert f.read() == 'Hello, UTF-8 world!\n' for path_parts in self._gen_resourcetxt_path_parts(): with resources.open_text( self.anchor02, *path_parts, encoding='utf-8', ) as f: - self.assertEqual(f.read(), 'a resource') + assert f.read() == 'a resource' # Use generic OSError, since e.g. attempting to read a directory can # fail with PermissionError rather than IsADirectoryError with self.assertRaises(OSError): resources.open_text(self.anchor01) - with self.assertRaises(OSError): + with self.assertRaises((OSError, resources.abc.TraversalError)): resources.open_text(self.anchor01, 'no-such-file') with resources.open_text(self.anchor01, 'utf-16.file') as f: with self.assertRaises(UnicodeDecodeError): @@ -129,71 +131,70 @@ def test_open_text(self): 'binary.file', encoding='latin1', ) as f: - self.assertEqual(f.read(), '\x00\x01\x02\x03') + assert f.read() == '\x00\x01\x02\x03' with resources.open_text( self.anchor01, 'utf-16.file', errors='backslashreplace', ) as f: - self.assertEndsWith( # ignore the BOM - f.read(), - 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( - errors='backslashreplace', - ), + assert self.remove_utf16_bom(f.read()) == 'Hello, UTF-16 world!\n'.encode( + 'utf-16-le' + ).decode( + errors='backslashreplace', ) def test_open_binary(self): with resources.open_binary(self.anchor01, 'utf-8.file') as f: - self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + assert f.read() == b'Hello, UTF-8 world!\n' for path_parts in self._gen_resourcetxt_path_parts(): with resources.open_binary( self.anchor02, *path_parts, ) as f: - self.assertEqual(f.read(), b'a resource') + assert f.read() == b'a resource' def test_path(self): with resources.path(self.anchor01, 'utf-8.file') as path: with open(str(path), encoding='utf-8') as f: - self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + assert f.read() == 'Hello, UTF-8 world!\n' with resources.path(self.anchor01) as path: with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: - self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + assert f.read() == 'Hello, UTF-8 world!\n' def test_is_resource(self): is_resource = resources.is_resource - self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) - self.assertFalse(is_resource(self.anchor01, 'no_such_file')) - self.assertFalse(is_resource(self.anchor01)) - self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + assert is_resource(self.anchor01, 'utf-8.file') + assert not is_resource(self.anchor01, 'no_such_file') + assert not is_resource(self.anchor01) + assert not is_resource(self.anchor01, 'subdirectory') for path_parts in self._gen_resourcetxt_path_parts(): - self.assertTrue(is_resource(self.anchor02, *path_parts)) + assert is_resource(self.anchor02, *path_parts) def test_contents(self): with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): c = resources.contents(self.anchor01) - self.assertGreaterEqual( - set(c), - {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, - ) - with self.assertRaises(OSError), warnings_helper.check_warnings(( - ".*contents.*", - DeprecationWarning, - )): + assert set(c) >= {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'} + with ( + self.assertRaises(OSError), + warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )), + ): list(resources.contents(self.anchor01, 'utf-8.file')) for path_parts in self._gen_resourcetxt_path_parts(): - with self.assertRaises(OSError), warnings_helper.check_warnings(( - ".*contents.*", - DeprecationWarning, - )): + with ( + self.assertRaises((OSError, resources.abc.TraversalError)), + warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )), + ): list(resources.contents(self.anchor01, *path_parts)) with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): c = resources.contents(self.anchor01, 'subdirectory') - self.assertGreaterEqual( - set(c), - {'binary.file'}, - ) + assert set(c) >= {'binary.file'} @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_common_errors(self): @@ -233,17 +234,28 @@ def test_text_errors(self): ) -class FunctionalAPITest_StringAnchor( +class FunctionalAPITest_StringAnchor_Disk( StringAnchorMixin, FunctionalAPIBase, + util.DiskSetup, unittest.TestCase, ): pass -class FunctionalAPITest_ModuleAnchor( +class FunctionalAPITest_ModuleAnchor_Disk( ModuleAnchorMixin, FunctionalAPIBase, + util.DiskSetup, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_StringAnchor_Memory( + StringAnchorMixin, + FunctionalAPIBase, + util.MemorySetup, unittest.TestCase, ): pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 8c00378ad3c..950f71db05a 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,6 +1,6 @@ +import importlib.resources as resources import unittest -from importlib import resources from . import util @@ -23,19 +23,19 @@ def test_open_binary(self): target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, bytes(range(4))) + assert result == bytes(range(4)) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' with target.open(encoding='utf-8') as fp: result = fp.read() - self.assertEqual(result, 'Hello, UTF-8 world!\n') + assert result == 'Hello, UTF-8 world!\n' def test_open_text_given_encoding(self): target = resources.files(self.data) / 'utf-16.file' with target.open(encoding='utf-16', errors='strict') as fp: result = fp.read() - self.assertEqual(result, 'Hello, UTF-16 world!\n') + assert result == 'Hello, UTF-16 world!\n' def test_open_text_with_errors(self): """ @@ -46,11 +46,10 @@ def test_open_text_with_errors(self): self.assertRaises(UnicodeError, fp.read) with target.open(encoding='utf-8', errors='ignore') as fp: result = fp.read() - self.assertEqual( - result, + assert result == ( 'H\x00e\x00l\x00l\x00o\x00,\x00 ' '\x00U\x00T\x00F\x00-\x001\x006\x00 ' - '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00', + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00' ) def test_open_binary_FileNotFoundError(self): diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 903911f57b3..162344e5d83 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,8 +1,8 @@ +import importlib.resources as resources import io import pathlib import unittest -from importlib import resources from . import util @@ -19,9 +19,9 @@ def test_reading(self): """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: - self.assertIsInstance(path, pathlib.Path) - self.assertEndsWith(path.name, "utf-8.file") - self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) + assert isinstance(path, pathlib.Path) + assert path.name.endswith("utf-8.file"), repr(path) + assert 'Hello, UTF-8 world!\n' == path.read_text(encoding='utf-8') class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 59c237d9641..4085a64b0ee 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,6 +1,6 @@ +import importlib.resources as resources import unittest - -from importlib import import_module, resources +from importlib import import_module from . import util @@ -18,23 +18,25 @@ def execute(self, package, path): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, bytes(range(4))) + assert result == bytes(range(4)) def test_read_text_default_encoding(self): result = ( - resources.files(self.data) + resources + .files(self.data) .joinpath('utf-8.file') .read_text(encoding='utf-8') ) - self.assertEqual(result, 'Hello, UTF-8 world!\n') + assert result == 'Hello, UTF-8 world!\n' def test_read_text_given_encoding(self): result = ( - resources.files(self.data) + resources + .files(self.data) .joinpath('utf-16.file') .read_text(encoding='utf-16') ) - self.assertEqual(result, 'Hello, UTF-16 world!\n') + assert result == 'Hello, UTF-16 world!\n' def test_read_text_with_errors(self): """ @@ -43,11 +45,10 @@ def test_read_text_with_errors(self): target = resources.files(self.data) / 'utf-16.file' self.assertRaises(UnicodeError, target.read_text, encoding='utf-8') result = target.read_text(encoding='utf-8', errors='ignore') - self.assertEqual( - result, + assert result == ( 'H\x00e\x00l\x00l\x00o\x00,\x00 ' '\x00U\x00T\x00F\x00-\x001\x006\x00 ' - '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00', + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00' ) @@ -59,13 +60,13 @@ class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, bytes(range(4, 8))) + assert result == bytes(range(4, 8)) def test_read_submodule_resource_by_name(self): result = ( resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, bytes(range(4, 8))) + assert result == bytes(range(4, 8)) class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): @@ -78,15 +79,16 @@ class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): submodule = import_module('namespacedata01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, bytes(range(12, 16))) + assert result == bytes(range(12, 16)) def test_read_submodule_resource_by_name(self): result = ( - resources.files('namespacedata01.subdirectory') + resources + .files('namespacedata01.subdirectory') .joinpath('binary.file') .read_bytes() ) - self.assertEqual(result, bytes(range(12, 16))) + assert result == bytes(range(12, 16)) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index ed5693ab416..691a78bb060 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,9 +1,8 @@ import os.path import pathlib import unittest - from importlib import import_module -from importlib.readers import MultiplexedPath, NamespaceReader +from importlib.resources.readers import MultiplexedPath, NamespaceReader from . import util @@ -31,9 +30,7 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual( - contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} - ) + assert contents == {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} def test_iterdir_duplicate(self): contents = { @@ -44,16 +41,19 @@ def test_iterdir_duplicate(self): contents.remove(remove) except (KeyError, ValueError): pass - self.assertEqual( - contents, - {'__init__.py', 'binary.file', 'subdirectory', 'utf-16.file', 'utf-8.file'}, - ) + assert contents == { + '__init__.py', + 'binary.file', + 'subdirectory', + 'utf-16.file', + 'utf-8.file', + } def test_is_dir(self): - self.assertEqual(MultiplexedPath(self.folder).is_dir(), True) + assert MultiplexedPath(self.folder).is_dir() def test_is_file(self): - self.assertEqual(MultiplexedPath(self.folder).is_file(), False) + assert not MultiplexedPath(self.folder).is_file() def test_open_file(self): path = MultiplexedPath(self.folder) @@ -67,19 +67,17 @@ def test_open_file(self): def test_join_path(self): prefix = str(self.folder.parent) path = MultiplexedPath(self.folder, self.data01) - self.assertEqual( - str(path.joinpath('binary.file'))[len(prefix) + 1 :], - os.path.join('namespacedata01', 'binary.file'), + assert str(path.joinpath('binary.file'))[len(prefix) + 1 :] == os.path.join( + 'namespacedata01', 'binary.file' ) sub = path.joinpath('subdirectory') assert isinstance(sub, MultiplexedPath) assert 'namespacedata01' in str(sub) assert 'data01' in str(sub) - self.assertEqual( - str(path.joinpath('imaginary'))[len(prefix) + 1 :], - os.path.join('namespacedata01', 'imaginary'), + assert str(path.joinpath('imaginary'))[len(prefix) + 1 :] == os.path.join( + 'namespacedata01', 'imaginary' ) - self.assertEqual(path.joinpath(), path) + assert path.joinpath() == path def test_join_path_compound(self): path = MultiplexedPath(self.folder) @@ -88,23 +86,16 @@ def test_join_path_compound(self): def test_join_path_common_subdir(self): prefix = str(self.data02.parent) path = MultiplexedPath(self.data01, self.data02) - self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) - self.assertEqual( - str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], - os.path.join('data02', 'subdirectory', 'subsubdir'), + assert isinstance(path.joinpath('subdirectory'), MultiplexedPath) + assert str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :] == ( + os.path.join('data02', 'subdirectory', 'subsubdir') ) def test_repr(self): - self.assertEqual( - repr(MultiplexedPath(self.folder)), - f"MultiplexedPath('{self.folder}')", - ) + assert repr(MultiplexedPath(self.folder)) == f"MultiplexedPath('{self.folder}')" def test_name(self): - self.assertEqual( - MultiplexedPath(self.folder).name, - os.path.basename(self.folder), - ) + assert MultiplexedPath(self.folder).name == os.path.basename(self.folder) class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): @@ -119,18 +110,14 @@ def test_resource_path(self): reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) root = self.data.__path__[0] - self.assertEqual( - reader.resource_path('binary.file'), os.path.join(root, 'binary.file') - ) - self.assertEqual( - reader.resource_path('imaginary'), os.path.join(root, 'imaginary') - ) + assert reader.resource_path('binary.file') == os.path.join(root, 'binary.file') + assert reader.resource_path('imaginary') == os.path.join(root, 'imaginary') def test_files(self): reader = NamespaceReader(self.data.__spec__.submodule_search_locations) root = self.data.__path__[0] - self.assertIsInstance(reader.files(), MultiplexedPath) - self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") + assert isinstance(reader.files(), MultiplexedPath) + assert repr(reader.files()) == f"MultiplexedPath('{root}')" if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index fcede14b891..b114b1f0d80 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,7 +1,9 @@ +import importlib.resources as resources +import types import unittest +from importlib import import_module from . import util -from importlib import resources, import_module class ResourceTests: @@ -9,16 +11,16 @@ class ResourceTests: def test_is_file_exists(self): target = resources.files(self.data) / 'binary.file' - self.assertTrue(target.is_file()) + assert target.is_file() def test_is_file_missing(self): target = resources.files(self.data) / 'not-a-file' - self.assertFalse(target.is_file()) + assert not target.is_file() def test_is_dir(self): target = resources.files(self.data) / 'subdirectory' - self.assertFalse(target.is_file()) - self.assertTrue(target.is_dir()) + assert not target.is_file() + assert target.is_dir() class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): @@ -38,7 +40,7 @@ def test_resource_contents(self): package = util.create_package( file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) - self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) + assert names(resources.files(package)) == {'A', 'B', 'C'} def test_is_file(self): package = util.create_package( @@ -46,7 +48,7 @@ def test_is_file(self): path=self.data.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertTrue(resources.files(package).joinpath('B').is_file()) + assert resources.files(package).joinpath('B').is_file() def test_is_dir(self): package = util.create_package( @@ -54,7 +56,7 @@ def test_is_dir(self): path=self.data.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertTrue(resources.files(package).joinpath('D').is_dir()) + assert resources.files(package).joinpath('D').is_dir() def test_resource_missing(self): package = util.create_package( @@ -62,7 +64,7 @@ def test_resource_missing(self): path=self.data.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertFalse(resources.files(package).joinpath('Z').is_file()) + assert not resources.files(package).joinpath('Z').is_file() class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): @@ -82,30 +84,26 @@ def test_package_has_no_reader_fallback(self): module.__file__ = '/path/which/shall/not/be/named' module.__spec__.loader = module.__loader__ module.__spec__.origin = module.__file__ - self.assertFalse(resources.files(module).joinpath('A').is_file()) + assert not resources.files(module).joinpath('A').is_file() class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): submodule = import_module('data01.subdirectory') - self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) + assert resources.files(submodule).joinpath('binary.file').is_file() def test_read_submodule_resource_by_name(self): - self.assertTrue( - resources.files('data01.subdirectory').joinpath('binary.file').is_file() - ) + assert resources.files('data01.subdirectory').joinpath('binary.file').is_file() def test_submodule_contents(self): submodule = import_module('data01.subdirectory') - self.assertEqual( - names(resources.files(submodule)), {'__init__.py', 'binary.file'} - ) + assert names(resources.files(submodule)) == {'__init__.py', 'binary.file'} def test_submodule_contents_by_name(self): - self.assertEqual( - names(resources.files('data01.subdirectory')), - {'__init__.py', 'binary.file'}, - ) + assert names(resources.files('data01.subdirectory')) == { + '__init__.py', + 'binary.file', + } def test_as_file_directory(self): with resources.as_file(resources.files('data01')) as data: @@ -124,14 +122,8 @@ def test_unrelated_contents(self): Test thata zip with two unrelated subpackages return distinct resources. Ref python/importlib_resources#44. """ - self.assertEqual( - names(resources.files('data02.one')), - {'__init__.py', 'resource1.txt'}, - ) - self.assertEqual( - names(resources.files('data02.two')), - {'__init__.py', 'resource2.txt'}, - ) + assert names(resources.files('data02.one')) == {'__init__.py', 'resource1.txt'} + assert names(resources.files('data02.two')) == {'__init__.py', 'resource2.txt'} class DeletingZipsTest(util.ZipSetup, unittest.TestCase): @@ -168,16 +160,15 @@ def test_read_text_does_not_keep_open(self): class ResourceFromNamespaceTests: def test_is_submodule_resource(self): - self.assertTrue( - resources.files(import_module('namespacedata01')) + assert ( + resources + .files(import_module('namespacedata01')) .joinpath('binary.file') .is_file() ) def test_read_submodule_resource_by_name(self): - self.assertTrue( - resources.files('namespacedata01').joinpath('binary.file').is_file() - ) + assert resources.files('namespacedata01').joinpath('binary.file').is_file() def test_submodule_contents(self): contents = names(resources.files(import_module('namespacedata01'))) @@ -185,9 +176,7 @@ def test_submodule_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual( - contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} - ) + assert contents == {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -195,9 +184,7 @@ def test_submodule_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual( - contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} - ) + assert contents == {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} def test_submodule_sub_contents(self): contents = names(resources.files(import_module('namespacedata01.subdirectory'))) @@ -205,7 +192,7 @@ def test_submodule_sub_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file'}) + assert contents == {'binary.file'} def test_submodule_sub_contents_by_name(self): contents = names(resources.files('namespacedata01.subdirectory')) @@ -213,7 +200,7 @@ def test_submodule_sub_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file'}) + assert contents == {'binary.file'} class ResourceFromNamespaceDiskTests( @@ -232,5 +219,24 @@ class ResourceFromNamespaceZipTests( MODULE = 'namespacedata01' +class MainModuleTests(unittest.TestCase): + def test_main_module_with_none_spec(self): + """ + __main__ module with no spec should raise TypeError (for clarity). + + See python/cpython#138531 for details. + """ + # construct a __main__ module with no __spec__. + mainmodule = types.ModuleType("__main__") + + assert mainmodule.__spec__ is None + + with self.assertRaises( + TypeError, + msg="Cannot access resources for '__main__' as it does not appear to correspond to an importable module (its __spec__ is None).", + ): + resources.files(mainmodule) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_util.py b/Lib/test/test_importlib/resources/test_util.py new file mode 100644 index 00000000000..de304b6f351 --- /dev/null +++ b/Lib/test/test_importlib/resources/test_util.py @@ -0,0 +1,29 @@ +import unittest + +from .util import MemorySetup, Traversable + + +class TestMemoryTraversableImplementation(unittest.TestCase): + def test_concrete_methods_are_not_overridden(self): + """`MemoryTraversable` must not override `Traversable` concrete methods. + + This test is not an attempt to enforce a particular `Traversable` protocol; + it merely catches changes in the `Traversable` abstract/concrete methods + that have not been mirrored in the `MemoryTraversable` subclass. + """ + + traversable_concrete_methods = { + method + for method, value in Traversable.__dict__.items() + if callable(value) and method not in Traversable.__abstractmethods__ + } + memory_traversable_concrete_methods = { + method + for method, value in MemorySetup.MemoryTraversable.__dict__.items() + if callable(value) and not method.startswith("__") + } + overridden_methods = ( + memory_traversable_concrete_methods & traversable_concrete_methods + ) + + assert not overridden_methods diff --git a/Lib/test/test_importlib/resources/update-zips.py b/Lib/test/test_importlib/resources/update-zips.py deleted file mode 100755 index 231334aa7e3..00000000000 --- a/Lib/test/test_importlib/resources/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index e2d995f5963..85b5c61518d 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -1,18 +1,18 @@ import abc +import contextlib +import functools import importlib import io +import pathlib import sys import types -import pathlib -import contextlib +from importlib.machinery import ModuleSpec +from importlib.resources.abc import ResourceReader, Traversable, TraversableResources -from importlib.resources.abc import ResourceReader from test.support import import_helper, os_helper -from . import zip as zip_ -from . import _path - -from importlib.machinery import ModuleSpec +from . import _path +from . import zip as zip_ class Reader(ResourceReader): @@ -122,7 +122,7 @@ def test_missing_path(self): bytes_data = io.BytesIO(b'Hello, world!') package = create_package(file=bytes_data, path=FileNotFoundError()) self.execute(package, 'utf-8.file') - self.assertEqual(package.__loader__._path, 'utf-8.file') + assert package.__loader__._path == 'utf-8.file' def test_extant_path(self): # Attempting to open or read or request the path when the @@ -133,7 +133,7 @@ def test_extant_path(self): path = __file__ package = create_package(file=bytes_data, path=path) self.execute(package, 'utf-8.file') - self.assertEqual(package.__loader__._path, 'utf-8.file') + assert package.__loader__._path == 'utf-8.file' def test_useless_loader(self): package = create_package(file=FileNotFoundError(), path=FileNotFoundError()) @@ -202,5 +202,108 @@ def tree_on_path(self, spec): self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) +class MemorySetup(ModuleSetup): + """Support loading a module in memory.""" + + MODULE = 'data01' + + def load_fixture(self, module): + self.fixtures.enter_context(self.augment_sys_metapath(module)) + return importlib.import_module(module) + + @contextlib.contextmanager + def augment_sys_metapath(self, module): + finder_instance = self.MemoryFinder(module) + sys.meta_path.append(finder_instance) + yield + sys.meta_path.remove(finder_instance) + + class MemoryFinder(importlib.abc.MetaPathFinder): + def __init__(self, module): + self._module = module + + def find_spec(self, fullname, path, target=None): + if fullname != self._module: + return None + + return importlib.machinery.ModuleSpec( + name=fullname, + loader=MemorySetup.MemoryLoader(self._module), + is_package=True, + ) + + class MemoryLoader(importlib.abc.Loader): + def __init__(self, module): + self._module = module + + def exec_module(self, module): + pass + + def get_resource_reader(self, fullname): + return MemorySetup.MemoryTraversableResources(self._module, fullname) + + class MemoryTraversableResources(TraversableResources): + def __init__(self, module, fullname): + self._module = module + self._fullname = fullname + + def files(self): + return MemorySetup.MemoryTraversable(self._module, self._fullname) + + class MemoryTraversable(Traversable): + """Implement only the abstract methods of `Traversable`. + + Besides `.__init__()`, no other methods may be implemented or overridden. + This is critical for validating the concrete `Traversable` implementations. + """ + + def __init__(self, module, fullname): + self._module = module + self._fullname = fullname + + def _resolve(self): + """ + Fully traverse the `fixtures` dictionary. + + This should be wrapped in a `try/except KeyError` + but it is not currently needed and lowers the code coverage numbers. + """ + path = pathlib.PurePosixPath(self._fullname) + return functools.reduce(lambda d, p: d[p], path.parts, fixtures) + + def iterdir(self): + directory = self._resolve() + if not isinstance(directory, dict): + # Filesystem openers raise OSError, and that exception is mirrored here. + raise OSError(f"{self._fullname} is not a directory") + for path in directory: + yield MemorySetup.MemoryTraversable( + self._module, f"{self._fullname}/{path}" + ) + + def is_dir(self) -> bool: + return isinstance(self._resolve(), dict) + + def is_file(self) -> bool: + return not self.is_dir() + + def open(self, mode='r', encoding=None, errors=None, *_, **__): + contents = self._resolve() + if isinstance(contents, dict): + # Filesystem openers raise OSError when attempting to open a directory, + # and that exception is mirrored here. + raise OSError(f"{self._fullname} is a directory") + if isinstance(contents, str): + contents = contents.encode("utf-8") + result = io.BytesIO(contents) + if "b" in mode: + return result + return io.TextIOWrapper(result, encoding=encoding, errors=errors) + + @property + def name(self): + return pathlib.PurePosixPath(self._fullname).name + + class CommonTests(DiskSetup, CommonTestsBase): pass diff --git a/Lib/test/test_importlib/resources/zipdata01/__init__.py b/Lib/test/test_importlib/resources/zipdata01/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739f8..00000000000 Binary files a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/resources/zipdata02/__init__.py b/Lib/test/test_importlib/resources/zipdata02/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d28..00000000000 Binary files a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index f35adec1a8e..ca2cc045fa6 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -5,24 +5,21 @@ machinery = util.import_importlib('importlib.machinery') importlib_util = util.import_importlib('importlib.util') -import errno import marshal import os import py_compile -import shutil import stat import sys import types import unittest -import warnings -from test.support.import_helper import make_legacy_pyc, unload +from test.support.import_helper import make_legacy_pyc -from test.test_py_compile import without_source_date_epoch -from test.test_py_compile import SourceDateEpochTestMeta +from test.support.os_helper import without_source_date_epoch +from test.support.os_helper import SourceDateEpochTestMeta -class SimpleTest(abc.LoaderTests): +class SimpleTest: """Should have no issue importing a source module [basic]. And if there is a syntax error, it should raise a SyntaxError [syntax error]. @@ -34,17 +31,6 @@ def setUp(self): self.filepath = os.path.join('ham', self.name + '.py') self.loader = self.machinery.SourceFileLoader(self.name, self.filepath) - def test_load_module_API(self): - class Tester(self.abc.FileLoader): - def get_source(self, _): return 'attr = 42' - def is_package(self, _): return False - - loader = Tester('blah', 'blah.py') - self.addCleanup(unload, 'blah') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module() # Should not raise an exception. - def test_get_filename_API(self): # If fullname is not set then assume self.path is desired. class Tester(self.abc.FileLoader): @@ -69,173 +55,11 @@ def test_inequality(self): other = self.machinery.SourceFileLoader('_' + self.name, self.filepath) self.assertNotEqual(self.loader, other) - # [basic] - def test_module(self): - with util.create_modules('_temp') as mapping: - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_temp') - self.assertIn('_temp', sys.modules) - check = {'__name__': '_temp', '__file__': mapping['_temp'], - '__package__': ''} - for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) - - def test_package(self): - with util.create_modules('_pkg.__init__') as mapping: - loader = self.machinery.SourceFileLoader('_pkg', - mapping['_pkg.__init__']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_pkg') - self.assertIn('_pkg', sys.modules) - check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'], - '__path__': [os.path.dirname(mapping['_pkg.__init__'])], - '__package__': '_pkg'} - for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) - - - def test_lacking_parent(self): - with util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: - loader = self.machinery.SourceFileLoader('_pkg.mod', - mapping['_pkg.mod']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_pkg.mod') - self.assertIn('_pkg.mod', sys.modules) - check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'], - '__package__': '_pkg'} - for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) def fake_mtime(self, fxn): """Fake mtime to always be higher than expected.""" return lambda name: fxn(name) + 1 - def test_module_reuse(self): - with util.create_modules('_temp') as mapping: - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_temp') - module_id = id(module) - module_dict_id = id(module.__dict__) - with open(mapping['_temp'], 'w', encoding='utf-8') as file: - file.write("testing_var = 42\n") - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_temp') - self.assertIn('testing_var', module.__dict__, - "'testing_var' not in " - "{0}".format(list(module.__dict__.keys()))) - self.assertEqual(module, sys.modules['_temp']) - self.assertEqual(id(module), module_id) - self.assertEqual(id(module.__dict__), module_dict_id) - - def test_state_after_failure(self): - # A failed reload should leave the original module intact. - attributes = ('__file__', '__path__', '__package__') - value = '' - name = '_temp' - with util.create_modules(name) as mapping: - orig_module = types.ModuleType(name) - for attr in attributes: - setattr(orig_module, attr, value) - with open(mapping[name], 'w', encoding='utf-8') as file: - file.write('+++ bad syntax +++') - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with self.assertRaises(SyntaxError): - loader.exec_module(orig_module) - for attr in attributes: - self.assertEqual(getattr(orig_module, attr), value) - with self.assertRaises(SyntaxError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module(name) - for attr in attributes: - self.assertEqual(getattr(orig_module, attr), value) - - # [syntax error] - def test_bad_syntax(self): - with util.create_modules('_temp') as mapping: - with open(mapping['_temp'], 'w', encoding='utf-8') as file: - file.write('=') - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with self.assertRaises(SyntaxError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module('_temp') - self.assertNotIn('_temp', sys.modules) - - def test_file_from_empty_string_dir(self): - # Loading a module found from an empty string entry on sys.path should - # not only work, but keep all attributes relative. - file_path = '_temp.py' - with open(file_path, 'w', encoding='utf-8') as file: - file.write("# test file for importlib") - try: - with util.uncache('_temp'): - loader = self.machinery.SourceFileLoader('_temp', file_path) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - mod = loader.load_module('_temp') - self.assertEqual(file_path, mod.__file__) - self.assertEqual(self.util.cache_from_source(file_path), - mod.__cached__) - finally: - os.unlink(file_path) - pycache = os.path.dirname(self.util.cache_from_source(file_path)) - if os.path.exists(pycache): - shutil.rmtree(pycache) - - @util.writes_bytecode_files - def test_timestamp_overflow(self): - # When a modification timestamp is larger than 2**32, it should be - # truncated rather than raise an OverflowError. - with util.create_modules('_temp') as mapping: - source = mapping['_temp'] - compiled = self.util.cache_from_source(source) - with open(source, 'w', encoding='utf-8') as f: - f.write("x = 5") - try: - os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5)) - except OverflowError: - self.skipTest("cannot set modification time to large integer") - except OSError as e: - if e.errno != getattr(errno, 'EOVERFLOW', None): - raise - self.skipTest("cannot set modification time to large integer ({})".format(e)) - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - # PEP 451 - module = types.ModuleType('_temp') - module.__spec__ = self.util.spec_from_loader('_temp', loader) - loader.exec_module(module) - self.assertEqual(module.x, 5) - self.assertTrue(os.path.exists(compiled)) - os.unlink(compiled) - # PEP 302 - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - mod = loader.load_module('_temp') - # Sanity checks. - self.assertEqual(mod.__cached__, compiled) - self.assertEqual(mod.x, 5) - # The pyc file was created. - self.assertTrue(os.path.exists(compiled)) - - def test_unloadable(self): - loader = self.machinery.SourceFileLoader('good name', {}) - module = types.ModuleType('bad name') - module.__spec__ = self.machinery.ModuleSpec('bad name', loader) - with self.assertRaises(ImportError): - loader.exec_module(module) - with self.assertRaises(ImportError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module('bad name') - @util.writes_bytecode_files def test_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -386,12 +210,21 @@ def manipulate_bytecode(self, del sys.modules['_temp'] except KeyError: pass - py_compile.compile(mapping[name], invalidation_mode=invalidation_mode) - if not del_source: - bytecode_path = self.util.cache_from_source(mapping[name]) + if sys.implementation.cache_tag is None: + if del_source: + bytecode_path = mapping[name] + 'c' + py_compile.compile(mapping[name], bytecode_path, + invalidation_mode=invalidation_mode) + os.unlink(mapping[name]) + else: + raise unittest.SkipTest('requires sys.implementation.cache_tag') else: - os.unlink(mapping[name]) - bytecode_path = make_legacy_pyc(mapping[name]) + py_compile.compile(mapping[name], invalidation_mode=invalidation_mode) + if not del_source: + bytecode_path = self.util.cache_from_source(mapping[name]) + else: + os.unlink(mapping[name]) + bytecode_path = make_legacy_pyc(mapping[name]) if manipulator: with open(bytecode_path, 'rb') as file: bc = file.read() @@ -511,16 +344,6 @@ def import_(self, file, module_name): loader.exec_module(module) -class BadBytecodeTestPEP302(BadBytecodeTest): - - def import_(self, file, module_name): - loader = self.loader(module_name, file) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module(module_name) - self.assertIn(module_name, sys.modules) - - class SourceLoaderBadBytecodeTest: @classmethod @@ -681,18 +504,6 @@ class SourceLoaderBadBytecodeTestPEP451( util=importlib_util) -class SourceLoaderBadBytecodeTestPEP302( - SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): - pass - - -(Frozen_SourceBadBytecodePEP302, - Source_SourceBadBytecodePEP302 - ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) - - class SourcelessLoaderBadBytecodeTest: @classmethod @@ -779,17 +590,5 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, util=importlib_util) -class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, - BadBytecodeTestPEP302): - pass - - -(Frozen_SourcelessBadBytecodePEP302, - Source_SourcelessBadBytecodePEP302 - ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 4de736a6bf3..91865b997ad 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -57,6 +57,8 @@ def run_test(self, test, create=None, *, compile_=None, unlink=None): """ if create is None: create = {test} + if (compile_ or unlink) and sys.implementation.cache_tag is None: + raise unittest.SkipTest('requires sys.implementation.cache_tag') with util.create_modules(*create) as mapping: if compile_: for name in compile_: @@ -73,7 +75,7 @@ def run_test(self, test, create=None, *, compile_=None, unlink=None): if error.errno != errno.ENOENT: raise loader = self.import_(mapping['.root'], test) - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') return loader def test_module(self): @@ -100,7 +102,7 @@ def test_module_in_package(self): with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') # [sub package] def test_package_in_package(self): @@ -108,7 +110,7 @@ def test_package_in_package(self): with context as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') # [package over modules] def test_package_over_module(self): @@ -129,7 +131,7 @@ def test_empty_string_for_dir(self): file.write("# test file for importlib") try: loader = self._find(finder, 'mod', loader_only=True) - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') finally: os.unlink('mod.py') diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index c09c9aa12b8..fada07877fc 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -102,19 +102,6 @@ def load(self, loader): ) = util.test_both(EncodingTestPEP451, machinery=machinery) -class EncodingTestPEP302(EncodingTest): - - def load(self, loader): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return loader.load_module(self.module_name) - - -(Frozen_EncodingTestPEP302, - Source_EncodingTestPEP302 - ) = util.test_both(EncodingTestPEP302, machinery=machinery) - - class LineEndingTest: r"""Source written with the three types of line endings (\n, \r\n, \r) @@ -158,18 +145,5 @@ def load(self, loader, module_name): ) = util.test_both(LineEndingTestPEP451, machinery=machinery) -class LineEndingTestPEP302(LineEndingTest): - - def load(self, loader, module_name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return loader.load_module(module_name) - - -(Frozen_LineEndingTestPEP302, - Source_LineEndingTestPEP302 - ) = util.test_both(LineEndingTestPEP302, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/stubs.py deleted file mode 100644 index e5b011c399f..00000000000 --- a/Lib/test/test_importlib/stubs.py +++ /dev/null @@ -1,10 +0,0 @@ -import unittest - - -class fake_filesystem_unittest: - """ - Stubbed version of the pyfakefs module - """ - class TestCase(unittest.TestCase): - def setUpPyfakefs(self): - self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index dd943210ffc..b24c5766468 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -194,10 +194,6 @@ def test_create_module(self): spec = 'a spec' self.assertIsNone(self.ins.create_module(spec)) - def test_load_module(self): - with self.assertRaises(ImportError): - self.ins.load_module('something') - def test_module_repr(self): mod = types.ModuleType('blah') with warnings.catch_warnings(): @@ -358,38 +354,6 @@ def is_package(self, fullname): return SpecLoader() - def test_fresh(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.loader() - name = 'blah' - with test_util.uncache(name): - loader.load_module(name) - module = loader.found - self.assertIs(sys.modules[name], module) - self.assertEqual(loader, module.__loader__) - self.assertEqual(loader, module.__spec__.loader) - self.assertEqual(name, module.__name__) - self.assertEqual(name, module.__spec__.name) - self.assertIsNotNone(module.__path__) - self.assertIsNotNone(module.__path__, - module.__spec__.submodule_search_locations) - - def test_reload(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - name = 'blah' - loader = self.loader() - module = types.ModuleType(name) - module.__spec__ = self.util.spec_from_loader(name, loader) - module.__loader__ = loader - with test_util.uncache(name): - sys.modules[name] = module - loader.load_module(name) - found = loader.found - self.assertIs(found, sys.modules[name]) - self.assertIs(module, sys.modules[name]) - (Frozen_LoaderLoadModuleTests, Source_LoaderLoadModuleTests @@ -478,59 +442,6 @@ def test_get_code_source_not_found(self): InspectLoaderSubclass=SPLIT_IL) -class InspectLoaderLoadModuleTests: - - """Test InspectLoader.load_module().""" - - module_name = 'blah' - - def setUp(self): - import_helper.unload(self.module_name) - self.addCleanup(import_helper.unload, self.module_name) - - def load(self, loader): - spec = self.util.spec_from_loader(self.module_name, loader) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return self.init._bootstrap._load_unlocked(spec) - - def mock_get_code(self): - return mock.patch.object(self.InspectLoaderSubclass, 'get_code') - - def test_get_code_ImportError(self): - # If get_code() raises ImportError, it should propagate. - with self.mock_get_code() as mocked_get_code: - mocked_get_code.side_effect = ImportError - with self.assertRaises(ImportError): - loader = self.InspectLoaderSubclass() - self.load(loader) - - def test_get_code_None(self): - # If get_code() returns None, raise ImportError. - with self.mock_get_code() as mocked_get_code: - mocked_get_code.return_value = None - with self.assertRaises(ImportError): - loader = self.InspectLoaderSubclass() - self.load(loader) - - def test_module_returned(self): - # The loaded module should be returned. - code = compile('attr = 42', '', 'exec') - with self.mock_get_code() as mocked_get_code: - mocked_get_code.return_value = code - loader = self.InspectLoaderSubclass() - module = self.load(loader) - self.assertEqual(module, sys.modules[self.module_name]) - - -(Frozen_ILLoadModuleTests, - Source_ILLoadModuleTests - ) = test_util.test_both(InspectLoaderLoadModuleTests, - InspectLoaderSubclass=SPLIT_IL, - init=init, - util=util) - - ##### ExecutionLoader concrete methods ######################################### class ExecutionLoaderGetCodeTests: @@ -599,8 +510,7 @@ def test_get_code_no_path(self): class SourceOnlyLoader: # Globals that should be defined for all modules. - source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " - b"repr(__loader__)])") + source = (b"_ = '::'.join([__name__, __file__, __package__, repr(__loader__)])") def __init__(self, path): self.path = path @@ -623,7 +533,10 @@ class SourceLoader(SourceOnlyLoader): def __init__(self, path, magic=None): super().__init__(path) - self.bytecode_path = self.util.cache_from_source(self.path) + try: + self.bytecode_path = self.util.cache_from_source(self.path) + except NotImplementedError: + self.bytecode_path = None self.source_size = len(self.source) if magic is None: magic = self.util.MAGIC_NUMBER @@ -669,26 +582,26 @@ def setUp(self, *, is_package=True, **kwargs): module_name = 'mod' self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) self.name = '.'.join([self.package, module_name]) - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = self.loader_mock(self.path, **kwargs) def verify_module(self, module): self.assertEqual(module.__name__, self.name) self.assertEqual(module.__file__, self.path) - self.assertEqual(module.__cached__, self.cached) self.assertEqual(module.__package__, self.package) self.assertEqual(module.__loader__, self.loader) values = module._.split('::') self.assertEqual(values[0], self.name) self.assertEqual(values[1], self.path) - self.assertEqual(values[2], self.cached) - self.assertEqual(values[3], self.package) - self.assertEqual(values[4], repr(self.loader)) + self.assertEqual(values[2], self.package) + self.assertEqual(values[3], repr(self.loader)) def verify_code(self, code_object): module = types.ModuleType(self.name) module.__file__ = self.path - module.__cached__ = self.cached module.__package__ = self.package module.__loader__ = self.loader module.__path__ = [] @@ -729,34 +642,6 @@ def test_source_to_code(self): code = self.loader.source_to_code(self.loader.source, self.path) self.verify_code(code) - def test_load_module(self): - # Loading a module should set __name__, __loader__, __package__, - # __path__ (for packages), __file__, and __cached__. - # The module should also be put into sys.modules. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - with test_util.uncache(self.name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertEqual(module.__path__, [os.path.dirname(self.path)]) - self.assertIn(self.name, sys.modules) - - def test_package_settings(self): - # __package__ needs to be set, while __path__ is set on if the module - # is a package. - # Testing the values for a package are covered by test_load_module. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - self.setUp(is_package=False) - with test_util.uncache(self.name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertNotHasAttr(module, '__path__') - def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise # specified by an encoding line. @@ -777,6 +662,8 @@ def test_get_source_encoding(self): @unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") +@unittest.skipIf(sys.implementation.cache_tag is None, + "sys.implementation.cache_tag is None") class SourceLoaderBytecodeTests(SourceLoaderTestHarness): """Test importlib.abc.SourceLoader's use of bytecode. @@ -904,7 +791,7 @@ def test_universal_newlines(self): mock = self.SourceOnlyLoaderMock('mod.file') source = "x = 42\r\ny = -13\r\n" mock.source = source.encode('utf-8') - expect = io.IncrementalNewlineDecoder(None, True).decode(source) + expect = io.StringIO(source, newline=None).getvalue() self.assertEqual(mock.get_source(name), expect) diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 1bc531a2fe3..70d93d693ae 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -231,11 +231,9 @@ def test_reload_location_changed(self): # Start as a plain module. self.init.invalidate_caches() path = os.path.join(cwd, name + '.py') - cached = self.util.cache_from_source(path) expected = {'__name__': name, '__package__': '', '__file__': path, - '__cached__': cached, '__doc__': None, } os_helper.create_empty_file(path) @@ -252,11 +250,9 @@ def test_reload_location_changed(self): # Change to a package. self.init.invalidate_caches() init_path = os.path.join(cwd, name, '__init__.py') - cached = self.util.cache_from_source(init_path) expected = {'__name__': name, '__package__': name, '__file__': init_path, - '__cached__': cached, '__path__': [os.path.dirname(init_path)], '__doc__': None, } @@ -283,7 +279,6 @@ def test_reload_namespace_changed(self): # Start as a namespace package. self.init.invalidate_caches() bad_path = os.path.join(cwd, name, '__init.py') - cached = self.util.cache_from_source(bad_path) expected = {'__name__': name, '__package__': name, '__doc__': None, @@ -312,11 +307,9 @@ def test_reload_namespace_changed(self): # Change to a regular package. self.init.invalidate_caches() init_path = os.path.join(cwd, name, '__init__.py') - cached = self.util.cache_from_source(init_path) expected = {'__name__': name, '__package__': name, '__file__': init_path, - '__cached__': cached, '__path__': [os.path.dirname(init_path)], '__doc__': None, 'eggs': None, diff --git a/Lib/test/test_importlib/test_discover.py b/Lib/test/test_importlib/test_discover.py new file mode 100644 index 00000000000..c4ab7b6982e --- /dev/null +++ b/Lib/test/test_importlib/test_discover.py @@ -0,0 +1,121 @@ +from unittest.mock import Mock + +from test.test_importlib import util + +importlib = util.import_importlib('importlib') +machinery = util.import_importlib('importlib.machinery') + + +class DiscoverableFinder: + def __init__(self, discover=[]): + self._discovered_values = discover + + def find_spec(self, fullname, path=None, target=None): + raise NotImplementedError + + def discover(self, parent=None): + yield from self._discovered_values + + +class TestPathFinder: + """PathFinder implements MetaPathFinder, which uses the PathEntryFinder(s) + registered in sys.path_hooks (and sys.path_importer_cache) to search + sys.path or the parent's __path__. + + PathFinder.discover() should redirect to the .discover() method of the + PathEntryFinder for each path entry. + """ + + def test_search_path_hooks_top_level(self): + modules = [ + self.machinery.ModuleSpec(name='example1', loader=None), + self.machinery.ModuleSpec(name='example2', loader=None), + self.machinery.ModuleSpec(name='example3', loader=None), + ] + + with util.import_state( + path_importer_cache={ + 'discoverable': DiscoverableFinder(discover=modules), + }, + path=['discoverable'], + ): + discovered = list(self.machinery.PathFinder.discover()) + + self.assertEqual(discovered, modules) + + + def test_search_path_hooks_parent(self): + parent = self.machinery.ModuleSpec(name='example', loader=None, is_package=True) + parent.submodule_search_locations.append('discoverable') + + children = [ + self.machinery.ModuleSpec(name='example.child1', loader=None), + self.machinery.ModuleSpec(name='example.child2', loader=None), + self.machinery.ModuleSpec(name='example.child3', loader=None), + ] + + with util.import_state( + path_importer_cache={ + 'discoverable': DiscoverableFinder(discover=children) + }, + path=[], + ): + discovered = list(self.machinery.PathFinder.discover(parent)) + + self.assertEqual(discovered, children) + + def test_invalid_parent(self): + parent = self.machinery.ModuleSpec(name='example', loader=None) + with self.assertRaises(ValueError): + list(self.machinery.PathFinder.discover(parent)) + + +( + Frozen_TestPathFinder, + Source_TestPathFinder, +) = util.test_both(TestPathFinder, importlib=importlib, machinery=machinery) + + +class TestFileFinder: + """FileFinder implements PathEntryFinder and provides the base finder + implementation to search the file system. + """ + + def get_finder(self, path): + loader_details = [ + (self.machinery.SourceFileLoader, self.machinery.SOURCE_SUFFIXES), + (self.machinery.SourcelessFileLoader, self.machinery.BYTECODE_SUFFIXES), + ] + return self.machinery.FileFinder(path, *loader_details) + + def test_discover_top_level(self): + modules = {'example1', 'example2', 'example3'} + with util.create_modules(*modules) as mapping: + finder = self.get_finder(mapping['.root']) + discovered = list(finder.discover()) + self.assertEqual({spec.name for spec in discovered}, modules) + + def test_discover_parent(self): + modules = { + 'example.child1', + 'example.child2', + 'example.child3', + } + with util.create_modules(*modules) as mapping: + example = self.get_finder(mapping['.root']).find_spec('example') + finder = self.get_finder(example.submodule_search_locations[0]) + discovered = list(finder.discover(example)) + self.assertEqual({spec.name for spec in discovered}, modules) + + def test_invalid_parent(self): + with util.create_modules('example') as mapping: + finder = self.get_finder(mapping['.root']) + example = finder.find_spec('example') + with self.assertRaises(ValueError): + list(finder.discover(example)) + + +( + Frozen_TestFileFinder, + Source_TestFileFinder, +) = util.test_both(TestFileFinder, importlib=importlib, machinery=machinery) diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py deleted file mode 100644 index 81f683799cb..00000000000 --- a/Lib/test/test_importlib/test_main.py +++ /dev/null @@ -1,427 +0,0 @@ -import re -import pickle -import unittest -import warnings -import importlib.metadata -import contextlib -import itertools - -try: - import pyfakefs.fake_filesystem_unittest as ffs -except ImportError: - from .stubs import fake_filesystem_unittest as ffs - -from . import fixtures -from ._context import suppress -from importlib.metadata import ( - Distribution, - EntryPoint, - PackageNotFoundError, - _unique, - distributions, - entry_points, - metadata, - packages_distributions, - version, -) - - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx - - -class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): - version_pattern = r'\d+\.\d+(\.\d)?' - - def test_retrieves_version_of_self(self): - dist = Distribution.from_name('distinfo-pkg') - assert isinstance(dist.version, str) - assert re.match(self.version_pattern, dist.version) - - def test_for_name_does_not_exist(self): - with self.assertRaises(PackageNotFoundError): - Distribution.from_name('does-not-exist') - - def test_package_not_found_mentions_metadata(self): - """ - When a package is not found, that could indicate that the - package is not installed or that it is installed without - metadata. Ensure the exception mentions metadata to help - guide users toward the cause. See #124. - """ - with self.assertRaises(PackageNotFoundError) as ctx: - Distribution.from_name('does-not-exist') - - assert "metadata" in str(ctx.exception) - - # expected to fail until ABC is enforced - @suppress(AssertionError) - @suppress_known_deprecation() - def test_abc_enforced(self): - with self.assertRaises(TypeError): - type('DistributionSubclass', (Distribution,), {})() - - @fixtures.parameterize( - dict(name=None), - dict(name=''), - ) - def test_invalid_inputs_to_from_name(self, name): - with self.assertRaises(ValueError): - Distribution.from_name(name) - - -class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): - def test_import_nonexistent_module(self): - # Ensure that the MetadataPathFinder does not crash an import of a - # non-existent module. - with self.assertRaises(ImportError): - importlib.import_module('does_not_exist') - - def test_resolve(self): - ep = entry_points(group='entries')['main'] - self.assertEqual(ep.load().__name__, "main") - - def test_entrypoint_with_colon_in_name(self): - ep = entry_points(group='entries')['ns:sub'] - self.assertEqual(ep.value, 'mod:main') - - def test_resolve_without_attr(self): - ep = EntryPoint( - name='ep', - value='importlib.metadata', - group='grp', - ) - assert ep.load() is importlib.metadata - - -class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): - @staticmethod - def make_pkg(name): - """ - Create minimal metadata for a dist-info package with - the indicated name on the file system. - """ - return { - f'{name}.dist-info': { - 'METADATA': 'VERSION: 1.0\n', - }, - } - - def test_dashes_in_dist_name_found_as_underscores(self): - """ - For a package with a dash in the name, the dist-info metadata - uses underscores in the name. Ensure the metadata loads. - """ - fixtures.build_files(self.make_pkg('my_pkg'), self.site_dir) - assert version('my-pkg') == '1.0' - - def test_dist_name_found_as_any_case(self): - """ - Ensure the metadata loads when queried with any case. - """ - pkg_name = 'CherryPy' - fixtures.build_files(self.make_pkg(pkg_name), self.site_dir) - assert version(pkg_name) == '1.0' - assert version(pkg_name.lower()) == '1.0' - assert version(pkg_name.upper()) == '1.0' - - def test_unique_distributions(self): - """ - Two distributions varying only by non-normalized name on - the file system should resolve as the same. - """ - fixtures.build_files(self.make_pkg('abc'), self.site_dir) - before = list(_unique(distributions())) - - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) - self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) - fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) - after = list(_unique(distributions())) - - assert len(after) == len(before) - - -class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): - @staticmethod - def pkg_with_non_ascii_description(site_dir): - """ - Create minimal metadata for a package with non-ASCII in - the description. - """ - contents = { - 'portend.dist-info': { - 'METADATA': 'Description: pôrˈtend', - }, - } - fixtures.build_files(contents, site_dir) - return 'portend' - - @staticmethod - def pkg_with_non_ascii_description_egg_info(site_dir): - """ - Create minimal metadata for an egg-info package with - non-ASCII in the description. - """ - contents = { - 'portend.dist-info': { - 'METADATA': """ - Name: portend - - pôrˈtend""", - }, - } - fixtures.build_files(contents, site_dir) - return 'portend' - - def test_metadata_loads(self): - pkg_name = self.pkg_with_non_ascii_description(self.site_dir) - meta = metadata(pkg_name) - assert meta['Description'] == 'pôrˈtend' - - def test_metadata_loads_egg_info(self): - pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) - meta = metadata(pkg_name) - assert meta['Description'] == 'pôrˈtend' - - -class DiscoveryTests( - fixtures.EggInfoPkg, - fixtures.EggInfoPkgPipInstalledNoToplevel, - fixtures.EggInfoPkgPipInstalledNoModules, - fixtures.EggInfoPkgSourcesFallback, - fixtures.DistInfoPkg, - unittest.TestCase, -): - def test_package_discovery(self): - dists = list(distributions()) - assert all(isinstance(dist, Distribution) for dist in dists) - assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'egg_with_module-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'egg_with_no_modules-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'sources_fallback-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) - - def test_invalid_usage(self): - with self.assertRaises(ValueError): - list(distributions(context='something', name='else')) - - -class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): - def test_egg_info(self): - # make an `EGG-INFO` directory that's unrelated - self.site_dir.joinpath('EGG-INFO').mkdir() - # used to crash with `IsADirectoryError` - with self.assertRaises(PackageNotFoundError): - version('unknown-package') - - def test_egg(self): - egg = self.site_dir.joinpath('foo-3.6.egg') - egg.mkdir() - with self.add_sys_path(egg): - with self.assertRaises(PackageNotFoundError): - version('foo') - - -class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): - site_dir = '/does-not-exist' - - def test_discovery(self): - """ - Discovering distributions should succeed even if - there is an invalid path on sys.path. - """ - importlib.metadata.distributions() - - -class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): - site_dir = '/access-denied' - - def setUp(self): - super().setUp() - self.setUpPyfakefs() - self.fs.create_dir(self.site_dir, perm_bits=000) - - def test_discovery(self): - """ - Discovering distributions should succeed even if - there is an invalid path on sys.path. - """ - list(importlib.metadata.distributions()) - - -class TestEntryPoints(unittest.TestCase): - def __init__(self, *args): - super().__init__(*args) - self.ep = importlib.metadata.EntryPoint( - name='name', value='value', group='group' - ) - - def test_entry_point_pickleable(self): - revived = pickle.loads(pickle.dumps(self.ep)) - assert revived == self.ep - - def test_positional_args(self): - """ - Capture legacy (namedtuple) construction, discouraged. - """ - EntryPoint('name', 'value', 'group') - - def test_immutable(self): - """EntryPoints should be immutable""" - with self.assertRaises(AttributeError): - self.ep.name = 'badactor' - - def test_repr(self): - assert 'EntryPoint' in repr(self.ep) - assert 'name=' in repr(self.ep) - assert "'name'" in repr(self.ep) - - def test_hashable(self): - """EntryPoints should be hashable""" - hash(self.ep) - - def test_module(self): - assert self.ep.module == 'value' - - def test_attr(self): - assert self.ep.attr is None - - def test_sortable(self): - """ - EntryPoint objects are sortable, but result is undefined. - """ - sorted( - [ - EntryPoint(name='b', value='val', group='group'), - EntryPoint(name='a', value='val', group='group'), - ] - ) - - -class FileSystem( - fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase -): - def test_unicode_dir_on_sys_path(self): - """ - Ensure a Unicode subdirectory of a directory on sys.path - does not crash. - """ - fixtures.build_files( - {self.unicode_filename(): {}}, - prefix=self.site_dir, - ) - list(distributions()) - - -class PackagesDistributionsPrebuiltTest(fixtures.ZipFixtures, unittest.TestCase): - def test_packages_distributions_example(self): - self._fixture_on_path('example-21.12-py3-none-any.whl') - assert packages_distributions()['example'] == ['example'] - - def test_packages_distributions_example2(self): - """ - Test packages_distributions on a wheel built - by trampolim. - """ - self._fixture_on_path('example2-1.0.0-py3-none-any.whl') - assert packages_distributions()['example2'] == ['example2'] - - -class PackagesDistributionsTest( - fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase -): - def test_packages_distributions_neither_toplevel_nor_files(self): - """ - Test a package built without 'top-level.txt' or a file list. - """ - fixtures.build_files( - { - 'trim_example-1.0.0.dist-info': { - 'METADATA': """ - Name: trim_example - Version: 1.0.0 - """, - } - }, - prefix=self.site_dir, - ) - packages_distributions() - - def test_packages_distributions_all_module_types(self): - """ - Test top-level modules detected on a package without 'top-level.txt'. - """ - suffixes = importlib.machinery.all_suffixes() - metadata = dict( - METADATA=""" - Name: all_distributions - Version: 1.0.0 - """, - ) - files = { - 'all_distributions-1.0.0.dist-info': metadata, - } - for i, suffix in enumerate(suffixes): - files.update( - { - f'importable-name {i}{suffix}': '', - f'in_namespace_{i}': { - f'mod{suffix}': '', - }, - f'in_package_{i}': { - '__init__.py': '', - f'mod{suffix}': '', - }, - } - ) - metadata.update(RECORD=fixtures.build_record(files)) - fixtures.build_files(files, prefix=self.site_dir) - - distributions = packages_distributions() - - for i in range(len(suffixes)): - assert distributions[f'importable-name {i}'] == ['all_distributions'] - assert distributions[f'in_namespace_{i}'] == ['all_distributions'] - assert distributions[f'in_package_{i}'] == ['all_distributions'] - - assert not any(name.endswith('.dist-info') for name in distributions) - - -class PackagesDistributionsEggTest( - fixtures.EggInfoPkg, - fixtures.EggInfoPkgPipInstalledNoToplevel, - fixtures.EggInfoPkgPipInstalledNoModules, - fixtures.EggInfoPkgSourcesFallback, - unittest.TestCase, -): - def test_packages_distributions_on_eggs(self): - """ - Test old-style egg packages with a variation of 'top_level.txt', - 'SOURCES.txt', and 'installed-files.txt', available. - """ - distributions = packages_distributions() - - def import_names_from_package(package_name): - return { - import_name - for import_name, package_names in distributions.items() - if package_name in package_names - } - - # egginfo-pkg declares one import ('mod') via top_level.txt - assert import_names_from_package('egginfo-pkg') == {'mod'} - - # egg_with_module-pkg has one import ('egg_with_module') inferred from - # installed-files.txt (top_level.txt is missing) - assert import_names_from_package('egg_with_module-pkg') == {'egg_with_module'} - - # egg_with_no_modules-pkg should not be associated with any import names - # (top_level.txt is empty, and installed-files.txt has no .py files) - assert import_names_from_package('egg_with_no_modules-pkg') == set() - - # sources_fallback-pkg has one import ('sources_fallback') inferred from - # SOURCES.txt (top_level.txt and installed-files.txt is missing) - assert import_names_from_package('sources_fallback-pkg') == {'sources_fallback'} diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py deleted file mode 100644 index 33c6e85ee94..00000000000 --- a/Lib/test/test_importlib/test_metadata_api.py +++ /dev/null @@ -1,322 +0,0 @@ -import re -import textwrap -import unittest -import warnings -import importlib -import contextlib - -from . import fixtures -from importlib.metadata import ( - Distribution, - PackageNotFoundError, - distribution, - entry_points, - files, - metadata, - requires, - version, -) - - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx - - -class APITests( - fixtures.EggInfoPkg, - fixtures.EggInfoPkgPipInstalledNoToplevel, - fixtures.EggInfoPkgPipInstalledNoModules, - fixtures.EggInfoPkgSourcesFallback, - fixtures.DistInfoPkg, - fixtures.DistInfoPkgWithDot, - fixtures.EggInfoFile, - unittest.TestCase, -): - version_pattern = r'\d+\.\d+(\.\d)?' - - def test_retrieves_version_of_self(self): - pkg_version = version('egginfo-pkg') - assert isinstance(pkg_version, str) - assert re.match(self.version_pattern, pkg_version) - - def test_retrieves_version_of_distinfo_pkg(self): - pkg_version = version('distinfo-pkg') - assert isinstance(pkg_version, str) - assert re.match(self.version_pattern, pkg_version) - - def test_for_name_does_not_exist(self): - with self.assertRaises(PackageNotFoundError): - distribution('does-not-exist') - - def test_name_normalization(self): - names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' - for name in names: - with self.subTest(name): - assert distribution(name).metadata['Name'] == 'pkg.dot' - - def test_prefix_not_matched(self): - prefixes = 'p', 'pkg', 'pkg.' - for prefix in prefixes: - with self.subTest(prefix): - with self.assertRaises(PackageNotFoundError): - distribution(prefix) - - def test_for_top_level(self): - tests = [ - ('egginfo-pkg', 'mod'), - ('egg_with_no_modules-pkg', ''), - ] - for pkg_name, expect_content in tests: - with self.subTest(pkg_name): - self.assertEqual( - distribution(pkg_name).read_text('top_level.txt').strip(), - expect_content, - ) - - def test_read_text(self): - tests = [ - ('egginfo-pkg', 'mod\n'), - ('egg_with_no_modules-pkg', '\n'), - ] - for pkg_name, expect_content in tests: - with self.subTest(pkg_name): - top_level = [ - path for path in files(pkg_name) if path.name == 'top_level.txt' - ][0] - self.assertEqual(top_level.read_text(), expect_content) - - def test_entry_points(self): - eps = entry_points() - assert 'entries' in eps.groups - entries = eps.select(group='entries') - assert 'main' in entries.names - ep = entries['main'] - self.assertEqual(ep.value, 'mod:main') - self.assertEqual(ep.extras, []) - - def test_entry_points_distribution(self): - entries = entry_points(group='entries') - for entry in ("main", "ns:sub"): - ep = entries[entry] - self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) - self.assertEqual(ep.dist.version, "1.0.0") - - def test_entry_points_unique_packages_normalized(self): - """ - Entry points should only be exposed for the first package - on sys.path with a given name (even when normalized). - """ - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) - self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) - alt_pkg = { - "DistInfo_pkg-1.1.0.dist-info": { - "METADATA": """ - Name: distinfo-pkg - Version: 1.1.0 - """, - "entry_points.txt": """ - [entries] - main = mod:altmain - """, - }, - } - fixtures.build_files(alt_pkg, alt_site_dir) - entries = entry_points(group='entries') - assert not any( - ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' - for ep in entries - ) - # ns:sub doesn't exist in alt_pkg - assert 'ns:sub' not in entries.names - - def test_entry_points_missing_name(self): - with self.assertRaises(KeyError): - entry_points(group='entries')['missing'] - - def test_entry_points_missing_group(self): - assert entry_points(group='missing') == () - - def test_entry_points_allows_no_attributes(self): - ep = entry_points().select(group='entries', name='main') - with self.assertRaises(AttributeError): - ep.foo = 4 - - def test_metadata_for_this_package(self): - md = metadata('egginfo-pkg') - assert md['author'] == 'Steven Ma' - assert md['LICENSE'] == 'Unknown' - assert md['Name'] == 'egginfo-pkg' - classifiers = md.get_all('Classifier') - assert 'Topic :: Software Development :: Libraries' in classifiers - - def test_missing_key_legacy(self): - """ - Requesting a missing key will still return None, but warn. - """ - md = metadata('distinfo-pkg') - with suppress_known_deprecation(): - assert md['does-not-exist'] is None - - def test_get_key(self): - """ - Getting a key gets the key. - """ - md = metadata('egginfo-pkg') - assert md.get('Name') == 'egginfo-pkg' - - def test_get_missing_key(self): - """ - Requesting a missing key will return None. - """ - md = metadata('distinfo-pkg') - assert md.get('does-not-exist') is None - - @staticmethod - def _test_files(files): - root = files[0].root - for file in files: - assert file.root == root - assert not file.hash or file.hash.value - assert not file.hash or file.hash.mode == 'sha256' - assert not file.size or file.size >= 0 - assert file.locate().exists() - assert isinstance(file.read_binary(), bytes) - if file.name.endswith('.py'): - file.read_text() - - def test_file_hash_repr(self): - util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0] - self.assertRegex(repr(util.hash), '') - - def test_files_dist_info(self): - self._test_files(files('distinfo-pkg')) - - def test_files_egg_info(self): - self._test_files(files('egginfo-pkg')) - self._test_files(files('egg_with_module-pkg')) - self._test_files(files('egg_with_no_modules-pkg')) - self._test_files(files('sources_fallback-pkg')) - - def test_version_egg_info_file(self): - self.assertEqual(version('egginfo-file'), '0.1') - - def test_requires_egg_info_file(self): - requirements = requires('egginfo-file') - self.assertIsNone(requirements) - - def test_requires_egg_info(self): - deps = requires('egginfo-pkg') - assert len(deps) == 2 - assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps) - - def test_requires_egg_info_empty(self): - fixtures.build_files( - { - 'requires.txt': '', - }, - self.site_dir.joinpath('egginfo_pkg.egg-info'), - ) - deps = requires('egginfo-pkg') - assert deps == [] - - def test_requires_dist_info(self): - deps = requires('distinfo-pkg') - assert len(deps) == 2 - assert all(deps) - assert 'wheel >= 1.0' in deps - assert "pytest; extra == 'test'" in deps - - def test_more_complex_deps_requires_text(self): - requires = textwrap.dedent( - """ - dep1 - dep2 - - [:python_version < "3"] - dep3 - - [extra1] - dep4 - dep6@ git+https://example.com/python/dep.git@v1.0.0 - - [extra2:python_version < "3"] - dep5 - """ - ) - deps = sorted(Distribution._deps_from_requires_text(requires)) - expected = [ - 'dep1', - 'dep2', - 'dep3; python_version < "3"', - 'dep4; extra == "extra1"', - 'dep5; (python_version < "3") and extra == "extra2"', - 'dep6@ git+https://example.com/python/dep.git@v1.0.0 ; extra == "extra1"', - ] - # It's important that the environment marker expression be - # wrapped in parentheses to avoid the following 'and' binding more - # tightly than some other part of the environment expression. - - assert deps == expected - - def test_as_json(self): - md = metadata('distinfo-pkg').json - assert 'name' in md - assert md['keywords'] == ['sample', 'package'] - desc = md['description'] - assert desc.startswith('Once upon a time\nThere was') - assert len(md['requires_dist']) == 2 - - def test_as_json_egg_info(self): - md = metadata('egginfo-pkg').json - assert 'name' in md - assert md['keywords'] == ['sample', 'package'] - desc = md['description'] - assert desc.startswith('Once upon a time\nThere was') - assert len(md['classifier']) == 2 - - def test_as_json_odd_case(self): - self.make_uppercase() - md = metadata('distinfo-pkg').json - assert 'name' in md - assert len(md['requires_dist']) == 2 - assert md['keywords'] == ['SAMPLE', 'PACKAGE'] - - -class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): - def test_name_normalization(self): - names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' - for name in names: - with self.subTest(name): - assert distribution(name).metadata['Name'] == 'pkg.dot' - - def test_name_normalization_versionless_egg_info(self): - names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot' - for name in names: - with self.subTest(name): - assert distribution(name).metadata['Name'] == 'pkg.lot' - - -class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): - def test_find_distributions_specified_path(self): - dists = Distribution.discover(path=[str(self.site_dir)]) - assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) - - def test_distribution_at_pathlib(self): - """Demonstrate how to load metadata direct from a directory.""" - dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' - dist = Distribution.at(dist_info_path) - assert dist.version == '1.0.0' - - def test_distribution_at_str(self): - dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' - dist = Distribution.at(str(dist_info_path)) - assert dist.version == '1.0.0' - - -class InvalidateCache(unittest.TestCase): - def test_invalidate_cache(self): - # No externally observable behavior, but ensures test coverage... - importlib.invalidate_caches() diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 6ca0978f9bc..522ed9fa43f 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -318,6 +318,17 @@ def test_module_before_namespace_package(self): self.assertEqual(a_test.attr, 'in module') +class NamespaceSubpackageSameName(NamespacePackageTest): + paths = [''] + + def test_namespace_subpackage_shares_name_with_directory(self): + submodule_path = 'project4.foo' + with self.assertRaises(ModuleNotFoundError) as cm: + importlib.machinery.PathFinder.find_spec(submodule_path) + + self.assertEqual(cm.exception.name, 'project4') + + class ReloadTests(NamespacePackageTest): paths = ['portion1'] diff --git a/Lib/test/test_importlib/test_pkg_import.py b/Lib/test/test_importlib/test_pkg_import.py index 5ffae6222ba..287684efc85 100644 --- a/Lib/test/test_importlib/test_pkg_import.py +++ b/Lib/test/test_importlib/test_pkg_import.py @@ -39,9 +39,12 @@ def tearDown(self): self.remove_modules() def rewrite_file(self, contents): - compiled_path = cache_from_source(self.module_path) - if os.path.exists(compiled_path): - os.remove(compiled_path) + try: + compiled_path = cache_from_source(self.module_path) + if os.path.exists(compiled_path): + os.remove(compiled_path) + except NotImplementedError: + pass with open(self.module_path, 'w', encoding='utf-8') as f: f.write(contents) diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index aebeabaf83f..77b6228940c 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -52,7 +52,10 @@ class ModuleSpecTests: def setUp(self): self.name = 'spam' self.path = 'spam.py' - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = TestLoader() self.spec = self.machinery.ModuleSpec(self.name, self.loader) self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader, @@ -184,6 +187,8 @@ def test_cached_with_origin_not_location(self): self.assertIs(spec.cached, None) + @unittest.skipIf(sys.implementation.cache_tag is None, + "sys.implementation.cache_tag is None") def test_cached_source(self): expected = self.util.cache_from_source(self.path) @@ -224,7 +229,10 @@ def bootstrap(self): def setUp(self): self.name = 'spam' self.path = 'spam.py' - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = TestLoader() self.spec = self.machinery.ModuleSpec(self.name, self.loader) self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader, @@ -287,20 +295,6 @@ def exec_module(self, module): loaded = self.bootstrap._load(self.spec) self.assertNotIn(self.spec.name, sys.modules) - def test_load_legacy_attributes_immutable(self): - module = object() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - class ImmutableLoader(TestLoader): - def load_module(self, name): - sys.modules[name] = module - return module - self.spec.loader = ImmutableLoader() - with CleanImport(self.spec.name): - loaded = self.bootstrap._load(self.spec) - - self.assertIs(sys.modules[self.spec.name], module) - # reload() def test_reload(self): @@ -350,7 +344,6 @@ def test_reload_init_module_attrs(self): self.assertIs(loaded.__spec__, self.spec) self.assertNotHasAttr(loaded, '__path__') self.assertNotHasAttr(loaded, '__file__') - self.assertNotHasAttr(loaded, '__cached__') (Frozen_ModuleSpecMethodsTests, @@ -364,7 +357,10 @@ class FactoryTests: def setUp(self): self.name = 'spam' self.path = os.path.abspath('spam.py') - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = TestLoader() self.fileloader = TestLoader(self.path) self.pkgloader = TestLoader(self.path, True) diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 0e0d0721eb5..f68f1d83af7 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -326,6 +326,143 @@ def do_import(delay=0): # Neither thread should have errors about stale modules self.assertEqual(errors, [], f"Race condition detected: {errors}") + def test_hierarchical_import_deadlock(self): + # Regression test for bpo-38884 / gh-83065 + # Tests that concurrent imports at different hierarchy levels + # don't deadlock when parent imports child in __init__.py + + # Create package structure: + # package/__init__.py: from package import subpackage + # package/subpackage/__init__.py: from package.subpackage.module import * + # package/subpackage/module.py: class SomeClass: pass + + pkg_dir = os.path.join(TESTFN, 'hier_deadlock_pkg') + os.makedirs(pkg_dir) + self.addCleanup(shutil.rmtree, TESTFN) + + subpkg_dir = os.path.join(pkg_dir, 'subpackage') + os.makedirs(subpkg_dir) + + with open(os.path.join(pkg_dir, "__init__.py"), "w") as f: + f.write("from hier_deadlock_pkg import subpackage\n") + + with open(os.path.join(subpkg_dir, "__init__.py"), "w") as f: + f.write("from hier_deadlock_pkg.subpackage.module import *\n") + + with open(os.path.join(subpkg_dir, "module.py"), "w") as f: + f.write("class SomeClass:\n pass\n") + + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + self.addCleanup(forget, 'hier_deadlock_pkg') + self.addCleanup(forget, 'hier_deadlock_pkg.subpackage') + self.addCleanup(forget, 'hier_deadlock_pkg.subpackage.module') + + importlib.invalidate_caches() + + errors = [] + results = [] + barrier = threading.Barrier(2) + + def t1(): + barrier.wait() + try: + import hier_deadlock_pkg.subpackage + results.append('t1_success') + except Exception as e: + errors.append(('t1', type(e).__name__, str(e))) + + def t2(): + barrier.wait() + try: + import hier_deadlock_pkg.subpackage.module + results.append('t2_success') + except Exception as e: + errors.append(('t2', type(e).__name__, str(e))) + + # Run multiple times to increase chance of hitting race condition + for i in range(10): + for mod in ['hier_deadlock_pkg', 'hier_deadlock_pkg.subpackage', + 'hier_deadlock_pkg.subpackage.module']: + sys.modules.pop(mod, None) + + errors.clear() + results.clear() + barrier.reset() + + thread1 = threading.Thread(target=t1) + thread2 = threading.Thread(target=t2) + + thread1.start() + thread2.start() + + thread1.join(timeout=5) + thread2.join(timeout=5) + + if thread1.is_alive() or thread2.is_alive(): + self.fail(f"Threads deadlocked on iteration {i}") + + self.assertEqual( + errors, [], + f"Import(s) failed on iteration {i}: {errors}") + self.assertEqual( + sorted(results), ['t1_success', 't2_success'], + f"Not all imports succeeded on iteration {i}: {results}") + + def test_cross_package_circular_import(self): + # Two packages whose __init__.py each import a submodule of the + # other. Concurrent imports of submodules of each must not raise + # _DeadlockError; the import system accepts a partially-initialised + # parent in this case (see _lock_unlock_module). + os.makedirs(os.path.join(TESTFN, "circ_a")) + os.makedirs(os.path.join(TESTFN, "circ_b")) + self.addCleanup(shutil.rmtree, TESTFN) + with open(os.path.join(TESTFN, "circ_a", "__init__.py"), "w") as f: + f.write("import time; time.sleep(0.03)\nimport circ_b.other\n") + with open(os.path.join(TESTFN, "circ_b", "__init__.py"), "w") as f: + f.write("import time; time.sleep(0.03)\nimport circ_a.other\n") + for pkg in ("circ_a", "circ_b"): + for mod in ("sub.py", "other.py"): + with open(os.path.join(TESTFN, pkg, mod), "w") as f: + f.write("X = 1\n") + + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + for mod in ("circ_a", "circ_a.sub", "circ_a.other", + "circ_b", "circ_b.sub", "circ_b.other"): + self.addCleanup(forget, mod) + importlib.invalidate_caches() + + errors = [] + barrier = threading.Barrier(2) + + def do_import(name): + barrier.wait() + try: + importlib.import_module(name) + except Exception as e: + errors.append((name, type(e).__name__, str(e))) + + for i in range(10): + for mod in ("circ_a", "circ_a.sub", "circ_a.other", + "circ_b", "circ_b.sub", "circ_b.other"): + sys.modules.pop(mod, None) + errors.clear() + barrier.reset() + + thread1 = threading.Thread(target=do_import, args=("circ_a.sub",)) + thread2 = threading.Thread(target=do_import, args=("circ_b.sub",)) + thread1.start() + thread2.start() + thread1.join(timeout=5) + thread2.join(timeout=5) + + if thread1.is_alive() or thread2.is_alive(): + self.fail(f"Threads deadlocked on iteration {i}") + self.assertEqual( + errors, [], + f"Import(s) failed on iteration {i}: {errors}") + def setUpModule(): thread_info = threading_helper.threading_setup() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 8c14b96271a..8a911c1b751 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -124,12 +124,6 @@ def test___file__(self): module = self.util.module_from_spec(spec) self.assertEqual(module.__file__, spec.origin) - def test___cached__(self): - spec = self.machinery.ModuleSpec('test', object()) - spec.cached = 'some/path' - spec.has_location = True - module = self.util.module_from_spec(spec) - self.assertEqual(module.__cached__, spec.cached) (Frozen_ModuleFromSpecTests, Source_ModuleFromSpecTests @@ -334,6 +328,17 @@ class PEP3147Tests: tag = sys.implementation.cache_tag + def setUp(self): + # Most of these tests assume the default (unset) pycache prefix, so + # clear it for the duration of the test (e.g. when the test suite is + # run with PYTHONPYCACHEPREFIX set). Tests that need a specific prefix + # set their own via util.temporary_pycache_prefix(). + self._orig_pycache_prefix = sys.pycache_prefix + sys.pycache_prefix = None + + def tearDown(self): + sys.pycache_prefix = self._orig_pycache_prefix + @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag not be None') def test_cache_from_source(self): @@ -351,6 +356,8 @@ def test_cache_from_source_no_cache_tag(self): with self.assertRaises(NotImplementedError): self.util.cache_from_source('whatever.py') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_no_dot(self): # Directory with a dot, filename without dot. path = os.path.join('foo.bar', 'file') @@ -359,47 +366,16 @@ def test_cache_from_source_no_dot(self): self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) - def test_cache_from_source_debug_override(self): - # Given the path to a .py file, return the path to its PEP 3147/PEP 488 - # defined .pyc file (i.e. under __pycache__). - path = os.path.join('foo', 'bar', 'baz', 'qux.py') - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self.assertEqual(self.util.cache_from_source(path, False), - self.util.cache_from_source(path, optimization=1)) - self.assertEqual(self.util.cache_from_source(path, True), - self.util.cache_from_source(path, optimization='')) - with warnings.catch_warnings(): - warnings.simplefilter('error') - with self.assertRaises(DeprecationWarning): - self.util.cache_from_source(path, False) - with self.assertRaises(DeprecationWarning): - self.util.cache_from_source(path, True) - + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_cwd(self): path = 'foo.py' expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag)) self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) - def test_cache_from_source_override(self): - # When debug_override is not None, it can be any true-ish or false-ish - # value. - path = os.path.join('foo', 'bar', 'baz.py') - # However if the bool-ishness can't be determined, the exception - # propagates. - class Bearish: - def __bool__(self): raise RuntimeError - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self.assertEqual(self.util.cache_from_source(path, []), - self.util.cache_from_source(path, optimization=1)) - self.assertEqual(self.util.cache_from_source(path, [17]), - self.util.cache_from_source(path, optimization='')) - with self.assertRaises(RuntimeError): - self.util.cache_from_source('/foo/bar/baz.py', Bearish()) - - + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_optimization_empty_string(self): # Setting 'optimization' to '' leads to no optimization tag (PEP 488). path = 'foo.py' @@ -407,6 +383,8 @@ def test_cache_from_source_optimization_empty_string(self): self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_optimization_None(self): # Setting 'optimization' to None uses the interpreter's optimization. # (PEP 488) @@ -423,6 +401,8 @@ def test_cache_from_source_optimization_None(self): self.assertEqual(self.util.cache_from_source(path, optimization=None), expect) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_optimization_set(self): # The 'optimization' parameter accepts anything that has a string repr # that passes str.alnum(). @@ -440,6 +420,8 @@ def test_cache_from_source_optimization_set(self): with self.assertRaises(ValueError): self.util.cache_from_source(path, optimization='path/is/bad') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_debug_override_optimization_both_set(self): # Can only set one of the optimization-related parameters. with warnings.catch_warnings(): @@ -449,6 +431,8 @@ def test_cache_from_source_debug_override_optimization_both_set(self): @unittest.skipUnless(os.sep == '\\' and os.altsep == '/', 'test meaningful only where os.altsep is defined') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_sep_altsep_and_sep_cache_from_source(self): # Windows path and PEP 3147 where sep is right of altsep. self.assertEqual( @@ -481,44 +465,60 @@ def test_source_from_cache_no_cache_tag(self): with self.assertRaises(NotImplementedError): self.util.source_from_cache(path) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_bad_path(self): # When the path to a pyc file is not in PEP 3147 format, a ValueError # is raised. self.assertRaises( ValueError, self.util.source_from_cache, '/foo/bar/bazqux.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_no_slash(self): # No slashes at all in path -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, 'foo.cpython-32.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_too_few_dots(self): # Too few dots in final path component -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, '__pycache__/foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_too_many_dots(self): with self.assertRaises(ValueError): self.util.source_from_cache( '__pycache__/foo.cpython-32.opt-1.foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_not_opt(self): # Non-`opt-` path component -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, '__pycache__/foo.cpython-32.foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_no__pycache__(self): # Another problem with the path -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, '/foo/bar/foo.cpython-32.foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_optimized_bytecode(self): # Optimized bytecode is not an issue. path = os.path.join('__pycache__', 'foo.{}.opt-1.pyc'.format(self.tag)) self.assertEqual(self.util.source_from_cache(path), 'foo.py') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_missing_optimization(self): # An empty optimization level is a no-no. path = os.path.join('__pycache__', 'foo.{}.opt-.pyc'.format(self.tag)) @@ -633,7 +633,7 @@ class MagicNumberTests(unittest.TestCase): 'only applies to candidate or final python release levels' ) def test_magic_number(self): - # Each python minor release should generally have a MAGIC_NUMBER + # Each Python feature release should generally have a MAGIC_NUMBER # that does not change once the release reaches candidate status. # Once a release reaches candidate status, the value of the constant @@ -643,11 +643,11 @@ def test_magic_number(self): # In exceptional cases, it may be required to change the MAGIC_NUMBER # for a maintenance release. In this case the change should be - # discussed in python-dev. If a change is required, community + # discussed in on Discourse. If a change is required, community # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3627 + EXPECTED_MAGIC_NUMBER = 3666 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( @@ -659,7 +659,7 @@ def test_magic_number(self): "magic number in this test to the current MAGIC_NUMBER to " "continue with the release.\n\n" "Changing the MAGIC_NUMBER for a maintenance release " - "requires discussion in python-dev and notification of " + "requires discussion on Discourse and notification of " "community stakeholders." ) self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/test_zip.py deleted file mode 100644 index 276f6288c91..00000000000 --- a/Lib/test/test_importlib/test_zip.py +++ /dev/null @@ -1,62 +0,0 @@ -import sys -import unittest - -from . import fixtures -from importlib.metadata import ( - PackageNotFoundError, - distribution, - distributions, - entry_points, - files, - version, -) - - -class TestZip(fixtures.ZipFixtures, unittest.TestCase): - def setUp(self): - super().setUp() - self._fixture_on_path('example-21.12-py3-none-any.whl') - - def test_zip_version(self): - self.assertEqual(version('example'), '21.12') - - def test_zip_version_does_not_match(self): - with self.assertRaises(PackageNotFoundError): - version('definitely-not-installed') - - def test_zip_entry_points(self): - scripts = entry_points(group='console_scripts') - entry_point = scripts['example'] - self.assertEqual(entry_point.value, 'example:main') - entry_point = scripts['Example'] - self.assertEqual(entry_point.value, 'example:main') - - def test_missing_metadata(self): - self.assertIsNone(distribution('example').read_text('does not exist')) - - def test_case_insensitive(self): - self.assertEqual(version('Example'), '21.12') - - def test_files(self): - for file in files('example'): - path = str(file.dist.locate_file(file)) - assert '.whl/' in path, path - - def test_one_distribution(self): - dists = list(distributions(path=sys.path[:1])) - assert len(dists) == 1 - - -class TestEgg(TestZip): - def setUp(self): - super().setUp() - self._fixture_on_path('example-21.12-py3.6.egg') - - def test_files(self): - for file in files('example'): - path = str(file.dist.locate_file(file)) - assert '.egg/' in path, path - - def test_normalized_name(self): - dist = distribution('example') - assert dist._normalized_name == 'example' diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 2eaefd68747..33f82e77396 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -294,6 +294,9 @@ def writes_bytecode_files(fxn): tests that require it to be set to False.""" if sys.dont_write_bytecode: return unittest.skip("relies on writing bytecode")(fxn) + if sys.implementation.cache_tag is None: + return unittest.skip("requires sys.implementation.cache_tag to not be None")(fxn) + @functools.wraps(fxn) def wrapper(*args, **kwargs): original = sys.dont_write_bytecode diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739f8..00000000000 Binary files a/Lib/test/test_importlib/zipdata01/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d28..00000000000 Binary files a/Lib/test/test_importlib/zipdata02/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_inspect/inspect_fodder3.py b/Lib/test/test_inspect/inspect_fodder3.py new file mode 100644 index 00000000000..ea2481edf93 --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder3.py @@ -0,0 +1,39 @@ +from functools import cached_property + +# docstring in parent, inherited in child +class ParentInheritDoc: + @cached_property + def foo(self): + """docstring for foo defined in parent""" + +class ChildInheritDoc(ParentInheritDoc): + pass + +class ChildInheritDefineDoc(ParentInheritDoc): + @cached_property + def foo(self): + pass + +# Redefine foo as something other than cached_property +class ChildPropertyFoo(ParentInheritDoc): + @property + def foo(self): + """docstring for the property foo""" + +class ChildMethodFoo(ParentInheritDoc): + def foo(self): + """docstring for the method foo""" + +# docstring in child but not parent +class ParentNoDoc: + @cached_property + def foo(self): + pass + +class ChildNoDoc(ParentNoDoc): + pass + +class ChildDefineDoc(ParentNoDoc): + @cached_property + def foo(self): + """docstring for foo defined in child""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c7c9a2930ad..41fc33b7c89 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -47,6 +47,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_fodder3 as mod3 from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_deferred_annotations @@ -173,6 +174,15 @@ def __get__(self, instance, owner): return self.func.__get__(instance, owner) +class TestImportTime(unittest.TestCase): + + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports( + "inspect", {"re", "tokenize"} + ) + + class TestPredicates(IsTestBase): def test_excluding_predicates(self): @@ -689,10 +699,56 @@ def test_getdoc_inherited(self): self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited_class_doc(self): + class A: + """Common base class""" + class B(A): + pass + + a = A() + self.assertEqual(inspect.getdoc(A), 'Common base class') + self.assertEqual(inspect.getdoc(A, inherit_class_doc=False), + 'Common base class') + self.assertEqual(inspect.getdoc(a), 'Common base class') + self.assertIsNone(inspect.getdoc(a, fallback_to_class_doc=False)) + a.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(a, fallback_to_class_doc=False), + 'Instance') + + b = B() + self.assertEqual(inspect.getdoc(B), 'Common base class') + self.assertIsNone(inspect.getdoc(B, inherit_class_doc=False)) + self.assertIsNone(inspect.getdoc(b)) + self.assertIsNone(inspect.getdoc(b, fallback_to_class_doc=False)) + b.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + + def test_getdoc_inherited_cached_property(self): + doc = inspect.getdoc(mod3.ParentInheritDoc.foo) + self.assertEqual(doc, 'docstring for foo defined in parent') + self.assertEqual(inspect.getdoc(mod3.ChildInheritDoc.foo), doc) + self.assertEqual(inspect.getdoc(mod3.ChildInheritDefineDoc.foo), doc) + + def test_getdoc_redefine_cached_property_as_other(self): + self.assertEqual(inspect.getdoc(mod3.ChildPropertyFoo.foo), + 'docstring for the property foo') + self.assertEqual(inspect.getdoc(mod3.ChildMethodFoo.foo), + 'docstring for the method foo') + + def test_getdoc_define_cached_property(self): + self.assertEqual(inspect.getdoc(mod3.ChildDefineDoc.foo), + 'docstring for foo defined in child') + + def test_getdoc_nodoc_inherited(self): + self.assertIsNone(inspect.getdoc(mod3.ChildNoDoc.foo)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc self.assertEqual(finddoc(int), int.__doc__) + self.assertIsNone(finddoc(int, search_in_class=False)) self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) @@ -1300,9 +1356,10 @@ def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, posonlyargs_e=[], kwonlyargs_e=[], kwonlydefaults_e=None, - ann_e={}): + ann_e={}, + annotation_format=Format.VALUE): args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - inspect.getfullargspec(routine) + inspect.getfullargspec(routine, annotation_format=annotation_format) self.assertEqual(args, args_e) self.assertEqual(varargs, varargs_e) self.assertEqual(varkw, varkw_e) @@ -1335,6 +1392,19 @@ def test_getfullargspec(self): kwonlyargs_e=['e', 'f'], kwonlydefaults_e={'e': 4, 'f': 5}) + def get_getfullargspec_with_undefined_names_in_annotations(self): + def my_func(a: undefined_name): + pass + + with self.assertRaises(NameError): + inspect.getfullargspec(my_func) + + self.assertFullArgSpecEquals(my_func, ['a'], ann_e={'a': 'undefined_name'}, + annotation_format=Format.STRING) + + arg_spec = inspect.getfullargspec(my_func, annotation_format=Format.FORWARDREF) + self.assertIsInstance(arg_spec.annotations['a'], ForwardRef) + def test_argspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) @@ -6081,7 +6151,8 @@ def _test_builtin_methods_have_signatures(self, cls, no_signature, unsupported_s self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) def test_builtins_have_signatures(self): - no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + no_signature = {'type', 'super', 'bytearray', 'bytes', + 'dict', 'frozendict', 'int', 'str'} # These need PEP 457 groups needs_groups = {"range", "slice", "dir", "getattr", "next", "iter", "vars"} @@ -6113,6 +6184,7 @@ def test_types_module_has_signatures(self): 'AsyncGeneratorType': {'athrow'}, 'CoroutineType': {'throw'}, 'GeneratorType': {'throw'}, + 'FrameLocalsProxyType': {'setdefault', 'pop', 'get'}, } self._test_module_has_signatures(types, unsupported_signature=unsupported_signature, @@ -6157,6 +6229,21 @@ def test_collections_abc_module_has_signatures(self): import collections.abc self._test_module_has_signatures(collections.abc) + def test_datetime_module_has_signatures(self): + # Only test if the C implementation is available. + import_helper.import_module('_datetime') + import datetime + no_signature = {'tzinfo'} + unsupported_signature = {'timezone'} + methods_unsupported_signature = { + 'date': {'replace'}, + 'time': {'replace'}, + 'datetime': {'replace', 'combine'}, + } + self._test_module_has_signatures(datetime, + no_signature, unsupported_signature, + methods_unsupported_signature=methods_unsupported_signature) + def test_errno_module_has_signatures(self): import errno self._test_module_has_signatures(errno) @@ -6169,8 +6256,7 @@ def test_faulthandler_module_has_signatures(self): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - unsupported_signature = {"reduce"} - self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + self._test_module_has_signatures(functools) def test_gc_module_has_signatures(self): import gc @@ -6214,7 +6300,10 @@ def test_pwd_module_has_signatures(self): def test_re_module_has_signatures(self): import re - methods_no_signature = {'Match': {'group'}} + methods_no_signature = { + 'Match': {'group'}, + 'Pattern': {'match'}, # It is now an alias for prefixmatch + } self._test_module_has_signatures(re, methods_no_signature=methods_no_signature, good_exceptions={'error', 'PatternError'}) @@ -6227,6 +6316,10 @@ def test_stat_module_has_signatures(self): import stat self._test_module_has_signatures(stat) + def test_struct_module_has_signatures(self): + import struct + self._test_module_has_signatures(struct) + def test_string_module_has_signatures(self): import string self._test_module_has_signatures(string) @@ -6284,7 +6377,7 @@ def test_typing_module_has_signatures(self): methods_unsupported_signature=methods_unsupported_signature) def test_warnings_module_has_signatures(self): - unsupported_signature = {'warn', 'warn_explicit'} + unsupported_signature = {'warn_explicit'} self._test_module_has_signatures(warnings, unsupported_signature=unsupported_signature) def test_weakref_module_has_signatures(self): @@ -6415,8 +6508,19 @@ def test_wrapped_descriptor(self): self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) +def _clean_object_ids(text): + # Helper to handle "" details in CLI output checks + import re + detect = r"object at 0x([0-9A-Fa-f]+)>" + replace = "object at 0x...>" + return re.sub(detect, replace, text) + +class TestModuleCLI(unittest.TestCase): + + BUILTIN_ERROR = "No source code available for builtin module" + NO_SOURCE_ERROR = "No source code available for defining module" + NO_SOURCE_TARGET_ERROR = "Failed to retrieve source code for given target" -class TestMain(unittest.TestCase): @staticmethod def _expected_cached(module): # assert_python_ok() runs the subprocess in isolated mode (-I), which @@ -6457,27 +6561,228 @@ def test_qualname_source(self): inspect.getsource(ThreadPoolExecutor).splitlines()) self.assertEqual(err, b'') - def test_builtins(self): + def test_error_builtins(self): _, out, err = assert_python_failure('-m', 'inspect', 'sys') lines = err.decode().splitlines() - self.assertEqual(lines, ["Can't get info for builtin modules."]) + self.assertEqual(lines, [self.BUILTIN_ERROR]) + + @unittest.skipIf(importlib.util.find_spec("_testcapi") is None, "TODO: RUSTPYTHON; _testcapi extension fixture is unavailable") + def test_error_extension(self): + module_name = "_testcapi" + if module_name in sys.builtin_module_names: + # WASI test environment has even _testcapi as a builtin module + expected_error = self.BUILTIN_ERROR + else: + expected_error = self.NO_SOURCE_ERROR + _, out, err = assert_python_failure('-m', 'inspect', + module_name) + lines = err.decode().splitlines() + self.assertEqual(lines, [expected_error]) - def test_details(self): - module = importlib.import_module('unittest') + def test_error_data(self): + _, out, err = assert_python_failure('-m', 'inspect', + 'importlib.machinery:SOURCE_SUFFIXES') + lines = err.decode().splitlines() + self.assertEqual(lines, [self.NO_SOURCE_TARGET_ERROR]) + + def test_details_option_with_package(self): + module_name = 'unittest' + module = importlib.import_module(module_name) args = support.optim_args_from_interpreter_flags() rc, out, err = assert_python_ok(*args, '-m', 'inspect', - 'unittest', '--details') - output = out.decode() + module_name, '--details') cached = self._expected_cached(module) - # Just a quick sanity check on the output - self.assertIn(module.__spec__.name, output) - self.assertIn(module.__name__, output) - self.assertIn(module.__spec__.origin, output) - self.assertIn(module.__file__, output) - self.assertIn(cached, output) + # Full rendering check on the expected output + expected_lines = [ + f"Target: {module.__name__}", # No aliasing + f"Origin: {module.__spec__.origin}", + f"Source: {module.__file__}", + f"Cached: {cached}", # None is still displayed + f"Loader: {_clean_object_ids(repr(module.__spec__.loader))}", + f"Submodule search paths: {module.__path__}", + "", + ] + output_lines = _clean_object_ids(out.decode()).splitlines() + self.assertEqual(output_lines, expected_lines) self.assertEqual(err, b'') + def test_details_option_with_builtin_module(self): + # Also an end-to-end test of non-package lookups + module_name = 'sys' + module = importlib.import_module(module_name) + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + module_name, '--details') + # Full rendering check on the expected output + # No error is reported when just fetching the module details + expected_lines = [ + f"Target: {module.__name__}", # No aliasing + f"Origin: {module.__spec__.origin}", + "Source: None", + "Cached: None", + f"Loader: {_clean_object_ids(repr(module.__spec__.loader))}", + "", + ] + output_lines = _clean_object_ids(out.decode()).splitlines() + self.assertEqual(output_lines, expected_lines) + self.assertEqual(err, b'') + + def test_details_option_with_data_target(self): + # Also an end-to-end test of non-module lookups without aliasing + module_name = 'importlib.machinery' + cli_target = f"{module_name}:SOURCE_SUFFIXES" + module = importlib.import_module(module_name) + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + cli_target, '--details') + cached = self._expected_cached(module) + # Full rendering check on the expected output + # The error is only informational when reading source details + expected_lines = [ + f"Target: {cli_target}", # No aliasing + f"Origin: {module.__spec__.origin}", + f"Source: {module.__file__}", + f"Cached: {cached}", # None is still displayed + self.NO_SOURCE_TARGET_ERROR, + "", + ] + output_lines = out.decode().splitlines() + self.assertEqual(output_lines, expected_lines) + self.assertEqual(err, b'') + + @unittest.skipIf(not os.path.exists(os.path.__file__), "Needs frozen source file") + def test_details_option_with_aliased_target(self): + # Also an end-to-end test of successful non-module lookups + module = importlib.import_module("os.path") + target = module.join + cli_target = "os:path.join" # Defining module is os.path, not os + defining_target = f"{target.__module__}:{target.__qualname__}" + + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + cli_target, '--details') + cached = self._expected_cached(module) + # Full rendering check on the expected output + expected_lines = [ + f'Target: {defining_target} (looked up as "{cli_target}")', + f"Origin: {module.__spec__.origin}", + f"Source: {module.__file__}", + f"Cached: {cached}", # None is still displayed + f"Line: {inspect.findsource(target)[1]}", + "", + ] + output_lines = out.decode().splitlines() + self.assertEqual(output_lines, expected_lines) + self.assertEqual(err, b'') + + def _check_details(self, module, details, other_expected_keys=(), *, alias=None, error=None): + expected_keys = {"target", "origin", "source", "cached"} + if other_expected_keys: + expected_keys |= other_expected_keys + if alias is not None: + expected_keys.add("alias") + if error is not None: + expected_keys.add("error") + self.assertEqual(set(details.keys()), expected_keys) + self.assertEqual(module.__spec__.origin, details["origin"]) + try: + expected_source = inspect.getsourcefile(module) + except Exception: + expected_source = None + if expected_source and expected_source.startswith("'. # The directory of argv[0] must match the directory of the Python # executable for the Popen() call to python to succeed as the directory - # path may be used by Py_GetPath() to build the default module search - # path. + # path may be used by PyConfig_Get("module_search_paths") to build the + # default module search path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "") cmd_line = [stdin_fname, '-E', '-i'] cmd_line.extend(args) @@ -6568,7 +6875,5 @@ def f(): self.assertIn(expected, output) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py index d2b724db40d..0a366e2a5bb 100644 --- a/Lib/test/test_pkg.py +++ b/Lib/test/test_pkg.py @@ -198,15 +198,15 @@ def test_5(self): import t5 self.assertEqual(fixdir(dir(t5)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__', - 'foo', 'string', 't5']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__', 'foo', + 'string', 't5']) self.assertEqual(fixdir(dir(t5.foo)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__spec__', 'string']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__spec__', 'string']) self.assertEqual(fixdir(dir(t5.string)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__spec__', 'spam']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__spec__', 'spam']) def test_6(self): hier = [ @@ -221,14 +221,13 @@ def test_6(self): import t6 self.assertEqual(fixdir(dir(t6)), - ['__all__', '__cached__', '__doc__', '__file__', - '__loader__', '__name__', '__package__', '__path__', - '__spec__']) + ['__all__', '__doc__', '__file__', '__loader__', + '__name__', '__package__', '__path__', '__spec__']) s = """ import t6 from t6 import * self.assertEqual(fixdir(dir(t6)), - ['__all__', '__cached__', '__doc__', '__file__', + ['__all__', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__path__', '__spec__', 'eggs', 'ham', 'spam']) self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6']) @@ -256,20 +255,19 @@ def test_7(self): t7, sub, subsub = None, None, None import t7 as tas self.assertEqual(fixdir(dir(tas)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__']) self.assertFalse(t7) from t7 import sub as subpar self.assertEqual(fixdir(dir(subpar)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__']) self.assertFalse(t7) self.assertFalse(sub) from t7.sub import subsub as subsubsub self.assertEqual(fixdir(dir(subsubsub)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__', - 'spam']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__', 'spam']) self.assertFalse(t7) self.assertFalse(sub) self.assertFalse(subsub) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index e07fa568a86..57b12609f71 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -8,7 +8,7 @@ from test.support.import_helper import import_fresh_module import collections.abc -from collections import namedtuple, UserDict +from collections import namedtuple, UserDict, OrderedDict import copy # XXX: RUSTPYTHON try: @@ -27,6 +27,7 @@ import typing import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import re c_types = import_fresh_module('types', fresh=['_types']) py_types = import_fresh_module('types', blocked=['_types']) @@ -45,8 +46,9 @@ def clear_typing_caches(): class TypesTests(unittest.TestCase): + @unittest.expectedFailureIf(not hasattr(c_types, "LazyImportType"), "TODO: RUSTPYTHON; PEP 810 LazyImportType is not implemented") def test_names(self): - c_only_names = {'CapsuleType'} + c_only_names = {'CapsuleType', 'LazyImportType'} ignored = {'new_class', 'resolve_bases', 'prepare_class', 'get_original_bases', 'DynamicClassAttribute', 'coroutine'} @@ -58,11 +60,13 @@ def test_names(self): 'AsyncGeneratorType', 'BuiltinFunctionType', 'BuiltinMethodType', 'CapsuleType', 'CellType', 'ClassMethodDescriptorType', 'CodeType', 'CoroutineType', 'EllipsisType', 'FrameType', 'FunctionType', + 'FrameLocalsProxyType', 'GeneratorType', 'GenericAlias', 'GetSetDescriptorType', - 'LambdaType', 'MappingProxyType', 'MemberDescriptorType', - 'MethodDescriptorType', 'MethodType', 'MethodWrapperType', - 'ModuleType', 'NoneType', 'NotImplementedType', 'SimpleNamespace', - 'TracebackType', 'UnionType', 'WrapperDescriptorType', + 'LambdaType', 'LazyImportType', 'MappingProxyType', + 'MemberDescriptorType', 'MethodDescriptorType', 'MethodType', + 'MethodWrapperType', 'ModuleType', 'NoneType', + 'NotImplementedType', 'SimpleNamespace', 'TracebackType', + 'UnionType', 'WrapperDescriptorType', } self.assertEqual(all_names, set(c_types.__all__)) self.assertEqual(all_names - c_only_names, set(py_types.__all__)) @@ -717,6 +721,19 @@ def call(part): """ assert_python_ok("-c", code) + def test_frame_locals_proxy_type(self): + self.assertIsInstance(types.FrameLocalsProxyType, type) + if MISSING_C_DOCSTRINGS: + self.assertIsNone(types.FrameLocalsProxyType.__doc__) + else: + self.assertIsInstance(types.FrameLocalsProxyType.__doc__, str) + self.assertEqual(types.FrameLocalsProxyType.__module__, 'builtins') + self.assertEqual(types.FrameLocalsProxyType.__name__, 'FrameLocalsProxy') + + frame = inspect.currentframe() + self.assertIsNotNone(frame) + self.assertIsInstance(frame.f_locals, types.FrameLocalsProxyType) + class UnionTests(unittest.TestCase): @@ -1382,6 +1399,128 @@ def __hash__(self): view = self.mappingproxy(mapping) self.assertEqual(hash(view), hash(mapping)) + def check_richcompare(self, mapping_type): + data1 = mapping_type({'a': 1}) + data2 = mapping_type({'a': 2}) + mp1 = self.mappingproxy(data1) + copy1 = self.mappingproxy(data1) + mp2 = self.mappingproxy(data2) + + self.assertTrue(mp1 == data1) + self.assertTrue(data1 == mp1) + self.assertTrue(copy1 == data1) + + self.assertTrue(mp1 == copy1) + self.assertFalse(mp1 != copy1) + + self.assertFalse(mp1 == mp2) + self.assertTrue(mp1 != mp2) + self.assertFalse(mp2 == data1) + self.assertTrue(mp2 != data1) + self.assertTrue(data1 != mp2) + + msg = "not supported between instances of 'mappingproxy' and 'mappingproxy'" + with self.assertRaisesRegex(TypeError, msg): + mp1 > mp2 + with self.assertRaisesRegex(TypeError, msg): + mp1 < copy1 + with self.assertRaisesRegex(TypeError, msg): + mp2 >= mp2 + with self.assertRaisesRegex(TypeError, msg): + copy1 <= mp1 + + if mapping_type.__module__ == 'collections': + mapping_name = f'{mapping_type.__module__}.{mapping_type.__name__}' + else: + mapping_name = mapping_type.__name__ + with self.assertRaisesRegex( + TypeError, + f"not supported between instances of 'mappingproxy' and '{mapping_name}'", + ): + copy1 <= data1 + + class Evil: + def __eq__(self, other): + return other + + result = (mp1 == Evil()) + self.assertIs(type(result), mapping_type) + if mapping_type == dict: + # Evil.__eq__() gets a copy of the mapping + self.assertEqual(result, data1) + else: + self.assertIs(result, data1) + + def test_richcompare_dict(self): + self.check_richcompare(dict) + + def test_richcompare_custom_mapping(self): + class CustomMapping(collections.abc.Mapping): + def __init__(self, data): + self._data = data + + def __getitem__(self, key): + return self._data[key] + + def __iter__(self): + return iter(self._data) + + def __len__(self): + return len(self._data) + + def __contains__(self, item): + return item in self._data + + self.check_richcompare(CustomMapping) + + class CustomMapping2(CustomMapping): + def __eq__(self, other): + return ( + isinstance(other, CustomMapping) + and self._data == other._data + ) + + mp1 = self.mappingproxy(CustomMapping({'a': 1})) + self.assertEqual(mp1, CustomMapping2({'a': 1})) + self.assertNotEqual(CustomMapping2({'a': 1}), mp1) + + def test_richcompare_odict(self): + self.check_richcompare(OrderedDict) + + def test_richcompare_frozendict(self): + self.check_richcompare(frozendict) + + def test_richcompare_evil(self): + # This test used to mutate the list dictionary, + # but MappingProxyType now creates a copy to call `__eq__`: + # https://github.com/python/cpython/issues/152405 + key = "__mappingproxy_crash_key__" + + class Evil: + def __eq__(self, other): + other[key] = 1 + return other + + # Checks that it does not mutate the internals of `MappingProxyType`: + dc = {} + leaked = self.mappingproxy(dc) == Evil() + self.assertIs(type(leaked), dict) + self.assertIn(key, leaked) + self.assertNotIn(key, dc) + + # Exposes the internals of `MappingProxyType` via richcompare: + leaked = vars(list) == Evil() + self.assertIs(type(leaked), dict) + self.assertIn(key, leaked) + name = "__mappingproxy_crash_probe__" + leaked[name] = lambda self: "probe" + self.assertIn(name, leaked) + self.assertNotHasAttr(list, key) + self.assertNotHasAttr(list, name) # it used to return `True` + del leaked[name] + self.assertNotIn(name, leaked) + self.assertNotHasAttr(list, name) # it used to crash + class ClassCreationTests(unittest.TestCase): @@ -2016,6 +2155,24 @@ def test_equal(self): self.assertEqual(ns1, ns2) self.assertNotEqual(ns2, types.SimpleNamespace()) + def test_richcompare_unsupported(self): + ns1 = types.SimpleNamespace(x=1) + ns2 = types.SimpleNamespace(y=2) + + msg = re.escape( + "not supported between instances of " + "'types.SimpleNamespace' and 'types.SimpleNamespace'" + ) + + with self.assertRaisesRegex(TypeError, msg): + ns1 > ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 >= ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 < ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 <= ns2 + def test_nested(self): ns1 = types.SimpleNamespace(a=1, b=2) ns2 = types.SimpleNamespace() diff --git a/Lib/types.py b/Lib/types.py index 60a4b22c200..6c069591ab2 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -58,7 +58,10 @@ def _m(self): pass raise TypeError except TypeError as exc: TracebackType = type(exc.__traceback__) - FrameType = type(exc.__traceback__.tb_frame) + + _f = (lambda: sys._getframe())() + FrameType = type(_f) + FrameLocalsProxyType = type(_f.f_locals) GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) @@ -73,6 +76,9 @@ def _m(self): pass # CapsuleType cannot be accessed from pure Python, # so there is no fallback definition. + # LazyImportType in pure Python cannot be guaranteed + # without overriding the filter, so there is no fallback. + del sys, _f, _g, _C, _c, _ag, _cell_factory # Not for export @@ -251,7 +257,6 @@ def deleter(self, fdel): class _GeneratorWrapper: - # TODO: Implement this in C. def __init__(self, gen): self.__wrapped = gen self.__isgen = gen.__class__ is GeneratorType @@ -278,6 +283,12 @@ def gi_yieldfrom(self): @property def gi_suspended(self): return self.__wrapped.gi_suspended + @property + def gi_state(self): + return self.__wrapped.gi_state + @property + def cr_state(self): + return self.__wrapped.gi_state.replace('GEN_', 'CORO_') cr_code = gi_code cr_frame = gi_frame cr_running = gi_running @@ -310,7 +321,6 @@ def coroutine(func): # Check if 'func' is a generator function. # (0x20 == CO_GENERATOR) if co_flags & 0x20: - # TODO: Implement this in C. co = func.__code__ # 0x100 == CO_ITERABLE_COROUTINE func.__code__ = co.replace(co_flags=co.co_flags | 0x100) diff --git a/crates/vm/src/builtins/asyncgenerator.rs b/crates/vm/src/builtins/asyncgenerator.rs index 91fb744dc39..81068337720 100644 --- a/crates/vm/src/builtins/asyncgenerator.rs +++ b/crates/vm/src/builtins/asyncgenerator.rs @@ -1,3 +1,5 @@ +// spell-checker: ignore AGEN + use super::{PyCode, PyGenerator, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, @@ -162,6 +164,15 @@ impl Py { self.inner.code() } #[pygetset] + fn ag_state(&self) -> &'static str { + self.inner.state_name([ + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", + "AGEN_CLOSED", + ]) + } + #[pygetset] fn ag_suspended(&self, _vm: &VirtualMachine) -> bool { self.inner.suspended() } diff --git a/crates/vm/src/builtins/coroutine.rs b/crates/vm/src/builtins/coroutine.rs index a0d4a581aa9..93481c41a1c 100644 --- a/crates/vm/src/builtins/coroutine.rs +++ b/crates/vm/src/builtins/coroutine.rs @@ -148,6 +148,15 @@ impl Py { self.inner.code() } #[pygetset] + fn cr_state(&self) -> &'static str { + self.inner.state_name([ + "CORO_CREATED", + "CORO_RUNNING", + "CORO_SUSPENDED", + "CORO_CLOSED", + ]) + } + #[pygetset] fn cr_suspended(&self, _vm: &VirtualMachine) -> bool { self.inner.suspended() } diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs index 28da2341f64..8cb98d77964 100644 --- a/crates/vm/src/builtins/frame_locals_proxy.rs +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -21,6 +21,13 @@ use crate::{ use rustpython_common::lock::LazyLock; use rustpython_common::wtf8::Wtf8Buf; +/// FrameLocalsProxy($frame) +/// -- +/// +/// Create a write-through view of the locals dictionary for a frame. +/// +/// frame +/// the frame object to wrap. #[pyclass( module = false, name = "FrameLocalsProxy", diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index feb95d49ca1..b1ea72b131b 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -527,6 +527,21 @@ impl Coro { self.started.load() } + pub(crate) fn state_name(&self, names: [&'static str; 4]) -> &'static str { + // Closing publishes the terminal state before releasing frame locals, + // whose finalizers can observe us while the running claim is held. + let index = if self.closed() { + 3 + } else if self.running() { + 1 + } else if self.started() { + 2 + } else { + 0 + }; + names[index] + } + pub fn suspended(&self) -> bool { !self.closed.load() && !self.running.load() && self.started() } diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 3fb6e5fdc2e..34911348b12 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -29,7 +29,7 @@ pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult PyObjectRef { + vm.ctx.types.frame_locals_proxy_type.to_owned().into() + } + #[pyattr] fn FrameType(vm: &VirtualMachine) -> PyObjectRef { vm.ctx.types.frame_type.to_owned().into() diff --git a/extra_tests/snippets/vm_frame_locals.py b/extra_tests/snippets/vm_frame_locals.py index 608167c2a17..4313b07dbda 100644 --- a/extra_tests/snippets/vm_frame_locals.py +++ b/extra_tests/snippets/vm_frame_locals.py @@ -230,6 +230,33 @@ def frame_with_overwritten_fast_local(): frame.clear() self.assertEqual(len(events), 1) + def test_closed_coroutine_state_is_visible_to_frame_local_finalizers(self): + events = [] + + class ReentrantFinalizer: + def __del__(self): + events.append(getattr(owner, state_attribute)) + + async def coroutine(value): + return value + + async def async_generator(value): + yield value + + for factory, state_attribute, closed in ( + (coroutine, "cr_state", "CORO_CLOSED"), + (async_generator, "ag_state", "AGEN_CLOSED"), + ): + with self.subTest(state_attribute=state_attribute): + events.clear() + owner = factory(ReentrantFinalizer()) + if state_attribute == "cr_state": + owner.close() + else: + with self.assertRaises(StopIteration): + owner.aclose().send(None) + self.assertEqual(events, [closed]) + def test_unoptimized_frame_locals_are_readable_from_another_thread(self): ready = threading.Event() release = threading.Event() diff --git a/scripts/update_lib/cmd_migrate.py b/scripts/update_lib/cmd_migrate.py index 6dea08a3c00..f9b8a5ca262 100644 --- a/scripts/update_lib/cmd_migrate.py +++ b/scripts/update_lib/cmd_migrate.py @@ -5,6 +5,7 @@ Usage: python scripts/update_lib migrate cpython/Lib/test/test_foo.py python scripts/update_lib migrate cpython/Lib/test/test_foo.py --patches remaining.json + python scripts/update_lib migrate cpython/Lib/test/test_foo --canonical-removals removals.json --dry-run Remaining metadata names the exact repository-relative target and its existing decorator patches. An optional "doctests" object selects existing guarded inputs @@ -12,6 +13,13 @@ this explicit selection; an empty object retires all their guards. Canonical expected output is never copied from the old file. +Canonical removals are a separate, explicit operation. JSON source metadata +pins path (Lib/test/test_*), commit and tree; removals maps explicit relative +filenames to sha256 and upstream_commit. The caller must first verify and +review historical provenance, including release-branch evidence. This command +checks pinned canonical absence and exact local bytes; it does not authenticate +historical evidence. --dry-run performs all validation without deleting files. + This will: 1. Extract patches from Lib/test/test_foo.py (if exists) 2. Apply them to cpython/Lib/test/test_foo.py @@ -25,8 +33,11 @@ import copy import hashlib import json +import os import pathlib +import re import shutil +import subprocess import sys import textwrap @@ -51,6 +62,33 @@ EXCEPTION_LOOP = """for excname in PYTHON2_EXCEPTIONS: NAME_MAPPING[("exceptions", excname)] = ("builtins", excname) """ +MULTIPHASE_UPSTREAM = ( + "# gh-116303: Skip test module dependent tests if test modules are unavailable\n" + 'import_helper.import_module("_testmultiphase")\n' +) +MULTIPHASE_LOCAL = """try: # TODO: RUSTPYTHON + import_helper.import_module("_testmultiphase") +except unittest.SkipTest: + _testmultiphase = None +""" +LOCK_SPECIALIZATION = """(Frozen_LifetimeTests, + Source_LifetimeTests + ) = test_util.test_both(LifetimeTests, init=init) +""" +LOCK_SKIP = """# TODO: RUSTPYTHON; dead weakref module locks not cleaned up in frozen bootstrap +Frozen_LifetimeTests.test_all_locks = unittest.skip("TODO: RUSTPYTHON")( + Frozen_LifetimeTests.test_all_locks +) +""" +TYPES_DATETIME_GUARD = """# XXX: RUSTPYTHON +try: + import _datetime +except ImportError: + _datetime = None +""" +TYPES_UNITTEST_IMPORT = ( + "import unittest # XXX: RUSTPYTHON; importing to be able to skip tests\n" +) BYTES_CAPI_UPSTREAM = ( " _testlimitedcapi = import_helper.import_module('_testlimitedcapi')\n" ) @@ -258,6 +296,78 @@ def _jit_block(contents: str) -> bool: ) +def _preserve_top_level_block(old, new, upstream, local): + """Require exact, consecutive top-level AST statements and comment bytes.""" + for contents, fragment in ((old, local), (new, upstream)): + tree, expected = ast.parse(contents), ast.parse(fragment).body + nodes = [ + _unique_anchor( + contents, + tree, + lambda node: ast.dump(node) == ast.dump(item), + direct=True, + ) + for item in expected + ] + start = tree.body.index(nodes[0]) + lines = contents.splitlines(keepends=True) + first = nodes[0].lineno - expected[0].lineno + if ( + tree.body[start : start + len(nodes)] != nodes + or contents.count(fragment) != 1 + or "".join(lines[first : nodes[-1].end_lineno]) != fragment + ): + raise ValueError("Changed or ambiguous migration customization anchor") + return new.replace(upstream, local, 1) + + +def _preserve_multiphase_guard(old, new): + for contents, local in ((old, True), (new, False)): + tree = ast.parse(contents) + if sum( + isinstance(n, ast.Constant) and n.value == "_testmultiphase" + for n in ast.walk(tree) + ) != 1 or sum( + isinstance(n, ast.Name) and n.id == "_testmultiphase" + for n in ast.walk(tree) + ) != int(local): + raise ValueError("Ambiguous _testmultiphase anchor") + return _preserve_top_level_block(old, new, MULTIPHASE_UPSTREAM, MULTIPHASE_LOCAL) + + +def _preserve_frozen_lock_skip(old, new): + methods = [] + for contents, local in ((old, True), (new, False)): + tree = ast.parse(contents) + owner = _unique_anchor( + contents, + tree, + lambda n: isinstance(n, ast.ClassDef) and n.name == "LifetimeTests", + direct=True, + ) + method = _unique_anchor( + contents, + owner, + lambda n: isinstance(n, ast.FunctionDef) and n.name == "test_all_locks", + direct=True, + ) + methods.append(ast.dump(method)) + if sum( + isinstance(n, ast.Name) and n.id == "Frozen_LifetimeTests" + for n in ast.walk(tree) + ) != (3 if local else 1): + raise ValueError("Ambiguous frozen lock specialization") + if methods[0] != methods[1]: + raise ValueError("Frozen lock test changed; review its customization") + setup = "def setUpModule():\n thread_info = threading_helper.threading_setup()\n unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)\n" + return _preserve_top_level_block( + old, + new, + LOCK_SPECIALIZATION + "\n\n" + setup, + LOCK_SPECIALIZATION + "\n" + LOCK_SKIP + "\n" + setup, + ) + + def preserve_migration_customizations(path: pathlib.Path, old: str, new: str) -> str: """Fail closed if either known local patch or its canonical anchor changes.""" parts = path.parts @@ -299,6 +409,31 @@ def preserve_migration_customizations(path: pathlib.Path, old: str, new: str) -> lines = new.splitlines(keepends=True) lines[anchor.lineno - 1 : anchor.end_lineno] = [BYTES_CAPI_LOCAL] return _preserve_bytes_allocations(old, "".join(lines)) + if parts == ("Lib", "test", "test_importlib", "util.py"): + return _preserve_multiphase_guard(old, new) + if parts == ("Lib", "test", "test_importlib", "test_locks.py"): + return _preserve_frozen_lock_skip(old, new) + if parts == ("Lib", "test", "test_types.py"): + for contents in (old, new): + if ( + sum( + isinstance(node, ast.alias) and node.name == "_datetime" + for node in ast.walk(ast.parse(contents)) + ) + != 1 + ): + raise ValueError("Ambiguous test_types _datetime import") + if any( + isinstance(node, ast.alias) and node.name == "unittest" + for node in ast.walk(ast.parse(new)) + ): + raise ValueError("Canonical test_types already imports unittest directly") + new = _preserve_top_level_block( + old, new, "import _datetime\n", TYPES_DATETIME_GUARD + ) + return _preserve_top_level_block( + old, new, "import typing\n", "import typing\n\n" + TYPES_UNITTEST_IMPORT + ) return new @@ -622,6 +757,8 @@ def patch_single_content( ) if selection is not None: src_content = apply_doctest_metadata(old_content, src_content, selection) + if not src_content and not patches: + return src_content return apply_patches(src_content, patches) @@ -849,6 +986,159 @@ def has_method(class_name, seen): legacy.unlink() +def remove_canonical_files(src_dir, manifest_path, *, dry_run=False): + """Remove only explicitly reviewed files, validating the whole batch first.""" + + def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate removal metadata key: {key}") + result[key] = value + return result + + def relative(name): + path = pathlib.PurePosixPath(name) if isinstance(name, str) else None + if ( + not path + or not path.parts + or path.is_absolute() + or ".." in path.parts + or "\\" in name + or path.as_posix() != name + or "\0" in name + ): + raise ValueError("Expected an exact relative POSIX filename") + return pathlib.Path(path) + + def no_symlinks(path): + if any(p.is_symlink() or p.is_junction() for p in (path, *path.parents)): + raise ValueError(f"Symlink or junction in removal path: {path}") + + def git(*args): + result = subprocess.run( + ["git", "-C", str(src_dir), *args], + env={ + **os.environ, + "GIT_NO_LAZY_FETCH": "1", + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_TERMINAL_PROMPT": "0", + }, + capture_output=True, + check=False, + ) + if result.returncode: + raise ValueError("Cannot verify pinned canonical source") + return result.stdout.decode().strip() + + document = json.loads( + manifest_path.read_text(encoding="utf-8"), object_pairs_hook=unique_object + ) + if not isinstance(document, dict) or set(document) != {"source", "removals"}: + raise ValueError("Expected source and removals metadata") + source, removals = document["source"], document["removals"] + if not isinstance(source, dict) or set(source) != {"path", "commit", "tree"}: + raise ValueError("Expected source path, commit and tree") + package = relative(source["path"]) + if ( + len(package.parts) != 3 + or package.parts[:2] != ("Lib", "test") + or not package.name.startswith("test_") + or not package.name.isidentifier() + ): + raise ValueError("Removals require an exact Lib/test/test_* package") + if any( + not isinstance(source[k], str) or not re.fullmatch(r"[0-9a-f]{40}", source[k]) + for k in ("commit", "tree") + ): + raise ValueError("Source commit and tree must be full Git hashes") + if not isinstance(removals, dict) or not removals: + raise ValueError("Removals must be a nonempty filename map") + src_dir = src_dir.absolute() + destination = pathlib.Path.cwd() / package + for path in (src_dir, destination): + no_symlinks(path) + if not path.is_dir(): + raise ValueError("Source and destination packages must exist") + if ( + src_dir == destination + or src_dir in destination.parents + or destination in src_dir.parents + ): + raise ValueError("Source and destination overlap") + if ( + src_dir != pathlib.Path(git("rev-parse", "--show-toplevel")) / package + or git("rev-parse", "HEAD") != source["commit"] + or git("rev-parse", f"HEAD:{package.as_posix()}") != source["tree"] + ): + raise ValueError("Canonical source identity differs from the pinned input") + canonical = set( + git( + "ls-tree", + "--full-tree", + "-r", + "--name-only", + "-z", + "HEAD", + "--", + package.as_posix(), + ).split("\0") + ) + snapshots, receipt = {}, {"source": source, "dry_run": dry_run, "removals": {}} + for name, entry in sorted(removals.items()): + path = relative(name) + if ( + not isinstance(entry, dict) + or set(entry) != {"sha256", "upstream_commit"} + or any( + not isinstance(entry[k], str) + or not re.fullmatch(f"[0-9a-f]{{{length}}}", entry[k]) + for k, length in (("sha256", 64), ("upstream_commit", 40)) + ) + ): + raise ValueError(f"Missing or invalid hash/provenance pins: {name}") + target = destination / path + no_symlinks(target) + no_symlinks(src_dir / path) + if (package / path).as_posix() in canonical or (src_dir / path).exists(): + raise ValueError(f"Selected file is still canonical: {name}") + if target.exists() and not target.is_file(): + raise ValueError(f"Removal target is not a regular file: {name}") + data = target.read_bytes() if target.exists() else None + if data is not None and ( + hashlib.sha256(data).hexdigest() != entry["sha256"] + or b"rustpython" in data.lower() + ): + raise ValueError(f"Changed bytes or RustPython customization: {name}") + snapshots[target] = data + receipt["removals"][name] = { + **entry, + "status": "already_absent" + if data is None + else "would_remove" + if dry_run + else "removed", + } + if git("rev-parse", "HEAD") != source["commit"]: + raise ValueError("Canonical source changed during validation") + for target, data in snapshots.items(): + no_symlinks(target) + canonical_path = src_dir / target.relative_to(destination) + no_symlinks(canonical_path) + if canonical_path.exists(): + raise ValueError("Canonical source changed during validation") + if (target.exists() and not target.is_file()) or ( + target.read_bytes() if target.exists() else None + ) != data: + raise ValueError(f"Removal target changed during validation: {target}") + if not dry_run: + for target, data in snapshots.items(): + if data is not None: + target.unlink() + print(json.dumps(receipt, sort_keys=True)) + return receipt + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=__doc__, @@ -865,6 +1155,16 @@ def main(argv: list[str] | None = None) -> int: type=pathlib.Path, help="Explicit legacy-file/class to package-file JSON mapping", ) + parser.add_argument( + "--canonical-removals", + type=pathlib.Path, + help="Remove only selected unchanged historical files absent from a pinned canonical tree", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate canonical removals without deleting", + ) parser.add_argument( "--patches", @@ -882,6 +1182,27 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) try: + if args.canonical_removals is not None: + if ( + args.legacy_map is not None + or args.patches is not None + or args.support_assignments + or args.support_exports + or args.source_sha256 + or args.destination_sha256 + ): + raise ValueError( + "Canonical removals cannot combine with other migration modes" + ) + try: + remove_canonical_files( + args.path, args.canonical_removals, dry_run=args.dry_run + ) + except (OSError, UnicodeError) as exc: + raise ValueError(str(exc)) from exc + return 0 + if args.dry_run: + raise ValueError("--dry-run requires --canonical-removals") if args.support_assignments: if args.legacy_map is not None or args.patches is not None: raise ValueError( diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py index 3f85165d53e..c6f0fa172f4 100644 --- a/scripts/update_lib/tests/test_migrate.py +++ b/scripts/update_lib/tests/test_migrate.py @@ -4,7 +4,9 @@ import hashlib import io import json +import os import pathlib +import subprocess import tempfile import unittest from pathlib import Path @@ -18,8 +20,14 @@ EXCEPTION_LOOP, JIT_ERROR, LOCAL_CALL, + LOCK_SKIP, + LOCK_SPECIALIZATION, + MULTIPHASE_LOCAL, + MULTIPHASE_UPSTREAM, SYSCONFIG_LOCAL, SYSCONFIG_UPSTREAM, + TYPES_DATETIME_GUARD, + TYPES_UNITTEST_IMPORT, copy_support_assignments, main, migrate_package, @@ -27,6 +35,7 @@ patch_file, patch_single_content, preserve_migration_customizations, + remove_canonical_files, ) from update_lib.patch_spec import ( CANONICAL_LOAD_TESTS, @@ -711,6 +720,17 @@ def test_one(self): class TestPatchDirectory(unittest.TestCase): """Tests for patch_directory function.""" + def test_empty_canonical_fixture_stays_byte_empty(self): + with tempfile.TemporaryDirectory() as temporary: + source, target = Path(temporary) / "canonical", Path(temporary) / "local" + source.mkdir() + target.mkdir() + (source / "empty.py").write_bytes(b"") + (target / "empty.py").write_bytes(b"\n") + for _ in range(2): + patch_directory(source, target, verbose=False) + self.assertEqual((target / "empty.py").read_bytes(), b"") + def test_excludes_cache_directories_but_preserves_canonical_assets(self): with tempfile.TemporaryDirectory() as temporary: root = pathlib.Path(temporary) @@ -1282,5 +1302,362 @@ def test_unmarked_syntax_error_prevents_all_writes(self): self.assert_rejected() +class TestCanonicalRemovals(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.source = self.root / "canonical/Lib/test/test_example" + self.target = self.root / "local/Lib/test/test_example" + self.source.mkdir(parents=True) + self.target.mkdir(parents=True) + (self.source / "__init__.py").write_bytes(b"") + self.git("init", "-q") + self.git("add", ".") + self.git("commit", "-qm", "canonical test fixture") + self.document = { + "source": { + "path": "Lib/test/test_example", + "commit": self.git("rev-parse", "HEAD"), + "tree": self.git("rev-parse", "HEAD:Lib/test/test_example"), + }, + "removals": {}, + } + for name, data in (("a.py", b"old\n"), ("data/z.bin", b"\x00\xff")): + path = self.target / name + path.parent.mkdir(exist_ok=True) + path.write_bytes(data) + self.document["removals"][name] = { + "sha256": hashlib.sha256(data).hexdigest(), + "upstream_commit": "1" * 40, + } + self.manifest = self.root / "removals.json" + + def git(self, *args): + return subprocess.check_output( + ["git", "-C", str(self.source.parents[2]), *args], + env={ + **os.environ, + "GIT_AUTHOR_NAME": "Fixture", + "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "Fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + }, + text=True, + ).strip() + + def run_removal(self, dry_run=False): + self.manifest.write_text(json.dumps(self.document)) + with ( + contextlib.chdir(self.target.parents[2]), + contextlib.redirect_stdout(io.StringIO()), + ): + return remove_canonical_files(self.source, self.manifest, dry_run=dry_run) + + def test_dry_run_explicit_scope_receipt_and_idempotence(self): + (self.target / "unselected").write_text("keep") + for dry_run, expected in ( + (True, "would_remove"), + (False, "removed"), + (False, "already_absent"), + ): + receipt = self.run_removal(dry_run) + self.assertEqual(receipt["source"], self.document["source"]) + self.assertEqual( + {v["status"] for v in receipt["removals"].values()}, {expected} + ) + self.assertEqual((self.target / "a.py").exists(), dry_run) + self.assertEqual((self.target / "unselected").read_text(), "keep") + + def test_junction_components_reject_the_whole_batch(self): + before = { + name: (self.target / name).read_bytes() + for name in self.document["removals"] + } + for junction in ( + self.source, + self.source.parent, + self.source / "data", + self.target, + self.target.parent, + self.target / "data", + ): + for dry_run in (True, False): + with self.subTest(junction=junction, dry_run=dry_run): + self.assertFalse(junction.is_symlink()) + with ( + patch.object( + Path, + "is_junction", + autospec=True, + side_effect=lambda path: path == junction, + ), + self.assertRaisesRegex(ValueError, "[Jj]unction"), + ): + self.run_removal(dry_run) + self.assertEqual( + {name: (self.target / name).read_bytes() for name in before}, + before, + ) + + def test_junction_revalidation_prevents_partial_removal(self): + for junction in (self.target / "data", self.source / "data"): + checks = 0 + + def becomes_junction(path): + nonlocal checks + if path == junction: + checks += 1 + return checks > 1 + return False + + with self.subTest(junction=junction): + with ( + patch.object( + Path, "is_junction", autospec=True, side_effect=becomes_junction + ), + self.assertRaisesRegex(ValueError, "[Jj]unction"), + ): + self.run_removal() + self.assertEqual(checks, 2) + self.assertEqual((self.target / "a.py").read_bytes(), b"old\n") + self.assertEqual((self.target / "data/z.bin").read_bytes(), b"\x00\xff") + + def test_utf8_manifest_ignores_non_utf8_default_encoding(self): + name = "caf\u00e9.py" + target = self.target / name + target.write_bytes(b"old\n") + self.document["removals"][name] = dict(self.document["removals"]["a.py"]) + self.manifest.write_bytes( + json.dumps(self.document, ensure_ascii=False).encode("utf-8") + ) + with ( + patch( + "pathlib.io.text_encoding", + side_effect=lambda encoding, *args: ( + "latin-1" if encoding is None else encoding + ), + ), + contextlib.chdir(self.target.parents[2]), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertNotEqual( + self.manifest.read_text(), self.manifest.read_text(encoding="utf-8") + ) + for dry_run, expected in ( + (True, "would_remove"), + (False, "removed"), + (False, "already_absent"), + ): + receipt = remove_canonical_files( + self.source, self.manifest, dry_run=dry_run + ) + self.assertEqual( + set(receipt["removals"]), set(self.document["removals"]) + ) + self.assertEqual(receipt["removals"][name]["status"], expected) + self.assertEqual(target.exists(), dry_run) + + def test_cli_dry_run_never_migrates_or_changes_destination_bytes(self): + def snapshot(): + return { + p.relative_to(self.target): p.read_bytes() + for p in self.target.rglob("*") + if p.is_file() + } + + before = snapshot() + self.manifest.write_text(json.dumps(self.document)) + with contextlib.ExitStack() as stack: + stack.enter_context(contextlib.chdir(self.target.parents[2])) + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + for name in ( + "patch_file", + "patch_directory", + "migrate_package", + "migrate_support_assignments", + ): + stack.enter_context( + patch( + f"update_lib.cmd_migrate.{name}", + side_effect=AssertionError("Dry-run must not migrate"), + ) + ) + self.assertEqual( + main( + [ + str(self.source), + "--canonical-removals", + str(self.manifest), + "--dry-run", + ] + ), + 0, + ) + self.assertEqual(snapshot(), before) + + def test_hash_customization_and_all_batch_preflight(self): + path = self.target / "data/z.bin" + for data, repin in ((b"changed", False), (b"# TODO: RUSTPYTHON", True)): + path.write_bytes(data) + if repin: + self.document["removals"]["data/z.bin"]["sha256"] = hashlib.sha256( + data + ).hexdigest() + with self.assertRaises(ValueError): + self.run_removal() + self.assertEqual((self.target / "a.py").read_bytes(), b"old\n") + + def test_canonical_source_identity_and_tracked_absence(self): + for key, value in ( + ("commit", "0" * 40), + ("tree", "0" * 40), + ("path", "Lib/test/test_other"), + ): + old = self.document["source"][key] + self.document["source"][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + self.run_removal() + self.document["source"][key] = old + (self.source / "__init__.py").unlink() + self.document["removals"]["__init__.py"] = self.document["removals"]["a.py"] + with self.assertRaisesRegex(ValueError, "still canonical"): + self.run_removal() + self.assertTrue((self.target / "a.py").exists()) + + def test_invalid_paths_symlinks_directories_and_empty_selection(self): + original = self.document["removals"] + for name in ("../a.py", "/a.py", "./a.py", "data//z.bin", "data\\z.bin"): + self.document["removals"] = {name: original["a.py"]} + with self.subTest(name=name), self.assertRaises(ValueError): + self.run_removal() + self.document["removals"] = {} + with self.assertRaises(ValueError): + self.run_removal() + self.document["removals"] = original + target = self.target / "a.py" + target.unlink() + target.symlink_to(self.target / "data/z.bin") + with self.assertRaisesRegex(ValueError, "Symlink"): + self.run_removal() + target.unlink() + target.mkdir() + with self.assertRaisesRegex(ValueError, "regular file"): + self.run_removal() + + def test_cli_refuses_combined_modes_and_duplicate_metadata(self): + for option in ("--patches", "--legacy-map", "--support-assignments"): + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual( + main( + [ + str(self.source), + "--canonical-removals", + "unused", + option, + "unused", + ] + ), + 1, + ) + self.manifest.write_text('{"source": {}, "source": {}}') + with self.assertRaisesRegex(ValueError, "Duplicate"): + remove_canonical_files(self.source, self.manifest) + + +class TestImportlibGuardMigration(unittest.TestCase): + def test_exact_guards_preserved_and_changed_ambiguous_anchors_refused(self): + setup = "def setUpModule():\n thread_info = threading_helper.threading_setup()\n unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)\n" + cases = ( + ("util.py", MULTIPHASE_LOCAL, MULTIPHASE_UPSTREAM), + ( + "test_locks.py", + "class LifetimeTests:\n def test_all_locks(self): pass\n\n" + + LOCK_SPECIALIZATION + + "\n" + + LOCK_SKIP + + "\n" + + setup, + "class LifetimeTests:\n def test_all_locks(self): pass\n\n" + + LOCK_SPECIALIZATION + + "\n\n" + + setup, + ), + ) + for filename, old, new in cases: + path = Path("Lib/test/test_importlib") / filename + result = preserve_migration_customizations(path, old, new + "VALUE = 2\n") + self.assertEqual(result, old + "VALUE = 2\n") + self.assertEqual( + preserve_migration_customizations(path, result, new + "VALUE = 2\n"), + result, + ) + for changed_old, changed_new in ( + (old + old, new), + (old, new + new), + (old.replace("TODO: RUSTPYTHON", "changed"), new), + ( + old, + "if True:\n" + + "".join(" " + line for line in new.splitlines(keepends=True)), + ), + ): + with self.subTest(filename=filename), self.assertRaises(ValueError): + preserve_migration_customizations(path, changed_old, changed_new) + + +class TestTypesGuardMigration(unittest.TestCase): + path = Path("Lib/test/test_types.py") + old = ( + "import copy\n" + + TYPES_DATETIME_GUARD + + "import gc\nimport unittest.mock\nimport typing\n\n" + + TYPES_UNITTEST_IMPORT + + "\nVALUE = 1\n" + ) + new = ( + "import copy\nimport _datetime\nimport gc\nimport unittest.mock\n" + "import typing\nimport re\n\nVALUE = 2\n" + ) + + def test_exact_guards_and_unrelated_canonical_changes_survive_idempotently(self): + expected = self.new.replace("import _datetime\n", TYPES_DATETIME_GUARD).replace( + "import typing\n", "import typing\n\n" + TYPES_UNITTEST_IMPORT + ) + result = preserve_migration_customizations(self.path, self.old, self.new) + self.assertEqual(result, expected) + self.assertEqual( + preserve_migration_customizations(self.path, result, self.new), expected + ) + + def test_changed_ambiguous_or_misplaced_local_guards_are_refused(self): + for old in ( + self.old.replace("except ImportError:", "except Exception:"), + self.old.replace("_datetime = None", "_datetime = False"), + self.old.replace("# XXX: RUSTPYTHON\n", "# changed\n"), + self.old.replace("importing to be able to skip tests", "changed reason"), + self.old + TYPES_DATETIME_GUARD, + self.old + "import _datetime\n", + self.old + TYPES_UNITTEST_IMPORT, + self.old.replace("import typing\n\n", "import typing\nVALUE = 3\n\n"), + ): + with self.subTest(old=old), self.assertRaises(ValueError): + preserve_migration_customizations(self.path, old, self.new) + + def test_changed_ambiguous_or_nested_canonical_anchors_are_refused(self): + for new in ( + self.new.replace("import _datetime", "import _datetime as other"), + self.new.replace("import _datetime\n", "if True:\n import _datetime\n"), + self.new.replace("import _datetime\n", 'TEXT = "import _datetime\\n"\n'), + self.new.replace("import typing", "import typing as other"), + self.new.replace("import typing\n", "if True:\n import typing\n"), + self.new + "import _datetime\n", + self.new + "import typing\n", + self.new + "import unittest\n", + ): + with self.subTest(new=new), self.assertRaises(ValueError): + preserve_migration_customizations(self.path, self.old, new) + + if __name__ == "__main__": unittest.main()