Skip to content

Update importlib to CPython 3.15 with native bootstrap callback #9169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 3 commits into from
Oct 17, 2026
Merged

Update importlib to CPython 3.15 with native bootstrap callback #9169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3c4989e
Refresh canonical importlib and preserve native import callback
youknowdot Oct 16, 2026
df0dec5
Complete canonical inspect and types closure for importlib metadata
youknowdot Oct 16, 2026
f038891
Reject canonical removal junctions and read UTF-8 manifests
youknowdot Oct 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 0 additions & 18 deletions Lib/importlib/_abc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,3 @@ def create_module(self, spec):

# We don't define exec_module() here since that would break
# hasattr checks we do to support backward compatibility.

def load_module(self, fullname):
"""Return the loaded module.

The module must be added to sys.modules and have import-related
attributes set properly. The fullname is a str.

ImportError is raised on failure.

This method is deprecated in favor of loader.exec_module(). If
exec_module() exists then it is used to provide a backwards-compatible
functionality for this method.

"""
if not hasattr(self, 'exec_module'):
raise ImportError
# Warning implemented in _load_module_shim().
return _bootstrap._load_module_shim(self, fullname)
214 changes: 99 additions & 115 deletions Lib/importlib/_bootstrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,64 @@ def __exit__(self, *args, **kwargs):
self._lock.release()


def _get_module_chain(name):
"""Return the chain of dotted-name prefixes from root to leaf.

For example: 'a.b.c' -> ['a', 'a.b', 'a.b.c']
"""
parts = name.split('.')
return ['.'.join(parts[:i+1]) for i in range(len(parts))]


class _HierarchicalLockManager:
"""Manages acquisition of multiple module locks in hierarchical order.

This prevents deadlocks by ensuring all threads acquire locks in the
same order (parent modules before child modules).
"""

def __init__(self, name):
self._name = name
self._module_chain = _get_module_chain(name)
self._locks = []

def __enter__(self):
try:
for module_name in self._module_chain:
# Only acquire lock if module is not already fully loaded
module = sys.modules.get(module_name)
if (module is None or
getattr(getattr(module, "__spec__", None),
"_initializing", False)):
lock = _get_module_lock(module_name)
try:
lock.acquire()
except _DeadlockError:
if module_name == self._name:
raise
# The parent is being initialised by a thread that
# is (transitively) waiting on a lock we hold.
# Apply the same policy as _lock_unlock_module():
# accept a partially-initialised parent for circular
# imports rather than failing the whole chain.
continue
self._locks.append((module_name, lock))
except:
# __exit__ is not called when __enter__ raises (e.g. _DeadlockError
# on the leaf lock, or KeyboardInterrupt), so release whatever we
# already hold to avoid permanently leaking held module locks.
for module_name, lock in reversed(self._locks):
lock.release()
self._locks.clear()
raise
return self

def __exit__(self, *args, **kwargs):
for module_name, lock in reversed(self._locks):
lock.release()
self._locks.clear()


# The following two functions are for consumption by Python/import.c.

def _get_module_lock(name):
Expand Down Expand Up @@ -521,24 +579,6 @@ def _requires_frozen_wrapper(self, fullname):
return _requires_frozen_wrapper


# Typically used by loader classes as a method replacement.
def _load_module_shim(self, fullname):
"""Load the specified module into sys.modules and return it.

This method is deprecated. Use loader.exec_module() instead.

"""
msg = ("the load_module() method is deprecated and slated for removal in "
"Python 3.15; use exec_module() instead")
_warnings.warn(msg, DeprecationWarning)
spec = spec_from_loader(fullname, self)
if fullname in sys.modules:
module = sys.modules[fullname]
_exec(spec, module)
return sys.modules[fullname]
else:
return _load(spec)

# Module specifications #######################################################

def _module_repr(module):
Expand Down Expand Up @@ -583,8 +623,7 @@ class ModuleSpec:
`has_location` indicates that a spec's "origin" reflects a location.
When this is True, `__file__` attribute of the module is set.

`cached` is the location of the cached bytecode file, if any. It
corresponds to the `__cached__` attribute.
`cached` is the location of the cached bytecode file, if any.

`submodule_search_locations` is the sequence of path entries to
search when importing submodules. If set, is_package should be
Expand Down Expand Up @@ -717,18 +756,14 @@ def _spec_from_module(module, loader=None, origin=None):
origin = getattr(loader, '_ORIGIN', None)
if not origin and location is not None:
origin = location
try:
cached = module.__cached__
except AttributeError:
cached = None
try:
submodule_search_locations = list(module.__path__)
except AttributeError:
submodule_search_locations = None

spec = ModuleSpec(name, loader, origin=origin)
spec._set_fileattr = False if location is None else (origin == location)
spec.cached = cached
spec.cached = None
spec.submodule_search_locations = submodule_search_locations
return spec

Expand Down Expand Up @@ -763,8 +798,7 @@ def _init_module_attrs(spec, module, *, override=False):
# should also be None for consistency. While a bit of a hack,
# this is the best place to ensure this consistency.
#
# See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module
# and bpo-32305
# See bpo-32305
module.__file__ = None
try:
module.__loader__ = loader
Expand All @@ -789,20 +823,14 @@ def _init_module_attrs(spec, module, *, override=False):
module.__path__ = spec.submodule_search_locations
except AttributeError:
pass
# __file__/__cached__
# __file__
if spec.has_location:
if override or getattr(module, '__file__', None) is None:
try:
module.__file__ = spec.origin
except AttributeError:
pass

if override or getattr(module, '__cached__', None) is None:
if spec.cached is not None:
try:
module.__cached__ = spec.cached
except AttributeError:
pass
return module


Expand Down Expand Up @@ -844,7 +872,7 @@ def _module_repr_from_spec(spec):
return f'<module {spec.name!r} ({spec.origin})>'


# Used by importlib.reload() and _load_module_shim().
# Used by importlib.reload().
def _exec(spec, module):
"""Execute the spec's specified module in an existing module's namespace."""
name = spec.name
Expand All @@ -860,67 +888,16 @@ def _exec(spec, module):
_init_module_attrs(spec, module, override=True)
else:
_init_module_attrs(spec, module, override=True)
if not hasattr(spec.loader, 'exec_module'):
msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
"falling back to load_module()")
_warnings.warn(msg, ImportWarning)
spec.loader.load_module(name)
else:
spec.loader.exec_module(module)
spec.loader.exec_module(module)
finally:
# Update the order of insertion into sys.modules for module
# clean-up at shutdown.
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
return module


def _load_backward_compatible(spec):
# It is assumed that all callers have been warned about using load_module()
# appropriately before calling this function.
try:
spec.loader.load_module(spec.name)
except:
if spec.name in sys.modules:
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
raise
# The module must be in sys.modules at this point!
# Move it to the end of sys.modules.
module = sys.modules.pop(spec.name)
sys.modules[spec.name] = module
if getattr(module, '__loader__', None) is None:
try:
module.__loader__ = spec.loader
except AttributeError:
pass
if getattr(module, '__package__', None) is None:
try:
# Since module.__path__ may not line up with
# spec.submodule_search_paths, we can't necessarily rely
# on spec.parent here.
module.__package__ = module.__name__
if not hasattr(module, '__path__'):
module.__package__ = spec.name.rpartition('.')[0]
except AttributeError:
pass
if getattr(module, '__spec__', None) is None:
try:
module.__spec__ = spec
except AttributeError:
pass
return module

def _load_unlocked(spec):
# A helper for direct use by the import system.
if spec.loader is not None:
# Not a namespace package.
if not hasattr(spec.loader, 'exec_module'):
msg = (f"{_object_name(spec.loader)}.exec_module() not found; "
"falling back to load_module()")
_warnings.warn(msg, ImportWarning)
return _load_backward_compatible(spec)

module = module_from_spec(spec)

# This must be done before putting the module in sys.modules
Expand Down Expand Up @@ -954,8 +931,7 @@ def _load_unlocked(spec):

return module

# A method used during testing of _load_unlocked() and by
# _load_module_shim().
# A method used during testing of _load_unlocked().
def _load(spec):
"""Return a new module object, loaded by the spec's loader.

Expand Down Expand Up @@ -1020,8 +996,6 @@ def is_package(cls, fullname):
"""Return False as built-in modules are never packages."""
return False

load_module = classmethod(_load_module_shim)


class FrozenImporter:

Expand Down Expand Up @@ -1178,25 +1152,6 @@ def exec_module(module):
code = _call_with_frames_removed(_imp.get_frozen_object, name)
exec(code, module.__dict__)

@classmethod
def load_module(cls, fullname):
"""Load a frozen module.

This method is deprecated. Use exec_module() instead.

"""
# Warning about deprecation implemented in _load_module_shim().
module = _load_module_shim(cls, fullname)
info = _imp.find_frozen(fullname)
assert info is not None
_, ispkg, origname = info
module.__origname__ = origname
vars(module).pop('__file__', None)
if ispkg:
module.__path__ = []
cls._fix_up_module(module)
return module

@classmethod
@_requires_frozen
def get_code(cls, fullname):
Expand Down Expand Up @@ -1305,8 +1260,16 @@ def _sanity_check(name, package, level):

_ERR_MSG_PREFIX = 'No module named '

def _find_and_load_unlocked(name, import_):
def _find_and_load_unlocked(name, import_, *, lazy_submodule=False):
path = None
sys.audit(
"import",
name,
path,
getattr(sys, "path", None),
getattr(sys, "meta_path", None),
getattr(sys, "path_hooks", None)
)
parent = name.rpartition('.')[0]
parent_spec = None
if parent:
Expand All @@ -1320,6 +1283,8 @@ def _find_and_load_unlocked(name, import_):
try:
path = parent_module.__path__
except AttributeError:
if lazy_submodule:
return None
msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package'
raise ModuleNotFoundError(msg, name=name) from None
parent_spec = parent_module.__spec__
Expand All @@ -1332,6 +1297,8 @@ def _find_and_load_unlocked(name, import_):
child = name.rpartition('.')[2]
spec = _find_spec(name, path)
if spec is None:
if lazy_submodule:
return None
raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name)
else:
if parent_spec:
Expand All @@ -1351,24 +1318,37 @@ def _find_and_load_unlocked(name, import_):
except AttributeError:
msg = f"Cannot set an attribute on {parent!r} for child module {child!r}"
_warnings.warn(msg, ImportWarning)
# Set attributes to lazy submodules on the module.
try:
_imp._set_lazy_attributes(module, name)
except Exception as e:
msg = f"Cannot set lazy attributes on {name!r}: {e!r}"
_warnings.warn(msg, ImportWarning)
return module


_NEEDS_LOADING = object()


def _find_and_load(name, import_):
def _find_and_load(name, import_, *, lazy_submodule=False):
"""Find and load the module."""

# Optimization: we avoid unneeded module locking if the module
# already exists in sys.modules and is fully initialized.
module = sys.modules.get(name, _NEEDS_LOADING)
if (module is _NEEDS_LOADING or
getattr(getattr(module, "__spec__", None), "_initializing", False)):
with _ModuleLockManager(name):

if '.' in name:
lock_manager = _HierarchicalLockManager(name)
else:
lock_manager = _ModuleLockManager(name)

with lock_manager:
module = sys.modules.get(name, _NEEDS_LOADING)
if module is _NEEDS_LOADING:
return _find_and_load_unlocked(name, import_)
return _find_and_load_unlocked(
name, import_, lazy_submodule=lazy_submodule)

# Optimization: only call _bootstrap._lock_unlock_module() if
# module.__spec__._initializing is True.
Expand All @@ -1382,7 +1362,7 @@ def _find_and_load(name, import_):
# to preserve normal semantics: the caller gets the exception from
# the actual import failure rather than a synthetic error.
if sys.modules.get(name) is not module:
return _find_and_load(name, import_)
return _find_and_load(name, import_, lazy_submodule=lazy_submodule)

if module is None:
message = f'import of {name} halted; None in sys.modules'
Expand All @@ -1391,6 +1371,10 @@ def _find_and_load(name, import_):
return module


def _find_and_load_lazy_submodule(name, import_):
return _find_and_load(name, import_, lazy_submodule=True)


def _gcd_import(name, package=None, level=0):
"""Import and return the module based on its name, the package the call is
being made from, and the level adjustment.
Expand Down
Loading
Loading