diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 80fd19f871f..bf648d55b9f 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -2432,6 +2432,9 @@ def magic_config_matches(self, text: str) -> list[str]: texts = text.strip().split() if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'): + # Magics classes are registered lazily and are only configurable + # once instantiated; load them so %config completes all of them. + self.shell.magics_manager.load_all_lazy_magics() # get all configuration classes classes = sorted({ c for c in self.shell.configurables if c.__class__.class_traits(config=True) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index b9812aee950..6b7cee36596 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2440,29 +2440,39 @@ def init_magics(self): # Expose as public API from the magics manager self.register_magics = self.magics_manager.register - self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics, - m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics, - m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics, - m.NamespaceMagics, m.OSMagics, m.PackagingMagics, - m.PylabMagics, m.ScriptMagics, - ) - self.register_magics(m.AsyncMagics) + mman = self.magics_manager + + # The built-in magics are registered lazily: only their *names* are + # known at startup, and the module implementing a magic is imported the + # first time that magic is used. The name -> class table is hand + # maintained in IPython.core.magics._table; tests/test_magic_table.py + # fails if it drifts out of sync. + for class_name, names in m.BUILTIN_MAGICS.items(): + spec = "{}:{}".format(m.MAGICS_CLASSES[class_name], class_name) + line_magics = names["line"] + cell_magics = names["cell"] + if class_name == "ScriptMagics": + # ScriptMagics generates one `%%` cell magic per + # entry of its (configurable) `script_magics` trait. + cell_magics = ( + *cell_magics, + *m.configured_script_magics(self.config), + ) + mman.register_lazy_class(spec, line_magics, cell_magics) # Register Magic Aliases - mman = self.magics_manager # FIXME: magic aliases should be defined by the Magics classes # or in MagicsManager, not here - mman.register_alias('ed', 'edit') - mman.register_alias('hist', 'history') - mman.register_alias('rep', 'recall') - mman.register_alias('SVG', 'svg', 'cell') - mman.register_alias('HTML', 'html', 'cell') - mman.register_alias('file', 'writefile', 'cell') + for alias, target, magic_kind in m.BUILTIN_MAGIC_ALIASES: + mman.register_alias(alias, target, magic_kind) # FIXME: Move the color initialization to the DisplayHook, which # should be split into a prompt manager and displayhook. We probably # even need a centralize colors management object. - self.run_line_magic('colors', self.colors) + # This used to go through `%colors`, but that would defeat the lazy + # registration above by importing the basic magics on every startup; + # all the magic does is assign `shell.colors`, whose observer this is. + self.init_syntax_highlighting() # Defined here so that it's included in the documentation @functools.wraps(magic.MagicsManager.register_function) @@ -2618,19 +2628,19 @@ def find_line_magic(self, magic_name): """Find and return a line magic by name. Returns None if the magic isn't found.""" - return self.magics_manager.magics['line'].get(magic_name) + return self.magics_manager.find("line", magic_name) def find_cell_magic(self, magic_name): """Find and return a cell magic by name. Returns None if the magic isn't found.""" - return self.magics_manager.magics['cell'].get(magic_name) + return self.magics_manager.find("cell", magic_name) def find_magic(self, magic_name, magic_kind='line'): """Find and return a magic of the given type by name. Returns None if the magic isn't found.""" - return self.magics_manager.magics[magic_kind].get(magic_name) + return self.magics_manager.find(magic_kind, magic_name) #------------------------------------------------------------------------- # Things related to macros diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 35bb9fd0ea9..31ad54aac98 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -304,6 +304,60 @@ def mark(func: _F, *a: Any, **kw: Any) -> _F: MAGIC_OUTPUT_CAN_BE_SILENCED = "_ipython_magic_output_can_be_silenced" +# Note: this class deliberately has no docstring of its own -- `__doc__` is a +# property so that anything asking a registered magic for its documentation +# gets the real magic's docstring instead of this placeholder's. +# +# A `LazyMagic` stands in the magics table for a magic whose implementing +# module has not been imported yet; see `MagicsManager.register_lazy_class`. +# Listing magics, completing them, or testing membership only ever looks at the +# *names* in that table, so they stay cheap. Anything that actually uses the +# magic -- calling it, reading its docstring, inspecting it -- goes through +# `_lazy_resolve` below, which imports the module, registers the real `Magics` +# instance (replacing this object in the table) and delegates to it. +class LazyMagic: + __slots__ = ("_lazy_kind", "_lazy_manager", "_lazy_name", "_lazy_spec") + + def __init__( + self, + manager: MagicsManager, + spec: str, + magic_kind: _MagicKind, + magic_name: str, + ) -> None: + self._lazy_manager = manager + self._lazy_spec = spec + self._lazy_kind = magic_kind + self._lazy_name = magic_name + + @property + def _lazy_class_name(self) -> str: + """Name of the ``Magics`` subclass that will provide this magic.""" + return self._lazy_spec.rpartition(":")[2] + + def _lazy_resolve(self) -> Callable[..., Any]: + """Import the implementing module and return the real magic.""" + return self._lazy_manager._resolve_lazy(self) + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return self._lazy_resolve()(*args, **kwargs) + + def __getattr__(self, name: str) -> Any: + if name.startswith("_lazy_"): + # Never resolve to answer for our own internals; a missing one is a + # genuine AttributeError, not a reason to import anything. + raise AttributeError(name) + return getattr(self._lazy_resolve(), name) + + @property + def __doc__(self) -> str | None: # type: ignore[override] + return self._lazy_resolve().__doc__ + + def __repr__(self) -> str: + escape = magic_escapes[self._lazy_kind] + return f"" + + def no_var_expand(magic_func: _F) -> _F: """Mark a magic function as not needing variable expansion @@ -349,6 +403,29 @@ def output_can_be_silenced(magic_func: _F) -> _F: # ----------------------------------------------------------------------------- +class _MagicsRegistry(dict[str, Any]): + """The ``MagicsManager.registry`` mapping, aware of unloaded magics. + + Looking a class up by name is a legitimate way to reach a magics instance + (``shell.magics_manager.registry["ExecutionMagics"]``), and with lazy + registration that lookup may well be the first thing that needs the class. + So a miss on a name we know how to load imports it rather than raising. + """ + + def __init__(self, manager: MagicsManager) -> None: + super().__init__() + self._manager = manager + + def __missing__(self, key: str) -> Any: + manager = self._manager + spec = manager._lazy_class_specs.get(key) + if spec is None or spec in manager._loaded_lazy_classes: + # Nothing left to try; a second miss must not loop back here. + raise KeyError(key) + manager.load_lazy_class(spec) + return self[key] + + class MagicsManager(Configurable): """Object that handles all magic-related functionality for IPython.""" @@ -421,6 +498,12 @@ def __init__( shell=shell, config=config, user_magics=user_magics, **traits ) self.magics = dict(line={}, cell={}) + # Class name -> ``module:ClassName`` spec, for the Magics classes + # registered through `register_lazy_class`... + self._lazy_class_specs: dict[str, str] = {} + # ... and the specs among those that have already been loaded. + self._loaded_lazy_classes: set[str] = set() + self.registry = _MagicsRegistry(self) # Let's add the user_magics to the registry for uniformity, so *all* # registered magic containers can be found there. if user_magics is not None: @@ -453,7 +536,7 @@ def lsmagic_docs( docs: dict[str, dict[str, str]] = {} for m_type in self.magics: m_docs: dict[str, str] = {} - for m_name, m_func in self.magics[m_type].items(): + for m_name, m_func in list(self.magics[m_type].items()): if m_func.__doc__: if brief: m_docs[m_name] = m_func.__doc__.split("\n", 1)[0] @@ -482,6 +565,116 @@ def register_lazy(self, name: str, fully_qualified_name: str) -> None: self.lazy_magics[name] = fully_qualified_name + def register_lazy_class( + self, + spec: str, + line_magics: t.Iterable[str] = (), + cell_magics: t.Iterable[str] = (), + ) -> None: + """Register magics whose module is only imported on first use. + + The named magics become visible to ``%lsmagic``, to completion and to + :meth:`find` right away, but nothing is imported until one of them is + actually used -- at which point the class is instantiated and + registered as if :meth:`register` had been called with it. + + This is how IPython registers its own magics; the name tables live in + :mod:`IPython.core.magics._table`. Unlike :attr:`lazy_magics`, which + loads an *extension* and trusts it to register something, this knows + exactly which class provides which name, and raises if loading the + module does not deliver it. + + Parameters + ---------- + spec : str + Import path of the class providing the magics, as + ``"package.module:ClassName"``. + line_magics : iterable of str + Names of the line magics the class provides. + cell_magics : iterable of str + Names of the cell magics the class provides. + """ + module_name, sep, class_name = spec.partition(":") + if not (sep and module_name and class_name): + raise ValueError( + f"spec must be of the form 'package.module:ClassName', got {spec!r}" + ) + if spec in self._loaded_lazy_classes: + # Already imported and registered; the real magics are in place and + # must not be shadowed by placeholders. + return + self._lazy_class_specs[class_name] = spec + by_kind: tuple[tuple[_MagicKind, t.Iterable[str]], ...] = ( + ("line", line_magics), + ("cell", cell_magics), + ) + for magic_kind, names in by_kind: + table = self.magics[magic_kind] + for magic_name in names: + table[magic_name] = LazyMagic(self, spec, magic_kind, magic_name) + + def load_lazy_class(self, spec: str) -> None: + """Import and register a class registered by :meth:`register_lazy_class`. + + Does nothing if it has already been loaded. + """ + if spec in self._loaded_lazy_classes: + return + module_name, _, class_name = spec.partition(":") + # Mark it loaded first: instantiating the class may itself look a magic + # up, and we must not recurse back into here. + self._loaded_lazy_classes.add(spec) + from importlib import import_module + + self.register(getattr(import_module(module_name), class_name)) + + def load_all_lazy_magics(self) -> None: + """Import and register every magic still waiting to be loaded. + + Only useful for the handful of things that need a complete picture of + the magics -- listing every configurable, for instance. Everything + else should go through :meth:`find` and pay for what it uses. + """ + specs = { + fn._lazy_spec + for table in self.magics.values() + for fn in table.values() + if isinstance(fn, LazyMagic) + } + for spec in sorted(specs): + self.load_lazy_class(spec) + + def _resolve_lazy(self, proxy: LazyMagic) -> Callable[..., Any]: + """Load `proxy`'s class and return the real magic it stands for.""" + magic_kind, magic_name = proxy._lazy_kind, proxy._lazy_name + self.load_lazy_class(proxy._lazy_spec) + fn = self.magics[magic_kind].get(magic_name) + if fn is None or fn is proxy: + # The name table is out of sync with the code it describes. Drop + # the stale entry so the magic reads as missing from now on. + if self.magics[magic_kind].get(magic_name) is proxy: + del self.magics[magic_kind][magic_name] + raise UsageError( + f"Magic `{magic_escapes[magic_kind]}{magic_name}` was declared to" + f" be provided by {proxy._lazy_spec}, but loading it did not" + " define that magic." + ) + if isinstance(fn, LazyMagic): + return fn._lazy_resolve() + return t.cast("Callable[..., Any]", fn) + + def find( + self, magic_kind: _MagicKind, magic_name: str + ) -> Callable[..., Any] | None: + """Return a registered magic, importing its implementation if needed. + + Returns None if there is no such magic. + """ + fn: Any = self.magics[magic_kind].get(magic_name) + if isinstance(fn, LazyMagic): + return fn._lazy_resolve() + return t.cast("Callable[..., Any] | None", fn) + def register(self, *magic_objects: type[Magics] | Magics) -> None: """Register one or more instances of Magics. diff --git a/IPython/core/magics/__init__.py b/IPython/core/magics/__init__.py index a6c5f474c15..4d0ae77c374 100644 --- a/IPython/core/magics/__init__.py +++ b/IPython/core/magics/__init__.py @@ -11,22 +11,66 @@ #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- +from __future__ import annotations + +import typing as t from ..magic import Magics, magics_class -from .auto import AutoMagics -from .basic import BasicMagics, AsyncMagics -from .code import CodeMagics, MacroToEdit -from .config import ConfigMagics -from .display import DisplayMagics -from .execution import ExecutionMagics -from .extension import ExtensionMagics -from .history import HistoryMagics -from .logging import LoggingMagics -from .namespace import NamespaceMagics -from .osm import OSMagics -from .packaging import PackagingMagics -from .pylab import PylabMagics -from .script import ScriptMagics +from ._table import ( + BUILTIN_MAGIC_ALIASES, + BUILTIN_MAGICS, + MAGICS_CLASSES, + configured_script_magics, + default_script_magics, +) + +# The submodules implementing the magics are *not* imported here: they, and the +# `Magics` subclasses they define, are only loaded the first time one of their +# magics is used. See `._table` and `IPython.core.magic.MagicsManager` for how +# the lazy registration works. The names below stay importable from this +# package through the module `__getattr__` further down. +if t.TYPE_CHECKING: + from .auto import AutoMagics + from .basic import AsyncMagics, BasicMagics + from .code import CodeMagics, MacroToEdit + from .config import ConfigMagics + from .display import DisplayMagics + from .execution import ExecutionMagics + from .extension import ExtensionMagics + from .history import HistoryMagics + from .logging import LoggingMagics + from .namespace import NamespaceMagics + from .osm import OSMagics + from .packaging import PackagingMagics + from .pylab import PylabMagics + from .script import ScriptMagics + +# Kept literal so that static analysers can see it; `tests/test_magic_table.py` +# checks it against MAGICS_CLASSES. +__all__ = [ + "BUILTIN_MAGICS", + "BUILTIN_MAGIC_ALIASES", + "MAGICS_CLASSES", + "AsyncMagics", + "AutoMagics", + "BasicMagics", + "CodeMagics", + "ConfigMagics", + "DisplayMagics", + "ExecutionMagics", + "ExtensionMagics", + "HistoryMagics", + "LoggingMagics", + "MacroToEdit", + "NamespaceMagics", + "OSMagics", + "PackagingMagics", + "PylabMagics", + "ScriptMagics", + "UserMagics", + "configured_script_magics", + "default_script_magics", +] #----------------------------------------------------------------------------- # Magic implementation classes @@ -40,3 +84,20 @@ class UserMagics(Magics): use this class to isolate the magics defined dynamically by the user into their own class. """ + + +def __getattr__(name: str) -> t.Any: + """Import the magics classes on first access (:pep:`562`).""" + module_name = MAGICS_CLASSES.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + obj = getattr(import_module(module_name), name) + # Cache it so subsequent lookups skip this function entirely. + globals()[name] = obj + return obj + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/IPython/core/magics/_table.py b/IPython/core/magics/_table.py new file mode 100644 index 00000000000..0f29d45060e --- /dev/null +++ b/IPython/core/magics/_table.py @@ -0,0 +1,224 @@ +"""Static description of the magics IPython ships with. + +Importing a module and instantiating the ``Magics`` subclasses it defines is +expensive, and most of the magics in a given session are never used. To keep +``import IPython`` and shell startup fast, IPython registers its own magics +lazily: only the tables below are consulted at startup, and the module +implementing a magic is imported the first time that magic is looked up. + +That means these tables are hand maintained and *must* be kept in sync with the +code. ``tests/test_magic_table.py`` imports every magics module, instantiates every +``Magics`` subclass, and fails if anything here is missing, stale, or pointing +at the wrong class -- and prints the corrected table. +""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +from __future__ import annotations + +import os +import typing as t + +if t.TYPE_CHECKING: + from traitlets.config import Config + + +#: Every public name re-exported by :mod:`IPython.core.magics`, mapped to the +#: module that defines it. Used by that package's module ``__getattr__`` so +#: that ``from IPython.core.magics import ExecutionMagics`` keeps working +#: without importing all the sibling modules. +MAGICS_CLASSES: dict[str, str] = { + "AsyncMagics": "IPython.core.magics.basic", + "AutoMagics": "IPython.core.magics.auto", + "BasicMagics": "IPython.core.magics.basic", + "CodeMagics": "IPython.core.magics.code", + "ConfigMagics": "IPython.core.magics.config", + "DisplayMagics": "IPython.core.magics.display", + "ExecutionMagics": "IPython.core.magics.execution", + "ExtensionMagics": "IPython.core.magics.extension", + "HistoryMagics": "IPython.core.magics.history", + "LoggingMagics": "IPython.core.magics.logging", + "MacroToEdit": "IPython.core.magics.code", + "NamespaceMagics": "IPython.core.magics.namespace", + "OSMagics": "IPython.core.magics.osm", + "PackagingMagics": "IPython.core.magics.packaging", + "PylabMagics": "IPython.core.magics.pylab", + "ScriptMagics": "IPython.core.magics.script", +} + +#: The magics each built-in ``Magics`` subclass provides, in the order the +#: classes are registered by ``InteractiveShell.init_magics``. Order matters: +#: a name defined by two classes resolves to the one registered last. +#: +#: ``ScriptMagics`` is a special case -- on top of the names listed here it +#: generates one cell magic per configured interpreter, see +#: :func:`configured_script_magics`. +BUILTIN_MAGICS: dict[str, dict[str, tuple[str, ...]]] = { + "AutoMagics": { + "line": ("autocall", "automagic"), + "cell": (), + }, + "BasicMagics": { + "line": ( + "alias_magic", + "colors", + "doctest_mode", + "gui", + "lsmagic", + "magic", + "notebook", + "page", + "pprint", + "precision", + "quickref", + "xmode", + ), + "cell": (), + }, + "CodeMagics": { + "line": ("edit", "load", "loadpy", "pastebin", "save"), + "cell": (), + }, + "ConfigMagics": { + "line": ("config",), + "cell": (), + }, + "DisplayMagics": { + "line": (), + "cell": ("html", "javascript", "js", "latex", "markdown", "svg"), + }, + "ExecutionMagics": { + "line": ( + "code_wrap", + "debug", + "macro", + "pdb", + "prun", + "run", + "tb", + "time", + "timeit", + ), + "cell": ("capture", "code_wrap", "debug", "prun", "time", "timeit"), + }, + "ExtensionMagics": { + "line": ("load_ext", "reload_ext", "unload_ext"), + "cell": (), + }, + "HistoryMagics": { + "line": ("history", "recall", "rerun"), + "cell": (), + }, + "LoggingMagics": { + "line": ("logoff", "logon", "logstart", "logstate", "logstop"), + "cell": (), + }, + "NamespaceMagics": { + "line": ( + "pdef", + "pdoc", + "pfile", + "pinfo", + "pinfo2", + "psearch", + "psource", + "reset", + "reset_selective", + "who", + "who_ls", + "whos", + "xdel", + ), + "cell": (), + }, + "OSMagics": { + "line": ( + "alias", + "bookmark", + "cd", + "dhist", + "dirs", + "env", + "popd", + "pushd", + "pwd", + "pycat", + "rehashx", + "sc", + "set_env", + "sx", + "system", + "unalias", + ), + "cell": ("!", "sx", "system", "writefile"), + }, + "PackagingMagics": { + "line": ("conda", "mamba", "micromamba", "pip", "uv"), + "cell": (), + }, + "PylabMagics": { + "line": ("matplotlib", "pylab"), + "cell": (), + }, + "ScriptMagics": { + "line": ("killbgscripts",), + "cell": ("script",), + }, + "AsyncMagics": { + "line": ("autoawait",), + "cell": (), + }, +} + +#: Aliases ``InteractiveShell.init_magics`` registers, as +#: ``(alias, target, kind)``. Aliases resolve their target when called, so +#: registering them does not force any magics module to be imported. +BUILTIN_MAGIC_ALIASES: tuple[tuple[str, str, str], ...] = ( + ("ed", "edit", "line"), + ("hist", "history", "line"), + ("rep", "recall", "line"), + ("SVG", "svg", "cell"), + ("HTML", "html", "cell"), + ("file", "writefile", "cell"), +) + + +def default_script_magics() -> list[str]: + """Interpreters ``%%script`` shortcuts are generated for by default. + + This is the default value of the ``ScriptMagics.script_magics`` trait; it + lives here so that the lazy registration can know the generated names + without importing :mod:`IPython.core.magics.script`. + """ + defaults = [ + "sh", + "bash", + "perl", + "ruby", + "python", + "python2", + "python3", + "pypy", + ] + if os.name == "nt": + defaults.extend( + [ + "cmd", + ] + ) + + return defaults + + +def configured_script_magics(config: Config | None) -> list[str]: + """Cell magic names ``ScriptMagics`` will provide for the given config. + + ``ScriptMagics.script_magics`` is configurable, so the generated names are + not known statically; peek at the config rather than instantiating the + class, which is what we are trying to avoid in the first place. + """ + section = getattr(config, "ScriptMagics", None) if config is not None else None + if section is not None and "script_magics" in section: + return list(section["script_magics"]) + return default_script_magics() diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 1ae8b67bfc3..4f52bf37847 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -13,7 +13,13 @@ from traitlets.utils.importstring import import_item from IPython.core import magic_arguments, page from IPython.core.error import UsageError -from IPython.core.magic import Magics, magics_class, line_magic, magic_escapes +from IPython.core.magic import ( + LazyMagic, + Magics, + magic_escapes, + magics_class, + line_magic, +) from IPython.utils.text import format_screen, dedent, indent from IPython.testing.skipdoctest import skip_doctest from IPython.utils.ipstruct import Struct @@ -60,10 +66,15 @@ def _jsonable(self): d = {} magic_dict[key] = d for name, obj in subdict.items(): - try: - classname = obj.__self__.__class__.__name__ - except AttributeError: - classname = 'Other' + if isinstance(obj, LazyMagic): + # Not imported yet, and asking for the class name is no + # reason to import it; the table already knows. + classname = obj._lazy_class_name + else: + try: + classname = obj.__self__.__class__.__name__ + except AttributeError: + classname = "Other" d[name] = classname return magic_dict diff --git a/IPython/core/magics/config.py b/IPython/core/magics/config.py index 56924de17eb..28d0a3fe29c 100644 --- a/IPython/core/magics/config.py +++ b/IPython/core/magics/config.py @@ -87,6 +87,11 @@ def config(self, s): """ from traitlets.config.loader import Config + + # Magics classes only become configurable once they are instantiated, + # and most of them are registered lazily; load them all so that every + # built-in magic shows up here whether or not it has been used yet. + self.shell.magics_manager.load_all_lazy_magics() # some IPython objects are Configurable, but do not yet have # any configurable traits. Exclude them from the effects of # this magic, as their presence is just noise: diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py index c0646b1191a..2f33a110dc0 100644 --- a/IPython/core/magics/script.py +++ b/IPython/core/magics/script.py @@ -23,6 +23,8 @@ from IPython.core.magic import Magics, cell_magic, line_magic, magics_class from IPython.utils.process import arg_split +from ._table import default_script_magics + #----------------------------------------------------------------------------- # Magic implementation classes #----------------------------------------------------------------------------- @@ -105,23 +107,9 @@ class ScriptMagics(Magics): @default('script_magics') def _script_magics_default(self): """default to a common list of programs""" - - defaults = [ - 'sh', - 'bash', - 'perl', - 'ruby', - 'python', - 'python2', - 'python3', - 'pypy', - ] - if os.name == 'nt': - defaults.extend([ - 'cmd', - ]) - - return defaults + # Kept in `_table` so that the lazy registration knows which + # `%%` magics this class provides without importing it. + return default_script_magics() script_paths = Dict( help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby' diff --git a/IPython/terminal/ipapp.py b/IPython/terminal/ipapp.py index 0104fbb5ef7..c6d45183df7 100755 --- a/IPython/terminal/ipapp.py +++ b/IPython/terminal/ipapp.py @@ -24,13 +24,9 @@ ProfileDir, BaseIPythonApplication, base_flags, base_aliases ) from IPython.core.magic import MagicsManager -from IPython.core.magics import ( - ScriptMagics, LoggingMagics -) from IPython.core.shellapp import ( InteractiveShellApp, shell_flags, shell_aliases ) -from IPython.extensions.storemagic import StoreMagics from .interactiveshell import TerminalInteractiveShell from IPython.paths import get_ipython_dir from traitlets import ( @@ -202,6 +198,12 @@ class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): @default('classes') def _classes_default(self): """This has to be in a method, for TerminalIPythonApp to be available.""" + # Imported here rather than at module level: these are only needed to + # generate config help, and importing them eagerly would defeat the + # lazy registration of the built-in magics. + from IPython.core.magics import LoggingMagics, ScriptMagics + from IPython.extensions.storemagic import StoreMagics + return [ InteractiveShellApp, # ShellApp comes before TerminalApp, because self.__class__, # it will also affect subclasses (e.g. QtConsole) diff --git a/docs/source/whatsnew/pr/lazy-builtin-magics.rst b/docs/source/whatsnew/pr/lazy-builtin-magics.rst new file mode 100644 index 00000000000..1a2e4ae9fe8 --- /dev/null +++ b/docs/source/whatsnew/pr/lazy-builtin-magics.rst @@ -0,0 +1,32 @@ +IPython's own magics are now registered lazily. Starting a shell used to import +all fifteen modules under :mod:`IPython.core.magics` and instantiate every +:class:`~IPython.core.magic.Magics` class in them, even though a given session +typically uses a handful of magics at most. Now only the magic *names* are known +up front -- from the hand-maintained tables in ``IPython.core.magics._table`` -- +and the module implementing a magic is imported the first time that magic is +looked up. This shaves roughly 35 ms off ``import IPython`` and shell startup. + +This is invisible in normal use: ``%lsmagic``, completion, ``%foo?`` and calling +a magic all behave as before. Two details are worth knowing if you poke at the +internals: + +* ``shell.magics_manager.magics['line']`` (and ``['cell']``) may hold + :class:`~IPython.core.magic.LazyMagic` placeholders rather than bound methods. + Calling one, or reading any attribute of one, transparently loads the real + magic. Use :meth:`~IPython.core.magic.MagicsManager.find` --- or + ``shell.find_line_magic`` / ``find_cell_magic`` / ``find_magic``, which go + through it --- to get the real callable. +* A magics class only appears in ``shell.configurables`` once it has been + loaded. ``%config`` (and its completions) load everything first, so the list + of configurable classes it shows is unchanged. + +Third-party code can use the same mechanism for its own magics:: + + shell.magics_manager.register_lazy_class( + "my_package.magics:MyMagics", + line_magics=["my_magic"], + cell_magics=["my_magic"], + ) + +which, unlike the existing ``MagicsManager.lazy_magics`` configuration, does not +require the magics to be packaged as an IPython extension. diff --git a/tests/test_magic_table.py b/tests/test_magic_table.py new file mode 100644 index 00000000000..0039d379c25 --- /dev/null +++ b/tests/test_magic_table.py @@ -0,0 +1,313 @@ +"""Check the hand-maintained table of built-in magics against the code. + +IPython registers its own magics lazily, from the static tables in +:mod:`IPython.core.magics._table`, so that starting a shell does not import +every magics module. Those tables are only correct as long as somebody keeps +them correct -- that is what this module is for. +""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import inspect +import pkgutil +import subprocess +import sys +import textwrap +from importlib import import_module + +import pytest +from traitlets.config import Config, Configurable + +import IPython.core.magics +from IPython.core.interactiveshell import InteractiveShellABC +from IPython.core.magic import Magics +from IPython.core.magics import _table +from IPython import get_ipython + + +def _shipped_magics_classes(): + """Every ``Magics`` subclass defined under ``IPython/core/magics``.""" + found = {} + for info in pkgutil.iter_modules(IPython.core.magics.__path__): + module_name = f"{IPython.core.magics.__name__}.{info.name}" + module = import_module(module_name) + for name, obj in vars(module).items(): + if ( + inspect.isclass(obj) + and issubclass(obj, Magics) + and obj is not Magics + # Only where it is defined, not where it is imported. + and obj.__module__ == module_name + ): + found[name] = obj + return found + + +class _UnconfiguredShell(Configurable): + """Just enough of a shell for a ``Magics`` class to be instantiated. + + The magics a class provides has to be compared against the *default* + configuration, not against whatever the session-wide test shell has been + configured with along the way (``tests/test_magic.py`` adds script magics + to it, for one). + """ + + def __init__(self): + super().__init__(config=Config()) + self.configurables = [] + + +# So that `MagicsManager.shell` accepts one of these. +InteractiveShellABC.register(_UnconfiguredShell) + + +def _actual_magics(cls): + """The line and cell magics an instance of `cls` registers.""" + instance = cls(shell=_UnconfiguredShell()) + return { + kind: tuple(sorted(instance.magics[kind])) for kind in ("line", "cell") + } + + +def _render(table): + """Render a name table as the source that should live in ``_table.py``.""" + lines = ["BUILTIN_MAGICS = {"] + for class_name, kinds in table.items(): + lines.append(f' "{class_name}": {{') + for kind in ("line", "cell"): + lines.append(f' "{kind}": {kinds[kind]!r},') + lines.append(" },") + lines.append("}") + return "\n".join(lines) + + +def test_every_shipped_class_is_in_the_table(): + """A new Magics class must be added to ``BUILTIN_MAGICS``.""" + shipped = set(_shipped_magics_classes()) + # UserMagics is a placeholder for magics defined at runtime; it never + # provides any magic of its own and is instantiated eagerly. + shipped.discard("UserMagics") + missing = shipped - set(_table.BUILTIN_MAGICS) + assert not missing, ( + f"{sorted(missing)} are not listed in IPython.core.magics._table." + "BUILTIN_MAGICS, so their magics would not be registered at startup." + ) + + +def test_every_shipped_class_is_importable_from_the_package(): + """``MAGICS_CLASSES`` drives ``IPython.core.magics.__getattr__``.""" + for class_name, cls in _shipped_magics_classes().items(): + if class_name == "UserMagics": + continue + assert _table.MAGICS_CLASSES.get(class_name) == cls.__module__, ( + f"IPython.core.magics._table.MAGICS_CLASSES[{class_name!r}] should" + f" be {cls.__module__!r}" + ) + assert getattr(IPython.core.magics, class_name) is cls + + +def test_all_covers_the_lazily_exported_names(): + """``__all__`` is written out by hand so tools can read it statically.""" + assert set(_table.MAGICS_CLASSES) <= set(IPython.core.magics.__all__) + for name in IPython.core.magics.__all__: + assert getattr(IPython.core.magics, name) is not None + assert set(IPython.core.magics.__all__) <= set(dir(IPython.core.magics)) + + +def test_magics_classes_entries_resolve(): + """Every entry of ``MAGICS_CLASSES`` points at something that exists.""" + for name, module_name in _table.MAGICS_CLASSES.items(): + assert hasattr(import_module(module_name), name), ( + f"{module_name} does not define {name}" + ) + assert getattr(IPython.core.magics, name) is getattr( + import_module(module_name), name + ) + + +def test_builtin_magics_table_is_accurate(): + """``BUILTIN_MAGICS`` lists exactly the magics each class provides.""" + expected = {} + for class_name in _table.BUILTIN_MAGICS: + cls = getattr(IPython.core.magics, class_name) + expected[class_name] = _actual_magics(cls) + + # ScriptMagics generates one cell magic per configured interpreter; the + # table only carries the fixed ones, `init_magics` adds the rest. + generated = set(_table.default_script_magics()) + expected["ScriptMagics"]["cell"] = tuple( + name for name in expected["ScriptMagics"]["cell"] if name not in generated + ) + + declared = { + class_name: { + kind: tuple(sorted(kinds[kind])) for kind in ("line", "cell") + } + for class_name, kinds in _table.BUILTIN_MAGICS.items() + } + + assert declared == expected, ( + "IPython.core.magics._table.BUILTIN_MAGICS is out of sync with the " + "magics classes. It should read:\n\n" + _render(expected) + ) + + +def test_no_magic_is_claimed_by_two_classes(): + """Lazy registration assumes one class per name. + + With eager registration a duplicated name resolved to whichever class was + registered last. Lazily, whichever class happens to be imported first + would win instead, so duplicates must not happen. + """ + for kind in ("line", "cell"): + owners = {} + for class_name, kinds in _table.BUILTIN_MAGICS.items(): + for magic_name in kinds[kind]: + owners.setdefault(magic_name, []).append(class_name) + duplicated = {name: cls for name, cls in owners.items() if len(cls) > 1} + assert not duplicated, f"duplicated {kind} magics: {duplicated}" + + +def test_registered_names_match_the_table(): + """A live shell knows every magic the table declares.""" + ip = get_ipython() + for kind in ("line", "cell"): + declared = { + name + for kinds in _table.BUILTIN_MAGICS.values() + for name in kinds[kind] + } + if kind == "cell": + declared |= set(_table.default_script_magics()) + missing = declared - set(ip.magics_manager.magics[kind]) + assert not missing + + +def test_aliases_point_at_known_magics(): + for alias, target, kind in _table.BUILTIN_MAGIC_ALIASES: + assert any( + target in kinds[kind] for kinds in _table.BUILTIN_MAGICS.values() + ), f"%{alias} aliases {target}, which no built-in class provides" + + +def test_configured_script_magics(): + from traitlets.config import Config + + assert _table.configured_script_magics(None) == _table.default_script_magics() + assert _table.configured_script_magics(Config()) == _table.default_script_magics() + + config = Config() + config.ScriptMagics.script_magics = ["nodejs"] + assert _table.configured_script_magics(config) == ["nodejs"] + + +EXECUTION_SPEC = "IPython.core.magics.execution:ExecutionMagics" + + +@pytest.fixture +def manager(): + """A standalone MagicsManager, so tests don't disturb the shared shell.""" + from IPython.core.magic import MagicsManager + + return MagicsManager(shell=_UnconfiguredShell()) + + +def test_register_lazy_class_validates_spec(manager): + with pytest.raises(ValueError): + manager.register_lazy_class("IPython.core.magics.execution") + + +def test_lazy_magic_resolves_and_replaces_itself(manager): + from IPython.core.magic import LazyMagic + + manager.register_lazy_class(EXECUTION_SPEC, line_magics=["timeit"]) + placeholder = manager.magics["line"]["timeit"] + assert isinstance(placeholder, LazyMagic) + assert placeholder._lazy_class_name == "ExecutionMagics" + assert "timeit" in repr(placeholder) + + # Resolving loads the class and puts the real magic in the table. + assert manager.find("line", "timeit") is not None + assert not isinstance(manager.magics["line"]["timeit"], LazyMagic) + # ... and the placeholder still works for anything holding on to it. + assert placeholder.__doc__ == manager.magics["line"]["timeit"].__doc__ + + +def test_lazy_magic_with_a_wrong_table_fails_loudly(manager): + from IPython.core.error import UsageError + + manager.register_lazy_class(EXECUTION_SPEC, line_magics=["dummy_lazy"]) + # ExecutionMagics provides no `%dummy_lazy`: resolving must raise and drop + # the stale entry rather than recurse or silently do nothing. + with pytest.raises(UsageError): + manager.magics["line"]["dummy_lazy"]._lazy_resolve() + assert "dummy_lazy" not in manager.magics["line"] + + +def test_register_lazy_class_does_not_shadow_a_loaded_class(manager): + from IPython.core.magic import LazyMagic + + manager.register_lazy_class(EXECUTION_SPEC, line_magics=["timeit"]) + manager.load_lazy_class(EXECUTION_SPEC) + manager.register_lazy_class(EXECUTION_SPEC, line_magics=["timeit"]) + assert not isinstance(manager.magics["line"]["timeit"], LazyMagic) + + +def test_registry_loads_lazily(manager): + manager.register_lazy_class(EXECUTION_SPEC, line_magics=["timeit"]) + assert "ExecutionMagics" not in dict(manager.registry) + + assert manager.registry["ExecutionMagics"].__class__.__name__ == "ExecutionMagics" + + with pytest.raises(KeyError): + manager.registry["NoSuchMagics"] + + +def test_load_all_lazy_magics(manager): + from IPython.core.magic import LazyMagic + + for class_name, kinds in _table.BUILTIN_MAGICS.items(): + manager.register_lazy_class( + f"{_table.MAGICS_CLASSES[class_name]}:{class_name}", + kinds["line"], + kinds["cell"], + ) + manager.load_all_lazy_magics() + left = [ + name + for table in manager.magics.values() + for name, fn in table.items() + if isinstance(fn, LazyMagic) + ] + assert left == [] + + +STARTUP_CHECK = textwrap.dedent( + """ + import sys + from IPython.core.interactiveshell import InteractiveShell + + shell = InteractiveShell.instance() + loaded = sorted( + name + for name in sys.modules + if name.startswith("IPython.core.magics.") + and not name.rsplit(".", 1)[1].startswith("_") + ) + print(",".join(loaded)) + """ +) + + +def test_startup_does_not_import_the_magics_modules(): + """The whole point: starting a shell imports no magics implementation.""" + result = subprocess.run( + [sys.executable, "-c", STARTUP_CHECK], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == "", ( + "starting a shell imported magics modules eagerly: " + result.stdout + )