diff --git a/IPython/terminal/embed.py b/IPython/terminal/embed.py index 0c8ede2bf4..1a80eacecf 100644 --- a/IPython/terminal/embed.py +++ b/IPython/terminal/embed.py @@ -20,6 +20,64 @@ from typing import Set + +class _EmbedGlobals(dict): + """Globals namespace for an embedded shell. + + Code typed in an embedded shell is compiled as module-level code, so + any new nested scope it creates (a lambda, a generator expression, a + comprehension, a function body...) looks its free variables up in + ``globals()``, not in the local namespace the shell was embedded in. + With a plain module ``__dict__`` as globals this makes the caller's + local variables invisible to those scopes (gh-136):: + + def f(): + x = 1 + embed() # then type: (lambda: x)() -> NameError + + This dict subclass keeps the interpreter's normal, C-level storage as + a snapshot of the caller module's globals, but resolves reads through + the caller's local namespace first, mimicking the closure lookup the + code would have had if it were written in place. The interpreter only + honors this ``__getitem__`` override because the shell passes a dict + *subclass* to ``exec``, which disables the exact-dict fast path of + ``LOAD_GLOBAL``. + + ``STORE_GLOBAL``/``DELETE_GLOBAL`` bypass ``__setitem__`` overrides + and mutate the C-level storage directly, so :meth:`sync_to_module` + propagates those (rare) mutations back to the real module on exit. + """ + + def __init__(self, module_dict, local_ns): + super().__init__(module_dict) + self._module_dict = module_dict + self._local_ns = local_ns + self._snapshot = dict(module_dict) + + def __getitem__(self, key): + try: + return self._local_ns[key] + except KeyError: + pass + try: + return dict.__getitem__(self, key) + except KeyError: + pass + # Fall back to the live module dict, so globals set in the real + # module while the shell is active are visible; raises KeyError, + # letting LOAD_GLOBAL continue to builtins. + return self._module_dict[key] + + def sync_to_module(self): + """Write back mutations of our snapshot to the real module.""" + for key, value in dict.items(self): + if key not in self._snapshot or self._snapshot[key] is not value: + self._module_dict[key] = value + for key in self._snapshot: + if not dict.__contains__(self, key): + self._module_dict.pop(key, None) + + class KillEmbedded(Exception):pass # kept for backward compatibility as IPython 6 was released with @@ -317,11 +375,19 @@ def mainloop( # data, but we also need the locals. We'll throw our hidden variables # like _ih and get_ipython() into the local namespace, but delete them # later. + embed_globals = None if local_ns is not None: reentrant_local_ns = {k: v for (k, v) in local_ns.items() if k not in self.user_ns_hidden.keys()} self.user_ns = reentrant_local_ns self.init_user_ns() + # Replace the module's globals with a namespace that falls back + # to the local one, so that nested scopes created interactively + # (lambdas, generator expressions, comprehensions, functions) + # can see the caller's local variables (gh-136). + embed_globals = _EmbedGlobals(self.user_global_ns, reentrant_local_ns) + self.user_module = make_main_module_type(embed_globals)() + # Compiler flags if compile_flags is not None: self.compile.flags = compile_flags @@ -336,6 +402,10 @@ def mainloop( # now, purge out the local namespace of IPython's hidden variables. if local_ns is not None: local_ns.update({k: v for (k, v) in self.user_ns.items() if k not in self.user_ns_hidden.keys()}) + # and propagate `global` assignments made by functions defined in + # the shell back to the real module. + if embed_globals is not None: + embed_globals.sync_to_module() # Restore original namespace so shell can shut down when we exit. diff --git a/tests/test_embed.py b/tests/test_embed.py index f76f2c0f0c..a65a00d0cf 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -15,6 +15,9 @@ import subprocess import sys +import pytest + +from IPython.terminal.embed import _EmbedGlobals from IPython.utils.tempdir import NamedFileInTemporaryDirectory from IPython.testing.decorators import skip_win32 from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE @@ -24,6 +27,58 @@ # ----------------------------------------------------------------------------- +def test_embed_globals_lookup_order(): + module_dict = {"g": 1, "shadowed": "global"} + local_ns = {"foo": 42, "shadowed": "local"} + g = _EmbedGlobals(module_dict, local_ns) + + # caller locals first, as regular closure lookup would + assert g["foo"] == 42 + assert g["shadowed"] == "local" + # then the module globals + assert g["g"] == 1 + # globals set in the real module after entry are still visible + module_dict["late"] = "added" + assert g["late"] == "added" + # missing names raise KeyError so LOAD_GLOBAL falls back to builtins + with pytest.raises(KeyError): + g["nowhere"] + + +def test_embed_globals_exec_nested_scopes(): + local_ns = {"foo": 42} + g = _EmbedGlobals({"g": 1}, local_ns) + + # nested scopes resolve caller locals through globals, and + # builtins (`range`) are still reachable + exec("result = (lambda: [foo + g for _ in range(1)][0])()", g, local_ns) + assert local_ns["result"] == 43 + + +def test_embed_globals_sync_to_module(): + module_dict = {"unchanged": 1, "rebound": 2, "removed": 3} + g = _EmbedGlobals(module_dict, {}) + + # STORE_GLOBAL/DELETE_GLOBAL from code defined in the shell bypass + # __setitem__ and mutate the C-level dict storage directly + exec( + "def s():\n" + " global added, rebound, removed\n" + " added = 5\n" + " rebound = 20\n" + " del removed\n" + "s()", + g, + {}, + ) + assert "added" not in module_dict + g.sync_to_module() + assert module_dict["added"] == 5 + assert module_dict["rebound"] == 20 + assert module_dict["unchanged"] == 1 + assert "removed" not in module_dict + + _sample_embed = """ import IPython @@ -39,6 +94,80 @@ _exit = "exit\r" +_sample_embed_locals = """ +import IPython + +shadowed = 'global' +seen_by_module = None + +def check_seen(): + return seen_by_module + +def bar(foo): + shadowed = 'local' + IPython.embed(banner1='', banner2='') + return foo * 2 + +print('RESULT:', bar(21)) +print('SYNCED:', check_seen()) +""" + +_embed_locals_commands = "\r".join( + [ + "print('genexp:', sum(foo for _ in range(1)))", + "print('lambda:', (lambda: foo)())", + "print('listcomp:', [foo for _ in range(1)][0])", + "def g():\r return foo\r", + "print('nested-def:', g())", + "print('shadow:', (lambda: shadowed)())", + "def s():\r global seen_by_module\r seen_by_module = 'set-in-shell'\r", + "s()", + "exit", + ] +) + "\r" + + +def test_ipython_embed_sees_locals_in_nested_scopes(): + """Nested scopes created in an embedded shell see the caller's locals. + + Generator expressions, lambdas, comprehensions and function bodies typed + into an embedded shell look up free variables through ``globals()``; + check they can still resolve variables local to the embedding frame. + See https://github.com/ipython/ipython/issues/136 + """ + with NamedFileInTemporaryDirectory("file_with_embed.py", "w") as f: + f.write(_sample_embed_locals) + f.flush() + f.close() + + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + p = subprocess.Popen( + [sys.executable, f.name], + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="UTF-8", + ) + std, err = p.communicate(_embed_locals_commands) + + assert p.returncode == 0, (p.returncode, std, err) + assert "NameError" not in std, std + assert "genexp: 21" in std + assert "lambda: 21" in std + assert "listcomp: 21" in std + assert "nested-def: 21" in std + # a local shadowing a module global wins, as in regular closures + assert "shadow: local" in std + assert "RESULT: 42" in std + # `global` assignments made by shell-defined functions reach the + # real module once the shell exits + assert "SYNCED: set-in-shell" in std + + def test_ipython_embed(): """test that `IPython.embed()` works""" with NamedFileInTemporaryDirectory("file_with_embed.py", "w") as f: