From d8ab3252214c39f53f04998f2f11af3ab59a2dbd Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Tue, 7 Jul 2026 10:14:05 +0200 Subject: [PATCH 1/2] Make caller locals visible to nested scopes in embedded shells Code typed into an embedded IPython shell is compiled as module-level code, so lambdas, generator expressions, comprehensions and function bodies defined interactively look their free variables up in globals() and could not see the local variables of the frame the shell was embedded in: def f(): x = 1 IPython.embed() # (lambda: x)() -> NameError Fix this by giving the embedded shell a globals namespace (a dict subclass installed via make_main_module_type) that resolves reads through the caller's local namespace first, then its own snapshot of the module globals, then the live module dict, mimicking the closure lookup the code would have had if written in place. Passing a dict subclass to exec disables the exact-dict LOAD_GLOBAL fast path, which is what makes the __getitem__ override effective. STORE_GLOBAL bypasses __setitem__ and mutates the C-level dict storage directly, so 'global' assignments made by shell-defined functions land in the snapshot; they are propagated back to the real module when the shell exits. Closes gh-136 --- IPython/terminal/embed.py | 70 ++++++++++++++++++++++++++++++++++++ tests/test_embed.py | 74 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/IPython/terminal/embed.py b/IPython/terminal/embed.py index 0c8ede2bf44..1a80eacecfc 100644 --- a/IPython/terminal/embed.py +++ b/IPython/terminal/embed.py @@ -20,6 +20,64 @@ from typing import Set + +class _EmbedGlobals(dict): + """Globals namespace for an embedded shell. + + Code typed in an embedded shell is compiled as module-level code, so + any new nested scope it creates (a lambda, a generator expression, a + comprehension, a function body...) looks its free variables up in + ``globals()``, not in the local namespace the shell was embedded in. + With a plain module ``__dict__`` as globals this makes the caller's + local variables invisible to those scopes (gh-136):: + + def f(): + x = 1 + embed() # then type: (lambda: x)() -> NameError + + This dict subclass keeps the interpreter's normal, C-level storage as + a snapshot of the caller module's globals, but resolves reads through + the caller's local namespace first, mimicking the closure lookup the + code would have had if it were written in place. The interpreter only + honors this ``__getitem__`` override because the shell passes a dict + *subclass* to ``exec``, which disables the exact-dict fast path of + ``LOAD_GLOBAL``. + + ``STORE_GLOBAL``/``DELETE_GLOBAL`` bypass ``__setitem__`` overrides + and mutate the C-level storage directly, so :meth:`sync_to_module` + propagates those (rare) mutations back to the real module on exit. + """ + + def __init__(self, module_dict, local_ns): + super().__init__(module_dict) + self._module_dict = module_dict + self._local_ns = local_ns + self._snapshot = dict(module_dict) + + def __getitem__(self, key): + try: + return self._local_ns[key] + except KeyError: + pass + try: + return dict.__getitem__(self, key) + except KeyError: + pass + # Fall back to the live module dict, so globals set in the real + # module while the shell is active are visible; raises KeyError, + # letting LOAD_GLOBAL continue to builtins. + return self._module_dict[key] + + def sync_to_module(self): + """Write back mutations of our snapshot to the real module.""" + for key, value in dict.items(self): + if key not in self._snapshot or self._snapshot[key] is not value: + self._module_dict[key] = value + for key in self._snapshot: + if not dict.__contains__(self, key): + self._module_dict.pop(key, None) + + class KillEmbedded(Exception):pass # kept for backward compatibility as IPython 6 was released with @@ -317,11 +375,19 @@ def mainloop( # data, but we also need the locals. We'll throw our hidden variables # like _ih and get_ipython() into the local namespace, but delete them # later. + embed_globals = None if local_ns is not None: reentrant_local_ns = {k: v for (k, v) in local_ns.items() if k not in self.user_ns_hidden.keys()} self.user_ns = reentrant_local_ns self.init_user_ns() + # Replace the module's globals with a namespace that falls back + # to the local one, so that nested scopes created interactively + # (lambdas, generator expressions, comprehensions, functions) + # can see the caller's local variables (gh-136). + embed_globals = _EmbedGlobals(self.user_global_ns, reentrant_local_ns) + self.user_module = make_main_module_type(embed_globals)() + # Compiler flags if compile_flags is not None: self.compile.flags = compile_flags @@ -336,6 +402,10 @@ def mainloop( # now, purge out the local namespace of IPython's hidden variables. if local_ns is not None: local_ns.update({k: v for (k, v) in self.user_ns.items() if k not in self.user_ns_hidden.keys()}) + # and propagate `global` assignments made by functions defined in + # the shell back to the real module. + if embed_globals is not None: + embed_globals.sync_to_module() # Restore original namespace so shell can shut down when we exit. diff --git a/tests/test_embed.py b/tests/test_embed.py index f76f2c0f0c9..ecb8408f755 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -39,6 +39,80 @@ _exit = "exit\r" +_sample_embed_locals = """ +import IPython + +shadowed = 'global' +seen_by_module = None + +def check_seen(): + return seen_by_module + +def bar(foo): + shadowed = 'local' + IPython.embed(banner1='', banner2='') + return foo * 2 + +print('RESULT:', bar(21)) +print('SYNCED:', check_seen()) +""" + +_embed_locals_commands = "\r".join( + [ + "print('genexp:', sum(foo for _ in range(1)))", + "print('lambda:', (lambda: foo)())", + "print('listcomp:', [foo for _ in range(1)][0])", + "def g():\r return foo\r", + "print('nested-def:', g())", + "print('shadow:', (lambda: shadowed)())", + "def s():\r global seen_by_module\r seen_by_module = 'set-in-shell'\r", + "s()", + "exit", + ] +) + "\r" + + +def test_ipython_embed_sees_locals_in_nested_scopes(): + """Nested scopes created in an embedded shell see the caller's locals. + + Generator expressions, lambdas, comprehensions and function bodies typed + into an embedded shell look up free variables through ``globals()``; + check they can still resolve variables local to the embedding frame. + See https://github.com/ipython/ipython/issues/136 + """ + with NamedFileInTemporaryDirectory("file_with_embed.py", "w") as f: + f.write(_sample_embed_locals) + f.flush() + f.close() + + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + p = subprocess.Popen( + [sys.executable, f.name], + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="UTF-8", + ) + std, err = p.communicate(_embed_locals_commands) + + assert p.returncode == 0, (p.returncode, std, err) + assert "NameError" not in std, std + assert "genexp: 21" in std + assert "lambda: 21" in std + assert "listcomp: 21" in std + assert "nested-def: 21" in std + # a local shadowing a module global wins, as in regular closures + assert "shadow: local" in std + assert "RESULT: 42" in std + # `global` assignments made by shell-defined functions reach the + # real module once the shell exits + assert "SYNCED: set-in-shell" in std + + def test_ipython_embed(): """test that `IPython.embed()` works""" with NamedFileInTemporaryDirectory("file_with_embed.py", "w") as f: From 52366fe30046197570b0eb4473bb186c6153e9f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 7 Jul 2026 08:39:51 +0000 Subject: [PATCH 2/2] Add in-process unit tests for _EmbedGlobals The embed integration tests exercise the new namespace machinery only in spawned subprocesses, where coverage is not collected. Test the lookup order, exec/nested-scope resolution, and module write-back of _EmbedGlobals directly in-process. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PkPMSsnbYhtE281smQ2sfT --- tests/test_embed.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_embed.py b/tests/test_embed.py index ecb8408f755..a65a00d0cf5 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -15,6 +15,9 @@ import subprocess import sys +import pytest + +from IPython.terminal.embed import _EmbedGlobals from IPython.utils.tempdir import NamedFileInTemporaryDirectory from IPython.testing.decorators import skip_win32 from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE @@ -24,6 +27,58 @@ # ----------------------------------------------------------------------------- +def test_embed_globals_lookup_order(): + module_dict = {"g": 1, "shadowed": "global"} + local_ns = {"foo": 42, "shadowed": "local"} + g = _EmbedGlobals(module_dict, local_ns) + + # caller locals first, as regular closure lookup would + assert g["foo"] == 42 + assert g["shadowed"] == "local" + # then the module globals + assert g["g"] == 1 + # globals set in the real module after entry are still visible + module_dict["late"] = "added" + assert g["late"] == "added" + # missing names raise KeyError so LOAD_GLOBAL falls back to builtins + with pytest.raises(KeyError): + g["nowhere"] + + +def test_embed_globals_exec_nested_scopes(): + local_ns = {"foo": 42} + g = _EmbedGlobals({"g": 1}, local_ns) + + # nested scopes resolve caller locals through globals, and + # builtins (`range`) are still reachable + exec("result = (lambda: [foo + g for _ in range(1)][0])()", g, local_ns) + assert local_ns["result"] == 43 + + +def test_embed_globals_sync_to_module(): + module_dict = {"unchanged": 1, "rebound": 2, "removed": 3} + g = _EmbedGlobals(module_dict, {}) + + # STORE_GLOBAL/DELETE_GLOBAL from code defined in the shell bypass + # __setitem__ and mutate the C-level dict storage directly + exec( + "def s():\n" + " global added, rebound, removed\n" + " added = 5\n" + " rebound = 20\n" + " del removed\n" + "s()", + g, + {}, + ) + assert "added" not in module_dict + g.sync_to_module() + assert module_dict["added"] == 5 + assert module_dict["rebound"] == 20 + assert module_dict["unchanged"] == 1 + assert "removed" not in module_dict + + _sample_embed = """ import IPython