diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index a00cd9e2584..7fc991670d2 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -222,6 +222,7 @@ pylifecycle pymain pymem pyrepl +pyrun pystate pystrcmp PYTHONTRACEMALLOC diff --git a/Cargo.toml b/Cargo.toml index eff494509e9..0b9c6af8d60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,6 @@ winresource = "0.1" [dependencies] rustpython-capi = { workspace = true, optional = true } -rustpython-compiler = { workspace = true } rustpython-pylib = { workspace = true, optional = true } rustpython-stdlib = { workspace = true, optional = true, default-features = false, features = ["compiler"] } rustpython-vm = { workspace = true, features = ["compiler", "gc"] } @@ -62,6 +61,7 @@ rustyline = { workspace = true } mimalloc = { version = "0.1", optional = true, default-features = false } [dev-dependencies] +rustpython-compiler = { workspace = true } criterion = { workspace = true } pyo3 = { workspace = true, features = ["auto-initialize"] } rustls-graviola = { workspace = true } diff --git a/Lib/runpy.py b/Lib/runpy.py index 1d5ecf0cf15..a535b4f651a 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -80,7 +80,6 @@ def _run_code(code, run_globals, init_globals=None, pkg_name = mod_spec.parent run_globals.update(__name__ = mod_name, __file__ = fname, - __cached__ = cached, __doc__ = None, __loader__ = loader, __package__ = pkg_name, @@ -185,7 +184,6 @@ def _run_module_as_main(mod_name, alter_argv=True): At the very least, these variables in __main__ will be overwritten: __name__ __file__ - __cached__ __loader__ __package__ """ @@ -254,7 +252,7 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(fname): +def _get_code_from_file(fname, module): # Check for a compiled file first from pkgutil import read_code code_path = os.path.abspath(fname) @@ -263,7 +261,7 @@ def _get_code_from_file(fname): if code is None: # That didn't work, so try it as normal source code with io.open_code(code_path) as f: - code = compile(f.read(), fname, 'exec') + code = compile(f.read(), fname, 'exec', module=module) return code def run_path(path_name, init_globals=None, run_name=None): @@ -290,7 +288,7 @@ def run_path(path_name, init_globals=None, run_name=None): if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code = _get_code_from_file(path_name, run_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=path_name) else: diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 6884ac0dbe6..8be5bf8aa4f 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -8,6 +8,8 @@ import contextlib import os import sys +import unittest.mock +from test.support import swap_item class TestHook: @@ -206,6 +208,16 @@ def rl(name): else: return None + try: + import _remote_debugging + except ImportError: + _remote_debugging = None + + def rd(name): + if _remote_debugging: + return getattr(_remote_debugging, name, None) + return None + # Try a range of "open" functions. # All of them should fail with TestHook(raise_on_events={"open"}) as hook: @@ -223,6 +235,8 @@ def rl(name): (rl("append_history_file"), 0, None), (rl("read_init_file"), testfn), (rl("read_init_file"), None), + (rd("BinaryWriter"), testfn, 1000, 0), + (rd("BinaryReader"), testfn), ]: if not fn: continue @@ -256,6 +270,8 @@ def rl(name): ("~/.history", "a") if rl("append_history_file") else None, (testfn, "r") if readline else None, ("", "r") if readline else None, + (testfn, "wb") if rd("BinaryWriter") else None, + (testfn, "rb") if rd("BinaryReader") else None, ] if i is not None ], @@ -672,6 +688,84 @@ def hook(event, args): assertEqual(event_script_path, tmp_file.name) assertEqual(remote_event_script_path, tmp_file.name) +def test_import_module(): + import importlib + + with TestHook() as hook: + importlib.import_module("importlib") # already imported, won't get logged + importlib.import_module("email") # standard library module + importlib.import_module("pythoninfo") # random module + importlib.import_module(".audit_test_data.submodule", "test") # relative import + importlib.import_module("test.audit_test_data.submodule2") # absolute import + importlib.import_module("_testcapi") # extension module + + actual = [a for e, a in hook.seen if e == "import"] + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + +def test_builtin__import__(): + import importlib # noqa: F401 + + with TestHook() as hook: + __import__("importlib") + __import__("email") + __import__("pythoninfo") + __import__("audit_test_data.submodule", level=1, globals={"__package__": "test"}) + __import__("test.audit_test_data.submodule2") + __import__("_testcapi") + + actual = [a for e, a in hook.seen if e == "import"] + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + +def test_import_statement(): + import importlib # noqa: F401 + # Set __package__ so relative imports work + with swap_item(globals(), "__package__", "test"): + with TestHook() as hook: + import importlib # noqa: F401 + import email # noqa: F401 + import pythoninfo # noqa: F401 + from .audit_test_data import submodule # noqa: F401 + import test.audit_test_data.submodule2 # noqa: F401 + import _testcapi # noqa: F401 + + actual = [a for e, a in hook.seen if e == "import"] + # Import statement ordering is different because the package is + # loaded first and then the submodule + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/audit_test_data/__init__.py b/Lib/test/audit_test_data/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/audit_test_data/submodule.py b/Lib/test/audit_test_data/submodule.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/audit_test_data/submodule2.py b/Lib/test/audit_test_data/submodule2.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 690a6e7434e..8c1658ca604 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -94,7 +94,6 @@ def test_posixsubprocess(self): import_helper.import_module("_posixsubprocess") self.do_test("test_posixsubprocess") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_excepthook(self): returncode, events, stderr = self.run_python("test_excepthook") if not returncode: @@ -345,5 +344,17 @@ def test_sys_remote_exec(self): if returncode: self.fail(stderr) + @unittest.expectedFailureIf(__import__("importlib.util", fromlist=["find_spec"]).find_spec("_testcapi") is None, "TODO: RUSTPYTHON; requires the absent _testcapi module for extension import audit events; recheck when discoverable") + def test_import_module(self): + self.do_test("test_import_module") + + @unittest.expectedFailureIf(__import__("importlib.util", fromlist=["find_spec"]).find_spec("_testcapi") is None, "TODO: RUSTPYTHON; requires the absent _testcapi module for extension import audit events; recheck when discoverable") + def test_builtin__import__(self): + self.do_test("test_builtin__import__") + + @unittest.expectedFailureIf(__import__("importlib.util", fromlist=["find_spec"]).find_spec("_testcapi") is None, "TODO: RUSTPYTHON; requires the absent _testcapi module for extension import audit events; recheck when discoverable") + def test_import_statement(self): + self.do_test("test_import_statement") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index d6fcfa3b471..248d6047f45 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -2,6 +2,7 @@ # Most tests are executed with environment variables ignored # See test_cmd_line_script.py for testing of script execution +import locale import os import re import subprocess @@ -32,6 +33,17 @@ def _kill_python_and_exit_code(p): return data, returncode +def presite_func(): + print("presite func") + +class Namespace: + pass + +presite = Namespace() +presite.attr = Namespace() +presite.attr.func = presite_func + + class CmdLineTest(unittest.TestCase): def test_directories(self): assert_python_failure('.') @@ -46,6 +58,7 @@ def verify_valid_flag(self, cmd_line): return out @support.cpython_only + @support.force_not_colorized def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -57,6 +70,7 @@ def test_help(self): self.assertLess(len(lines), 50) @support.cpython_only + @support.force_not_colorized def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) @@ -70,6 +84,7 @@ def test_help_env(self): "env vars should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) @@ -78,6 +93,7 @@ def test_help_xoptions(self): "options should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -89,6 +105,25 @@ def test_help_all(self): # but the rest should be ASCII-only b''.join(lines[1:]).decode('ascii') + @support.cpython_only + @support.force_colorized + def test_help_colorized(self): + rc, out, err = assert_python_ok("--help", FORCE_COLOR="1") + # Check ANSI color codes are present + self.assertIn(b"\x1b[", out) + # Check that key text elements are still present + self.assertIn(b"usage:", out) + self.assertIn(b"-h", out) + self.assertIn(b"--help-all", out) + self.assertIn(b"cmd", out) + self.assertIn(b"Arguments:", out) + + @support.cpython_only + @support.force_not_colorized + def test_help_not_colorized(self): + rc, out, err = assert_python_ok("--help") + self.assertNotIn(b"\x1b[", out) + def test_optimize(self): self.verify_valid_flag('-O') self.verify_valid_flag('-OO') @@ -322,6 +357,10 @@ def run_utf8_mode(arg): cmd = [sys.executable, '-X', 'utf8', '-c', code, arg] return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + def run_no_utf8_mode(arg): + cmd = [sys.executable, '-X', 'utf8=0', '-c', code, arg] + return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + valid_utf8 = 'e:\xe9, euro:\u20ac, non-bmp:\U0010ffff'.encode('utf-8') # invalid UTF-8 byte sequences with a valid UTF-8 sequence # in the middle. @@ -334,8 +373,27 @@ def run_utf8_mode(arg): ) test_args = [valid_utf8, invalid_utf8] - for run_cmd in (run_default, run_c_locale, run_utf8_mode): - with self.subTest(run_cmd=run_cmd): + for run_cmd, encoding in ( + (run_default, sys.getfilesystemencoding()), + (run_c_locale, None), + (run_utf8_mode, None), + (run_no_utf8_mode, locale.getencoding()) + ): + with self.subTest(run_cmd=run_cmd.__name__): + # Arbitrary bytes round-trip through surrogateescape only in + # UTF-8 and single-byte encodings, not in a multibyte encoding + # such as EUC-JP. + if encoding is not None: + try: + lossless = len(bytes(range(256)).decode( + encoding, 'surrogateescape')) == 256 + except UnicodeError: + lossless = False + else: + lossless = True + if not lossless: + self.skipTest(f'{encoding} cannot losslessly ' + f'round-trip arbitrary bytes') for arg in test_args: proc = run_cmd(arg) self.assertEqual(proc.stdout.rstrip(), ascii(arg)) @@ -433,7 +491,6 @@ def test_empty_PYTHONPATH_issue16309(self): path = ":".join(sys.path) path = path.encode("ascii", "backslashreplace") sys.stdout.buffer.write(path)""" - # TODO: RUSTPYTHON we must unset RUSTPYTHONPATH as well rc1, out1, err1 = assert_python_ok('-c', code, PYTHONPATH="", RUSTPYTHONPATH="") rc2, out2, err2 = assert_python_ok('-c', code, __isolated=False) # regarding to Posix specification, outputs should be equal @@ -509,6 +566,7 @@ def test_unmached_quote(self): self.assertEqual(b'', out) @unittest.expectedFailure # TODO: RUSTPYTHON + @force_not_colorized def test_stdout_flush_at_shutdown(self): # Issue #5319: if stdout.flush() fails at shutdown, an error should # be printed out. @@ -1066,6 +1124,7 @@ def test_argv0_normalization(self): self.assertEqual(proc.stdout.strip(), b'0') @support.cpython_only + @support.force_not_colorized def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, @@ -1226,6 +1285,25 @@ def test_import_time(self): assert_python_failure('-X', 'importtime=-1', '-c', code) assert_python_failure('-X', 'importtime=3', '-c', code) + @unittest.expectedFailure # TODO: RUSTPYTHON; -X importtime=2 timing output and surrogate module import exception semantics are not implemented correctly + def test_import_time_unencodable_module_name(self): + code = textwrap.dedent(""" + import sys, types + name = 'mod\\ud800' + sys.modules[name] = types.ModuleType(name) + __import__(name) + try: + __import__('nonexistent\\ud800') + except ModuleNotFoundError: + pass + """) + res = assert_python_ok('-X', 'importtime=2', '-c', code) + res_err = res.err.decode('utf-8') + self.assertRegex(res_err, + r'import time: cached\s* \| cached\s* \| mod\\ud800') + self.assertRegex(res_err, + r'import time: \s*\d+ \| \s*\d+ \| \s*nonexistent\\ud800') + def res2int(self, res): out = res.out.strip().decode("utf-8") return tuple(int(i) for i in out.split()) @@ -1282,6 +1360,29 @@ def test_invalid_thread_local_bytecode(self): rc, out, err = assert_python_failure(PYTHON_TLBC="2") self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + @unittest.skipUnless(support.Py_DEBUG, + '-X presite requires a Python debug build') + def test_presite(self): + entrypoint = "test.test_cmd_line:presite_func" + proc = assert_python_ok("-X", f"presite={entrypoint}", "-c", "pass") + self.assertEqual(proc.out.rstrip(), b"presite func") + + entrypoint = "test.test_cmd_line:presite.attr.func" + proc = assert_python_ok("-X", f"presite={entrypoint}", "-c", "pass") + self.assertEqual(proc.out.rstrip(), b"presite func") + + @unittest.expectedFailure # TODO: RUSTPYTHON; explicit PYTHONHOME startup path configuration and fatal encodings failure reporting are not implemented + def test_dump_path_config(self): + # gh-151253: At the first import (import encodings) during Python + # startup, if the import fails, dump the Python path configuration. + nonexistent = '/nonexistent-python-path' + # Use -X frozen_modules=off to disable frozen encodings module + # on release build. + cmd = ["-X", "frozen_modules=off", "-c", "pass"] + proc = assert_python_failure(*cmd, PYTHONHOME=nonexistent) + self.assertIn(b'Python path configuration:', proc.err) + self.assertIn(f"PYTHONHOME = '{nonexistent}'".encode(), proc.err) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') @@ -1330,11 +1431,9 @@ def check_string(self, code): self.assertNotEqual(proc.stderr, None) self.assertIn(b"\nSyntaxError", proc.stderr) - @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON") def test_tokenizer_error_with_stdin(self): self.check_string(b"(1+2+3") - @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON") def test_decoding_error_at_the_end_of_the_line(self): self.check_string(br"'\u1f'") diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 71dab8c3876..a0abcc0f0ba 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -17,7 +17,6 @@ SHORT_TIMEOUT, ) from test.support.script_helper import kill_python -from test.support.import_helper import import_module try: import pty @@ -40,8 +39,8 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, custom=F # line option '-i' and the process name set to ''. # The directory of argv[0] must match the directory of the Python # executable for the Popen() call to python to succeed as the directory - # path may be used by Py_GetPath() to build the default module search - # path. + # path may be used by PyConfig_Get("module_search_paths") to build the + # default module search path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "") cmd_line = [stdin_fname] # Isolated mode implies -EPs and ignores PYTHON* variables. @@ -99,12 +98,8 @@ def run_on_interactive_mode(source): @support.force_not_colorized_test_class class TestInteractiveInterpreter(unittest.TestCase): - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @support.nomemtest def test_no_memory(self): - import_module("_testcapi") # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive # loop when an exception is raised. @@ -189,7 +184,6 @@ def test_close_stdin(self): self.assertEqual(process.returncode, 0) self.assertIn('before close', output) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 != 0 def test_interactive_traceback_reporting(self): user_input = "1 / 0 / 3 / 4" p = spawn_repl() @@ -232,7 +226,6 @@ def foo(x): ] self.assertEqual(traceback_lines, expected_lines) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pythonstartup_error_reporting(self): # errors based on https://github.com/python/cpython/issues/137576 @@ -303,7 +296,7 @@ def test_runsource_show_syntax_error_location(self): expected_lines = [ ' def f(x, x): ...', ' ^', - "SyntaxError: duplicate argument 'x' in function definition" + "SyntaxError: duplicate parameter 'x' in function definition" ] self.assertEqual(output.splitlines()[4:-1], expected_lines) @@ -443,6 +436,13 @@ def test_toplevel_contextvars_async(self): p = spawn_asyncio_repl() p.stdin.write(user_input) user_input2 = "async def set_var(): var.set('ok')\n" + try: + import _pyrepl # noqa: F401 + except ModuleNotFoundError: + # If we're going to be forced into the regular REPL, then we need an + # extra newline here. Omit it by default to catch any breakage to + # the new REPL's behavior. + user_input2 += "\n" p.stdin.write(user_input2) user_input3 = "await set_var()\n" p.stdin.write(user_input3) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 0fb7f73da11..55b9673ef6c 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -20,9 +20,11 @@ requires_subprocess, verbose, ) +from test import support from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script +from test.test_importlib.util import temporary_pycache_prefix import runpy @@ -55,7 +57,6 @@ def f(): implicit_namespace = { "__name__": None, "__file__": None, - "__cached__": None, "__package__": None, "__doc__": None, "__spec__": None @@ -303,7 +304,6 @@ def _del_pkg(self, top): def _fix_ns_for_legacy_pyc(self, ns, alter_sys): char_to_add = "c" ns["__file__"] += char_to_add - ns["__cached__"] = ns["__file__"] spec = ns["__spec__"] new_spec = importlib.util.spec_from_file_location(spec.name, ns["__file__"]) @@ -323,7 +323,6 @@ def _check_module(self, depth, alter_sys=False, expected_ns.update({ "__name__": mod_name, "__file__": mod_fname, - "__cached__": mod_spec.cached, "__package__": mod_name.rpartition(".")[0], "__spec__": mod_spec, }) @@ -340,14 +339,16 @@ def create_ns(init_globals): self.check_code_execution(create_ns, expected_ns) importlib.invalidate_caches() __import__(mod_name) - os.remove(mod_fname) if not sys.dont_write_bytecode: - make_legacy_pyc(mod_fname) + make_legacy_pyc(mod_fname, allow_compile=True) unload(mod_name) # In case loader caches paths + os.remove(mod_fname) importlib.invalidate_caches() if verbose > 1: print("Running from compiled:", mod_name) self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) self.check_code_execution(create_ns, expected_ns) + else: + os.remove(mod_fname) finally: self._del_pkg(pkg_dir) if verbose > 1: print("Module executed successfully") @@ -364,7 +365,6 @@ def _check_package(self, depth, alter_sys=False, expected_ns.update({ "__name__": mod_name, "__file__": mod_fname, - "__cached__": importlib.util.cache_from_source(mod_fname), "__package__": pkg_name, "__spec__": mod_spec, }) @@ -381,14 +381,16 @@ def create_ns(init_globals): self.check_code_execution(create_ns, expected_ns) importlib.invalidate_caches() __import__(mod_name) - os.remove(mod_fname) if not sys.dont_write_bytecode: - make_legacy_pyc(mod_fname) + make_legacy_pyc(mod_fname, allow_compile=True) unload(mod_name) # In case loader caches paths + os.remove(mod_fname) if verbose > 1: print("Running from compiled:", pkg_name) importlib.invalidate_caches() self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) self.check_code_execution(create_ns, expected_ns) + else: + os.remove(mod_fname) finally: self._del_pkg(pkg_dir) if verbose > 1: print("Package executed successfully") @@ -441,7 +443,7 @@ def _check_relative_imports(self, depth, run_name=None): importlib.invalidate_caches() __import__(mod_name) os.remove(mod_fname) - if not sys.dont_write_bytecode: + if not sys.dont_write_bytecode and sys.implementation.cache_tag: make_legacy_pyc(mod_fname) unload(mod_name) # In case the loader caches paths if verbose > 1: print("Running from compiled:", mod_name) @@ -569,7 +571,6 @@ def test_run_name(self): expected_ns.update({ "__name__": run_name, "__file__": mod_fname, - "__cached__": importlib.util.cache_from_source(mod_fname), "__package__": mod_name.rpartition(".")[0], "__spec__": mod_spec, }) @@ -649,7 +650,6 @@ def create_ns(init_globals): expected_ns.update({ "__name__": expected_name, "__file__": expected_file, - "__cached__": mod_cached, "__package__": "", "__spec__": mod_spec, "run_argv0": expected_argv0, @@ -699,6 +699,8 @@ def test_basic_script_no_suffix(self): self._check_script(script_name, "", script_name, script_name, expect_spec=False) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag') def test_script_compiled(self): with temp_dir() as script_dir: mod_name = 'script' @@ -719,12 +721,10 @@ def test_directory_compiled(self): with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = py_compile.compile(script_name, doraise=True) + legacy_pyc = make_legacy_pyc(script_name, allow_compile=True) os.remove(script_name) - if not sys.dont_write_bytecode: - legacy_pyc = make_legacy_pyc(script_name) - self._check_script(script_dir, "", legacy_pyc, - script_dir, mod_name=mod_name) + self._check_script(script_dir, "", legacy_pyc, + script_dir, mod_name=mod_name) def test_directory_error(self): with temp_dir() as script_dir: @@ -756,7 +756,8 @@ def test_zipfile_compiled(self): with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = py_compile.compile(script_name, doraise=True) + compiled_name = script_name + 'c' + py_compile.compile(script_name, compiled_name, doraise=True) zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name) self._check_script(zip_name, "", fname, zip_name, @@ -793,6 +794,47 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + def test_run_module_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with (temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + ns = run_module('test.test_import.data.syntax_warnings') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = ns['__file__'] + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + def test_run_path_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.script\z') + warnings.filterwarnings('error', module='') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename, run_name='package.script') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + @force_not_colorized_test_class class TestExit(unittest.TestCase): @@ -867,7 +909,6 @@ def test_pymain_run_command_run_module(self): def test_pymain_run_command(self): self.assertSigInt(["-c", "import ham"], cwd=self.ham.parent) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pymain_run_stdin(self): self.assertSigInt([], input="import ham", cwd=self.ham.parent) diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index ea99242005f..aeeff3537d7 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -4,7 +4,7 @@ use crate::pyerrors::init_exception_statics; use crate::pylifecycle::{MAIN_INTERP, MAIN_INTERP_PTR}; use core::sync::atomic::Ordering; pub use rustpython_vm::PyObject; -use rustpython_vm::{Context, Interpreter}; +use rustpython_vm::{Context, Interpreter, VirtualMachine}; use std::sync::MutexGuard; extern crate alloc; @@ -71,3 +71,33 @@ pub fn init_main_interpreter(interpreter: Interpreter) { Ordering::Release, ); } + +/// Py_RunMain for an interpreter that C API threads can attach to. +/// +/// `f` reports its own errors and returns the exit code. +pub fn run_main_interpreter( + mut interpreter: Interpreter, + f: impl FnOnce(&VirtualMachine) -> u32, +) -> u32 { + let main_thread = interpreter.take_main_thread(); + init_main_interpreter(interpreter); + + struct RegistrationGuard; + impl Drop for RegistrationGuard { + fn drop(&mut self) { + let mut registered = MAIN_INTERP + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + MAIN_INTERP_PTR.store(core::ptr::null_mut(), Ordering::Release); + let interpreter = registered.take(); + drop(registered); + drop(interpreter); + } + } + let _registration = RegistrationGuard; + + // The registered interpreter is an inactive worker-VM template. Running + // the original VM locally preserves main-thread state without keeping the + // global mutex locked across Python execution or finalization callbacks. + main_thread.run_main(f) +} diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 82f881d9598..636182c7708 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -37,6 +37,60 @@ pub struct ParseError { pub is_unclosed_bracket: bool, /// Set when a string is still open at EOF and more input could close it. pub is_unclosed_string: bool, + /// Set when the parser failed on a token made up at the end of the input. + pub eof_token: Option>, +} + +/// The tokenizer state at the end of the input, where it makes up the +/// NEWLINE, DEDENT and ENDMARKER tokens that close it. Those tokens have no +/// column, so an error on one of them is placed where the tokenizer stopped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EofToken { + /// `tok->lineno`: the number of lines read, 0 for empty input. + pub lineno: usize, + /// The last line read, which the tokenizer stopped at the end of. + pub line: String, + /// The error is raised at the token itself, as the generic syntax error + /// and a missing forced token are, so its columns are 0. + pub known_location: bool, +} + +impl EofToken { + fn new(source: &str, mode: Mode, known_location: bool) -> Self { + let lineno = source.split_inclusive('\n').count(); + let mut line = source + .split_inclusive('\n') + .next_back() + .unwrap_or_default() + .to_owned(); + // translate_newlines: exec input that does not end with a newline + // gets one. + if matches!(mode, Mode::Exec | Mode::BlockExpr) && !line.is_empty() && !line.ends_with('\n') + { + line.push('\n'); + } + Self { + lineno, + line, + known_location, + } + } + + /// `SyntaxError.offset` + #[must_use] + pub fn offset(&self) -> usize { + if self.known_location { + 0 + } else { + self.line.chars().count() + } + } + + /// `SyntaxError.end_offset` + #[must_use] + pub const fn end_offset(&self) -> isize { + if self.known_location { 0 } else { -1 } + } } impl ::core::fmt::Display for ParseError { @@ -61,6 +115,7 @@ impl CompileError { mode: Mode, ) -> Self { let raw_location = error.location; + let at_eof_token = fails_at_eof_token(&error, source_file.source_text(), mode); let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) { Some(diagnostic) => diagnostic, None => { @@ -80,6 +135,11 @@ impl CompileError { } }; + let eof_token = at_eof_token.then(|| { + let source = source_file.source_text(); + let known_location = is_raised_at_token(&diagnostic.error, source, raw_location); + Box::new(EofToken::new(source, mode, known_location)) + }); Self::Parse(ParseError { error: diagnostic.error, raw_location, @@ -88,6 +148,7 @@ impl CompileError { source_path: source_file.name().to_owned(), is_unclosed_bracket: diagnostic.is_unclosed_bracket, is_unclosed_string: diagnostic.is_unclosed_string, + eof_token, }) } @@ -105,6 +166,7 @@ impl CompileError { source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) } @@ -117,8 +179,14 @@ impl CompileError { } #[must_use] - pub const fn python_location(&self) -> (usize, usize) { - if let Some(location) = self.location() { + pub fn python_location(&self) -> (usize, usize) { + if let Self::Parse(ParseError { + eof_token: Some(eof_token), + .. + }) = self + { + (eof_token.lineno, eof_token.offset()) + } else if let Some(location) = self.location() { (location.line.get(), location.character_offset.get()) } else { (0, 0) @@ -316,6 +384,94 @@ fn cpython_parse_diagnostic_override( None } +/// Whether the parser fails on a token made up at the end of the input. +fn fails_at_eof_token(error: &parser::ParseError, source: &str, mode: Mode) -> bool { + let start = error.location.start().to_usize(); + match &error.error { + parser::ParseErrorType::Lexical(_) => false, + // `NEWLINE !INDENT` fails on the token after the newline, and the + // error is reported at the newline when only blank lines and comments + // follow it. + parser::ParseErrorType::ExpectedIndentedBlock { .. } => { + let end = error.location.end().to_usize(); + source + .get(start..end) + .is_some_and(|text| text.chars().all(|c| c.is_ascii_whitespace())) + && source.get(end..).is_some_and(is_blank_python_source) + } + _ if start < source.len() => false, + // Exec input gets a newline appended, which is a real NEWLINE token, + // so the error is at the end only when it stays there once the + // newline is added. + _ if matches!(mode, Mode::Exec | Mode::BlockExpr) + && !source.is_empty() + && !source.ends_with('\n') => + { + let source = format!("{source}\n"); + let parsed = + parser::parse_unchecked(&source, parser::ParseOptions::from(parser::Mode::Module)); + parsed + .errors() + .first() + .is_none_or(|error| error.location.start().to_usize() >= source.len()) + } + _ => true, + } +} + +/// Whether the error is raised at the token the parser failed on, as the +/// generic syntax error and a missing forced token are, rather than where the +/// tokenizer stopped. +fn is_raised_at_token( + error: &parser::ParseErrorType, + source: &str, + range: ruff_text_size::TextRange, +) -> bool { + match error { + // Only `else`, `try`, `finally` and `def` force their `:`; the other + // clause headers report a missing one from an `invalid_*` rule. + parser::ParseErrorType::ExpectedToken { + expected: TokenKind::Colon, + found: TokenKind::Newline, + } => matches!( + clause_keyword(source, range.start()), + Some(TokenKind::Else | TokenKind::Try | TokenKind::Finally | TokenKind::Def) + ), + parser::ParseErrorType::ExpectedToken { .. } + | parser::ParseErrorType::ExpectedExpression + | parser::ParseErrorType::ExpectedIdentifier + | parser::ParseErrorType::UnexpectedExpressionToken + | parser::ParseErrorType::SimpleStatementsOnSameLine + | parser::ParseErrorType::SimpleAndCompoundStatementOnSameLine + | parser::ParseErrorType::InvalidStarredExpressionUsage => true, + parser::ParseErrorType::OtherError(message) => message == "invalid syntax", + _ => false, + } +} + +/// The keyword that starts the logical line `offset` is in, after `async`. +fn clause_keyword(source: &str, offset: TextSize) -> Option { + let parsed = parser::parse_unchecked(source, parser::ParseOptions::from(parser::Mode::Module)); + let mut keyword = None; + let mut at_line_start = true; + for token in parsed + .tokens() + .iter() + .take_while(|token| token.start() < offset) + { + match token.kind() { + TokenKind::Newline | TokenKind::Indent | TokenKind::Dedent => at_line_start = true, + TokenKind::NonLogicalNewline | TokenKind::Comment | TokenKind::Async => {} + kind if at_line_start => { + keyword = Some(kind); + at_line_start = false; + } + _ => {} + } + } + keyword +} + fn eof_parse_diagnostic( error: &parser::ParseError, source_file: &SourceFile, @@ -775,6 +931,7 @@ fn single_mode_blank_source_error(source_file: &SourceFile) -> Option= next.start() && !token.range().is_empty())?; + return Some(CompileError::from_source_error( + source_file, + CpythonDiagnostic::new( + "invalid syntax".to_owned(), + token.start().to_usize(), + token.end().to_usize(), + ), + )); } let tokens = parsed.tokens(); let newline_index = tokens @@ -1604,6 +1783,56 @@ pub fn prepare_barry_as_flufl_source( } } +/// Parse-only state for a native interactive record. This never runs codegen, +/// emits Python warnings, or changes the caller's future flags. +pub struct InteractiveInput { + pub future_features: core::bytecode::CodeFlags, + pub empty: bool, +} + +/// The native interactive grammar terminates compound statements with an empty +/// physical line (or EOF), unlike the module grammar used by the Ruff parser. +/// Use the parsed statement shape rather than a second ad-hoc source lexer. +pub fn interactive_input( + source: &str, + source_path: &str, + future_features: core::bytecode::CodeFlags, + terminated: bool, +) -> Result, CompileError> { + let source_file = SourceFileBuilder::new(source_path, source).finish(); + pre_parse_source_error(&source_file)?; + let options = parser::ParseOptions::from(parser::Mode::Module); + let prepared = prepare_barry_as_flufl_source( + source, + options.clone(), + future_features.contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), + ); + let parsed = parser::parse_unchecked(prepared.source(), options); + if let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) { + return Err(error); + } + if let Some(error) = prepared.diagnostic(parsed.errors().first(), &source_file) { + return Err(error); + } + let parsed = first_parse_error(parsed, &source_file, Mode::Single)?; + let ast::Mod::Module(module) = parsed.syntax() else { + unreachable!() + }; + // A compound statement ends with an empty line or EOF. + if !terminated && module.body.first().is_some_and(is_compound_stmt) { + return Ok(None); + } + if let Some(error) = unsupported_grammar_error(parsed.syntax(), &source_file) { + return Err(error); + } + let future_features = codegen::preprocess::checked_future_features_in_body(&module.body) + .map_err(|error| future_feature_error(error, &source_file))?; + Ok(Some(InteractiveInput { + future_features, + empty: module.body.is_empty(), + })) +} + pub fn compile_with_syntax_warning_handler<'a>( source: &str, mode: Mode, diff --git a/crates/host_env/src/errno.rs b/crates/host_env/src/errno.rs index e816a04a55f..c0a122ddbcc 100644 --- a/crates/host_env/src/errno.rs +++ b/crates/host_env/src/errno.rs @@ -2,7 +2,7 @@ /// Return the platform `strerror(errno)` message as an owned `String`. /// Returns `None` when the runtime gives no description for `errno`. -#[cfg(any(unix, windows))] +#[cfg(any(unix, windows, target_os = "wasi"))] #[must_use] pub fn strerror_string(errno: i32) -> Option { let ptr = unsafe { libc::strerror(errno) }; diff --git a/crates/host_env/src/os.rs b/crates/host_env/src/os.rs index f716d2770c1..71469890587 100644 --- a/crates/host_env/src/os.rs +++ b/crates/host_env/src/os.rs @@ -519,8 +519,24 @@ pub fn device_encoding(_fd: i32) -> Option { } } +/// Terminate the process through the C `exit()`, which runs the C atexit +/// handlers and flushes C stdio. pub fn exit(code: i32) -> ! { - // These callers require immediate termination without C exit handlers. + cfg_select! { + // std::process::exit ends in ExitProcess here, which skips the CRT + // atexit handlers and the stdio flush. + windows => { + use std::io::Write; + let _ = std::io::stdout().flush(); + unsafe { libc::exit(code) } + } + _ => std::process::exit(code), + } +} + +/// Terminate the process through `_exit()`, without running the C atexit +/// handlers or flushing C stdio. +pub fn exit_immediately(code: i32) -> ! { cfg_select! { any(unix, windows, target_os = "wasi") => unsafe { libc::_exit(code) }, _ => std::process::exit(code), diff --git a/crates/host_env/src/readline.rs b/crates/host_env/src/readline.rs index f3f468cba56..c37d3714de0 100644 --- a/crates/host_env/src/readline.rs +++ b/crates/host_env/src/readline.rs @@ -196,3 +196,129 @@ impl Readline { self.0.readline(prompt) } } + +/// Borrow the shared buffered stdin for one physical read. The caller retains +/// partial bytes across EINTR and checks signals after this lock is released. +pub fn stdio_readline_bytes(prompt: &str, line: &mut Vec) -> io::Result { + use std::io::Write; + let mut stderr = io::stderr().lock(); + stderr.write_all(prompt.as_bytes())?; + stderr.flush()?; + drop(stderr); + read_line_bytes(&mut io::stdin().lock(), line) +} + +fn read_line_bytes(reader: &mut impl io::BufRead, line: &mut Vec) -> io::Result { + let mut read = 0; + loop { + // Unlike read_until, return Interrupted to the VM before retrying. + let available = reader.fill_buf()?; + if available.is_empty() { + return Ok(read); + } + let newline = available.iter().position(|&byte| byte == b'\n'); + let used = newline.map_or(available.len(), |index| index + 1); + line.extend_from_slice(&available[..used]); + reader.consume(used); + read += used; + if newline.is_some() { + return Ok(read); + } + } +} + +#[cfg(test)] +mod tests { + use super::read_line_bytes; + use std::io::{self, BufRead, Read}; + + struct ScriptedReader { + chunks: alloc::collections::VecDeque>>, + consumed: usize, + } + + impl ScriptedReader { + fn new(chunks: impl IntoIterator>>) -> Self { + Self { + chunks: chunks.into_iter().collect(), + consumed: 0, + } + } + } + + impl Read for ScriptedReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + unreachable!("the transport uses BufRead directly") + } + } + + impl BufRead for ScriptedReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + if self.chunks.front().is_some_and(Result::is_err) { + return Err(self.chunks.pop_front().unwrap().unwrap_err()); + } + Ok(self + .chunks + .front() + .map_or(&[], |chunk| &chunk.as_ref().unwrap()[self.consumed..])) + } + + fn consume(&mut self, amount: usize) { + self.consumed += amount; + if self.consumed == self.chunks.front().unwrap().as_ref().unwrap().len() { + self.chunks.pop_front(); + self.consumed = 0; + } + } + } + + #[test] + fn physical_lines_preserve_buffered_remainder_and_count_only_new_bytes() { + let mut reader = ScriptedReader::new([Ok(b"one\ntwo\ntail".to_vec())]); + let mut line = b"prefix".to_vec(); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"prefixone\n"); + line.clear(); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"two\n"); + line.clear(); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"tail"); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 0); + assert_eq!(line, b"tail"); + } + + #[test] + fn errors_return_immediately_and_leave_partial_bytes_for_the_caller() { + for kind in [io::ErrorKind::Interrupted, io::ErrorKind::Other] { + let mut reader = ScriptedReader::new([ + Ok(b"part".to_vec()), + Err(io::Error::from(kind)), + Ok(b"ial\nnext".to_vec()), + ]); + let mut line = Vec::new(); + assert_eq!( + read_line_bytes(&mut reader, &mut line).unwrap_err().kind(), + kind + ); + assert_eq!(line, b"part"); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"partial\n"); + } + } + + #[test] + fn eof_after_interruption_keeps_partial_bytes() { + let mut reader = ScriptedReader::new([ + Ok(b"partial".to_vec()), + Err(io::Error::from(io::ErrorKind::Interrupted)), + ]); + let mut line = Vec::new(); + assert_eq!( + read_line_bytes(&mut reader, &mut line).unwrap_err().kind(), + io::ErrorKind::Interrupted + ); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 0); + assert_eq!(line, b"partial"); + } +} diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 05229ddd304..c017f0bcd92 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -408,7 +408,7 @@ mod decl { host_faulthandler::signal_default_and_raise(signum); // Fallback - rustpython_host_env::os::exit(1); + rustpython_host_env::os::exit_immediately(1); } // Windows vectored exception handler (faulthandler.c:417-480) @@ -687,7 +687,7 @@ mod decl { } if exit { - rustpython_host_env::os::exit(1); + rustpython_host_env::os::exit_immediately(1); } } } diff --git a/crates/stdlib/src/posixsubprocess.rs b/crates/stdlib/src/posixsubprocess.rs index f2ee7bbc2fe..268131e28bc 100644 --- a/crates/stdlib/src/posixsubprocess.rs +++ b/crates/stdlib/src/posixsubprocess.rs @@ -283,7 +283,7 @@ fn exec(args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>, vm: &VirtualMachine) -> let errno = e.raw_os_error().unwrap_or(0); let _ = write!(pipe, "OSError:{errno:x}:{}", ctx.as_msg()); } - rustpython_host_env::os::exit(255) + rustpython_host_env::os::exit_immediately(255) } } } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 8adee7270e9..b6f0cad9303 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -101,26 +101,115 @@ impl VirtualMachine { /// Print exception chain by calling sys.excepthook pub fn print_exception(&self, exc: &Py) { let vm = self; - let write_fallback = |exc, errstr| { - if let Ok(stderr) = sys::get_stderr(vm) { - let mut stderr = py_io::PyWriter(stderr, vm); - // if this fails stderr might be closed -- ignore it - let _ = writeln!(stderr, "{errstr}"); - let _ = self.write_exception(&mut stderr, exc); - } else { - eprintln!("{errstr}\nlost sys.stderr"); - let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), exc); - } - }; if let Ok(excepthook) = vm.sys_module.get_attr("excepthook", vm) { let (exc_type, exc_val, exc_tb) = vm.split_exception(exc.to_owned()); if let Err(eh_exc) = excepthook.call((exc_type, exc_val, exc_tb), vm) { - write_fallback(&eh_exc, "Error in sys.excepthook:"); - write_fallback(exc, "Original exception was:"); + writeln!(sys::PyStderr(vm), "Error in sys.excepthook:"); + vm.display_exception(&eh_exc); + writeln!(sys::PyStderr(vm), "Original exception was:"); + vm.display_exception(exc); } } else { - write_fallback(exc, "missing sys.excepthook"); + writeln!(sys::PyStderr(vm), "missing sys.excepthook"); + vm.display_exception(exc); + } + } + + /// PyErr_Print + /// + /// Record the exception as `sys.last_exc` and display it through + /// `sys.excepthook`. A `SystemExit` exits the process unless inspect mode + /// is enabled. + pub fn print_uncaught_exception(&self, exc: &Py) { + self.handle_system_exit(exc); + let (exc_type, exc_val, exc_tb) = self.split_exception(exc.to_owned()); + for (name, value) in [ + ("last_exc", &exc_val), + ("last_type", &exc_type), + ("last_value", &exc_val), + ("last_traceback", &exc_tb), + ] { + let _ = self.sys_module.set_attr(name, value.clone(), self); + } + let hook = self.sys_module.get_attr("excepthook", self).ok(); + if let Err(audit_exc) = self.audit("sys.excepthook", || { + ( + hook.clone().unwrap_or_else(|| self.ctx.none()), + exc_type.clone(), + exc_val.clone(), + exc_tb.clone(), + ) + }) { + if audit_exc.fast_isinstance(self.ctx.exceptions.runtime_error) { + return; + } + self.run_unraisable( + audit_exc, + Some("Exception ignored in audit hook".to_owned()), + self.ctx.none(), + ); + } + if let Some(hook) = hook { + if let Err(hook_exc) = hook.call((exc_type, exc_val, exc_tb), self) { + self.handle_system_exit(&hook_exc); + let _ = io::Write::flush(&mut io::stdout()); + writeln!(sys::PyStderr(self), "Error in sys.excepthook:"); + self.display_exception(&hook_exc); + writeln!(sys::PyStderr(self), "\nOriginal exception was:"); + self.display_exception(exc); + } + } else { + writeln!(sys::PyStderr(self), "sys.excepthook is missing"); + self.display_exception(exc); + } + } + + /// handle_system_exit + fn handle_system_exit(&self, exc: &Py) { + if let Some(exit_code) = self.handle_system_exit_and_keyboard_interrupt(exc) { + self.exit_process(exit_code); + } + } + + /// PyErr_Display + /// + /// Display `exc` on `sys.stderr`; nothing is written when it is `None`. + pub fn display_exception(&self, exc: &Py) { + let file = self.sys_module.dict().get_item_opt("stderr", self); + if let Ok(Some(file)) = &file { + if !self.is_none(file) { + self.display_exception_to(file, exc); + } + return; } + self.dump_object(exc.as_object()); + eprintln!("lost sys.stderr"); + if let Err(err) = file { + self.dump_object(err.as_object()); + } + } + + /// _PyErr_Display + /// + /// Display `exc` through `traceback._print_exception_bltin`, falling back + /// to the native printer writing to `file`. + pub(crate) fn display_exception_to(&self, file: &PyObject, exc: &Py) { + if let Ok(print_exception) = self + .import("traceback", 0) + .and_then(|traceback| traceback.get_attr("_print_exception_bltin", self)) + && print_exception.is_callable() + && print_exception.call((exc.to_owned(),), self).is_ok() + { + return; + } + if self + .write_exception(&mut py_io::PyWriter(file.to_owned(), self), exc) + .is_err() + { + self.dump_object(exc.as_object()); + eprintln!("lost sys.stderr"); + } + let _ = self.call_method(file, "flush", ()); } pub fn write_exception( @@ -185,10 +274,7 @@ impl VirtualMachine { let traceback = exc.traceback.read().clone(); if let Some(tb) = traceback { - writeln!(output, "Traceback (most recent call last):")?; - for tb in tb.iter() { - write_traceback_entry(output, &tb)?; - } + write_traceback(output, &tb)?; } let varargs = exc.args(); @@ -448,6 +534,17 @@ fn print_source_line( Ok(()) } +pub(crate) fn write_traceback( + output: &mut W, + traceback: &PyTracebackRef, +) -> Result<(), W::Error> { + writeln!(output, "Traceback (most recent call last):")?; + for entry in traceback.iter() { + write_traceback_entry(output, &entry)?; + } + Ok(()) +} + /// Print exception occurrence location from traceback element fn write_traceback_entry( output: &mut W, diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 07d5a020983..20759fb5f8b 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -2729,7 +2729,7 @@ impl Py { /// 0 doubles as "no thread", which is what `attached_tid` wants for a build /// that cannot have a frame running anywhere else. #[inline] -fn current_thread_ident() -> u64 { +pub(crate) fn current_thread_ident() -> u64 { #[cfg(feature = "threading")] { crate::stdlib::_thread::get_ident() diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs index 02357f12660..68e95583a13 100644 --- a/crates/vm/src/getpath.rs +++ b/crates/vm/src/getpath.rs @@ -19,6 +19,7 @@ mod platform { pub(super) const BUILD_LANDMARK: &str = "Modules/Setup.local"; pub(super) const VENV_LANDMARK: &str = "pyvenv.cfg"; pub(super) const BUILDSTDLIB_LANDMARK: &str = "Lib/os.py"; + pub(super) const DEFAULT_PROGRAM_NAME: &str = "python3"; pub(super) fn stdlib_subdir() -> String { format!("lib/python{}.{}", version::MAJOR, version::MINOR) @@ -51,6 +52,7 @@ mod platform { pub(super) const VENV_LANDMARK: &str = "pyvenv.cfg"; pub(super) const BUILDSTDLIB_LANDMARK: &str = "Lib\\os.py"; pub(super) const STDLIB_SUBDIR: &str = "Lib"; + pub(super) const DEFAULT_PROGRAM_NAME: &str = "python"; pub(super) fn stdlib_landmarks() -> [String; 2] { ["Lib\\os.py".into(), "Lib\\os.pyc".into()] @@ -106,7 +108,10 @@ fn search_up_dir>(start: P, landmarks: &[&str]) -> Option Paths { - let mut paths = Paths::default(); + let mut paths = Paths { + program_name: program_name(), + ..Paths::default() + }; // Step 0: Get executable path let executable = get_executable_path(); @@ -488,6 +493,22 @@ fn env_executable_override() -> Option { None } +/// CALCULATE program_name: `argv[0]`, with the executable suffix added when +/// only the suffixed file exists. +fn program_name() -> String { + let program_name = env::args() + .next() + .filter(|arg| !arg.is_empty()) + .unwrap_or_else(|| platform::DEFAULT_PROGRAM_NAME.to_owned()); + if cfg!(windows) && !program_name.to_lowercase().ends_with(".exe") { + let suffixed = format!("{program_name}.exe"); + if Path::new(&suffixed).is_file() { + return suffixed; + } + } + program_name +} + /// Get the current executable path fn get_executable_path() -> Option { #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index 8bef84d33da..4ea023505a8 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -58,6 +58,37 @@ pub(crate) static TRIGGERS: [AtomicBool; NSIG] = [ATOMIC_FALSE; NSIG]; #[cfg(windows)] static SIGINT_EVENT: AtomicIsize = AtomicIsize::new(0); +/// `_PyRuntime.signals.unhandled_keyboard_interrupt` +static UNHANDLED_KEYBOARD_INTERRUPT: AtomicBool = AtomicBool::new(false); + +pub(crate) fn set_unhandled_keyboard_interrupt() { + UNHANDLED_KEYBOARD_INTERRUPT.store(true, Ordering::Relaxed); +} + +pub(crate) fn clear_unhandled_keyboard_interrupt() { + UNHANDLED_KEYBOARD_INTERRUPT.store(false, Ordering::Relaxed); +} + +pub(crate) fn unhandled_keyboard_interrupt() -> bool { + UNHANDLED_KEYBOARD_INTERRUPT.load(Ordering::Relaxed) +} + +/// exit_sigint: exit through the default SIGINT handler so that the parent +/// process sees that a KeyboardInterrupt went unhandled. +pub(crate) fn exit_sigint() -> u32 { + cfg_select! { + unix => { + if crate::host_env::signal::set_sigint_default_onstack().is_ok() { + let _ = crate::host_env::signal::send_sigint_to_self(); + } + libc::SIGINT as u32 + 128 + } + // STATUS_CONTROL_C_EXIT + windows => 0xC000013A, + _ => 2 + 128, + } +} + thread_local! { /// Prevent recursive signal handler invocation. When a Python signal /// handler is running, new signals are deferred until it completes. diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index a899e248115..83375b8ab01 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -996,6 +996,7 @@ fn type_comment_parse_error( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, } .into() } @@ -1726,6 +1727,7 @@ fn ipython_escape_command_syntax_error( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, } .into(), ) @@ -1810,6 +1812,7 @@ pub(crate) fn parse>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) .into()); } @@ -1840,6 +1843,7 @@ pub(crate) fn parse>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) .into()); } @@ -1974,6 +1978,7 @@ pub(crate) fn parse_func_type>( source_path: filename.to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, } .into() }; @@ -2003,6 +2008,7 @@ pub(crate) fn parse_func_type>( source_path: filename.to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) .into()); }; @@ -2024,6 +2030,7 @@ pub(crate) fn parse_func_type>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) })?; let ast::Mod::Expression(expression) = parsed.into_syntax() else { @@ -2059,6 +2066,7 @@ pub(crate) fn parse_func_type>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) })?; let ast::Mod::Expression(expression) = parsed.into_syntax() else { diff --git a/crates/vm/src/stdlib/_testinternalcapi.rs b/crates/vm/src/stdlib/_testinternalcapi.rs index 7250de54f0a..9e368d4462b 100644 --- a/crates/vm/src/stdlib/_testinternalcapi.rs +++ b/crates/vm/src/stdlib/_testinternalcapi.rs @@ -423,7 +423,7 @@ mod _testinternalcapi { global.set_item("dev_mode", vm.ctx.new_bool(settings.dev_mode).into(), vm)?; global.set_item("verbose", vm.ctx.new_int(settings.verbose).into(), vm)?; global.set_item("quiet", vm.ctx.new_bool(settings.quiet).into(), vm)?; - global.set_item("inspect", vm.ctx.new_bool(settings.inspect).into(), vm)?; + global.set_item("inspect", vm.ctx.new_bool(vm.config_inspect()).into(), vm)?; global.set_item( "interactive", vm.ctx.new_bool(settings.interactive).into(), diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 7a51f81ae0f..3a69305d2d0 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -1746,7 +1746,7 @@ pub(super) mod _os { #[pyfunction] fn _exit(status: ExitStatus) { - crate::host_env::os::exit(status.status) + crate::host_env::os::exit_immediately(status.status) } #[pyfunction] diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index ae70b7001bb..8e142c41793 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -755,6 +755,8 @@ pub mod module { crate::object::qsbr::QSBR.reset_after_fork() }; + vm.state.reinit_running_main(); + // Phase 3: Clean up thread state. Locks are now reinit'd so we can // acquire them normally instead of using try_lock(). #[cfg(feature = "threading")] diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 7e746b5f521..6051723b19a 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -56,7 +56,7 @@ pub mod sys { use num_traits::ToPrimitive; use std::{ env, - io::{IsTerminal, Read, Write}, + io::{IsTerminal, Write}, }; // Rust target triple (e.g., "x86_64-unknown-linux-gnu") @@ -760,17 +760,11 @@ pub mod sys { #[cfg(feature = "rustpython-compiler")] #[pyfunction] - fn _baserepl(vm: &VirtualMachine) -> PyResult<()> { - // read stdin to end - let stdin = std::io::stdin(); - let mut handle = stdin.lock(); - let mut source = String::new(); - handle - .read_to_string(&mut source) - .map_err(|e| vm.new_os_error(format!("Error reading from stdin: {e}")))?; - vm.compile(&source, crate::compiler::Mode::Single, "") - .map_err(|e| e.into_pyexception(vm, Some(&source)))?; - Ok(()) + fn _baserepl(vm: &VirtualMachine) { + // PyRun_AnyFileExFlags(stdin, "", 0, &cf) + if let Err(exc) = vm.run_any_stdin() { + vm.print_uncaught_exception(&exc); + } } #[pyfunction] @@ -867,7 +861,6 @@ pub mod sys { traceback: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - let stderr = super::get_stderr(vm)?; // Keep runtime SyntaxErrors on the normal traceback path. let has_traceback = !vm.is_none(&traceback); match vm.normalize_exception(exctype, value.clone(), traceback) { @@ -887,19 +880,11 @@ pub mod sys { .and_then(|text| text.downcast::().ok()) .is_some_and(|text| text.to_string_lossy().trim_end() == "\\"); if native_syntax_error_display { + let stderr = super::get_stderr(vm)?; return vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc); } - // PyErr_Display: try traceback._print_exception_bltin first - if let Ok(tb_mod) = vm.import("traceback", 0) - && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) - && print_exc_builtin - .call((exc.as_object().to_owned(),), vm) - .is_ok() - { - return Ok(()); - } - // Fallback to Rust-level exception printing - vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc) + vm.display_exception(&exc); + Ok(()) } Err(_) => { let type_name = value.class().name(); @@ -907,6 +892,7 @@ pub mod sys { "TypeError: print_exception(): Exception expected for value, {type_name} found\n" ); use crate::py_io::Write; + let stderr = super::get_stderr(vm)?; write!(&mut crate::py_io::PyWriter(stderr, vm), "{msg}")?; Ok(()) } @@ -1039,6 +1025,36 @@ pub mod sys { PyFlags::from_data(FlagsData::from_settings(&vm.state.config.settings), vm) } + /// _PySys_SetFlagObj + pub(crate) fn set_flag_obj( + vm: &VirtualMachine, + name: &str, + value: PyObjectRef, + ) -> PyResult<()> { + use crate::{class::StaticType, types::PyStructSequenceData}; + + let old_flags = vm + .sys_module + .get_attr("flags", vm) + .map_err(|_| vm.new_runtime_error("lost sys.flags"))?; + let old_flags = old_flags + .downcast_ref::() + .filter(|flags| { + flags.len() + == FlagsData::REQUIRED_FIELD_NAMES.len() + FlagsData::OPTIONAL_FIELD_NAMES.len() + }) + .ok_or_else(|| vm.new_type_error("sys.flags must be a sys.flags instance"))?; + let pos = FlagsData::REQUIRED_FIELD_NAMES + .iter() + .position(|field| *field == name) + .expect("unknown sys.flags field"); + let mut values = old_flags.as_slice().to_vec(); + values[pos] = value; + let new_flags = PyTuple::new_unchecked(values.into_boxed_slice()) + .into_ref_with_type(vm, PyFlags::static_type().to_owned())?; + vm.sys_module.set_attr("flags", new_flags, vm) + } + #[pyattr] fn float_info(vm: &VirtualMachine) -> PyTupleRef { PyFloatInfo::from_data(FloatInfoData::INFO, vm) diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 575e954d034..95dbcac5b8f 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -11,6 +11,16 @@ use crate::{ vm::compile_mode::{CompileStart, CompilerFlags, compile_future_features_from_flags}, }; +/// Warnings already emitted while reading one interactive statement. +/// +/// The source is warned about again after each line is read, but the +/// tokenizer warns about each token only once. +#[cfg(feature = "parser")] +#[derive(Default)] +pub(super) struct InteractiveWarnings { + emitted: alloc::collections::BTreeSet<(usize, String)>, +} + #[derive(Debug)] pub enum VmCompileError { Compile(CompileError), @@ -595,6 +605,20 @@ impl VirtualMachine { self.emit_string_escape_warnings(source, &source_path, module, emitted_escapes) .map_err(VmCompileError::Warning)?; } + self.compile_after_parser_warnings(source, mode, source_path, opts, module) + } + + pub(super) fn compile_after_parser_warnings( + &self, + source: &str, + mode: compiler::Mode, + source_path: impl Into, + opts: CompileOpts, + module: Option<&Py>, + ) -> Result, VmCompileError> { + let source_path = source_path.into(); + #[cfg(not(feature = "parser"))] + let _ = module; #[cfg(feature = "parser")] let code = { // A warning the filter escalates to an exception is stashed here so @@ -724,14 +748,21 @@ mod escape_warnings { } enum ParserWarning { - Escape(InvalidEscape, EscapeOrigin), - Numeric { offset: usize, message: String }, + /// The escape, where it is reported from, and for a bytes literal + /// escape on the literal's first line, the error column that + /// string_parser.c reports. + Escape(InvalidEscape, EscapeOrigin, Option), + Numeric { + offset: usize, + last_offset: usize, + message: String, + }, } impl ParserWarning { fn offset(&self) -> usize { match self { - Self::Escape(escape, _) => escape.offset(), + Self::Escape(escape, ..) => escape.offset(), Self::Numeric { offset, .. } => *offset, } } @@ -1206,7 +1237,19 @@ mod escape_warnings { content_bounds(source, text_range_from_bounds(literal_start, end)) && let Some(escape) = first_invalid_escape(source, start, content_end, is_bytes) { - warnings.push(ParserWarning::Escape(escape, EscapeOrigin::Parser)); + let before_escape = &source[start..escape.offset()]; + // string_parser.c counts leading quotes from the token start. + // A bytes prefix contributes none to its first-line error range. + // Keep the actual escape offset for warning order and deduplication. + let error_column = (is_bytes && !before_escape.contains(['\r', '\n'])).then(|| { + let (_, column) = line_offset_at(source, escape.offset()); + column.saturating_sub(start - literal_start) + }); + warnings.push(ParserWarning::Escape( + escape, + EscapeOrigin::Parser, + error_column, + )); } } @@ -1225,7 +1268,7 @@ mod escape_warnings { } else { EscapeOrigin::Parser }; - warnings.push(ParserWarning::Escape(escape, origin)); + warnings.push(ParserWarning::Escape(escape, origin, None)); return; } // In CPython, _PyTokenizer_warn_invalid_escape_sequence handles @@ -1249,6 +1292,7 @@ mod escape_warnings { offset: end - 1, }, EscapeOrigin::Tokenizer, + None, )); } } @@ -1474,6 +1518,7 @@ mod escape_warnings { if end > index && numeric_keyword_suffix(&bytes[end..]) { warnings.push(ParserWarning::Numeric { offset: index, + last_offset: end - 1, message: format!("invalid {kind} literal"), }); } @@ -1704,10 +1749,12 @@ mod escape_warnings { warnings.sort_by_key(ParserWarning::offset); // Interpolated tokenizer warnings have their own stop/error boundary. // Numeric-only recovery must not skip across one of those events. - if warnings - .iter() - .any(|warning| matches!(warning, ParserWarning::Escape(_, EscapeOrigin::Tokenizer))) - { + if warnings.iter().any(|warning| { + matches!( + warning, + ParserWarning::Escape(_, EscapeOrigin::Tokenizer, _) + ) + }) { ast_tokenizer_recovery = false; } (warnings, ast_tokenizer_recovery) @@ -1716,11 +1763,12 @@ mod escape_warnings { enum WarningEmission<'a> { Standard { emitted_escapes: &'a [usize] }, Ast { tokenizer_recovery: bool }, + Interactive(&'a mut super::InteractiveWarnings), } impl VirtualMachine { - /// Emit tokenizer-level SyntaxWarnings raised before - /// code generation. + // Keep ordinary source compilation's existing warning order and + // recovery surface; interactive records use the bounded parser below. pub(super) fn emit_tokenizer_syntax_warnings( &self, source: &str, @@ -1741,8 +1789,6 @@ mod escape_warnings { .map(|_| ()) } - /// Walk all string literals in `source` and emit `SyntaxWarning` for - /// each that contains an invalid escape sequence. pub(super) fn emit_string_escape_warnings( &self, source: &str, @@ -1750,8 +1796,6 @@ mod escape_warnings { module: Option<&Py>, emitted_escapes: &[usize], ) -> Result<(), CompileWarningError> { - // The compile that follows rejects this source; parsing it here - // would build a tree that exhausts the stack when dropped. let source_file = compiler::core::SourceFileBuilder::new(filename, source).finish(); if compiler::pre_parse_source_error(&source_file).is_err() { return Ok(()); @@ -1821,6 +1865,31 @@ mod escape_warnings { ) } + /// Emit the tokenizer warnings for the interactive input read so far. + pub(in crate::vm) fn emit_interactive_parser_warnings( + &self, + source: &str, + filename: &str, + module: Option<&Py>, + inherited_barry_as_flufl: bool, + emitted: &mut super::InteractiveWarnings, + ) -> Result<(), CompileWarningError> { + let (warnings, _) = collect_parser_warnings( + source, + filename, + Some(ruff_python_parser::Mode::Module), + inherited_barry_as_flufl, + ); + self.emit_parser_warning_batch( + source, + filename, + module, + warnings, + WarningEmission::Interactive(emitted), + ) + .map(|_| ()) + } + fn emit_parser_warning_batch( &self, source: &str, @@ -1829,9 +1898,10 @@ mod escape_warnings { warnings: Vec, emission: WarningEmission<'_>, ) -> Result, CompileWarningError> { - let (emitted_escapes, ast_tokenizer_recovery) = match emission { - WarningEmission::Standard { emitted_escapes } => (emitted_escapes, false), - WarningEmission::Ast { tokenizer_recovery } => (&[][..], tokenizer_recovery), + let (emitted_escapes, ast_tokenizer_recovery, mut interactive) = match emission { + WarningEmission::Standard { emitted_escapes } => (emitted_escapes, false, None), + WarningEmission::Ast { tokenizer_recovery } => (&[][..], tokenizer_recovery, None), + WarningEmission::Interactive(emitted) => (&[][..], false, Some(emitted)), }; let mut considered_escapes = Vec::new(); let mut parser_error = None; @@ -1841,8 +1911,17 @@ mod escape_warnings { if parser_error.is_some() && !matches!(warning, ParserWarning::Numeric { .. }) { continue; } + if let Some(state) = interactive.as_deref_mut() { + let message = match &warning { + ParserWarning::Escape(escape, ..) => escape.warning_message(), + ParserWarning::Numeric { message, .. } => message.clone(), + }; + if !state.emitted.insert((warning.offset(), message)) { + continue; + } + } match warning { - ParserWarning::Escape(escape, origin) => { + ParserWarning::Escape(escape, origin, error_column) => { let offset = escape.offset(); if considered_escapes.last() == Some(&offset) { continue; @@ -1851,7 +1930,18 @@ mod escape_warnings { if emitted_escapes.binary_search(&offset).is_err() { let result = warn_invalid_escape_sequence( source, escape, filename, module, self, - ); + ) + .map_err(|mut error| { + if interactive.is_some() + && let Some(column) = error_column + { + error.offset = column; + if let Some(replacement) = &mut error.replacement { + replacement.end_offset = column + 2; + } + } + error + }); if let Err(error) = result { if ast_tokenizer_recovery && origin == EscapeOrigin::Parser @@ -1866,8 +1956,35 @@ mod escape_warnings { } } } - ParserWarning::Numeric { offset, message } => { - warn_syntax_at_offset(source, filename, module, offset, message, self)?; + ParserWarning::Numeric { + offset, + last_offset, + message, + } => { + warn_syntax_at_offset( + source, + filename, + module, + if interactive.is_some() { + last_offset + } else { + offset + }, + message.clone(), + self, + ) + .map_err(|mut error| { + // Tokenizer numeric warnings become a zero-width + // error at the literal's last character, in character + // columns rather than grammar-action byte columns. + if interactive.is_some() { + error.replacement = Some(SyntaxErrorReplacement { + message, + end_offset: error.offset, + }); + } + error + })?; } } } diff --git a/crates/vm/src/vm/crossinterp.rs b/crates/vm/src/vm/crossinterp.rs index 70ea5104827..1fb46a64356 100644 --- a/crates/vm/src/vm/crossinterp.rs +++ b/crates/vm/src/vm/crossinterp.rs @@ -932,16 +932,7 @@ where )); } - if state - .running_main - .compare_exchange( - false, - true, - core::sync::atomic::Ordering::AcqRel, - core::sync::atomic::Ordering::Acquire, - ) - .is_err() - { + if !state.set_running_main() { return Err(crate::stdlib::_interpreters::interpreter_error( caller, "interpreter already running", @@ -951,9 +942,7 @@ where struct RunningGuard<'a>(&'a crate::vm::PyGlobalState); impl Drop for RunningGuard<'_> { fn drop(&mut self) { - self.0 - .running_main - .store(false, core::sync::atomic::Ordering::Release); + self.0.set_not_running_main(); } } let _guard = RunningGuard(&state); @@ -965,6 +954,5 @@ where #[must_use] pub fn is_running(id: i64) -> bool { - crate::vm::runtime::lookup_interpreter(id) - .is_some_and(|s| s.is_main || s.running_main.load(core::sync::atomic::Ordering::Acquire)) + crate::vm::runtime::lookup_interpreter(id).is_some_and(|s| s.is_running_main()) } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index d6141223704..b703da7d3ae 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -74,7 +74,7 @@ where use crate::codecs::CodecsRegistry; use crate::common::lock::PyMutex; use crate::warn::WarningsState; - use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64}; + use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize}; use crossbeam_utils::atomic::AtomicCell; // Before any lock this interpreter's threads can contend on exists. @@ -84,8 +84,10 @@ where let (config, all_module_defs, frozen, int_max_str_digits) = if let Some(parent) = parent_state { // Subinterpreter: clone config and module tables from parent, fresh runtime state. let int_max_str_digits = AtomicCell::new(parent.int_max_str_digits.load()); + let mut config = parent.config.clone(); + config.settings.inspect = parent.inspect.load(Ordering::Relaxed); ( - parent.config.clone(), + config, parent.module_defs.clone(), parent.frozen.clone(), int_max_str_digits, @@ -154,6 +156,8 @@ where runtime_root_id, whence, is_main, + inspect: AtomicBool::new(config.settings.inspect), + interactive_src_count: AtomicUsize::new(0), config, module_defs: all_module_defs, frozen, @@ -193,7 +197,7 @@ where stop_the_world: StopTheWorldState::new(), feature_flags, own_gil, - running_main: AtomicBool::new(false), + running_main: AtomicCell::new(None), ready: AtomicBool::new(false), id_refcount: AtomicI64::new(0), require_idref: AtomicBool::new(false), @@ -587,6 +591,24 @@ impl Interpreter { self.vm.new_thread() } + /// Move the main execution context into a separate owner, leaving this + /// handle available to create worker VMs for the same interpreter. + /// + /// The returned interpreter must run and finalize on the original thread. + /// This handle should only create worker VMs and must not be finalized. + /// Unlike `new_thread()`, this preserves the main VM's signal handlers and + /// all of its other thread-local state. + #[cfg(feature = "threading")] + #[must_use] + pub fn take_main_thread(&mut self) -> Self { + let replacement = self.vm.new_thread().vm; + let vm = core::mem::replace(&mut self.vm, replacement); + Self { + global_state: self.global_state.clone(), + vm, + } + } + /// Run a function with the main virtual machine and return a PyResult of the result. /// /// To enter vm context multiple times or to avoid buffer/exception management, this function is preferred. @@ -635,6 +657,25 @@ impl Interpreter { self.finalize(res.err()) } + /// Py_RunMain + /// + /// Run a function that reports its own errors and returns the exit code, + /// then finalize with that exit code. An unhandled `KeyboardInterrupt` + /// exits through the default SIGINT handler instead. + pub fn run_main(self, f: F) -> u32 + where + F: FnOnce(&VirtualMachine) -> u32, + { + crate::signal::clear_unhandled_keyboard_interrupt(); + let exit_code = self.enter(f); + let exit_code = self.finalize_with_exit_code(exit_code); + if crate::signal::unhandled_keyboard_interrupt() { + crate::signal::exit_sigint() + } else { + exit_code + } + } + /// Finalize vm and turns an exception to exit code. /// /// Finalization steps (matching Py_FinalizeEx): @@ -650,93 +691,122 @@ impl Interpreter { /// /// Note that calling `finalize` is not necessary by purpose though. pub fn finalize(self, exc: Option) -> u32 { + self.finalize_inner(|vm| exc.map_or(0, |exc| vm.handle_exit_exception(exc))) + } + + /// Finalize vm with an exit code the caller has already determined. + /// + /// See [`Interpreter::finalize`] for the finalization steps. + pub fn finalize_with_exit_code(self, exit_code: u32) -> u32 { + self.finalize_inner(|_| exit_code) + } + + fn finalize_inner(self, exit_code: impl FnOnce(&VirtualMachine) -> u32) -> u32 { self.enter(|vm| { - let mut flush_status = vm.flush_std(); + let flush_status = vm.flush_std(); // See if any exception leaked out: - let exit_code = if let Some(exc) = exc { - vm.handle_exit_exception(exc) + let exit_code = exit_code(vm); + + // Match CPython: if exit_code is 0 and stdout flush failed, exit 120 + if vm.finalize_ex(flush_status) < 0 && exit_code == 0 { + EXITCODE_FLUSH_FAILURE } else { - 0 - }; - - // Wait for thread shutdown - call threading._shutdown() if available. - // This waits for all non-daemon threads to complete. - // threading module may not be imported, so ignore import errors. - #[cfg(feature = "threading")] - if let Ok(threading) = vm.import("threading", 0) - && let Ok(shutdown) = threading.get_attr("_shutdown", vm) - && let Err(e) = shutdown.call((), vm) - { - vm.run_unraisable( - e, - Some("Exception ignored in threading shutdown".to_owned()), - threading, - ); + exit_code } + }) + } +} - // Run atexit handlers before setting finalizing flag. - // This allows unraisable exceptions from atexit handlers to be reported. - atexit::_run_exitfuncs(vm); - - // Clean up any lingering subinterpreters. This has to happen before - // the finalizing flag is set, or else threads might get prematurely - // blocked. - #[cfg(feature = "threading")] - finalize_subinterpreters(vm); - - // Remaining daemon threads hang in `_PyThreadState_HangThread` when - // they next attach (or at the eval-breaker if they are already - // running). Their `_ThreadHandle` stays not-done so `is_alive()` - // is still true for `join()` during the GC that follows. - #[cfg(feature = "threading")] - { - vm.state.stop_the_world.stop_the_world(&vm.state); - vm.state - .finalizing_thread_ident - .store(crate::stdlib::_thread::get_ident()); - } - vm.state.finalizing.store(true, Ordering::Release); - #[cfg(feature = "threading")] - { - thread::set_other_threads_shutting_down(&vm.state); - vm.state.stop_the_world.start_the_world(&vm.state); - crate::signal::set_finalizing_bit(); - } +impl VirtualMachine { + /// Py_Exit + /// + /// Finalize the interpreter and exit the process with `status`. + pub fn exit_process(&self, status: u32) -> ! { + if self.state.thread_is_running_main() { + self.state.set_not_running_main(); + } + let status = if self.finalize_ex(self.flush_std()) < 0 { + EXITCODE_FLUSH_FAILURE + } else { + status + }; + crate::host_env::os::exit(status as i32) + } - // GC pass - collect cycles before module cleanup - vm.state.gc.collect_force(2); + /// The steps of Py_FinalizeEx after the first flush of stdout and stderr. + /// Return a negative value if any flush failed. + fn finalize_ex(&self, mut flush_status: i32) -> i32 { + let vm = self; + // Wait for thread shutdown - call threading._shutdown() if available. + // This waits for all non-daemon threads to complete. + // threading module may not be imported, so ignore import errors. + #[cfg(feature = "threading")] + if let Ok(threading) = vm.import("threading", 0) + && let Ok(shutdown) = threading.get_attr("_shutdown", vm) + && let Err(e) = shutdown.call((), vm) + { + vm.run_unraisable( + e, + Some("Exception ignored in threading shutdown".to_owned()), + threading, + ); + } - // Module finalization: remove modules from sys.modules, GC collect - // (while builtins is still available for __del__), then clear module dicts. - vm.finalize_modules(); + // Run atexit handlers before setting finalizing flag. + // This allows unraisable exceptions from atexit handlers to be reported. + atexit::_run_exitfuncs(vm); - // CPython clears low-level cross-interpreter container data from - // _PyAtExit_Fini(), after Python atexit callbacks and module - // finalization. In particular, values sent by an atexit callback - // must also become unbound when this interpreter goes away. - let interpreter_id = vm.state.interpreter_id; - crate::stdlib::_interpchannels::clear_interpreter(interpreter_id); - crate::stdlib::_interpqueues::clear_interpreter(interpreter_id); + // Clean up any lingering subinterpreters. This has to happen before + // the finalizing flag is set, or else threads might get prematurely + // blocked. + #[cfg(feature = "threading")] + finalize_subinterpreters(vm); - if vm.flush_std() < 0 && flush_status == 0 { - flush_status = -1; - } + // Remaining daemon threads hang in `_PyThreadState_HangThread` when + // they next attach (or at the eval-breaker if they are already + // running). Their `_ThreadHandle` stays not-done so `is_alive()` + // is still true for `join()` during the GC that follows. + #[cfg(feature = "threading")] + { + vm.state.stop_the_world.stop_the_world(&vm.state); + vm.state + .finalizing_thread_ident + .store(crate::stdlib::_thread::get_ident()); + } + vm.state.finalizing.store(true, Ordering::Release); + #[cfg(feature = "threading")] + { + thread::set_other_threads_shutting_down(&vm.state); + vm.state.stop_the_world.start_the_world(&vm.state); + crate::signal::set_finalizing_bit(); + } - // Match CPython: if exit_code is 0 and stdout flush failed, exit 120 - let exit_code = if exit_code == 0 && flush_status < 0 { - EXITCODE_FLUSH_FAILURE - } else { - exit_code - }; + // GC pass - collect cycles before module cleanup + vm.state.gc.collect_force(2); - // Daemon threads may still exist, so use the safe `process()`, - // not `drain_all()`. - #[cfg(feature = "threading")] - crate::object::qsbr::QSBR.process(); + // Module finalization: remove modules from sys.modules, GC collect + // (while builtins is still available for __del__), then clear module dicts. + vm.finalize_modules(); - exit_code - }) + // CPython clears low-level cross-interpreter container data from + // _PyAtExit_Fini(), after Python atexit callbacks and module + // finalization. In particular, values sent by an atexit callback + // must also become unbound when this interpreter goes away. + let interpreter_id = vm.state.interpreter_id; + crate::stdlib::_interpchannels::clear_interpreter(interpreter_id); + crate::stdlib::_interpqueues::clear_interpreter(interpreter_id); + + if vm.flush_std() < 0 && flush_status == 0 { + flush_status = -1; + } + + // Daemon threads may still exist, so use the safe `process()`, + // not `drain_all()`. + #[cfg(feature = "threading")] + crate::object::qsbr::QSBR.process(); + + flush_status } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index d34ded1bca9..0ef153b7248 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -14,6 +14,8 @@ mod interpreter; mod method; #[cfg(feature = "rustpython-compiler")] mod python_run; +#[cfg(feature = "rustpython-compiler")] +pub use python_run::InteractiveLine; pub mod runtime; mod setting; pub mod thread; @@ -52,7 +54,7 @@ use alloc::{borrow::Cow, collections::BTreeMap}; use core::ptr::NonNull; use core::{ cell::{Cell, OnceCell, RefCell}, - sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering}, }; use crossbeam_utils::atomic::AtomicCell; use std::{ @@ -858,6 +860,10 @@ pub struct PyGlobalState { /// becomes *the* process main — see [`runtime::main_interpreter_id`]. pub is_main: bool, pub config: PyConfig, + /// `config.inspect`, which `PyConfig_Set` can change at runtime. + pub(crate) inspect: AtomicBool, + /// `_interactive_src_count`: numbers the code filenames of interactive inputs. + pub(crate) interactive_src_count: AtomicUsize, pub module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef>, pub frozen: HashMap<&'static str, FrozenModule, rapidhash::quality::RandomState>, pub stacksize: AtomicCell, @@ -918,8 +924,8 @@ pub struct PyGlobalState { pub feature_flags: runtime::InterpFeatureFlags, /// Whether the interpreter was configured with `gil="own"`. pub own_gil: bool, - /// Whether `__main__` is currently executing via `_interpreters.exec` / `run_*`. - pub running_main: AtomicBool, + /// `threads.main`: the ident of the thread running `__main__`, if any. + pub running_main: AtomicCell>, /// True after `initialize()` has finished (CPython "ready"). pub ready: AtomicBool, /// Optional ID refcount used by `_interpreters.create(reqrefs=True)`. @@ -959,6 +965,45 @@ impl PyGlobalState { self.feature_flags.allow_daemon_threads } + /// _PyInterpreterState_SetRunningMain + /// + /// Record the calling thread as the one running `__main__`. Return `false` + /// when a thread already does. + pub fn set_running_main(&self) -> bool { + self.running_main + .compare_exchange(None, Some(crate::frame::current_thread_ident())) + .is_ok() + } + + /// _PyInterpreterState_SetNotRunningMain + pub fn set_not_running_main(&self) { + debug_assert!(self.thread_is_running_main()); + self.running_main.store(None); + } + + /// _PyInterpreterState_IsRunningMain + #[must_use] + pub fn is_running_main(&self) -> bool { + self.running_main.load().is_some() + } + + /// _PyThreadState_IsRunningMain + #[must_use] + pub fn thread_is_running_main(&self) -> bool { + self.running_main.load() == Some(crate::frame::current_thread_ident()) + } + + /// _PyInterpreterState_ReinitRunningMain + /// + /// After a fork only the forking thread survives, so `__main__` is no + /// longer running unless that thread was running it. + #[cfg(all(feature = "host_env", unix))] + pub(crate) fn reinit_running_main(&self) { + if !self.thread_is_running_main() { + self.running_main.store(None); + } + } + /// The config this interpreter was created with, rebuilt from the flags it /// kept (`_PyInterpreterConfig_InitFromState`). #[must_use] @@ -1705,33 +1750,52 @@ impl VirtualMachine { #[cold] pub fn run_unraisable(&self, e: PyBaseExceptionRef, msg: Option, object: PyObjectRef) { - // During interpreter finalization, sys.unraisablehook may not be available, - // but we still need to report exceptions (especially from atexit callbacks). - // Write directly to stderr like PyErr_FormatUnraisable. - if self.state.finalizing.load(Ordering::Acquire) { + let msg = msg.map(|msg| self.ctx.new_str(msg)); + let write_original = || { + let msg = msg.as_ref().map(|msg| msg.to_string_lossy()); self.write_unraisable_to_stderr(&e, msg.as_deref(), &object); + }; + // During finalization the hook may no longer exist. + if self.state.finalizing.load(Ordering::Acquire) { + write_original(); return; } - - let sys_module = self.import("sys", 0).unwrap(); - let unraisablehook = sys_module.get_attr("unraisablehook", self).unwrap(); - - let exc_type = e.class().to_owned(); - let exc_traceback = e.traceback().to_pyobject(self); // TODO: actual traceback - let exc_value = e.into(); - let args = stdlib::sys::UnraisableHookArgsData { - exc_type, - exc_value, - exc_traceback, - err_msg: self.new_pyobj(msg), - object, + let Some(unraisablehook) = self.sys_module.get_attr("unraisablehook", self).ok() else { + write_original(); + return; }; - if let Err(e) = unraisablehook.call((args,), self) { - println!("{}", e.as_object().repr(self).unwrap()); + let args = stdlib::sys::UnraisableHookArgsData { + exc_type: e.class().to_owned(), + exc_value: e.clone().into(), + exc_traceback: e.traceback().to_pyobject(self), + err_msg: msg.clone().to_pyobject(self), + object: object.clone(), + } + .to_pyobject(self); + if let Err(error) = self.audit("sys.unraisablehook", || { + (unraisablehook.clone(), args.clone()) + }) { + self.write_unraisable_to_stderr( + &error, + Some("Exception ignored in audit hook"), + &self.ctx.none(), + ); + return; + } + if self.is_none(&unraisablehook) { + write_original(); + return; + } + if let Err(error) = unraisablehook.call((args,), self) { + self.write_unraisable_to_stderr( + &error, + Some("Exception ignored in sys.unraisablehook"), + &unraisablehook, + ); } } - /// Write unraisable exception to stderr during finalization. + /// Write an unraisable exception when its hook cannot be used. /// Similar to _PyErr_WriteUnraisableDefaultHook in CPython. fn write_unraisable_to_stderr( &self, @@ -1769,15 +1833,29 @@ impl VirtualMachine { write_to_stderr(&format!("{repr_wtf8}\n"), &stderr, self); } - // Write exception type and message - let exc_type_name = e.class().name(); - let msg = match e.as_object().str(self) { - Ok(exc_str) if !exc_str.as_wtf8().is_empty() => { - format!("{}: {}\n", exc_type_name, exc_str.as_wtf8()) + // Print the existing traceback without importing Python modules; + // this path must also work while the interpreter is finalizing. + if let Some(traceback) = e.traceback() { + if let Some(stderr) = &stderr { + let mut writer = crate::py_io::PyWriter(stderr.clone(), self); + let _ = crate::exceptions::write_traceback(&mut writer, &traceback); + } else { + let mut writer = crate::py_io::IoWriter(std::io::stderr()); + let _ = crate::exceptions::write_traceback(&mut writer, &traceback); + } + } + let exc_type_name = e + .class() + .fully_qualified_name(self) + .unwrap_or_else(|_| e.class().name().to_string()); + let message = match e.as_object().str(self) { + Ok(value) if !value.as_wtf8().is_empty() => { + format!("{exc_type_name}: {}\n", value.as_wtf8()) } - _ => format!("{exc_type_name}\n"), + Ok(_) => format!("{exc_type_name}\n"), + Err(_) => format!("{exc_type_name}: \n"), }; - write_to_stderr(&msg, &stderr, self); + write_to_stderr(&message, &stderr, self); // Flush stderr to ensure output is visible if let Some(ref stderr) = stderr { @@ -2308,9 +2386,7 @@ impl VirtualMachine { path: &str, run: impl FnOnce(&Py) -> PyResult<()>, ) -> PyResult<()> { - let sys_modules = self.sys_module.get_attr(identifier!(self, modules), self)?; - let main_module = sys_modules.get_item(identifier!(self, __main__), self)?; - let module_dict = main_module.dict().expect("main module must have __dict__"); + let module_dict = self.add_main_module()?.dict(); // Track whether we set __file__ (for cleanup) let set_file_name = !module_dict.contains_key(identifier!(self, __file__), self); @@ -3874,58 +3950,77 @@ impl VirtualMachine { excs.stack.iter().rev().find_map(|e| e.clone()) } + /// The current `config.inspect`. + pub fn config_inspect(&self) -> bool { + self.state.inspect.load(Ordering::Relaxed) + } + + /// `PyConfig_Set("inspect", value)`: update the config and `sys.flags.inspect`. + pub fn config_set_inspect(&self, inspect: bool) -> PyResult<()> { + let value: PyObjectRef = self.ctx.new_int(u8::from(inspect)).into(); + self.audit("cpython.PyConfig_Set", || ("inspect", value.clone()))?; + self.state.inspect.store(inspect, Ordering::Relaxed); + stdlib::sys::sys::set_flag_obj(self, "inspect", value) + } + + /// pymain_exit_err_print + /// + /// Report an exception that reached the top level and return the exit code. pub fn handle_exit_exception(&self, exc: PyBaseExceptionRef) -> u32 { - if exc.fast_isinstance(self.ctx.exceptions.system_exit) { - let code = exc - .as_object() - .get_attr("code", self) - .unwrap_or_else(|_| exc.as_object().to_owned()); - let msg = match_class!(match code { - ref i @ PyInt => { - use num_traits::cast::ToPrimitive; - // Try u32 first, then i32 (for negative values), else -1 for overflow - let code = i - .as_bigint() - .to_u32() - .or_else(|| i.as_bigint().to_i32().map(|v| v as u32)) - .unwrap_or(-1i32 as u32); - return code; - } - code => { - if self.is_none(&code) { - return 0; - } - code.str(self).ok() - } - }); - if let Some(msg) = msg { - // Write using Python's write() to use stderr's error handler (backslashreplace) - if let Ok(stderr) = stdlib::sys::get_stderr(self) { - let _ = self.call_method(&stderr, "write", (msg,)); - let _ = self.call_method(&stderr, "write", ("\n",)); - } - } - 1 - } else if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { - self.print_exception(&exc); - cfg_select! { - unix => { - if crate::host_env::signal::set_sigint_default_onstack().is_ok() { - self.flush_std(); - crate::host_env::signal::send_sigint_to_self() - .expect("Expect to be killed."); - } + if let Some(exit_code) = self.handle_system_exit_and_keyboard_interrupt(&exc) { + return exit_code; + } + self.print_uncaught_exception(&exc); + 1 + } - (libc::SIGINT as u32) + 128 + /// _Py_HandleSystemExitAndKeyboardInterrupt + /// + /// Return the exit code of a `SystemExit` that should end the process. + /// A `KeyboardInterrupt` is remembered so that the process exits through SIGINT. + pub fn handle_system_exit_and_keyboard_interrupt( + &self, + exc: &Py, + ) -> Option { + if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { + crate::signal::set_unhandled_keyboard_interrupt(); + return None; + } + // Don't exit if -i flag was given. This flag is set to 0 + // when entering interactive mode for inspecting. + if self.config_inspect() || !exc.fast_isinstance(self.ctx.exceptions.system_exit) { + return None; + } + let code = exc + .as_object() + .get_attr("code", self) + .unwrap_or_else(|_| exc.as_object().to_owned()); + let msg = match_class!(match code { + ref i @ PyInt => { + use num_traits::cast::ToPrimitive; + // Try u32 first, then i32 (for negative values), else -1 for overflow + let code = i + .as_bigint() + .to_u32() + .or_else(|| i.as_bigint().to_i32().map(|v| v as u32)) + .unwrap_or(-1i32 as u32); + return Some(code); + } + code => { + if self.is_none(&code) { + return Some(0); } - // STATUS_CONTROL_C_EXIT - same as CPython - windows => 0xC000013A, - _ => 1, + code.str(self).ok() + } + }); + if let Some(msg) = msg { + // Write using Python's write() to use stderr's error handler (backslashreplace) + if let Ok(stderr) = stdlib::sys::get_stderr(self) { + let _ = self.call_method(&stderr, "write", (msg,)); + let _ = self.call_method(&stderr, "write", ("\n",)); } - } else { - self.print_exception(&exc); - 1 } + Some(1) } #[doc(hidden)] diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index b8b83a240d1..a9fcb56ca63 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -1,11 +1,14 @@ //! Python code execution functions. use crate::{ - Py, PyObjectRef, PyResult, VirtualMachine, - builtins::PyCode, + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyBaseException, PyBaseExceptionRef, PyCode, PyModule, PyStr}, + bytecode::CodeFlags, compiler::{self}, scope::Scope, + stdlib, }; +use core::sync::atomic::Ordering; impl VirtualMachine { /// PyRun_SimpleString @@ -58,8 +61,8 @@ impl VirtualMachine { #[cfg(feature = "host_env")] mod file_run { use crate::{ - AsObject, Py, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyCode, PyDict}, + Py, PyResult, VirtualMachine, + builtins::{PyCode, PyDict}, compiler::{self}, scope::Scope, }; @@ -67,8 +70,8 @@ mod file_run { impl VirtualMachine { /// _PyRun_AnyFileObject (internal) /// - /// Execute a Python file. Currently always delegates to run_simple_file - /// (interactive mode is handled separately in shell.rs). + /// Execute a Python file. A named file is not interactive, so this + /// always delegates to run_simple_file; see `run_any_stdin` for stdin. /// /// Note: This is an internal function. Use `run_file` for the public interface. #[doc(hidden)] @@ -111,7 +114,12 @@ mod file_run { match crate::host_env::fs::read(path) { Ok(source_bytes) => { if let Some(null_at) = source_bytes.iter().position(|&b| b == 0) { - return Err(null_byte_syntax_error(self, path, &source_bytes, null_at)); + return Err(super::null_byte_syntax_error( + self, + path, + &source_bytes, + null_at, + )); } #[cfg(feature = "parser")] // Match compile() by honoring BOMs and encoding cookies in files. @@ -138,31 +146,6 @@ mod file_run { } } - fn null_byte_syntax_error( - vm: &VirtualMachine, - path: &str, - source_bytes: &[u8], - null_at: usize, - ) -> PyBaseExceptionRef { - let before = &source_bytes[..null_at]; - let lineno = before.iter().filter(|&&b| b == b'\n').count() + 1; - let line_start = before - .iter() - .rposition(|&b| b == b'\n') - .map_or(0, |i| i + 1); - let line = String::from_utf8_lossy(&source_bytes[line_start..null_at]); - let syntax_error = vm.new_exception_msg( - vm.ctx.exceptions.syntax_error.to_owned(), - "source code cannot contain null bytes".into(), - ); - let obj = syntax_error.as_object(); - obj.set_attr("filename", vm.ctx.new_str(path), vm).unwrap(); - obj.set_attr("lineno", vm.ctx.new_int(lineno), vm).unwrap(); - obj.set_attr("text", vm.ctx.new_str(format!("{line}\n")), vm) - .unwrap(); - syntax_error - } - fn set_main_loader( module_dict: &Py, filename: &str, @@ -216,6 +199,398 @@ mod file_run { } } +/// One line of interactive input: raw bytes read from stdin, or text from a +/// line editor. +pub enum InteractiveLine { + Bytes(Vec), + Text(String), +} + +/// Source of one interactive statement, ready to be compiled. +struct InteractiveSource { + source: String, + parsed: compiler::InteractiveInput, +} + +impl VirtualMachine { + /// _Py_FdIsInteractive(stdin, "") + pub fn stdin_is_interactive(&self) -> bool { + use std::io::IsTerminal; + std::io::stdin().is_terminal() || self.state.config.settings.interactive + } + + /// _PyRun_AnyFile(stdin, "") + /// + /// Run stdin as an interactive session or as a script. + pub fn run_any_stdin(&self) -> PyResult<()> { + if self.stdin_is_interactive() { + self.run_interactive_loop(|prompt| self.stdio_readline(prompt)) + } else { + self.run_simple_stdin() + } + } + + /// _PyRun_InteractiveLoop + /// + /// `readline` plays the role of `PyOS_Readline`: it returns `Ok(None)` at + /// the end of input. + pub fn run_interactive_loop( + &self, + mut readline: impl FnMut(&str) -> PyResult>, + ) -> PyResult<()> { + for (name, default) in [("ps1", ">>> "), ("ps2", "... ")] { + if self.sys_module.dict().get_item_opt(name, self)?.is_none() { + let _ = self + .sys_module + .set_attr(name, self.ctx.new_str(default), self); + } + } + let mut flags = CodeFlags::empty(); + let mut nomem_count = 0; + loop { + match self.run_interactive_one(&mut readline, &mut flags) { + Ok(true) => nomem_count = 0, + // E_EOF + Ok(false) => return Ok(()), + Err(exc) => { + // Prevent an endless loop after multiple consecutive + // MemoryErrors while still allowing an interactive + // command to fail with a MemoryError. + if exc.fast_isinstance(self.ctx.exceptions.memory_error) { + nomem_count += 1; + if nomem_count > 16 { + return Err(exc); + } + } else { + nomem_count = 0; + } + if !self.config_inspect() + && exc.fast_isinstance(self.ctx.exceptions.system_exit) + { + return Err(exc); + } + self.print_uncaught_exception(&exc); + } + } + } + } + + /// _PyRun_InteractiveOne + /// + /// Read, compile and run one interactive statement in `__main__`. + /// Return `Ok(false)` at the end of input. + fn run_interactive_one( + &self, + readline: &mut impl FnMut(&str) -> PyResult>, + flags: &mut CodeFlags, + ) -> PyResult { + let Some(InteractiveSource { source, parsed }) = + self.interactive_parse(readline, *flags)? + else { + return Ok(false); + }; + let main_module = self.add_main_module()?; + // _PyAST_Compile merges the future features of the parsed statement + // into the flags before code generation. + *flags |= parsed.future_features; + self.run_interactive_mod(&source, parsed.empty, *flags, &main_module) + .inspect_err(|exc| self.fix_syntax_error_text(exc, &source))?; + self.flush_io(); + Ok(true) + } + + /// pyrun_one_parse_ast + /// + /// Read lines with `sys.ps1` and `sys.ps2` prompts until they form one + /// complete interactive statement. Return `None` at the end of input. + fn interactive_parse( + &self, + readline: &mut impl FnMut(&str) -> PyResult>, + flags: CodeFlags, + ) -> PyResult> { + let encoding = self + .sys_module + .get_attr("stdin", self) + .and_then(|stdin| stdin.get_attr("encoding", self)) + .ok() + .and_then(|encoding| encoding.downcast::().ok()) + .and_then(|encoding| encoding.to_str().map(str::to_owned)) + .unwrap_or_else(|| "utf-8".to_owned()); + let prompt = |name| { + self.sys_module + .get_attr(name, self) + .and_then(|prompt| prompt.str(self)) + .ok() + .and_then(|prompt| prompt.to_str().map(str::to_owned)) + .unwrap_or_default() + }; + let ps1 = prompt("ps1"); + let ps2 = prompt("ps2"); + + // _PyParser_InteractiveASTFromFile + self.audit("compile", || (self.ctx.none(), self.ctx.new_str("")))?; + let mut source = String::new(); + let mut raw_source = Vec::new(); + #[cfg(feature = "parser")] + let mut warnings = super::compile::InteractiveWarnings::default(); + loop { + let line = readline(if source.is_empty() { &ps1 } else { &ps2 }); + // tok_underflow_interactive ends the prompt line at EOF or on an + // interrupted read. + if !matches!(line, Ok(Some(_))) { + writeln!(stdlib::sys::PyStderr(self)); + } + let (terminated, unterminated_fragment, eof) = match line? { + None if source.is_empty() => return Ok(None), + None => { + if !source.ends_with('\n') { + source.push('\n'); + } + (true, false, true) + } + Some(InteractiveLine::Text(mut line)) => { + if !line.ends_with('\n') { + line.push('\n'); + } + let line = line.replace("\r\n", "\n"); + let terminated = line == "\n"; + source.push_str(&line); + (terminated, false, false) + } + Some(InteractiveLine::Bytes(line)) => { + let unterminated_fragment = !line.ends_with(b"\n"); + let terminated = line == b"\n" || line == b"\r\n"; + raw_source.extend_from_slice(&line); + source = self + .state + .codec_registry + .decode_text( + self.ctx.new_bytes(raw_source.clone()).into(), + &encoding, + None, + self, + )? + .to_str() + .ok_or_else(|| { + self.new_exception_msg( + self.ctx.exceptions.unicode_error.to_owned(), + "stdin source contains surrogates".into(), + ) + })? + .replace("\r\n", "\n"); + (terminated, unterminated_fragment, false) + } + }; + #[cfg(feature = "parser")] + self.emit_interactive_parser_warnings( + &source, + "", + Some(identifier!(self, __main__)), + flags.contains(CodeFlags::FUTURE_BARRY_AS_BDFL), + &mut warnings, + ) + .map_err(|error| { + super::compile::VmCompileError::Warning(error).into_pyexception(self, Some(&source)) + })?; + let parsed = match compiler::interactive_input(&source, "", flags, terminated) { + Ok(Some(parsed)) => parsed, + Ok(None) => continue, + Err(error) => { + let unclosed = matches!(&error, compiler::CompileError::Parse(error) + if error.is_unclosed_bracket || error.is_unclosed_string); + let error = super::compile::VmCompileError::Compile(error) + .into_pyexception_maybe_incomplete(self, Some(&source), !terminated); + if !eof + && (unclosed + || (!terminated + && error + .fast_isinstance(self.ctx.exceptions.incomplete_input_error))) + { + continue; + } + return Err(error); + } + }; + // A final fragment without a newline is not a complete line yet. + // Its warnings and definitive syntax errors are reported now, but + // it is only accepted once the next read reports EOF. + if unterminated_fragment { + continue; + } + return Ok(Some(InteractiveSource { source, parsed })); + } + } + + /// run_mod for interactive input + fn run_interactive_mod( + &self, + source: &str, + empty: bool, + flags: CodeFlags, + main_module: &Py, + ) -> PyResult<()> { + let filename = get_interactive_filename( + "", + self.state + .interactive_src_count + .fetch_add(1, Ordering::Relaxed), + ); + let mode = if empty { + compiler::Mode::Exec + } else { + compiler::Mode::Single + }; + let opts = compiler::CompileOpts { + future_features: flags, + ..self.compile_opts() + }; + let code = self + .compile_after_parser_warnings( + source, + mode, + filename, + opts, + Some(identifier!(self, __main__)), + ) + .map_err(|error| error.into_pyexception(self, Some(source)))?; + let linecache = self.import("linecache", 0)?; + linecache.get_attr("_register_code", self)?.call( + ( + code.clone(), + self.ctx.new_str(source), + self.ctx.new_str(""), + ), + self, + )?; + self.audit("exec", || (code.clone(),))?; + let globals = main_module.dict(); + self.run_code_obj(code, Scope::with_builtins(None, globals, self))?; + Ok(()) + } + + /// Fix the `text` attribute of a SyntaxError raised by interactive input. + fn fix_syntax_error_text(&self, exc: &Py, source: &str) { + if !exc.fast_isinstance(self.ctx.exceptions.syntax_error) { + return; + } + let source = self.ctx.new_str(source); + let lines = source.splitlines(crate::anystr::SplitLinesArgs { keepends: true }, self); + let Ok(lineno) = exc + .as_object() + .get_attr("lineno", self) + .and_then(|lineno| lineno.try_into_value::(self)) + else { + return; + }; + if lineno > 0 + && let Some(line) = lines.get((lineno - 1) as usize) + { + let _ = exc.as_object().set_attr("text", line.clone(), self); + } + } + + /// PyOS_StdioReadline + /// + /// Write the prompt to stderr and read one line from stdin. + fn stdio_readline(&self, prompt: &str) -> PyResult> { + let mut line = Vec::new(); + let mut prompt = prompt; + loop { + let result = crate::readline::stdio_readline_bytes(prompt, &mut line); + prompt = ""; + match result { + Ok(0) if line.is_empty() => return Ok(None), + Ok(_) => return Ok(Some(InteractiveLine::Bytes(line))), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { + self.check_signals()?; + } + // A read error ends the input like EOF. + Err(_) if line.is_empty() => return Ok(None), + Err(_) => return Ok(Some(InteractiveLine::Bytes(line))), + } + } + } + + /// _PyRun_SimpleFile(stdin, "") + fn run_simple_stdin(&self) -> PyResult<()> { + use std::io::Read; + self.with_simple_run("", |module_dict| { + // _PyParser_ASTFromFile + self.audit("compile", || (self.ctx.none(), self.ctx.new_str("")))?; + let mut bytes = Vec::new(); + let mut buf = [0; 8192]; + loop { + match std::io::stdin().read(&mut buf) { + Ok(0) => break, + Ok(n) => bytes.extend_from_slice(&buf[..n]), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { + self.check_signals()?; + } + // A read error ends the input like EOF. + Err(_) => break, + } + } + if let Some(null_at) = bytes.iter().position(|&b| b == 0) { + return Err(null_byte_syntax_error(self, "", &bytes, null_at)); + } + #[cfg(feature = "parser")] + let source = self.decode_source_bytes(&bytes, "", false)?; + #[cfg(not(feature = "parser"))] + let source = + String::from_utf8(bytes).map_err(|error| self.new_os_error(error.to_string()))?; + let code = self + .compile_with_opts_and_module( + &source, + compiler::Mode::Exec, + "", + self.compile_opts(), + Some(identifier!(self, __main__)), + &[], + ) + .map_err(|error| error.into_pyexception(self, Some(&source)))?; + self.audit("exec", || (code.clone(),))?; + let scope = Scope::with_builtins(None, module_dict.to_owned(), self); + self.run_code_obj(code, scope).map(drop) + }) + } +} + +fn null_byte_syntax_error( + vm: &VirtualMachine, + path: &str, + source_bytes: &[u8], + null_at: usize, +) -> PyBaseExceptionRef { + let before = &source_bytes[..null_at]; + let lineno = before.iter().filter(|&&b| b == b'\n').count() + 1; + let line_start = before + .iter() + .rposition(|&b| b == b'\n') + .map_or(0, |i| i + 1); + let line = String::from_utf8_lossy(&source_bytes[line_start..null_at]); + let syntax_error = vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + "source code cannot contain null bytes".into(), + ); + let obj = syntax_error.as_object(); + obj.set_attr("filename", vm.ctx.new_str(path), vm).unwrap(); + obj.set_attr("lineno", vm.ctx.new_int(lineno), vm).unwrap(); + obj.set_attr("text", vm.ctx.new_str(format!("{line}\n")), vm) + .unwrap(); + syntax_error +} + +/// get_interactive_filename +fn get_interactive_filename(filename: &str, count: usize) -> String { + match filename + .strip_prefix('<') + .and_then(|name| name.strip_suffix('>')) + { + Some(middle) => format!("<{middle}-{count}>"), + None => format!("{filename}-{count}"), + } +} + #[cfg(test)] mod tests { use crate::object::AsObject; diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index df07adf0fd1..9f3ef85f7d4 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -5,6 +5,8 @@ use std::ffi::OsString; /// Path configuration computed at runtime (like PyConfig path outputs) #[derive(Debug, Clone, Default)] pub struct Paths { + /// program_name: the name the interpreter was invoked as + pub program_name: String, /// sys.executable pub executable: String, /// sys._base_executable (original interpreter in venv) diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index d611cae2acc..217d63ab267 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -114,6 +114,19 @@ impl VirtualMachine { Ok(module) } + /// PyImport_AddModuleRef("__main__") + pub fn add_main_module(&self) -> PyResult> { + let modules = self.sys_module.get_attr(identifier!(self, modules), self)?; + if let Ok(module) = modules.get_item(identifier!(self, __main__), self) + && let Ok(module) = module.downcast::() + { + return Ok(module); + } + let module = self.new_module("__main__", self.ctx.new_dict(), None); + modules.set_item(identifier!(self, __main__), module.clone().into(), self)?; + Ok(module) + } + /// Set `__main__.__loader__` to BuiltinImporter when it is missing or None. pub fn set_main_builtin_importer( &self, @@ -547,6 +560,19 @@ impl VirtualMachine { self.ctx.exceptions.indentation_error } } + // _is_end_of_source: the parser failed at the end of the input, + // where more input can still follow. + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error, + raw_location, + .. + }) if !matches!(error, ParseErrorType::Lexical(_)) + && source + .is_some_and(|source| raw_location.start().to_usize() >= source.len()) => + { + incomplete_or_syntax(allow_incomplete) + } #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::OtherError(s), @@ -588,7 +614,16 @@ impl VirtualMachine { // Tokenizer/parser errors attach the current source line. Compiler // errors load that line with ProgramText, which is None when the // filename cannot be opened (compile of a string). - let mut statement = if matches!(error, crate::compiler::CompileError::Codegen(_)) { + let eof_token = match error { + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + eof_token: Some(eof_token), + .. + }) => Some(eof_token), + _ => None, + }; + let mut statement = if let Some(eof_token) = eof_token { + Some(eof_token.line.clone()) + } else if matches!(error, crate::compiler::CompileError::Codegen(_)) { self.program_text(error.source_path(), error.python_location().0) } else { source.and_then(|src| get_statement(src, error.location())) @@ -701,7 +736,9 @@ impl VirtualMachine { .as_deref() .and_then(|line| line.chars().next()) .is_some_and(|ch| ch.is_ascii_whitespace())); - let (end_lineno, end_offset) = if no_end_offset { + let (end_lineno, end_offset) = if let Some(eof_token) = eof_token { + (eof_token.lineno, eof_token.end_offset()) + } else if no_end_offset { (end_lineno, -1) } else if let Some(end_offset) = tokenizer_end_offset { (end_lineno, end_offset) diff --git a/crates/vm/src/vm/vm_object.rs b/crates/vm/src/vm/vm_object.rs index baa0c8095c6..db9459b9b75 100644 --- a/crates/vm/src/vm/vm_object.rs +++ b/crates/vm/src/vm/vm_object.rs @@ -9,6 +9,23 @@ use crate::{ /// PyObject support impl VirtualMachine { + /// PyObject_Dump + /// + /// Write a description of `obj` to the process stderr, bypassing + /// `sys.stderr`. + pub fn dump_object(&self, obj: &PyObject) { + let class = obj.class(); + eprintln!("object address : {obj:p}"); + eprintln!("object refcount : {}", obj.strong_count()); + eprintln!("object type : {class:p}"); + eprintln!("object type name: {}", class.slot_name()); + eprint!("object repr : "); + if let Ok(repr) = obj.repr(self) { + eprint!("{}", repr.as_wtf8()); + } + eprintln!(); + } + #[track_caller] #[cold] fn _py_panic_failed(&self, exc: PyBaseExceptionRef, msg: &str) -> ! { diff --git a/extra_tests/snippets/stdlib_signal.py b/extra_tests/snippets/stdlib_signal.py index c03bc70ab30..ba333f90032 100644 --- a/extra_tests/snippets/stdlib_signal.py +++ b/extra_tests/snippets/stdlib_signal.py @@ -218,3 +218,131 @@ def raise_wakeup_signal(): check_windows_wakeup_reporting() + + +def check_native_stdin_eintr(): + if sys.platform in ("win32", "wasi", "emscripten"): + return + if not sys.executable or not hasattr(sys, "_baserepl"): + return + + import os + import select + import subprocess + + if os.name != "posix": + return + + prompt = b"__NATIVE_STDIN_PROMPT__" + handled = b"__NATIVE_STDIN_HANDLED__" + executed = b"__NATIVE_STDIN_EXECUTED__" + + # This explicitly exercises the native adapter rather than an editor REPL. + # Lib/test/test_repl.py covers ordinary input, not a blocked native read + # interrupted while its pipe remains open and no complete line is supplied. + setup = ( + "import signal, sys\n" + f"sys.ps1 = {prompt.decode()!r}\n" + "sys.ps2 = '__NATIVE_STDIN_CONTINUATION__'\n" + ) + setup += ( + "def handler(signum, frame):\n" + " print('__NATIVE_' + 'STDIN_HANDLED__', flush=True)\n" + "signal.signal(signal.SIGINT, handler)\n" + ) + setup += "sys._baserepl()\n" + process = subprocess.Popen( + [sys.executable, "-i", "-S", "-u", "-c", setup], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + ) + output = bytearray() + output_eof = False + deadline = time.monotonic() + 10.0 + + def remaining(): + timeout = deadline - time.monotonic() + assert timeout > 0, ("native stdin child timed out", output) + return timeout + + def read_output(timeout): + nonlocal output_eof + ready, _, _ = select.select([process.stdout], [], [], min(timeout, remaining())) + if ready: + try: + chunk = os.read(process.stdout.fileno(), 4096) + except BlockingIOError: + return + if chunk: + output.extend(chunk) + assert len(output) <= 65536, ("child output limit", output) + else: + output_eof = True + + def expect(marker, start=0, interrupt=False): + observed = len(output) + retry_at = time.monotonic() + attempts = 0 + while marker not in output[start:]: + assert not output_eof, ("child exited before marker", marker, output) + timeout = remaining() + if interrupt and len(output) == observed and attempts < 10: + # Drain bytes already available before deciding to retry. + read_output(0) + if len(output) != observed or output_eof: + continue + now = time.monotonic() + if now >= retry_at: + # Prompt emission precedes read(). Retry only this child + # to cover that race; never send input to wake the read. + assert process.poll() is None, ("child exited", output) + process.send_signal(signal.SIGINT) + attempts += 1 + retry_at = now + 0.5 + timeout = min(timeout, max(0, retry_at - time.monotonic())) + # Any response stops retries, including a partial traceback or + # marker, so another SIGINT does not interrupt its reporting. + read_output(timeout) + + def write_input(data): + offset = 0 + while offset < len(data): + _, ready, _ = select.select([], [process.stdin], [], remaining()) + if ready: + try: + written = os.write(process.stdin.fileno(), data[offset:]) + except BlockingIOError: + continue + assert written > 0 + offset += written + + try: + os.set_blocking(process.stdin.fileno(), False) + os.set_blocking(process.stdout.fileno(), False) + expect(prompt) + after_prompt = len(output) + source = b"print('__NATIVE_' + 'STDIN_EXECUTED__', flush=True)" + # Keep the pipe empty until the native reader dispatches the signal. + # Only then provide a complete line and EOF; this does not assert + # CPython's behavior for a partial libc fgets interrupted by a signal. + expect(handled, after_prompt, interrupt=True) + write_input(source + b"\n") + process.stdin.close() + expect(executed, after_prompt) + while not output_eof: + read_output(remaining()) + returncode = process.wait(timeout=remaining()) + assert returncode == 0, (returncode, output) + assert output.count(executed) == 1, output + assert b"KeyboardInterrupt" not in output, output + finally: + process.stdin.close() + process.stdout.close() + if process.poll() is None: + process.kill() + process.wait(timeout=5.0) + + +check_native_stdin_eintr() diff --git a/scripts/update_lib/cmd_migrate.py b/scripts/update_lib/cmd_migrate.py index c509dcfb769..f2db1bf0cf3 100644 --- a/scripts/update_lib/cmd_migrate.py +++ b/scripts/update_lib/cmd_migrate.py @@ -856,7 +856,12 @@ def patch_single_content( if patches_document is not None: from update_lib.patch_spec import select_remaining_patches - patches = select_remaining_patches(patches, patches_document, relative_path) + patches = select_remaining_patches( + patches, + patches_document, + relative_path, + old_content=old_content if lib_path.exists() else None, + ) # Apply patches to source content src_content = src_path.read_text(encoding="utf-8") diff --git a/scripts/update_lib/patch_spec.py b/scripts/update_lib/patch_spec.py index 4dee842a8f7..35828e685d8 100644 --- a/scripts/update_lib/patch_spec.py +++ b/scripts/update_lib/patch_spec.py @@ -551,9 +551,65 @@ def temp_attr(obj: object, attr: str, value: object): setattr(obj, attr, old) -def select_remaining_patches(current, document, relative_path: Path): +def _conditional_retirements(document, old_content): + entries = document.get("retire_conditional", []) + if not isinstance(entries, list): + raise ValueError("Conditional retirements must be a list") + if not entries: + return {} + if not isinstance(old_content, str): + raise ValueError("Conditional retirements require original source") + tree = ast.parse(old_content) + lines = old_content.splitlines(keepends=True) + result = {} + for entry in entries: + if not isinstance(entry, dict) or set(entry) != { + "class", + "method", + "decorator", + }: + raise ValueError("Invalid conditional retirement metadata") + if not all(isinstance(value, str) and value for value in entry.values()): + raise ValueError("Conditional retirement fields must be nonempty strings") + key = (entry["class"], entry["method"]) + if key in result: + raise ValueError("Duplicate conditional retirement") + classes = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) and node.name == key[0] + ] + if len(classes) != 1: + raise ValueError("Missing or ambiguous conditional retirement class") + methods = [ + node for node in _iter_class_methods(classes[0]) if node.name == key[1] + ] + if len(methods) != 1: + raise ValueError("Missing or ambiguous conditional retirement method") + matches = [ + node + for node in methods[0].decorator_list + if "".join(lines[node.lineno - 1 : node.end_lineno]) == entry["decorator"] + ] + if len(matches) != 1: + raise ValueError( + "Original conditional decorator does not match exactly once" + ) + marker = PatchSpec.try_from_ast_node(matches[0], old_content.splitlines()) + if marker is None or marker.ut_method != UtMethod.ExpectedFailureIf: + raise ValueError( + "Only explicit expectedFailureIf retirements are supported" + ) + result[key] = marker + return result + + +def select_remaining_patches( + current, document, relative_path: Path, *, old_content=None +): if not isinstance(document, dict) or ( - set(document) not in ({"target", "patches"}, {"target", "patches", "doctests"}) + not {"target", "patches"} <= set(document) + or not set(document) <= {"target", "patches", "doctests", "retire_conditional"} ): raise ValueError("Expected target and patches metadata") if ( @@ -568,6 +624,8 @@ def select_remaining_patches(current, document, relative_path: Path): remaining = patches_from_json(document["patches"]) except (TypeError, KeyError, AttributeError, ValueError) as error: raise ValueError("Invalid remaining marker metadata") from error + retirements = _conditional_retirements(document, old_content) + consumed = set() for cls, methods in remaining.items(): if cls not in current or any(method not in current[cls] for method in methods): raise ValueError("Marker override cannot add targets") @@ -584,10 +642,21 @@ def select_remaining_patches(current, document, relative_path: Path): omitted.extend(pending[:index]) pending = pending[index + 1 :] omitted.extend(pending) - if any(marker.ut_method != UtMethod.ExpectedFailure for marker in omitted): - raise ValueError( - "Only unconditional expectedFailure markers may be removed" - ) + for marker in omitted: + if marker.ut_method == UtMethod.ExpectedFailure: + continue + key = (cls, method) + if ( + key in consumed + or retirements.get(key) != marker + or existing.count(marker) != 1 + ): + raise ValueError( + "Conditional removal requires an exact explicit retirement" + ) + consumed.add(key) + if consumed != set(retirements): + raise ValueError("Conditional retirement was not removed exactly once") return remaining diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py index 1baf66538cf..1a138742012 100644 --- a/scripts/update_lib/tests/test_migrate.py +++ b/scripts/update_lib/tests/test_migrate.py @@ -1246,6 +1246,114 @@ def test_added_changed_or_reordered_markers_fail(self): select_remaining_patches(current, document, Path(document["target"])) +class ConditionalMarkerOverrideTests(unittest.TestCase): + decorator = ( + ' @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON")\n' + ) + + def fixture(self): + old = OVERRIDE_CLEAN.replace( + " def test_known", self.decorator + " def test_known" + ) + current = extract_patches(old) + document = { + "target": "Lib/test/test_example.py", + "patches": {}, + "retire_conditional": [ + { + "class": "Example", + "method": "test_known", + "decorator": self.decorator, + } + ], + } + return old, current, document + + def select(self, old, current, document): + return select_remaining_patches( + current, + document, + Path(document["target"]), + old_content=old, + ) + + def test_explicit_conditional_retirement(self): + old, current, document = self.fixture() + self.assertEqual(self.select(old, current, document), {}) + + def test_omitted_conditional_without_authorization_fails(self): + old, current, document = self.fixture() + del document["retire_conditional"] + with self.assertRaises(ValueError): + self.select(old, current, document) + + def test_absent_duplicate_modified_and_retained_fail_closed(self): + for change in ( + "absent", + "duplicate", + "text", + "condition", + "retained", + "duplicate_source", + ): + old, current, document = self.fixture() + with self.subTest(change=change), self.assertRaises(ValueError): + if change == "absent": + document["retire_conditional"][0]["method"] = "missing" + elif change == "duplicate": + document["retire_conditional"] *= 2 + elif change == "text": + document["retire_conditional"][0]["decorator"] = ( + self.decorator.replace('"', "'") + ) + elif change == "condition": + old = old.replace("not support.is_android", "support.is_android") + current = extract_patches(old) + elif change == "retained": + document["patches"] = patches_to_json(current) + elif change == "duplicate_source": + old = old.replace(self.decorator, self.decorator * 2) + current = extract_patches(old) + self.select(old, current, document) + + def test_other_guards_and_original_source_required(self): + for replacement in ("skipIf", "expectedFailureIfWindows"): + old, current, document = self.fixture() + old = old.replace("expectedFailureIf", replacement) + document["retire_conditional"][0]["decorator"] = self.decorator.replace( + "expectedFailureIf", replacement + ) + with self.subTest(replacement=replacement), self.assertRaises(ValueError): + self.select(old, extract_patches(old), document) + old, current, document = self.fixture() + with self.assertRaises(ValueError): + self.select(None, current, document) + + def test_cli_conditional_retirement_preserves_remaining_markers(self): + old = OVERRIDE_OLD.replace( + " @unittest.expectedFailure # TODO: RUSTPYTHON; old failure\n", + self.decorator, + ) + + def authorize(document): + document["retire_conditional"] = [ + { + "class": "Example", + "method": "test_known", + "decorator": self.decorator, + } + ] + + with patch(__name__ + ".OVERRIDE_OLD", old): + status, result = MarkerOverrideTests().invoke(authorize) + self.assertEqual(status, 0) + self.assertNotIn("expectedFailureIf", result) + self.assertIn('unittest.skip("TODO: RUSTPYTHON; retained guard")', result) + status, result = MarkerOverrideTests().invoke() + self.assertEqual(status, 1) + self.assertEqual(result, old) + + PACKAGE_MARKED = """import unittest class Example: @unittest.expectedFailure # TODO: RUSTPYTHON; retained reason diff --git a/src/lib.rs b/src/lib.rs index e1047a08f3d..2f6ee077f53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,7 +59,9 @@ mod shell; #[cfg(feature = "rustpython-pylib")] pub use rustpython_pylib as pylib; -use rustpython_vm::{AsObject, PyObjectRef, PyResult, VirtualMachine, scope::Scope}; +use rustpython_vm::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, scope::Scope, +}; use std::env; use std::io::IsTerminal; use std::process::ExitCode; @@ -121,16 +123,8 @@ pub fn run(mut builder: InterpreterBuilder) -> ExitCode { let interp = builder.interpreter(); let exitcode = cfg_select! { - feature = "capi" => {{ - let local_vm = interp.enter(|vm| vm.new_thread()); - rustpython_capi::init_main_interpreter(interp); - let result = local_vm.run(|vm| run_rustpython(vm, run_mode)); - rustpython_capi::get_main_interpreter() - .take() - .unwrap() - .finalize(result.err()) - }}, - _ => interp.run(move |vm| run_rustpython(vm, run_mode)), + feature = "capi" => rustpython_capi::run_main_interpreter(interp, move |vm| run_rustpython(vm, run_mode)), + _ => interp.run_main(move |vm| run_rustpython(vm, run_mode)), }; rustpython_vm::host_env::os::exit_code(exitcode) @@ -221,7 +215,7 @@ fn script_sys_path0(argv0: &str) -> String { } // pymain_run_file_obj in Modules/main.c -fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { +fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult { // config_run_filename_abspath: __file__ and co_filename are absolute even // when the script was named relative to the working directory. let path = &abs_path(argv0); @@ -232,7 +226,7 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { let runpy = vm.import("runpy", 0)?; let run_module_as_main = runpy.get_attr("_run_module_as_main", vm)?; run_module_as_main.call((vm::identifier!(vm, __main__).to_owned(), false), vm)?; - return Ok(()); + return Ok(0); } // Add script directory to sys.path[0] @@ -242,7 +236,7 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { cfg_select! { feature = "host_env" => match rustpython_vm::host_env::fs::metadata(path) { - Ok(_) => vm.run_any_file(scope, path), + Ok(_) => vm.run_any_file(scope, path).map(|()| 0), Err(err) => cant_open_file(vm, path, &err), }, _ => { @@ -250,19 +244,23 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { // The VM itself has no filesystem access. let path = if path.is_empty() { "???" } else { path }; match std::fs::read_to_string(path) { - Ok(source) => vm.run_string(scope, &source, path).map(drop), + Ok(source) => vm.run_string(scope, &source, path).map(|_| 0), Err(err) => cant_open_file(vm, path, &err), } } } } -fn cant_open_file(vm: &VirtualMachine, path: &str, err: &std::io::Error) -> PyResult<()> { - let program = &vm.state.config.paths.executable; +fn cant_open_file(vm: &VirtualMachine, path: &str, err: &std::io::Error) -> PyResult { + let program_name = &vm.state.config.paths.program_name; let filename_repr = vm.ctx.new_str(path).as_object().repr(vm)?; let errno = err.raw_os_error().unwrap_or(2); - eprintln!("{program}: can't open file {filename_repr}: [Errno {errno}] {err}"); - Err(vm.new_system_exit(vec![vm.ctx.new_int(2).into()].into())) + let strerror = vm::host_env::errno::strerror_string(errno).unwrap_or_default(); + writeln!( + vm::stdlib::sys::PyStderr(vm), + "{program_name}: can't open file {filename_repr}: [Errno {errno}] {strerror}" + ); + Ok(2) } fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult> { @@ -296,11 +294,26 @@ fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult }) } -// pymain_run_python -fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { +fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> u32 { #[cfg(feature = "flame-it")] let main_guard = flame::start_guard("RustPython main"); + let exitcode = + pymain_run_python(vm, run_mode).unwrap_or_else(|exc| vm.handle_exit_exception(exc)); + + #[cfg(feature = "flame-it")] + { + main_guard.end(); + if let Err(e) = write_profile(&vm.state.as_ref().config.settings) { + error!("Error writing profile information: {}", e); + } + } + + exitcode +} + +// pymain_run_python +fn pymain_run_python(vm: &VirtualMachine, run_mode: RunMode) -> PyResult { let scope = vm.new_scope_with_main()?; // Initialize warnings module to process sys.warnoptions @@ -345,8 +358,9 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { } let is_repl = matches!(run_mode, RunMode::Repl); + // pymain_header if !vm.state.config.settings.quiet - && (vm.state.config.settings.verbose > 0 || (is_repl && std::io::stdin().is_terminal())) + && (vm.state.config.settings.verbose > 0 || (is_repl && vm.stdin_is_interactive())) { eprintln!( "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", @@ -361,58 +375,220 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { eprintln!("Type \"help\", \"copyright\", \"credits\" or \"license\" for more information."); } - let res = match run_mode { + + let set_running_main = vm.state.set_running_main(); + let exitcode = + pymain_run_main(vm, scope, run_mode).unwrap_or_else(|exc| vm.handle_exit_exception(exc)); + if set_running_main { + vm.state.set_not_running_main(); + } + Ok(exitcode) +} + +/// The part of pymain_run_python that runs while the main thread owns +/// `__main__`. +fn pymain_run_main(vm: &VirtualMachine, scope: Scope, run_mode: RunMode) -> PyResult { + let is_repl = matches!(run_mode, RunMode::Repl); + vm.check_signals()?; + let result = match run_mode { RunMode::Command(command) => { debug!("Running command {command}"); - vm.run_string(scope.clone(), &command, "").map(drop) + vm.run_string(scope, &command, "").map(|_| 0) } RunMode::Module(module) => { debug!("Running module {module}"); - vm.run_module(&module) + vm.run_module(&module).map(|()| 0) } - RunMode::InstallPip(installer) => install_pip(installer, scope.clone(), vm), + RunMode::InstallPip(installer) => install_pip(installer, scope, vm).map(|()| 0), RunMode::Script(script_path) => { // pymain_run_file_obj debug!("Running script {script_path}"); - run_file(vm, scope.clone(), &script_path) + run_file(vm, scope, &script_path) } - RunMode::Repl => Ok(()), + RunMode::Repl => Ok(pymain_run_stdin(vm, scope)), }; - let result = if is_repl || vm.state.config.settings.inspect { - if std::io::stdin().is_terminal() { - warn_if_pyrepl_unavailable(vm); - } - shell::run_shell(vm, scope) - } else { - res + let mut exitcode = result.unwrap_or_else(|exc| vm.handle_exit_exception(exc)); + vm.check_signals()?; + + pymain_repl(vm, !is_repl, &mut exitcode); + Ok(exitcode) +} + +/// pymain_err_print +/// +/// Report `exc` and return the exit code if it ends the process. +fn pymain_err_print(vm: &VirtualMachine, exc: PyBaseExceptionRef) -> Option { + if let Some(exitcode) = vm.handle_system_exit_and_keyboard_interrupt(&exc) { + return Some(exitcode); + } + vm.print_uncaught_exception(&exc); + None +} + +// pymain_set_inspect +fn pymain_set_inspect(vm: &VirtualMachine, inspect: bool) { + if let Err(exc) = vm.config_set_inspect(inspect) { + eprintln!("Could not set the inspect flag"); + vm.print_uncaught_exception(&exc); + } +} + +// pymain_run_startup +fn pymain_run_startup(vm: &VirtualMachine, scope: Scope) -> Option { + let exc = run_startup(vm, scope).err()?; + pymain_err_print(vm, exc) +} + +fn run_startup(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { + if vm.state.config.settings.ignore_environment { + return Ok(()); + } + let Some(startup) = env::var_os("PYTHONSTARTUP").filter(|value| !value.is_empty()) else { + return Ok(()); }; + let startup = vm.fsdecode(startup); + vm.audit("cpython.run_startup", || (startup.clone(),))?; + let could_not_open = |err: std::io::Error| { + use vm::convert::ToPyException; + writeln!( + vm::stdlib::sys::PyStderr(vm), + "Could not open PYTHONSTARTUP" + ); + let exc = err.to_pyexception(vm); + exc.as_object().set_attr("filename", startup.clone(), vm)?; + Err(exc) + }; + let path = startup.expect_str(); + cfg_select! { + feature = "host_env" => match vm::host_env::fs::open(path) { + Ok(_) => vm.run_any_file(scope, path), + Err(err) => could_not_open(err), + }, + _ => match std::fs::read_to_string(path) { + Ok(source) => vm.run_string(scope, &source, path).map(drop), + Err(err) => could_not_open(err), + }, + } +} - #[cfg(feature = "flame-it")] +// pymain_run_interactive_hook +fn pymain_run_interactive_hook(vm: &VirtualMachine) -> Option { + let exc = run_interactive_hook(vm).err()?; + writeln!( + vm::stdlib::sys::PyStderr(vm), + "Failed calling sys.__interactivehook__" + ); + pymain_err_print(vm, exc) +} + +fn run_interactive_hook(vm: &VirtualMachine) -> PyResult<()> { + let Some(hook) = vm + .sys_module + .dict() + .get_item_opt("__interactivehook__", vm)? + else { + return Ok(()); + }; + vm.audit("cpython.run_interactivehook", || (hook.clone(),))?; + hook.call((), vm)?; + Ok(()) +} + +// pymain_run_stdin +fn pymain_run_stdin(vm: &VirtualMachine, scope: Scope) -> u32 { + if vm.stdin_is_interactive() { + // do exit on SystemExit + pymain_set_inspect(vm, false); + + if let Some(exitcode) = pymain_run_startup(vm, scope) { + return exitcode; + } + if let Some(exitcode) = pymain_run_interactive_hook(vm) { + return exitcode; + } + } + pymain_run_repl(vm, false) +} + +// pymain_repl +fn pymain_repl(vm: &VirtualMachine, run_code: bool, exitcode: &mut u32) { + // Check this environment variable at the end, to give programs the + // opportunity to set it from Python. + if !vm.config_inspect() + && !vm.state.config.settings.ignore_environment + && env::var_os("PYTHONINSPECT").is_some_and(|value| !value.is_empty()) { - main_guard.end(); - if let Err(e) = write_profile(&vm.state.as_ref().config.settings) { - error!("Error writing profile information: {}", e); + pymain_set_inspect(vm, true); + } + + if !(vm.config_inspect() && vm.stdin_is_interactive() && run_code) { + return; + } + + pymain_set_inspect(vm, false); + if let Some(code) = pymain_run_interactive_hook(vm) { + *exitcode = code; + return; + } + + *exitcode = pymain_run_repl(vm, true); +} + +// _pymain_run_repl +fn pymain_run_repl(vm: &VirtualMachine, startup: bool) -> u32 { + run_repl(vm, startup).unwrap_or_else(|exc| vm.handle_exit_exception(exc)) +} + +fn run_repl(vm: &VirtualMachine, startup: bool) -> PyResult { + vm.check_signals()?; + vm.audit("cpython.run_stdin", || ())?; + + let tty = std::io::stdin().is_terminal(); + if tty + && (vm.state.config.settings.ignore_environment + || env::var_os("PYTHON_BASIC_REPL").is_none_or(|value| value.is_empty())) + { + match vm.import("_pyrepl", 0) { + Ok(_) => return Ok(pymain_run_pyrepl(vm, startup)), + Err(exc) if !exc.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => { + eprintln!("Could not import _pyrepl.main"); + return Err(exc); + } + Err(_) => {} } } - result + // _PyRun_AnyFile(stdin, "") + vm.run_any_stdin()?; + Ok(0) } -/// When the fancy REPL cannot start (for example `TERM=dumb`), print the same -/// warning `_pyrepl.main` would have printed, then keep the rustyline shell. -fn warn_if_pyrepl_unavailable(vm: &VirtualMachine) { - let _ = vm.run_simple_string( - r" -import os, sys -if not os.getenv('PYTHON_BASIC_REPL'): - try: - from _pyrepl.main import CAN_USE_PYREPL, FAIL_REASON - if not CAN_USE_PYREPL and FAIL_REASON: - print(FAIL_REASON, file=sys.stderr) - except Exception: - pass -", - ); +// pymain_run_pyrepl +fn pymain_run_pyrepl(vm: &VirtualMachine, pythonstartup: bool) -> u32 { + run_pyrepl(vm, pythonstartup).map_or_else(|exc| vm.handle_exit_exception(exc), |()| 0) +} + +fn run_pyrepl(vm: &VirtualMachine, pythonstartup: bool) -> PyResult<()> { + let pyrepl = vm.import("_pyrepl.main", 0).inspect_err(|_| { + eprintln!("Could not import _pyrepl.main"); + })?; + let console = pyrepl + .get_attr("main", vm) + .and_then(|main| main.get_attr("interactive_console", vm)) + .inspect_err(|_| { + eprintln!("Could not access _pyrepl.main.interactive_console"); + })?; + let kwargs: vm::function::KwArgs = [ + ("mainmodule".to_owned(), vm.add_main_module()?.into()), + ( + "pythonstartup".to_owned(), + vm.ctx.new_bool(pythonstartup).into(), + ), + ] + .into_iter() + .collect(); + console.call(vm::function::FuncArgs::new(vec![], kwargs), vm)?; + Ok(()) } #[cfg(feature = "flame-it")] diff --git a/src/shell.rs b/src/shell.rs index 0d5ab590faf..7d206f28e9e 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,238 +1,41 @@ mod helper; -use rustpython_compiler::{ - CompileError, ParseError, parser::InterpolatedStringErrorType, parser::LexicalErrorType, - parser::ParseErrorType, -}; use rustpython_vm::{ - AsObject, PyResult, VirtualMachine, - builtins::PyBaseExceptionRef, - bytecode::CodeFlags, - compiler::{self}, + PyResult, VirtualMachine, readline::{Readline, ReadlineResult}, scope::Scope, - vm::VmCompileError, + vm::InteractiveLine, }; -enum ShellExecResult { - Ok, - PyErr(PyBaseExceptionRef), - ContinueBlock, - ContinueLine, -} - -fn shell_exec( - vm: &VirtualMachine, - source: &str, - scope: Scope, - empty_line_given: bool, - continuing_block: bool, - future_features: &mut CodeFlags, -) -> ShellExecResult { - // compiling expects only UNIX style line endings, and will replace windows line endings - // internally. Since we might need to analyze the source to determine if an error could be - // resolved by future input, we need the location from the error to match the source code that - // was actually compiled. - #[cfg(windows)] - let source = &source.replace("\r\n", "\n"); - let opts = compiler::CompileOpts { - future_features: *future_features, - ..vm.compile_opts() - }; - match vm.compile_with_opts(source, compiler::Mode::Single, "", opts) { - Ok(code) => { - *future_features |= code.code.flags & CodeFlags::FUTURE_MASK; - let _ = vm.register_code_in_linecache(&code, source); - if empty_line_given || !continuing_block { - match vm.run_code_obj(code, scope) { - Ok(_val) => ShellExecResult::Ok, - Err(err) => ShellExecResult::PyErr(err), - } - } else { - // We can just return an ok result - ShellExecResult::Ok - } - } - Err(err) => { - if matches!( - &err, - VmCompileError::Compile(CompileError::Parse( - ParseError { - error: ParseErrorType::Lexical( - LexicalErrorType::Eof - | LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, - ) - | LexicalErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, - ) - | LexicalErrorType::UnclosedStringError { - triple_quoted: true, - .. - } - ), - .. - } | ParseError { - is_unclosed_bracket: true, - .. - } - )) - ) { - return ShellExecResult::ContinueLine; - } - - // An unfinished suite is _IncompleteInputError, not IndentationError. - let exc = err.into_pyexception_maybe_incomplete(vm, Some(source), !empty_line_given); - if !empty_line_given && exc.fast_isinstance(vm.ctx.exceptions.incomplete_input_error) { - ShellExecResult::ContinueBlock - } else { - ShellExecResult::PyErr(exc) - } - } - } -} - -fn flush_stdio(vm: &VirtualMachine) { - for name in ["stdout", "stderr"] { - if let Ok(stream) = vm.sys_module.get_attr(name, vm) { - let _ = vm.call_method(&stream, "flush", ()); - } - } -} - /// Enter a repl loop +/// +/// rustyline takes the role of `PyOS_Readline` with the readline module, +/// adding line editing, completion and history. pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { - let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals.clone())); - let mut full_input = String::new(); - - // Retrieve a `history_path_str` dependent on the OS - let repl_history_path = match dirs::config_dir() { - Some(mut path) => { + let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals)); + let history = dirs::config_dir().map_or_else( + || ".repl_history.txt".into(), + |mut path| { path.push("rustpython"); path.push("repl_history.txt"); path + }, + ); + let _ = repl.load_history(&history); + let result = vm.run_interactive_loop(|prompt| match repl.readline(prompt) { + ReadlineResult::Line(line) => { + let _ = repl.add_history_entry(line.trim_end()); + Ok(Some(InteractiveLine::Text(line))) } - None => ".repl_history.txt".into(), - }; - - let _ = repl.load_history(&repl_history_path); - - // We might either be waiting to know if a block is complete, or waiting to know if a multiline - // statement is complete. In the former case, we need to ensure that we read one extra new line - // to know that the block is complete. In the latter, we can execute as soon as the statement is - // valid. - let mut continuing_block = false; - let mut continuing_line = false; - let mut future_features = CodeFlags::empty(); - - loop { - let prompt_name = if continuing_block || continuing_line { - "ps2" - } else { - "ps1" - }; - let prompt = vm - .sys_module - .get_attr(prompt_name, vm) - .and_then(|prompt| prompt.str(vm)); - let prompt = match prompt { - Ok(ref s) => s.expect_str(), - Err(_) => "", - }; - - continuing_line = false; - let result = match repl.readline(prompt) { - ReadlineResult::Line(line) => { - #[cfg(debug_assertions)] - debug!("You entered {line:?}"); - - repl.add_history_entry(line.trim_end()).unwrap(); - - let empty_line_given = line.is_empty(); - - if full_input.is_empty() { - full_input = line; - } else { - full_input.push_str(&line); - } - full_input.push('\n'); - - match shell_exec( - vm, - &full_input, - scope.clone(), - empty_line_given, - continuing_block, - &mut future_features, - ) { - ShellExecResult::Ok => { - if continuing_block { - if empty_line_given { - // We should exit continue mode since the block successfully executed - continuing_block = false; - full_input.clear(); - } - } else { - // We aren't in continue mode so proceed normally - full_input.clear(); - } - Ok(()) - } - // Continue, but don't change the mode - ShellExecResult::ContinueLine => { - continuing_line = true; - Ok(()) - } - ShellExecResult::ContinueBlock => { - continuing_block = true; - Ok(()) - } - ShellExecResult::PyErr(err) => { - continuing_block = false; - full_input.clear(); - Err(err) - } - } - } - ReadlineResult::Interrupt => { - continuing_block = false; - full_input.clear(); - let keyboard_interrupt = - vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned()); - Err(keyboard_interrupt) - } - ReadlineResult::Eof => { - break; - } - #[cfg(unix)] - ReadlineResult::OsError(num) => { - let os_error = vm.new_exception_msg( - vm.ctx.exceptions.os_error.to_owned(), - format!("{num:?}").into(), - ); - vm.print_exception(&os_error); - break; - } - ReadlineResult::Other(err) => { - eprintln!("Readline error: {err:?}"); - break; - } - ReadlineResult::Io(err) => { - eprintln!("IO error: {err:?}"); - break; - } - }; - - if let Err(exc) = result { - if exc.fast_isinstance(vm.ctx.exceptions.system_exit) { - let _ = repl.save_history(&repl_history_path); - return Err(exc); - } - vm.print_exception(&exc); + ReadlineResult::Eof => Ok(None), + ReadlineResult::Interrupt => { + Err(vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned())) } - flush_stdio(vm); - } - let _ = repl.save_history(&repl_history_path); - - Ok(()) + ReadlineResult::Io(_) => Ok(None), + #[cfg(unix)] + ReadlineResult::OsError(_) => Ok(None), + ReadlineResult::Other(_) => Ok(None), + }); + let _ = repl.save_history(&history); + result }