From 8a7e0ffb2ef59c4388433b5d9b7be7e834efe47c Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 15:09:35 +0900 Subject: [PATCH 01/19] Preserve original main VM through C API execution Run and finalize the original main VM while keeping the registered attachment template available without holding its mutex across Python callbacks. Assisted-by: OpenAI Codex:model version unavailable --- crates/capi/src/lib.rs | 30 +++++++++++++++++++++++++++++- crates/vm/src/vm/interpreter.rs | 18 ++++++++++++++++++ src/lib.rs | 10 +--------- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index ea99242005f..5d8e4cb5d1a 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -4,7 +4,7 @@ use crate::pyerrors::init_exception_statics; use crate::pylifecycle::{MAIN_INTERP, MAIN_INTERP_PTR}; use core::sync::atomic::Ordering; pub use rustpython_vm::PyObject; -use rustpython_vm::{Context, Interpreter}; +use rustpython_vm::{Context, Interpreter, PyResult, VirtualMachine}; use std::sync::MutexGuard; extern crate alloc; @@ -71,3 +71,31 @@ pub fn init_main_interpreter(interpreter: Interpreter) { Ordering::Release, ); } + +/// Run and finalize the main VM while leaving C API thread attachment available. +pub fn run_main_interpreter( + mut interpreter: Interpreter, + f: impl FnOnce(&VirtualMachine) -> PyResult<()>, +) -> u32 { + let main_thread = interpreter.take_main_thread(); + init_main_interpreter(interpreter); + + struct RegistrationGuard; + impl Drop for RegistrationGuard { + fn drop(&mut self) { + let mut registered = MAIN_INTERP + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + MAIN_INTERP_PTR.store(core::ptr::null_mut(), Ordering::Release); + let interpreter = registered.take(); + drop(registered); + drop(interpreter); + } + } + let _registration = RegistrationGuard; + + // The registered interpreter is an inactive worker-VM template. Running + // the original VM locally preserves main-thread state without keeping the + // global mutex locked across Python execution or finalization callbacks. + main_thread.run(f) +} diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index d6141223704..6812edf9e3b 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -587,6 +587,24 @@ impl Interpreter { self.vm.new_thread() } + /// Move the main execution context into a separate owner, leaving this + /// handle available to create worker VMs for the same interpreter. + /// + /// The returned interpreter must run and finalize on the original thread. + /// This handle should only create worker VMs and must not be finalized. + /// Unlike `new_thread()`, this preserves the main VM's signal handlers and + /// all of its other thread-local state. + #[cfg(feature = "threading")] + #[must_use] + pub fn take_main_thread(&mut self) -> Self { + let replacement = self.vm.new_thread().vm; + let vm = core::mem::replace(&mut self.vm, replacement); + Self { + global_state: self.global_state.clone(), + vm, + } + } + /// Run a function with the main virtual machine and return a PyResult of the result. /// /// To enter vm context multiple times or to avoid buffer/exception management, this function is preferred. diff --git a/src/lib.rs b/src/lib.rs index e1047a08f3d..f5445fdc563 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,15 +121,7 @@ pub fn run(mut builder: InterpreterBuilder) -> ExitCode { let interp = builder.interpreter(); let exitcode = cfg_select! { - feature = "capi" => {{ - let local_vm = interp.enter(|vm| vm.new_thread()); - rustpython_capi::init_main_interpreter(interp); - let result = local_vm.run(|vm| run_rustpython(vm, run_mode)); - rustpython_capi::get_main_interpreter() - .take() - .unwrap() - .finalize(result.err()) - }}, + feature = "capi" => rustpython_capi::run_main_interpreter(interp, move |vm| run_rustpython(vm, run_mode)), _ => interp.run(move |vm| run_rustpython(vm, run_mode)), }; From 4da4895d343d966de1e974322c747d5502bb1e8b Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 15:09:46 +0900 Subject: [PATCH 02/19] Make unraisable reporting robust at hook boundaries Keep the existing message API and traceback formatting while handling missing, disabled, audited and failing hooks without panicking. Assisted-by: OpenAI Codex:model version unavailable --- crates/vm/src/exceptions.rs | 16 +++++-- crates/vm/src/vm/mod.rs | 87 +++++++++++++++++++++++++------------ 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 8adee7270e9..a94f37cb08a 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -185,10 +185,7 @@ impl VirtualMachine { let traceback = exc.traceback.read().clone(); if let Some(tb) = traceback { - writeln!(output, "Traceback (most recent call last):")?; - for tb in tb.iter() { - write_traceback_entry(output, &tb)?; - } + write_traceback(output, &tb)?; } let varargs = exc.args(); @@ -448,6 +445,17 @@ fn print_source_line( Ok(()) } +pub(crate) fn write_traceback( + output: &mut W, + traceback: &PyTracebackRef, +) -> Result<(), W::Error> { + writeln!(output, "Traceback (most recent call last):")?; + for entry in traceback.iter() { + write_traceback_entry(output, &entry)?; + } + Ok(()) +} + /// Print exception occurrence location from traceback element fn write_traceback_entry( output: &mut W, diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index d34ded1bca9..b5f85dc5fca 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1705,33 +1705,52 @@ impl VirtualMachine { #[cold] pub fn run_unraisable(&self, e: PyBaseExceptionRef, msg: Option, object: PyObjectRef) { - // During interpreter finalization, sys.unraisablehook may not be available, - // but we still need to report exceptions (especially from atexit callbacks). - // Write directly to stderr like PyErr_FormatUnraisable. - if self.state.finalizing.load(Ordering::Acquire) { + let msg = msg.map(|msg| self.ctx.new_str(msg)); + let write_original = || { + let msg = msg.as_ref().map(|msg| msg.to_string_lossy()); self.write_unraisable_to_stderr(&e, msg.as_deref(), &object); + }; + // During finalization the hook may no longer exist. + if self.state.finalizing.load(Ordering::Acquire) { + write_original(); return; } - - let sys_module = self.import("sys", 0).unwrap(); - let unraisablehook = sys_module.get_attr("unraisablehook", self).unwrap(); - - let exc_type = e.class().to_owned(); - let exc_traceback = e.traceback().to_pyobject(self); // TODO: actual traceback - let exc_value = e.into(); - let args = stdlib::sys::UnraisableHookArgsData { - exc_type, - exc_value, - exc_traceback, - err_msg: self.new_pyobj(msg), - object, + let Some(unraisablehook) = self.sys_module.get_attr("unraisablehook", self).ok() else { + write_original(); + return; }; - if let Err(e) = unraisablehook.call((args,), self) { - println!("{}", e.as_object().repr(self).unwrap()); + let args = stdlib::sys::UnraisableHookArgsData { + exc_type: e.class().to_owned(), + exc_value: e.clone().into(), + exc_traceback: e.traceback().to_pyobject(self), + err_msg: msg.clone().to_pyobject(self), + object: object.clone(), + } + .to_pyobject(self); + if let Err(error) = self.audit("sys.unraisablehook", || { + (unraisablehook.clone(), args.clone()) + }) { + self.write_unraisable_to_stderr( + &error, + Some("Exception ignored in audit hook"), + &self.ctx.none(), + ); + return; + } + if self.is_none(&unraisablehook) { + write_original(); + return; + } + if let Err(error) = unraisablehook.call((args,), self) { + self.write_unraisable_to_stderr( + &error, + Some("Exception ignored in sys.unraisablehook"), + &unraisablehook, + ); } } - /// Write unraisable exception to stderr during finalization. + /// Write an unraisable exception when its hook cannot be used. /// Similar to _PyErr_WriteUnraisableDefaultHook in CPython. fn write_unraisable_to_stderr( &self, @@ -1769,15 +1788,29 @@ impl VirtualMachine { write_to_stderr(&format!("{repr_wtf8}\n"), &stderr, self); } - // Write exception type and message - let exc_type_name = e.class().name(); - let msg = match e.as_object().str(self) { - Ok(exc_str) if !exc_str.as_wtf8().is_empty() => { - format!("{}: {}\n", exc_type_name, exc_str.as_wtf8()) + // Print the existing traceback without importing Python modules; + // this path must also work while the interpreter is finalizing. + if let Some(traceback) = e.traceback() { + if let Some(stderr) = &stderr { + let mut writer = crate::py_io::PyWriter(stderr.clone(), self); + let _ = crate::exceptions::write_traceback(&mut writer, &traceback); + } else { + let mut writer = crate::py_io::IoWriter(std::io::stderr()); + let _ = crate::exceptions::write_traceback(&mut writer, &traceback); + } + } + let exc_type_name = e + .class() + .fully_qualified_name(self) + .unwrap_or_else(|_| e.class().name().to_string()); + let message = match e.as_object().str(self) { + Ok(value) if !value.as_wtf8().is_empty() => { + format!("{exc_type_name}: {}\n", value.as_wtf8()) } - _ => format!("{exc_type_name}\n"), + Ok(_) => format!("{exc_type_name}\n"), + Err(_) => format!("{exc_type_name}: \n"), }; - write_to_stderr(&msg, &stderr, self); + write_to_stderr(&message, &stderr, self); // Flush stderr to ensure output is visible if let Some(ref stderr) = stderr { From 3a4ad2995c11e39c13358df7e68525e8aaf0cf97 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 15:09:56 +0900 Subject: [PATCH 03/19] Separate interactive warning collection from compilation Retain ordinary warning behavior and typed module APIs while exposing record-local parser warnings and post-warning code generation. Add a source-only lazy future-import guard. Assisted-by: OpenAI Codex:model version unavailable --- crates/compiler/src/lib.rs | 49 +++++++++++ crates/vm/src/vm/compile.rs | 161 +++++++++++++++++++++++++++++++----- 2 files changed, 188 insertions(+), 22 deletions(-) diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 82f881d9598..96a5317f707 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -950,6 +950,55 @@ pub fn too_deeply_nested_error( })) } +/// Source-only rejection performed by CPython's checked_from_import parser action. +#[doc(hidden)] +#[must_use] +pub fn lazy_future_import_error(ast: &ast::Mod, source_file: &SourceFile) -> Option { + use ast::statement_visitor::StatementVisitor; + + struct Checker<'a> { + pending: Vec<&'a ast::Stmt>, + } + + impl<'a> StatementVisitor<'a> for Checker<'a> { + fn visit_stmt(&mut self, stmt: &'a ast::Stmt) { + self.pending.push(stmt); + } + } + + let ast::Mod::Module(module) = ast else { + return None; + }; + let mut checker = Checker { + pending: module.body.iter().collect(), + }; + let mut first = None; + while let Some(stmt) = checker.pending.pop() { + if let ast::Stmt::ImportFrom(import) = stmt + && import.is_lazy + && import.level == 0 + && import + .module + .as_ref() + .is_some_and(|name| name.as_str() == "__future__") + { + let start = import.range.start().to_usize(); + first = Some(first.map_or(start, |previous: usize| previous.min(start))); + } + ast::statement_visitor::walk_stmt(&mut checker, stmt); + } + first.map(|start| { + CompileError::from_source_error( + source_file, + CpythonDiagnostic::new( + "lazy from __future__ import is not allowed".to_owned(), + start, + start + 4, + ), + ) + }) +} + /// Syntax the reference grammar has no rule for, but that this parser accepts. /// /// A bare generator expression is one: `f(x for x in y)` is the `primary diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 575e954d034..2c05f4e37c4 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -11,6 +11,19 @@ use crate::{ vm::compile_mode::{CompileStart, CompilerFlags, compile_future_features_from_flags}, }; +/// Warnings delivered while collecting one physical interactive record. +#[cfg(feature = "parser")] +#[derive(Default)] +pub(super) struct InteractiveWarnings { + emitted: alloc::collections::BTreeSet<(usize, String)>, +} + +#[cfg(feature = "parser")] +pub(super) struct ParserWarningState<'a> { + pub emitted_escapes: &'a [usize], + pub interactive: Option<&'a mut InteractiveWarnings>, +} + #[derive(Debug)] pub enum VmCompileError { Compile(CompileError), @@ -595,6 +608,20 @@ impl VirtualMachine { self.emit_string_escape_warnings(source, &source_path, module, emitted_escapes) .map_err(VmCompileError::Warning)?; } + self.compile_after_parser_warnings(source, mode, source_path, opts, module) + } + + pub(super) fn compile_after_parser_warnings( + &self, + source: &str, + mode: compiler::Mode, + source_path: impl Into, + opts: CompileOpts, + module: Option<&Py>, + ) -> Result, VmCompileError> { + let source_path = source_path.into(); + #[cfg(not(feature = "parser"))] + let _ = module; #[cfg(feature = "parser")] let code = { // A warning the filter escalates to an exception is stashed here so @@ -724,14 +751,21 @@ mod escape_warnings { } enum ParserWarning { - Escape(InvalidEscape, EscapeOrigin), - Numeric { offset: usize, message: String }, + /// The escape, where it is reported from, and for a bytes literal + /// escape on the literal's first line, the error column that + /// string_parser.c reports. + Escape(InvalidEscape, EscapeOrigin, Option), + Numeric { + offset: usize, + last_offset: usize, + message: String, + }, } impl ParserWarning { fn offset(&self) -> usize { match self { - Self::Escape(escape, _) => escape.offset(), + Self::Escape(escape, ..) => escape.offset(), Self::Numeric { offset, .. } => *offset, } } @@ -1206,7 +1240,19 @@ mod escape_warnings { content_bounds(source, text_range_from_bounds(literal_start, end)) && let Some(escape) = first_invalid_escape(source, start, content_end, is_bytes) { - warnings.push(ParserWarning::Escape(escape, EscapeOrigin::Parser)); + let before_escape = &source[start..escape.offset()]; + // string_parser.c counts leading quotes from the token start. + // A bytes prefix contributes none to its first-line error range. + // Keep the actual escape offset for warning order and deduplication. + let error_column = (is_bytes && !before_escape.contains(['\r', '\n'])).then(|| { + let (_, column) = line_offset_at(source, escape.offset()); + column.saturating_sub(start - literal_start) + }); + warnings.push(ParserWarning::Escape( + escape, + EscapeOrigin::Parser, + error_column, + )); } } @@ -1225,7 +1271,7 @@ mod escape_warnings { } else { EscapeOrigin::Parser }; - warnings.push(ParserWarning::Escape(escape, origin)); + warnings.push(ParserWarning::Escape(escape, origin, None)); return; } // In CPython, _PyTokenizer_warn_invalid_escape_sequence handles @@ -1249,6 +1295,7 @@ mod escape_warnings { offset: end - 1, }, EscapeOrigin::Tokenizer, + None, )); } } @@ -1474,6 +1521,7 @@ mod escape_warnings { if end > index && numeric_keyword_suffix(&bytes[end..]) { warnings.push(ParserWarning::Numeric { offset: index, + last_offset: end - 1, message: format!("invalid {kind} literal"), }); } @@ -1704,10 +1752,12 @@ mod escape_warnings { warnings.sort_by_key(ParserWarning::offset); // Interpolated tokenizer warnings have their own stop/error boundary. // Numeric-only recovery must not skip across one of those events. - if warnings - .iter() - .any(|warning| matches!(warning, ParserWarning::Escape(_, EscapeOrigin::Tokenizer))) - { + if warnings.iter().any(|warning| { + matches!( + warning, + ParserWarning::Escape(_, EscapeOrigin::Tokenizer, _) + ) + }) { ast_tokenizer_recovery = false; } (warnings, ast_tokenizer_recovery) @@ -1716,11 +1766,12 @@ mod escape_warnings { enum WarningEmission<'a> { Standard { emitted_escapes: &'a [usize] }, Ast { tokenizer_recovery: bool }, + Interactive(super::ParserWarningState<'a>), } impl VirtualMachine { - /// Emit tokenizer-level SyntaxWarnings raised before - /// code generation. + // Keep ordinary source compilation's existing warning order and + // recovery surface; interactive records use the bounded parser below. pub(super) fn emit_tokenizer_syntax_warnings( &self, source: &str, @@ -1741,8 +1792,6 @@ mod escape_warnings { .map(|_| ()) } - /// Walk all string literals in `source` and emit `SyntaxWarning` for - /// each that contains an invalid escape sequence. pub(super) fn emit_string_escape_warnings( &self, source: &str, @@ -1750,8 +1799,6 @@ mod escape_warnings { module: Option<&Py>, emitted_escapes: &[usize], ) -> Result<(), CompileWarningError> { - // The compile that follows rejects this source; parsing it here - // would build a tree that exhausts the stack when dropped. let source_file = compiler::core::SourceFileBuilder::new(filename, source).finish(); if compiler::pre_parse_source_error(&source_file).is_err() { return Ok(()); @@ -1821,6 +1868,26 @@ mod escape_warnings { ) } + pub(in crate::vm) fn emit_interactive_parser_warnings( + &self, + source: &str, + filename: &str, + module: Option<&Py>, + mode: Option, + inherited_barry_as_flufl: bool, + state: super::ParserWarningState<'_>, + ) -> Result, CompileWarningError> { + let (warnings, _) = + collect_parser_warnings(source, filename, mode, inherited_barry_as_flufl); + self.emit_parser_warning_batch( + source, + filename, + module, + warnings, + WarningEmission::Interactive(state), + ) + } + fn emit_parser_warning_batch( &self, source: &str, @@ -1829,9 +1896,12 @@ mod escape_warnings { warnings: Vec, emission: WarningEmission<'_>, ) -> Result, CompileWarningError> { - let (emitted_escapes, ast_tokenizer_recovery) = match emission { - WarningEmission::Standard { emitted_escapes } => (emitted_escapes, false), - WarningEmission::Ast { tokenizer_recovery } => (&[][..], tokenizer_recovery), + let (emitted_escapes, ast_tokenizer_recovery, mut interactive) = match emission { + WarningEmission::Standard { emitted_escapes } => (emitted_escapes, false, None), + WarningEmission::Ast { tokenizer_recovery } => (&[][..], tokenizer_recovery, None), + WarningEmission::Interactive(state) => { + (state.emitted_escapes, false, state.interactive) + } }; let mut considered_escapes = Vec::new(); let mut parser_error = None; @@ -1841,8 +1911,17 @@ mod escape_warnings { if parser_error.is_some() && !matches!(warning, ParserWarning::Numeric { .. }) { continue; } + if let Some(state) = interactive.as_deref_mut() { + let message = match &warning { + ParserWarning::Escape(escape, ..) => escape.warning_message(), + ParserWarning::Numeric { message, .. } => message.clone(), + }; + if !state.emitted.insert((warning.offset(), message)) { + continue; + } + } match warning { - ParserWarning::Escape(escape, origin) => { + ParserWarning::Escape(escape, origin, error_column) => { let offset = escape.offset(); if considered_escapes.last() == Some(&offset) { continue; @@ -1851,7 +1930,18 @@ mod escape_warnings { if emitted_escapes.binary_search(&offset).is_err() { let result = warn_invalid_escape_sequence( source, escape, filename, module, self, - ); + ) + .map_err(|mut error| { + if interactive.is_some() + && let Some(column) = error_column + { + error.offset = column; + if let Some(replacement) = &mut error.replacement { + replacement.end_offset = column + 2; + } + } + error + }); if let Err(error) = result { if ast_tokenizer_recovery && origin == EscapeOrigin::Parser @@ -1866,8 +1956,35 @@ mod escape_warnings { } } } - ParserWarning::Numeric { offset, message } => { - warn_syntax_at_offset(source, filename, module, offset, message, self)?; + ParserWarning::Numeric { + offset, + last_offset, + message, + } => { + warn_syntax_at_offset( + source, + filename, + module, + if interactive.is_some() { + last_offset + } else { + offset + }, + message.clone(), + self, + ) + .map_err(|mut error| { + // Tokenizer numeric warnings become a zero-width + // error at the literal's last character, in character + // columns rather than grammar-action byte columns. + if interactive.is_some() { + error.replacement = Some(SyntaxErrorReplacement { + message, + end_offset: error.offset, + }); + } + error + })?; } } } From 5649b19a5f4c313531fb0ab0e09a02b30264489e Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 15:10:09 +0900 Subject: [PATCH 04/19] Evaluate native stdin with shared VM record semantics Use one evaluator for basic REPL and root transports, preserve effective inspect snapshots and future flags, and carry owned host exit status through original-main finalization. Assisted-by: OpenAI Codex:model version unavailable --- crates/capi/src/lib.rs | 14 +- crates/compiler/src/lib.rs | 75 ++++++ crates/host_env/src/readline.rs | 11 + crates/vm/src/dict_inner.rs | 18 ++ crates/vm/src/exceptions.rs | 60 +++++ crates/vm/src/stdlib/_testinternalcapi.rs | 6 +- crates/vm/src/stdlib/posix.rs | 2 + crates/vm/src/stdlib/sys.rs | 68 ++++- crates/vm/src/vm/interpreter.rs | 44 +++- crates/vm/src/vm/mod.rs | 44 +++- crates/vm/src/vm/python_run.rs | 305 ++++++++++++++++++++++ src/lib.rs | 156 +++++++++-- src/shell.rs | 252 ++---------------- 13 files changed, 784 insertions(+), 271 deletions(-) diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index 5d8e4cb5d1a..4db968dbb9e 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -74,8 +74,18 @@ pub fn init_main_interpreter(interpreter: Interpreter) { /// Run and finalize the main VM while leaving C API thread attachment available. pub fn run_main_interpreter( - mut interpreter: Interpreter, + interpreter: Interpreter, f: impl FnOnce(&VirtualMachine) -> PyResult<()>, +) -> u32 { + run_main_interpreter_with_exit_status(interpreter, |vm| { + f(vm).map(|()| rustpython_vm::vm::InterpreterExit::Code(0)) + }) +} + +/// Preserve the main VM and an already-reported result through C API shutdown. +pub fn run_main_interpreter_with_exit_status( + mut interpreter: Interpreter, + f: impl FnOnce(&VirtualMachine) -> PyResult, ) -> u32 { let main_thread = interpreter.take_main_thread(); init_main_interpreter(interpreter); @@ -97,5 +107,5 @@ pub fn run_main_interpreter( // The registered interpreter is an inactive worker-VM template. Running // the original VM locally preserves main-thread state without keeping the // global mutex locked across Python execution or finalization callbacks. - main_thread.run(f) + main_thread.run_with_exit_status(f) } diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 96a5317f707..c6f8a4e103d 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -1653,6 +1653,81 @@ pub fn prepare_barry_as_flufl_source( } } +/// Parse-only state for a native interactive record. This never runs codegen, +/// emits Python warnings, or changes the caller's future flags. +pub struct InteractiveInput { + pub future_features: core::bytecode::CodeFlags, + pub empty: bool, +} + +/// The native interactive grammar terminates compound statements with an empty +/// physical line (or EOF), unlike the module grammar used by the Ruff parser. +/// Use the parsed statement shape rather than a second ad-hoc source lexer. +pub fn interactive_input( + source: &str, + source_path: &str, + future_features: core::bytecode::CodeFlags, + terminated: bool, +) -> Result, CompileError> { + let source_file = SourceFileBuilder::new(source_path, source).finish(); + pre_parse_source_error(&source_file)?; + let options = parser::ParseOptions::from(parser::Mode::Module); + let prepared = prepare_barry_as_flufl_source( + source, + options.clone(), + future_features.contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), + ); + let parsed = parser::parse_unchecked(prepared.source(), options); + if let Some(error) = prepared.diagnostic(parsed.errors().first(), &source_file) { + return Err(error); + } + let parsed = first_parse_error(parsed, &source_file, Mode::Single)?; + let ast::Mod::Module(module) = parsed.syntax() else { + unreachable!() + }; + if let Some(first) = module.body.first() + && is_compound_stmt(first) + { + if let Some(next) = module.body.get(1) { + let range = parsed + .tokens() + .iter() + .find(|token| { + token.start() >= next.start() + && token.start() < next.end() + && !token.range().is_empty() + }) + .map_or_else( + || ruff_text_size::TextRange::new(next.start(), next.start()), + Ranged::range, + ); + return Err(CompileError::from_source_error( + &source_file, + CpythonDiagnostic::new( + "invalid syntax".to_owned(), + range.start().to_usize(), + range.end().to_usize(), + ), + )); + } + if !terminated { + return Ok(None); + } + } + if let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) { + return Err(error); + } + if let Some(error) = lazy_future_import_error(parsed.syntax(), &source_file) { + return Err(error); + } + let future_features = codegen::preprocess::checked_future_features_in_body(&module.body) + .map_err(|error| future_feature_error(error, &source_file))?; + Ok(Some(InteractiveInput { + future_features, + empty: module.body.is_empty(), + })) +} + pub fn compile_with_syntax_warning_handler<'a>( source: &str, mode: Mode, diff --git a/crates/host_env/src/readline.rs b/crates/host_env/src/readline.rs index f3f468cba56..a7de2f50880 100644 --- a/crates/host_env/src/readline.rs +++ b/crates/host_env/src/readline.rs @@ -196,3 +196,14 @@ impl Readline { self.0.readline(prompt) } } + +/// Borrow the shared buffered stdin for one physical read. The caller retains +/// partial bytes across EINTR and checks signals after this lock is released. +pub fn stdio_readline_bytes(prompt: &str, line: &mut Vec) -> io::Result { + use std::io::{BufRead, Write}; + let mut stderr = io::stderr().lock(); + stderr.write_all(prompt.as_bytes())?; + stderr.flush()?; + drop(stderr); + io::stdin().lock().read_until(b'\n', line) +} diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 8b4a8eb6def..0c496a36f45 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -424,6 +424,24 @@ impl DictInner { type PopInnerResult = ControlFlow>>; impl Dict { + /// Replace a retained entry without hashing, equality calls, or releasing owners. + /// The caller must keep key, expected, and replacement alive until its own + /// outer locks are released, and drop the returned owner only afterwards. + pub(crate) fn replace_by_identity( + &self, + key: &PyObject, + matches: impl FnOnce(&T) -> bool, + replacement: &T, + ) -> Option { + let mut inner = self.write(); + let entry = inner + .entries + .iter_mut() + .flatten() + .find(|entry| entry.key.is(key))?; + matches(&entry.value).then(|| core::mem::replace(&mut entry.value, replacement.clone())) + } + /// Current keys-version stamp, or 0 if none has been assigned since the /// last key-set change. Equal nonzero stamps guarantee an unchanged key /// set (values may differ). diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index a94f37cb08a..856ff75d104 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -123,6 +123,66 @@ impl VirtualMachine { } } + /// Report an uncaught exception, returning an unconsumed SystemExit to the host. + /// Unlike `print_exception`, this updates the interactive last-exception state + /// and performs the sys.excepthook audit. + pub fn report_uncaught_exception(&self, exc: &Py) -> PyResult<()> { + if exc.fast_isinstance(self.ctx.exceptions.system_exit) && !self.inspect_requested() { + return Err(exc.to_owned()); + } + let (exc_type, exc_val, exc_tb) = self.split_exception(exc.to_owned()); + for (name, value) in [ + ("last_exc", exc_val.clone()), + ("last_type", exc_type.clone()), + ("last_value", exc_val.clone()), + ("last_traceback", exc_tb.clone()), + ] { + let _ = self.sys_module.set_attr(name, value, self); + } + let hook = self.sys_module.get_attr("excepthook", self).ok(); + if let Err(audit_exc) = self.audit("sys.excepthook", || { + ( + hook.clone().unwrap_or_else(|| self.ctx.none()), + exc_type.clone(), + exc_val.clone(), + exc_tb.clone(), + ) + }) { + if audit_exc.fast_isinstance(self.ctx.exceptions.runtime_error) { + return Ok(()); + } + self.run_unraisable( + audit_exc, + Some("Exception ignored in audit hook".to_owned()), + self.ctx.none(), + ); + } + let write_fallback = |error, heading| { + if let Ok(stderr) = sys::get_stderr(self) { + let mut stderr = py_io::PyWriter(stderr, self); + let _ = writeln!(stderr, "{heading}"); + let _ = self.write_exception(&mut stderr, error); + } else { + eprintln!("{heading}\nlost sys.stderr"); + let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), error); + } + }; + if let Some(hook) = hook { + if let Err(hook_exc) = hook.call((exc_type, exc_val, exc_tb), self) { + if hook_exc.fast_isinstance(self.ctx.exceptions.system_exit) + && !self.inspect_requested() + { + return Err(hook_exc); + } + write_fallback(&hook_exc, "Error in sys.excepthook:"); + write_fallback(exc, "Original exception was:"); + } + } else { + write_fallback(exc, "sys.excepthook is missing"); + } + Ok(()) + } + pub fn write_exception( &self, output: &mut W, diff --git a/crates/vm/src/stdlib/_testinternalcapi.rs b/crates/vm/src/stdlib/_testinternalcapi.rs index 7250de54f0a..7581229b74a 100644 --- a/crates/vm/src/stdlib/_testinternalcapi.rs +++ b/crates/vm/src/stdlib/_testinternalcapi.rs @@ -423,7 +423,11 @@ mod _testinternalcapi { global.set_item("dev_mode", vm.ctx.new_bool(settings.dev_mode).into(), vm)?; global.set_item("verbose", vm.ctx.new_int(settings.verbose).into(), vm)?; global.set_item("quiet", vm.ctx.new_bool(settings.quiet).into(), vm)?; - global.set_item("inspect", vm.ctx.new_bool(settings.inspect).into(), vm)?; + global.set_item( + "inspect", + vm.ctx.new_bool(vm.inspect_requested()).into(), + vm, + )?; global.set_item( "interactive", vm.ctx.new_bool(settings.interactive).into(), diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index ae70b7001bb..b6abf4246e3 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -788,6 +788,7 @@ pub mod module { unsafe { // PyGlobalState PyMutex locks + reinit_mutex_after_fork(&vm.state.inspect_requested); reinit_mutex_after_fork(&vm.state.before_forkers); reinit_mutex_after_fork(&vm.state.after_forkers_child); reinit_mutex_after_fork(&vm.state.after_forkers_parent); @@ -838,6 +839,7 @@ pub mod module { } unsafe { + reinit_mutex_after_fork(&state.inspect_requested); reinit_mutex_after_fork(&state.before_forkers); reinit_mutex_after_fork(&state.after_forkers_child); reinit_mutex_after_fork(&state.after_forkers_parent); diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 7e746b5f521..4efddb8a9c7 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -56,7 +56,7 @@ pub mod sys { use num_traits::ToPrimitive; use std::{ env, - io::{IsTerminal, Read, Write}, + io::{IsTerminal, Write}, }; // Rust target triple (e.g., "x86_64-unknown-linux-gnu") @@ -761,16 +761,7 @@ pub mod sys { #[cfg(feature = "rustpython-compiler")] #[pyfunction] fn _baserepl(vm: &VirtualMachine) -> PyResult<()> { - // read stdin to end - let stdin = std::io::stdin(); - let mut handle = stdin.lock(); - let mut source = String::new(); - handle - .read_to_string(&mut source) - .map_err(|e| vm.new_os_error(format!("Error reading from stdin: {e}")))?; - vm.compile(&source, crate::compiler::Mode::Single, "") - .map_err(|e| e.into_pyexception(vm, Some(&source)))?; - Ok(()) + vm.run_basic_interactive() } #[pyfunction] @@ -1039,6 +1030,61 @@ pub mod sys { PyFlags::from_data(FlagsData::from_settings(&vm.state.config.settings), vm) } + pub(crate) fn set_inspect_requested(vm: &VirtualMachine, requested: bool) -> PyResult<()> { + use crate::class::StaticType; + use crate::types::PyStructSequenceData; + + let dict = vm.sys_module.dict(); + loop { + // Retain every encountered key/value before taking the publication + // lock: a concurrent deletion must never finalize one under it. + let entries = dict.entries.items(); + let (key, current) = entries + .iter() + .find(|(key, _)| { + key.class().is(vm.ctx.types.str_type) + && key + .downcast_ref::() + .is_some_and(|name| name.as_bytes() == b"flags") + }) + .ok_or_else(|| vm.new_runtime_error("sys.flags is missing"))?; + if !current.class().is(PyFlags::static_type()) { + return Err(vm.new_type_error("sys.flags must be a sys.flags instance")); + } + let tuple = current + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("invalid sys.flags payload"))?; + let fields = FlagsData::REQUIRED_FIELD_NAMES; + let field_count = fields.len() + FlagsData::OPTIONAL_FIELD_NAMES.len(); + if tuple.as_slice().len() != field_count { + return Err(vm.new_type_error("invalid sys.flags field count")); + } + let inspect_index = fields + .iter() + .position(|name| *name == "inspect") + .expect("sys.flags has an inspect field"); + let mut values = tuple.as_slice().to_vec(); + values[inspect_index] = vm.ctx.new_int(u8::from(requested)).into(); + let replacement: PyObjectRef = PyTuple::new_unchecked(values.into_boxed_slice()) + .into_ref_with_type(vm, PyFlags::static_type().to_owned())? + .into(); + + // No Python, hashing, equality, or owner destruction in this section. + // A racing publisher or direct sys.flags assignment causes a retry. + let mut effective = vm.state.inspect_requested.lock(); + let removed = + dict.entries + .replace_by_identity(key, |value| value.is(current), &replacement); + if removed.is_some() { + *effective = requested; + } + drop(effective); + if removed.is_some() { + return Ok(()); + } + } + } + #[pyattr] fn float_info(vm: &VirtualMachine) -> PyTupleRef { PyFloatInfo::from_data(FloatInfoData::INFO, vm) diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 6812edf9e3b..f36b8780e7d 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -19,6 +19,13 @@ type InitFunc = Box; /// Matches CPython's behavior (see cpython/Python/pylifecycle.c). const EXITCODE_FLUSH_FAILURE: u32 = 120; +/// An execution result whose Python exception has already been reported. +/// Unreported exceptions continue to travel separately through `PyResult`. +pub enum InterpreterExit { + Code(u32), + ReportedException(PyBaseExceptionRef), +} + /// Configuration builder for constructing an Interpreter. /// /// This is the preferred way to configure and create an interpreter with custom modules. @@ -84,8 +91,10 @@ where let (config, all_module_defs, frozen, int_max_str_digits) = if let Some(parent) = parent_state { // Subinterpreter: clone config and module tables from parent, fresh runtime state. let int_max_str_digits = AtomicCell::new(parent.int_max_str_digits.load()); + let mut config = parent.config.clone(); + config.settings.inspect = *parent.inspect_requested.lock(); ( - parent.config.clone(), + config, parent.module_defs.clone(), parent.frozen.clone(), int_max_str_digits, @@ -154,6 +163,8 @@ where runtime_root_id, whence, is_main, + inspect_requested: PyMutex::new(config.settings.inspect), + interactive_code_counter: AtomicU64::new(0), config, module_defs: all_module_defs, frozen, @@ -649,8 +660,20 @@ impl Interpreter { where F: FnOnce(&VirtualMachine) -> PyResult<()>, { - let res = self.enter(|vm| f(vm)); - self.finalize(res.err()) + self.run_with_exit_status(|vm| f(vm).map(|()| InterpreterExit::Code(0))) + } + + /// Run and finalize while preserving a host status for already-reported errors. + /// Errors returned by the callback are still reported during finalization. + pub fn run_with_exit_status(self, f: F) -> u32 + where + F: FnOnce(&VirtualMachine) -> PyResult, + { + let (exc, exit_code) = match self.enter(f) { + Ok(exit_code) => (None, exit_code), + Err(exc) => (Some(exc), InterpreterExit::Code(0)), + }; + self.finalize_with_exit_status(exc, exit_code) } /// Finalize vm and turns an exception to exit code. @@ -668,6 +691,16 @@ impl Interpreter { /// /// Note that calling `finalize` is not necessary by purpose though. pub fn finalize(self, exc: Option) -> u32 { + self.finalize_with_exit_status(exc, InterpreterExit::Code(0)) + } + + /// Finalize with an explicit status when no unreported exception remains. + /// A pending exception takes precedence over the supplied status. + pub fn finalize_with_exit_status( + self, + exc: Option, + status: InterpreterExit, + ) -> u32 { self.enter(|vm| { let mut flush_status = vm.flush_std(); @@ -675,7 +708,10 @@ impl Interpreter { let exit_code = if let Some(exc) = exc { vm.handle_exit_exception(exc) } else { - 0 + match status { + InterpreterExit::Code(code) => code, + InterpreterExit::ReportedException(exc) => vm.handle_reported_exception(&exc), + } }; // Wait for thread shutdown - call threading._shutdown() if available. diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index b5f85dc5fca..7321f4bcadf 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -14,6 +14,8 @@ mod interpreter; mod method; #[cfg(feature = "rustpython-compiler")] mod python_run; +#[cfg(feature = "rustpython-compiler")] +pub use python_run::InteractiveLine; pub mod runtime; mod setting; pub mod thread; @@ -61,7 +63,7 @@ use std::{ }; pub use context::Context; -pub use interpreter::{Interpreter, InterpreterBuilder}; +pub use interpreter::{Interpreter, InterpreterBuilder, InterpreterExit}; pub(crate) use method::PyMethod; pub use runtime::{ InterpFeatureFlags, InterpreterConfig, InterpreterGil, InterpreterInfo, InterpreterWhence, @@ -858,6 +860,10 @@ pub struct PyGlobalState { /// becomes *the* process main — see [`runtime::main_interpreter_id`]. pub is_main: bool, pub config: PyConfig, + /// Effective runtime inspect state, serialized with sys.flags publication. + pub(crate) inspect_requested: PyMutex, + /// Unique source identities shared by all interactive entries in this interpreter. + pub(crate) interactive_code_counter: AtomicU64, pub module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef>, pub frozen: HashMap<&'static str, FrozenModule, rapidhash::quality::RandomState>, pub stacksize: AtomicCell, @@ -3907,7 +3913,31 @@ impl VirtualMachine { excs.stack.iter().rev().find_map(|e| e.clone()) } + pub fn inspect_requested(&self) -> bool { + *self.state.inspect_requested.lock() + } + + /// Publish a fresh immutable sys.flags snapshot and the effective inspect bit. + pub fn set_inspect_requested(&self, requested: bool) -> PyResult<()> { + stdlib::sys::sys::set_inspect_requested(self, requested) + } + + pub fn next_interactive_code_filename(&self) -> PyResult { + let serial = self + .state + .interactive_code_counter + .try_update(Ordering::Relaxed, Ordering::Relaxed, |serial| { + serial.checked_add(1) + }) + .map_err(|_| self.new_overflow_error("interactive code filename counter exhausted"))?; + Ok(format!("")) + } + pub fn handle_exit_exception(&self, exc: PyBaseExceptionRef) -> u32 { + let exc = match self.report_uncaught_exception(&exc) { + Err(exit) => exit, + Ok(()) => return self.handle_reported_exception(&exc), + }; if exc.fast_isinstance(self.ctx.exceptions.system_exit) { let code = exc .as_object() @@ -3939,8 +3969,15 @@ impl VirtualMachine { } } 1 - } else if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { - self.print_exception(&exc); + } else { + self.handle_reported_exception(&exc) + } + } + + /// Apply the host exit policy after an exception has already been reported. + /// Call only after deciding not to continue into an interactive session. + pub fn handle_reported_exception(&self, exc: &Py) -> u32 { + if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { cfg_select! { unix => { if crate::host_env::signal::set_sigint_default_onstack().is_ok() { @@ -3956,7 +3993,6 @@ impl VirtualMachine { _ => 1, } } else { - self.print_exception(&exc); 1 } } diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index b8b83a240d1..9d7e0270421 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -216,6 +216,311 @@ mod file_run { } } +/// A physical stdin line or an already decoded editor line. +pub enum InteractiveLine { + Bytes(Vec), + Text(String), +} + +impl VirtualMachine { + /// Resolve the current main module without rewriting an existing loader. + pub fn stdin_main_module(&self) -> PyResult> { + let modules = self.sys_module.get_attr("modules", self)?; + if let Ok(module) = modules.get_item("__main__", self) + && let Ok(module) = module.downcast::() + { + return Ok(module); + } + self.ensure_main_module() + } + + /// Evaluate native interactive records. Input policy belongs to the caller; + /// compilation, exception handling and namespace ownership stay in the VM. + pub fn run_interactive( + &self, + mut readline: impl FnMut(&str) -> PyResult>, + ) -> PyResult<()> { + use crate::{AsObject, bytecode::CodeFlags}; + for (name, default) in [("ps1", ">>> "), ("ps2", "... ")] { + if self.sys_module.dict().get_item_opt(name, self)?.is_none() { + self.sys_module + .set_attr(name, self.ctx.new_str(default), self)?; + } + } + let mut future_features = CodeFlags::empty(); + let mut memory_errors = 0; + loop { + let mut eof = false; + let mut empty_eof = false; + let result = (|| { + let encoding = self + .sys_module + .get_attr("stdin", self) + .and_then(|stdin| stdin.get_attr("encoding", self)) + .ok() + .and_then(|encoding| encoding.downcast::().ok()) + .and_then(|encoding| encoding.to_str().map(str::to_owned)) + .unwrap_or_else(|| "utf-8".to_owned()); + let prompt = |name| { + self.sys_module + .get_attr(name, self) + .and_then(|prompt| prompt.str(self)) + .ok() + .and_then(|prompt| prompt.to_str().map(str::to_owned)) + .unwrap_or_default() + }; + let ps1 = prompt("ps1"); + let ps2 = prompt("ps2"); + self.audit("compile", || (self.ctx.none(), self.ctx.new_str("")))?; + let mut source = String::new(); + let mut raw_source = Vec::new(); + #[cfg(feature = "parser")] + let mut warnings = super::compile::InteractiveWarnings::default(); + loop { + let line = readline(if source.is_empty() { &ps1 } else { &ps2 })?; + let mut terminated = line.is_none(); + let mut unterminated_fragment = false; + match line { + None => { + eof = true; + if source.is_empty() { + empty_eof = true; + return Ok(()); + } + if !source.ends_with('\n') { + source.push('\n'); + } + } + Some(InteractiveLine::Text(mut line)) => { + if !line.ends_with('\n') { + line.push('\n'); + } + let line = line.replace("\r\n", "\n"); + terminated = line == "\n"; + source.push_str(&line); + } + Some(InteractiveLine::Bytes(line)) => { + unterminated_fragment = !line.ends_with(b"\n"); + terminated = line == b"\n" || line == b"\r\n"; + raw_source.extend_from_slice(&line); + source = self + .state + .codec_registry + .decode_text( + self.ctx.new_bytes(raw_source.clone()).into(), + &encoding, + None, + self, + )? + .to_str() + .ok_or_else(|| { + self.new_exception_msg( + self.ctx.exceptions.unicode_error.to_owned(), + "stdin source contains surrogates".into(), + ) + })? + .replace("\r\n", "\n"); + } + } + #[cfg(feature = "parser")] + self.emit_interactive_parser_warnings( + &source, + "", + Some(identifier!(self, __main__)), + Some(compiler::parser::Mode::Module), + future_features.contains(CodeFlags::FUTURE_BARRY_AS_BDFL), + super::compile::ParserWarningState { + emitted_escapes: &[], + interactive: Some(&mut warnings), + }, + ) + .map_err(|error| { + super::compile::VmCompileError::Warning(error) + .into_pyexception(self, Some(&source)) + })?; + let parsed = match compiler::interactive_input( + &source, + "", + future_features, + terminated, + ) { + Ok(Some(parsed)) => parsed, + Ok(None) => continue, + Err(error) => { + let unclosed = matches!(&error, compiler::CompileError::Parse(error) + if error.is_unclosed_bracket || error.is_unclosed_string); + let error = super::compile::VmCompileError::Compile(error) + .into_pyexception_maybe_incomplete( + self, + Some(&source), + !terminated, + ); + if !eof + && (unclosed + || (!terminated + && error.fast_isinstance( + self.ctx.exceptions.incomplete_input_error, + ))) + { + continue; + } + return Err(error); + } + }; + // A raw final fragment is not a completed physical line. + // Warnings and definitive parser errors are observable now, + // but acceptance waits for the transport's EOF transaction. + if unterminated_fragment { + continue; + } + let main = self.stdin_main_module()?; + future_features |= parsed.future_features; + return (|| { + let filename = self.next_interactive_code_filename()?; + let code = self + .compile_after_parser_warnings( + &source, + if parsed.empty { + compiler::Mode::Exec + } else { + compiler::Mode::Single + }, + filename, + compiler::CompileOpts { + future_features, + ..self.compile_opts() + }, + Some(identifier!(self, __main__)), + ) + .map_err(|error| error.into_pyexception(self, Some(&source)))?; + let linecache = self.import("linecache", 0)?; + linecache.get_attr("_register_code", self)?.call( + ( + code.clone(), + self.ctx.new_str(source.as_str()), + self.ctx.new_str(""), + ), + self, + )?; + self.audit("exec", || (code.clone(),))?; + self.run_code_obj(code, Scope::with_builtins(None, main.dict(), self))?; + Ok(()) + })() + .inspect_err(|error| { + self.repair_interactive_syntax_error_text(error, &source) + }); + } + })(); + if let Err(error) = result { + if error.fast_isinstance(self.ctx.exceptions.memory_error) { + memory_errors += 1; + if memory_errors > 16 { + return Err(error); + } + } else { + memory_errors = 0; + } + self.report_uncaught_exception(&error)?; + } else { + memory_errors = 0; + } + self.flush_io(); + if empty_eof { + return Ok(()); + } + } + } + + fn repair_interactive_syntax_error_text( + &self, + error: &crate::builtins::PyBaseExceptionRef, + source: &str, + ) { + use crate::AsObject; + + if !error.fast_isinstance(self.ctx.exceptions.syntax_error) { + return; + } + // Match the interactive run_mod boundary: repair only text and keep + // the original exception even if attribute access/conversion fails. + let source = self.ctx.new_str(source); + let lines = source.splitlines(crate::anystr::SplitLinesArgs { keepends: true }, self); + let Ok(lineno) = error + .as_object() + .get_attr("lineno", self) + .and_then(|lineno| lineno.try_into_value::(self)) + else { + return; + }; + if lineno > 0 + && let Some(line) = lines.get((lineno - 1) as usize) + { + let _ = error.as_object().set_attr("text", line.clone(), self); + } + } + + /// Native fd0 adapter used by sys._baserepl and non-terminal CLI inspection. + pub fn run_basic_interactive(&self) -> PyResult<()> { + self.run_interactive(|prompt| { + let mut line = Vec::new(); + let mut prompt = prompt; + loop { + let result = crate::readline::stdio_readline_bytes(prompt, &mut line); + prompt = ""; + match result { + Ok(0) => { + eprintln!(); + return Ok(None); + } + Ok(_) => return Ok(Some(InteractiveLine::Bytes(line))), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { + self.check_signals()?; + } + Err(_) if !line.is_empty() => return Ok(Some(InteractiveLine::Bytes(line))), + Err(_) => return Ok(None), + } + } + }) + } + + /// Evaluate ordinary piped stdin as one file, without prompts or startup. + pub fn run_stdin_file(&self) -> PyResult<()> { + use std::io::Read; + let main = self.stdin_main_module()?; + self.with_simple_run("", |_| { + self.audit("compile", || (self.ctx.none(), self.ctx.new_str("")))?; + let mut bytes = Vec::new(); + loop { + let result = std::io::stdin().lock().read_to_end(&mut bytes); + match result { + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { + self.check_signals()? + } + _ => break, + } + } + #[cfg(feature = "parser")] + let source = self.decode_source_bytes(&bytes, "", false)?; + #[cfg(not(feature = "parser"))] + let source = + String::from_utf8(bytes).map_err(|error| self.new_os_error(error.to_string()))?; + let code = self + .compile_with_opts_and_module( + &source, + compiler::Mode::Exec, + "", + self.compile_opts(), + Some(identifier!(self, __main__)), + &[], + ) + .map_err(|error| error.into_pyexception(self, Some(&source)))?; + self.audit("exec", || (code.clone(),))?; + self.run_code_obj(code, Scope::with_builtins(None, main.dict(), self)) + .map(drop) + }) + } +} + #[cfg(test)] mod tests { use crate::object::AsObject; diff --git a/src/lib.rs b/src/lib.rs index f5445fdc563..09df89f682a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,8 +121,8 @@ pub fn run(mut builder: InterpreterBuilder) -> ExitCode { let interp = builder.interpreter(); let exitcode = cfg_select! { - feature = "capi" => rustpython_capi::run_main_interpreter(interp, move |vm| run_rustpython(vm, run_mode)), - _ => interp.run(move |vm| run_rustpython(vm, run_mode)), + feature = "capi" => rustpython_capi::run_main_interpreter_with_exit_status(interp, move |vm| run_rustpython(vm, run_mode)), + _ => interp.run_with_exit_status(move |vm| run_rustpython(vm, run_mode)), }; rustpython_vm::host_env::os::exit_code(exitcode) @@ -289,7 +289,8 @@ fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult } // pymain_run_python -fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { +fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult { + use vm::vm::InterpreterExit; #[cfg(feature = "flame-it")] let main_guard = flame::start_guard("RustPython main"); @@ -368,15 +369,30 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { debug!("Running script {script_path}"); run_file(vm, scope.clone(), &script_path) } - RunMode::Repl => Ok(()), + RunMode::Repl => run_interactive(vm, scope.clone(), true), }; - let result = if is_repl || vm.state.config.settings.inspect { - if std::io::stdin().is_terminal() { - warn_if_pyrepl_unavailable(vm); - } - shell::run_shell(vm, scope) + let result = if is_repl { + res.map(|()| InterpreterExit::Code(0)) } else { - res + let reported = if let Err(error) = res { + vm.report_uncaught_exception(&error)?; + Some(error) + } else { + None + }; + if !vm.inspect_requested() + && !vm.state.config.settings.ignore_environment + && env::var_os("PYTHONINSPECT").is_some_and(|value| !value.is_empty()) + { + vm.set_inspect_requested(true)?; + } + if vm.inspect_requested() + && (std::io::stdin().is_terminal() || vm.state.config.settings.interactive) + { + run_interactive(vm, scope, false).map(|()| InterpreterExit::Code(0)) + } else { + Ok(reported.map_or(InterpreterExit::Code(0), InterpreterExit::ReportedException)) + } }; #[cfg(feature = "flame-it")] @@ -390,21 +406,111 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { result } -/// When the fancy REPL cannot start (for example `TERM=dumb`), print the same -/// warning `_pyrepl.main` would have printed, then keep the rustyline shell. -fn warn_if_pyrepl_unavailable(vm: &VirtualMachine) { - let _ = vm.run_simple_string( - r" -import os, sys -if not os.getenv('PYTHON_BASIC_REPL'): - try: - from _pyrepl.main import CAN_USE_PYREPL, FAIL_REASON - if not CAN_USE_PYREPL and FAIL_REASON: - print(FAIL_REASON, file=sys.stderr) - except Exception: - pass -", - ); +// Follow the CLI entry points in CPython's Modules/main.c: initial +// interactive startup runs before the hook; inspection runs startup in pyrepl. +fn run_interactive(vm: &VirtualMachine, scope: Scope, initial: bool) -> PyResult<()> { + let tty = std::io::stdin().is_terminal(); + let interactive = tty || vm.state.config.settings.interactive; + if interactive { + vm.set_inspect_requested(false)?; + } + if interactive + && initial + && !vm.state.config.settings.ignore_environment + && let Err(exc) = run_interactive_startup(vm, scope.clone()) + { + if exc.fast_isinstance(vm.ctx.exceptions.system_exit) { + return Err(exc); + } + vm.report_uncaught_exception(&exc)?; + } + if interactive && let Err(exc) = run_interactive_hook(vm) { + writeln!( + vm::stdlib::sys::PyStderr(vm), + "Failed calling sys.__interactivehook__" + ); + if exc.fast_isinstance(vm.ctx.exceptions.system_exit) { + return Err(exc); + } + vm.report_uncaught_exception(&exc)?; + } + vm.check_signals()?; + vm.audit("cpython.run_stdin", || ())?; + let basic = !vm.state.config.settings.ignore_environment + && env::var_os("PYTHON_BASIC_REPL").is_some_and(|value| !value.is_empty()); + if tty && !basic { + let available = match vm.import("_pyrepl", 0) { + Ok(_) => true, + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => false, + Err(exc) => return Err(exc), + }; + if available { + let package = vm.import("_pyrepl.main", 0)?; + let pyrepl = package.get_attr("main", vm)?; + let console = pyrepl.get_attr("interactive_console", vm)?; + let kwargs: vm::function::KwArgs = [ + ("mainmodule".to_owned(), vm.stdin_main_module()?.into()), + ("pythonstartup".to_owned(), vm.ctx.new_bool(!initial).into()), + ] + .into_iter() + .collect(); + return console + .call(vm::function::FuncArgs::new(vec![], kwargs), vm) + .map(drop); + } + } + if !interactive { + return vm.run_stdin_file(); + } + shell::run_shell(vm, scope) +} + +fn run_interactive_hook(vm: &VirtualMachine) -> PyResult<()> { + if let Some(hook) = vm + .sys_module + .dict() + .get_item_opt("__interactivehook__", vm)? + { + vm.audit("cpython.run_interactivehook", || (hook.clone(),))?; + hook.call((), vm)?; + } + Ok(()) +} + +fn run_interactive_startup(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { + let Some(path) = env::var_os("PYTHONSTARTUP").filter(|value| !value.is_empty()) else { + return Ok(()); + }; + let path = vm.fsdecode(path); + vm.audit("cpython.run_startup", || (path.clone(),))?; + #[cfg(feature = "host_env")] + if let Some(filename) = path.to_str() { + // Opening precedes __main__ setup in pymain_run_startup. In + // particular, a missing startup file must not replace its loader. + if let Err(err) = vm::host_env::fs::open(filename) { + use vm::convert::ToPyException; + writeln!( + vm::stdlib::sys::PyStderr(vm), + "Could not open PYTHONSTARTUP" + ); + let exc = err.to_pyexception(vm); + exc.as_object().set_attr("filename", path.clone(), vm)?; + return Err(exc); + } + return vm.run_any_file(scope, filename); + } + let tokenize = vm.import("tokenize", 0)?; + let file = tokenize.get_attr("open", vm)?.call((path.clone(),), vm)?; + let source = file.get_attr("read", vm)?.call((), vm); + file.get_attr("close", vm)?.call((), vm)?; + let code = vm + .builtins + .get_attr("compile", vm)? + .call((source?, path, "exec"), vm)?; + vm.builtins + .get_attr("exec", vm)? + .call((code, scope.globals), vm) + .map(drop) } #[cfg(feature = "flame-it")] diff --git a/src/shell.rs b/src/shell.rs index 0d5ab590faf..9413a2e05bf 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,238 +1,42 @@ mod helper; -use rustpython_compiler::{ - CompileError, ParseError, parser::InterpolatedStringErrorType, parser::LexicalErrorType, - parser::ParseErrorType, -}; use rustpython_vm::{ - AsObject, PyResult, VirtualMachine, - builtins::PyBaseExceptionRef, - bytecode::CodeFlags, - compiler::{self}, + PyResult, VirtualMachine, readline::{Readline, ReadlineResult}, scope::Scope, - vm::VmCompileError, + vm::InteractiveLine, }; -enum ShellExecResult { - Ok, - PyErr(PyBaseExceptionRef), - ContinueBlock, - ContinueLine, -} - -fn shell_exec( - vm: &VirtualMachine, - source: &str, - scope: Scope, - empty_line_given: bool, - continuing_block: bool, - future_features: &mut CodeFlags, -) -> ShellExecResult { - // compiling expects only UNIX style line endings, and will replace windows line endings - // internally. Since we might need to analyze the source to determine if an error could be - // resolved by future input, we need the location from the error to match the source code that - // was actually compiled. - #[cfg(windows)] - let source = &source.replace("\r\n", "\n"); - let opts = compiler::CompileOpts { - future_features: *future_features, - ..vm.compile_opts() - }; - match vm.compile_with_opts(source, compiler::Mode::Single, "", opts) { - Ok(code) => { - *future_features |= code.code.flags & CodeFlags::FUTURE_MASK; - let _ = vm.register_code_in_linecache(&code, source); - if empty_line_given || !continuing_block { - match vm.run_code_obj(code, scope) { - Ok(_val) => ShellExecResult::Ok, - Err(err) => ShellExecResult::PyErr(err), - } - } else { - // We can just return an ok result - ShellExecResult::Ok - } - } - Err(err) => { - if matches!( - &err, - VmCompileError::Compile(CompileError::Parse( - ParseError { - error: ParseErrorType::Lexical( - LexicalErrorType::Eof - | LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, - ) - | LexicalErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, - ) - | LexicalErrorType::UnclosedStringError { - triple_quoted: true, - .. - } - ), - .. - } | ParseError { - is_unclosed_bracket: true, - .. - } - )) - ) { - return ShellExecResult::ContinueLine; - } - - // An unfinished suite is _IncompleteInputError, not IndentationError. - let exc = err.into_pyexception_maybe_incomplete(vm, Some(source), !empty_line_given); - if !empty_line_given && exc.fast_isinstance(vm.ctx.exceptions.incomplete_input_error) { - ShellExecResult::ContinueBlock - } else { - ShellExecResult::PyErr(exc) - } - } - } -} - -fn flush_stdio(vm: &VirtualMachine) { - for name in ["stdout", "stderr"] { - if let Ok(stream) = vm.sys_module.get_attr(name, vm) { - let _ = vm.call_method(&stream, "flush", ()); - } - } -} - -/// Enter a repl loop +/// Root-only editor/history policy, adapting into the VM's shared evaluator. pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { - let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals.clone())); - let mut full_input = String::new(); - - // Retrieve a `history_path_str` dependent on the OS - let repl_history_path = match dirs::config_dir() { - Some(mut path) => { + use std::io::IsTerminal; + if !std::io::stdin().is_terminal() { + return vm.run_basic_interactive(); + } + let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals)); + let history = dirs::config_dir().map_or_else( + || ".repl_history.txt".into(), + |mut path| { path.push("rustpython"); path.push("repl_history.txt"); path + }, + ); + let _ = repl.load_history(&history); + let result = vm.run_interactive(|prompt| match repl.readline(prompt) { + ReadlineResult::Line(line) => { + let _ = repl.add_history_entry(line.trim_end()); + Ok(Some(InteractiveLine::Text(line))) } - None => ".repl_history.txt".into(), - }; - - let _ = repl.load_history(&repl_history_path); - - // We might either be waiting to know if a block is complete, or waiting to know if a multiline - // statement is complete. In the former case, we need to ensure that we read one extra new line - // to know that the block is complete. In the latter, we can execute as soon as the statement is - // valid. - let mut continuing_block = false; - let mut continuing_line = false; - let mut future_features = CodeFlags::empty(); - - loop { - let prompt_name = if continuing_block || continuing_line { - "ps2" - } else { - "ps1" - }; - let prompt = vm - .sys_module - .get_attr(prompt_name, vm) - .and_then(|prompt| prompt.str(vm)); - let prompt = match prompt { - Ok(ref s) => s.expect_str(), - Err(_) => "", - }; - - continuing_line = false; - let result = match repl.readline(prompt) { - ReadlineResult::Line(line) => { - #[cfg(debug_assertions)] - debug!("You entered {line:?}"); - - repl.add_history_entry(line.trim_end()).unwrap(); - - let empty_line_given = line.is_empty(); - - if full_input.is_empty() { - full_input = line; - } else { - full_input.push_str(&line); - } - full_input.push('\n'); - - match shell_exec( - vm, - &full_input, - scope.clone(), - empty_line_given, - continuing_block, - &mut future_features, - ) { - ShellExecResult::Ok => { - if continuing_block { - if empty_line_given { - // We should exit continue mode since the block successfully executed - continuing_block = false; - full_input.clear(); - } - } else { - // We aren't in continue mode so proceed normally - full_input.clear(); - } - Ok(()) - } - // Continue, but don't change the mode - ShellExecResult::ContinueLine => { - continuing_line = true; - Ok(()) - } - ShellExecResult::ContinueBlock => { - continuing_block = true; - Ok(()) - } - ShellExecResult::PyErr(err) => { - continuing_block = false; - full_input.clear(); - Err(err) - } - } - } - ReadlineResult::Interrupt => { - continuing_block = false; - full_input.clear(); - let keyboard_interrupt = - vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned()); - Err(keyboard_interrupt) - } - ReadlineResult::Eof => { - break; - } - #[cfg(unix)] - ReadlineResult::OsError(num) => { - let os_error = vm.new_exception_msg( - vm.ctx.exceptions.os_error.to_owned(), - format!("{num:?}").into(), - ); - vm.print_exception(&os_error); - break; - } - ReadlineResult::Other(err) => { - eprintln!("Readline error: {err:?}"); - break; - } - ReadlineResult::Io(err) => { - eprintln!("IO error: {err:?}"); - break; - } - }; - - if let Err(exc) = result { - if exc.fast_isinstance(vm.ctx.exceptions.system_exit) { - let _ = repl.save_history(&repl_history_path); - return Err(exc); - } - vm.print_exception(&exc); + ReadlineResult::Eof => Ok(None), + ReadlineResult::Interrupt => { + Err(vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned())) } - flush_stdio(vm); - } - let _ = repl.save_history(&repl_history_path); - - Ok(()) + ReadlineResult::Io(_) => Ok(None), + #[cfg(unix)] + ReadlineResult::OsError(_) => Ok(None), + ReadlineResult::Other(_) => Ok(None), + }); + let _ = repl.save_history(&history); + result } From 7e1fe213bcd9c33bbf47264e3a8e9af456016fc8 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 15:29:14 +0900 Subject: [PATCH 05/19] Refresh canonical REPL tests and retire fixed failures Run update_lib migrate from pinned CPython 3.15.0, preserving three remaining platform/runtime markers and retiring the two traceback/startup failures verified on the native evaluator. Assisted-by: OpenAI Codex:model version unavailable --- Lib/test/test_repl.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 71dab8c3876..a0abcc0f0ba 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -17,7 +17,6 @@ SHORT_TIMEOUT, ) from test.support.script_helper import kill_python -from test.support.import_helper import import_module try: import pty @@ -40,8 +39,8 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, custom=F # line option '-i' and the process name set to ''. # The directory of argv[0] must match the directory of the Python # executable for the Popen() call to python to succeed as the directory - # path may be used by Py_GetPath() to build the default module search - # path. + # path may be used by PyConfig_Get("module_search_paths") to build the + # default module search path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "") cmd_line = [stdin_fname] # Isolated mode implies -EPs and ignores PYTHON* variables. @@ -99,12 +98,8 @@ def run_on_interactive_mode(source): @support.force_not_colorized_test_class class TestInteractiveInterpreter(unittest.TestCase): - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @support.nomemtest def test_no_memory(self): - import_module("_testcapi") # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive # loop when an exception is raised. @@ -189,7 +184,6 @@ def test_close_stdin(self): self.assertEqual(process.returncode, 0) self.assertIn('before close', output) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 != 0 def test_interactive_traceback_reporting(self): user_input = "1 / 0 / 3 / 4" p = spawn_repl() @@ -232,7 +226,6 @@ def foo(x): ] self.assertEqual(traceback_lines, expected_lines) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pythonstartup_error_reporting(self): # errors based on https://github.com/python/cpython/issues/137576 @@ -303,7 +296,7 @@ def test_runsource_show_syntax_error_location(self): expected_lines = [ ' def f(x, x): ...', ' ^', - "SyntaxError: duplicate argument 'x' in function definition" + "SyntaxError: duplicate parameter 'x' in function definition" ] self.assertEqual(output.splitlines()[4:-1], expected_lines) @@ -443,6 +436,13 @@ def test_toplevel_contextvars_async(self): p = spawn_asyncio_repl() p.stdin.write(user_input) user_input2 = "async def set_var(): var.set('ok')\n" + try: + import _pyrepl # noqa: F401 + except ModuleNotFoundError: + # If we're going to be forced into the regular REPL, then we need an + # extra newline here. Omit it by default to catch any breakage to + # the new REPL's behavior. + user_input2 += "\n" p.stdin.write(user_input2) user_input3 = "await set_var()\n" p.stdin.write(user_input3) From bddb59f79c8f4f8801c67a1d99b193c79c333be0 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 16:04:44 +0900 Subject: [PATCH 06/19] Keep the root compiler dependency for development targets The native evaluator now compiles through the VM. Only root benchmarks and the parse_folder example use rustpython-compiler directly, so declare it as a dev dependency while preserving VM compiler features. Assisted-by: OpenAI Codex:model version unavailable --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index eff494509e9..0b9c6af8d60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,6 @@ winresource = "0.1" [dependencies] rustpython-capi = { workspace = true, optional = true } -rustpython-compiler = { workspace = true } rustpython-pylib = { workspace = true, optional = true } rustpython-stdlib = { workspace = true, optional = true, default-features = false, features = ["compiler"] } rustpython-vm = { workspace = true, features = ["compiler", "gc"] } @@ -62,6 +61,7 @@ rustyline = { workspace = true } mimalloc = { version = "0.1", optional = true, default-features = false } [dev-dependencies] +rustpython-compiler = { workspace = true } criterion = { workspace = true } pyo3 = { workspace = true, features = ["auto-initialize"] } rustls-graviola = { workspace = true } From 371728d22815522c14e297aa06e89a3b7ba07cf1 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 16:31:31 +0900 Subject: [PATCH 07/19] Validate explicit conditional failure marker retirement Require exact selector and original decorator evidence for each conditional expected-failure marker retired by migrate. Preserve undeclared guards and reject missing, duplicate, modified or unconsumed declarations. Assisted-by: OpenAI Codex:model version unavailable --- scripts/update_lib/cmd_migrate.py | 7 +- scripts/update_lib/patch_spec.py | 81 +++++++++++++++-- scripts/update_lib/tests/test_migrate.py | 108 +++++++++++++++++++++++ 3 files changed, 189 insertions(+), 7 deletions(-) diff --git a/scripts/update_lib/cmd_migrate.py b/scripts/update_lib/cmd_migrate.py index c509dcfb769..f2db1bf0cf3 100644 --- a/scripts/update_lib/cmd_migrate.py +++ b/scripts/update_lib/cmd_migrate.py @@ -856,7 +856,12 @@ def patch_single_content( if patches_document is not None: from update_lib.patch_spec import select_remaining_patches - patches = select_remaining_patches(patches, patches_document, relative_path) + patches = select_remaining_patches( + patches, + patches_document, + relative_path, + old_content=old_content if lib_path.exists() else None, + ) # Apply patches to source content src_content = src_path.read_text(encoding="utf-8") diff --git a/scripts/update_lib/patch_spec.py b/scripts/update_lib/patch_spec.py index 4dee842a8f7..35828e685d8 100644 --- a/scripts/update_lib/patch_spec.py +++ b/scripts/update_lib/patch_spec.py @@ -551,9 +551,65 @@ def temp_attr(obj: object, attr: str, value: object): setattr(obj, attr, old) -def select_remaining_patches(current, document, relative_path: Path): +def _conditional_retirements(document, old_content): + entries = document.get("retire_conditional", []) + if not isinstance(entries, list): + raise ValueError("Conditional retirements must be a list") + if not entries: + return {} + if not isinstance(old_content, str): + raise ValueError("Conditional retirements require original source") + tree = ast.parse(old_content) + lines = old_content.splitlines(keepends=True) + result = {} + for entry in entries: + if not isinstance(entry, dict) or set(entry) != { + "class", + "method", + "decorator", + }: + raise ValueError("Invalid conditional retirement metadata") + if not all(isinstance(value, str) and value for value in entry.values()): + raise ValueError("Conditional retirement fields must be nonempty strings") + key = (entry["class"], entry["method"]) + if key in result: + raise ValueError("Duplicate conditional retirement") + classes = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) and node.name == key[0] + ] + if len(classes) != 1: + raise ValueError("Missing or ambiguous conditional retirement class") + methods = [ + node for node in _iter_class_methods(classes[0]) if node.name == key[1] + ] + if len(methods) != 1: + raise ValueError("Missing or ambiguous conditional retirement method") + matches = [ + node + for node in methods[0].decorator_list + if "".join(lines[node.lineno - 1 : node.end_lineno]) == entry["decorator"] + ] + if len(matches) != 1: + raise ValueError( + "Original conditional decorator does not match exactly once" + ) + marker = PatchSpec.try_from_ast_node(matches[0], old_content.splitlines()) + if marker is None or marker.ut_method != UtMethod.ExpectedFailureIf: + raise ValueError( + "Only explicit expectedFailureIf retirements are supported" + ) + result[key] = marker + return result + + +def select_remaining_patches( + current, document, relative_path: Path, *, old_content=None +): if not isinstance(document, dict) or ( - set(document) not in ({"target", "patches"}, {"target", "patches", "doctests"}) + not {"target", "patches"} <= set(document) + or not set(document) <= {"target", "patches", "doctests", "retire_conditional"} ): raise ValueError("Expected target and patches metadata") if ( @@ -568,6 +624,8 @@ def select_remaining_patches(current, document, relative_path: Path): remaining = patches_from_json(document["patches"]) except (TypeError, KeyError, AttributeError, ValueError) as error: raise ValueError("Invalid remaining marker metadata") from error + retirements = _conditional_retirements(document, old_content) + consumed = set() for cls, methods in remaining.items(): if cls not in current or any(method not in current[cls] for method in methods): raise ValueError("Marker override cannot add targets") @@ -584,10 +642,21 @@ def select_remaining_patches(current, document, relative_path: Path): omitted.extend(pending[:index]) pending = pending[index + 1 :] omitted.extend(pending) - if any(marker.ut_method != UtMethod.ExpectedFailure for marker in omitted): - raise ValueError( - "Only unconditional expectedFailure markers may be removed" - ) + for marker in omitted: + if marker.ut_method == UtMethod.ExpectedFailure: + continue + key = (cls, method) + if ( + key in consumed + or retirements.get(key) != marker + or existing.count(marker) != 1 + ): + raise ValueError( + "Conditional removal requires an exact explicit retirement" + ) + consumed.add(key) + if consumed != set(retirements): + raise ValueError("Conditional retirement was not removed exactly once") return remaining diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py index 1baf66538cf..1a138742012 100644 --- a/scripts/update_lib/tests/test_migrate.py +++ b/scripts/update_lib/tests/test_migrate.py @@ -1246,6 +1246,114 @@ def test_added_changed_or_reordered_markers_fail(self): select_remaining_patches(current, document, Path(document["target"])) +class ConditionalMarkerOverrideTests(unittest.TestCase): + decorator = ( + ' @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON")\n' + ) + + def fixture(self): + old = OVERRIDE_CLEAN.replace( + " def test_known", self.decorator + " def test_known" + ) + current = extract_patches(old) + document = { + "target": "Lib/test/test_example.py", + "patches": {}, + "retire_conditional": [ + { + "class": "Example", + "method": "test_known", + "decorator": self.decorator, + } + ], + } + return old, current, document + + def select(self, old, current, document): + return select_remaining_patches( + current, + document, + Path(document["target"]), + old_content=old, + ) + + def test_explicit_conditional_retirement(self): + old, current, document = self.fixture() + self.assertEqual(self.select(old, current, document), {}) + + def test_omitted_conditional_without_authorization_fails(self): + old, current, document = self.fixture() + del document["retire_conditional"] + with self.assertRaises(ValueError): + self.select(old, current, document) + + def test_absent_duplicate_modified_and_retained_fail_closed(self): + for change in ( + "absent", + "duplicate", + "text", + "condition", + "retained", + "duplicate_source", + ): + old, current, document = self.fixture() + with self.subTest(change=change), self.assertRaises(ValueError): + if change == "absent": + document["retire_conditional"][0]["method"] = "missing" + elif change == "duplicate": + document["retire_conditional"] *= 2 + elif change == "text": + document["retire_conditional"][0]["decorator"] = ( + self.decorator.replace('"', "'") + ) + elif change == "condition": + old = old.replace("not support.is_android", "support.is_android") + current = extract_patches(old) + elif change == "retained": + document["patches"] = patches_to_json(current) + elif change == "duplicate_source": + old = old.replace(self.decorator, self.decorator * 2) + current = extract_patches(old) + self.select(old, current, document) + + def test_other_guards_and_original_source_required(self): + for replacement in ("skipIf", "expectedFailureIfWindows"): + old, current, document = self.fixture() + old = old.replace("expectedFailureIf", replacement) + document["retire_conditional"][0]["decorator"] = self.decorator.replace( + "expectedFailureIf", replacement + ) + with self.subTest(replacement=replacement), self.assertRaises(ValueError): + self.select(old, extract_patches(old), document) + old, current, document = self.fixture() + with self.assertRaises(ValueError): + self.select(None, current, document) + + def test_cli_conditional_retirement_preserves_remaining_markers(self): + old = OVERRIDE_OLD.replace( + " @unittest.expectedFailure # TODO: RUSTPYTHON; old failure\n", + self.decorator, + ) + + def authorize(document): + document["retire_conditional"] = [ + { + "class": "Example", + "method": "test_known", + "decorator": self.decorator, + } + ] + + with patch(__name__ + ".OVERRIDE_OLD", old): + status, result = MarkerOverrideTests().invoke(authorize) + self.assertEqual(status, 0) + self.assertNotIn("expectedFailureIf", result) + self.assertIn('unittest.skip("TODO: RUSTPYTHON; retained guard")', result) + status, result = MarkerOverrideTests().invoke() + self.assertEqual(status, 1) + self.assertEqual(result, old) + + PACKAGE_MARKED = """import unittest class Example: @unittest.expectedFailure # TODO: RUSTPYTHON; retained reason From 0efd645683d4b3b04b1c400e663fbc5739aa62c9 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 16:36:01 +0900 Subject: [PATCH 08/19] Refresh canonical stdin and exception audit tests Migrate the pinned CPython 3.15.0 modules and their canonical helper fixtures through update_lib. Retire exactly four failure markers proven obsolete by hosted CI, preserving all other existing guards. Assisted-by: OpenAI Codex:model version unavailable --- Lib/test/audit-tests.py | 94 ++++++++++++++++++++++ Lib/test/audit_test_data/__init__.py | 0 Lib/test/audit_test_data/submodule.py | 0 Lib/test/audit_test_data/submodule2.py | 0 Lib/test/test_audit.py | 10 ++- Lib/test/test_cmd_line.py | 107 +++++++++++++++++++++++-- Lib/test/test_runpy.py | 77 +++++++++++++----- 7 files changed, 264 insertions(+), 24 deletions(-) create mode 100644 Lib/test/audit_test_data/__init__.py create mode 100644 Lib/test/audit_test_data/submodule.py create mode 100644 Lib/test/audit_test_data/submodule2.py diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 6884ac0dbe6..8be5bf8aa4f 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -8,6 +8,8 @@ import contextlib import os import sys +import unittest.mock +from test.support import swap_item class TestHook: @@ -206,6 +208,16 @@ def rl(name): else: return None + try: + import _remote_debugging + except ImportError: + _remote_debugging = None + + def rd(name): + if _remote_debugging: + return getattr(_remote_debugging, name, None) + return None + # Try a range of "open" functions. # All of them should fail with TestHook(raise_on_events={"open"}) as hook: @@ -223,6 +235,8 @@ def rl(name): (rl("append_history_file"), 0, None), (rl("read_init_file"), testfn), (rl("read_init_file"), None), + (rd("BinaryWriter"), testfn, 1000, 0), + (rd("BinaryReader"), testfn), ]: if not fn: continue @@ -256,6 +270,8 @@ def rl(name): ("~/.history", "a") if rl("append_history_file") else None, (testfn, "r") if readline else None, ("", "r") if readline else None, + (testfn, "wb") if rd("BinaryWriter") else None, + (testfn, "rb") if rd("BinaryReader") else None, ] if i is not None ], @@ -672,6 +688,84 @@ def hook(event, args): assertEqual(event_script_path, tmp_file.name) assertEqual(remote_event_script_path, tmp_file.name) +def test_import_module(): + import importlib + + with TestHook() as hook: + importlib.import_module("importlib") # already imported, won't get logged + importlib.import_module("email") # standard library module + importlib.import_module("pythoninfo") # random module + importlib.import_module(".audit_test_data.submodule", "test") # relative import + importlib.import_module("test.audit_test_data.submodule2") # absolute import + importlib.import_module("_testcapi") # extension module + + actual = [a for e, a in hook.seen if e == "import"] + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + +def test_builtin__import__(): + import importlib # noqa: F401 + + with TestHook() as hook: + __import__("importlib") + __import__("email") + __import__("pythoninfo") + __import__("audit_test_data.submodule", level=1, globals={"__package__": "test"}) + __import__("test.audit_test_data.submodule2") + __import__("_testcapi") + + actual = [a for e, a in hook.seen if e == "import"] + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + +def test_import_statement(): + import importlib # noqa: F401 + # Set __package__ so relative imports work + with swap_item(globals(), "__package__", "test"): + with TestHook() as hook: + import importlib # noqa: F401 + import email # noqa: F401 + import pythoninfo # noqa: F401 + from .audit_test_data import submodule # noqa: F401 + import test.audit_test_data.submodule2 # noqa: F401 + import _testcapi # noqa: F401 + + actual = [a for e, a in hook.seen if e == "import"] + # Import statement ordering is different because the package is + # loaded first and then the submodule + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/audit_test_data/__init__.py b/Lib/test/audit_test_data/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/audit_test_data/submodule.py b/Lib/test/audit_test_data/submodule.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/audit_test_data/submodule2.py b/Lib/test/audit_test_data/submodule2.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 690a6e7434e..2a43c8f2663 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -94,7 +94,6 @@ def test_posixsubprocess(self): import_helper.import_module("_posixsubprocess") self.do_test("test_posixsubprocess") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_excepthook(self): returncode, events, stderr = self.run_python("test_excepthook") if not returncode: @@ -345,5 +344,14 @@ def test_sys_remote_exec(self): if returncode: self.fail(stderr) + def test_import_module(self): + self.do_test("test_import_module") + + def test_builtin__import__(self): + self.do_test("test_builtin__import__") + + def test_import_statement(self): + self.do_test("test_import_statement") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index d6fcfa3b471..c7716fb7191 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -2,6 +2,7 @@ # Most tests are executed with environment variables ignored # See test_cmd_line_script.py for testing of script execution +import locale import os import re import subprocess @@ -32,6 +33,17 @@ def _kill_python_and_exit_code(p): return data, returncode +def presite_func(): + print("presite func") + +class Namespace: + pass + +presite = Namespace() +presite.attr = Namespace() +presite.attr.func = presite_func + + class CmdLineTest(unittest.TestCase): def test_directories(self): assert_python_failure('.') @@ -46,6 +58,7 @@ def verify_valid_flag(self, cmd_line): return out @support.cpython_only + @support.force_not_colorized def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -57,6 +70,7 @@ def test_help(self): self.assertLess(len(lines), 50) @support.cpython_only + @support.force_not_colorized def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) @@ -70,6 +84,7 @@ def test_help_env(self): "env vars should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) @@ -78,6 +93,7 @@ def test_help_xoptions(self): "options should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -89,6 +105,25 @@ def test_help_all(self): # but the rest should be ASCII-only b''.join(lines[1:]).decode('ascii') + @support.cpython_only + @support.force_colorized + def test_help_colorized(self): + rc, out, err = assert_python_ok("--help", FORCE_COLOR="1") + # Check ANSI color codes are present + self.assertIn(b"\x1b[", out) + # Check that key text elements are still present + self.assertIn(b"usage:", out) + self.assertIn(b"-h", out) + self.assertIn(b"--help-all", out) + self.assertIn(b"cmd", out) + self.assertIn(b"Arguments:", out) + + @support.cpython_only + @support.force_not_colorized + def test_help_not_colorized(self): + rc, out, err = assert_python_ok("--help") + self.assertNotIn(b"\x1b[", out) + def test_optimize(self): self.verify_valid_flag('-O') self.verify_valid_flag('-OO') @@ -322,6 +357,10 @@ def run_utf8_mode(arg): cmd = [sys.executable, '-X', 'utf8', '-c', code, arg] return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + def run_no_utf8_mode(arg): + cmd = [sys.executable, '-X', 'utf8=0', '-c', code, arg] + return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + valid_utf8 = 'e:\xe9, euro:\u20ac, non-bmp:\U0010ffff'.encode('utf-8') # invalid UTF-8 byte sequences with a valid UTF-8 sequence # in the middle. @@ -334,8 +373,27 @@ def run_utf8_mode(arg): ) test_args = [valid_utf8, invalid_utf8] - for run_cmd in (run_default, run_c_locale, run_utf8_mode): - with self.subTest(run_cmd=run_cmd): + for run_cmd, encoding in ( + (run_default, sys.getfilesystemencoding()), + (run_c_locale, None), + (run_utf8_mode, None), + (run_no_utf8_mode, locale.getencoding()) + ): + with self.subTest(run_cmd=run_cmd.__name__): + # Arbitrary bytes round-trip through surrogateescape only in + # UTF-8 and single-byte encodings, not in a multibyte encoding + # such as EUC-JP. + if encoding is not None: + try: + lossless = len(bytes(range(256)).decode( + encoding, 'surrogateescape')) == 256 + except UnicodeError: + lossless = False + else: + lossless = True + if not lossless: + self.skipTest(f'{encoding} cannot losslessly ' + f'round-trip arbitrary bytes') for arg in test_args: proc = run_cmd(arg) self.assertEqual(proc.stdout.rstrip(), ascii(arg)) @@ -433,7 +491,6 @@ def test_empty_PYTHONPATH_issue16309(self): path = ":".join(sys.path) path = path.encode("ascii", "backslashreplace") sys.stdout.buffer.write(path)""" - # TODO: RUSTPYTHON we must unset RUSTPYTHONPATH as well rc1, out1, err1 = assert_python_ok('-c', code, PYTHONPATH="", RUSTPYTHONPATH="") rc2, out2, err2 = assert_python_ok('-c', code, __isolated=False) # regarding to Posix specification, outputs should be equal @@ -509,6 +566,7 @@ def test_unmached_quote(self): self.assertEqual(b'', out) @unittest.expectedFailure # TODO: RUSTPYTHON + @force_not_colorized def test_stdout_flush_at_shutdown(self): # Issue #5319: if stdout.flush() fails at shutdown, an error should # be printed out. @@ -1066,6 +1124,7 @@ def test_argv0_normalization(self): self.assertEqual(proc.stdout.strip(), b'0') @support.cpython_only + @support.force_not_colorized def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, @@ -1226,6 +1285,24 @@ def test_import_time(self): assert_python_failure('-X', 'importtime=-1', '-c', code) assert_python_failure('-X', 'importtime=3', '-c', code) + def test_import_time_unencodable_module_name(self): + code = textwrap.dedent(""" + import sys, types + name = 'mod\\ud800' + sys.modules[name] = types.ModuleType(name) + __import__(name) + try: + __import__('nonexistent\\ud800') + except ModuleNotFoundError: + pass + """) + res = assert_python_ok('-X', 'importtime=2', '-c', code) + res_err = res.err.decode('utf-8') + self.assertRegex(res_err, + r'import time: cached\s* \| cached\s* \| mod\\ud800') + self.assertRegex(res_err, + r'import time: \s*\d+ \| \s*\d+ \| \s*nonexistent\\ud800') + def res2int(self, res): out = res.out.strip().decode("utf-8") return tuple(int(i) for i in out.split()) @@ -1282,6 +1359,28 @@ def test_invalid_thread_local_bytecode(self): rc, out, err = assert_python_failure(PYTHON_TLBC="2") self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + @unittest.skipUnless(support.Py_DEBUG, + '-X presite requires a Python debug build') + def test_presite(self): + entrypoint = "test.test_cmd_line:presite_func" + proc = assert_python_ok("-X", f"presite={entrypoint}", "-c", "pass") + self.assertEqual(proc.out.rstrip(), b"presite func") + + entrypoint = "test.test_cmd_line:presite.attr.func" + proc = assert_python_ok("-X", f"presite={entrypoint}", "-c", "pass") + self.assertEqual(proc.out.rstrip(), b"presite func") + + def test_dump_path_config(self): + # gh-151253: At the first import (import encodings) during Python + # startup, if the import fails, dump the Python path configuration. + nonexistent = '/nonexistent-python-path' + # Use -X frozen_modules=off to disable frozen encodings module + # on release build. + cmd = ["-X", "frozen_modules=off", "-c", "pass"] + proc = assert_python_failure(*cmd, PYTHONHOME=nonexistent) + self.assertIn(b'Python path configuration:', proc.err) + self.assertIn(f"PYTHONHOME = '{nonexistent}'".encode(), proc.err) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') @@ -1330,11 +1429,9 @@ def check_string(self, code): self.assertNotEqual(proc.stderr, None) self.assertIn(b"\nSyntaxError", proc.stderr) - @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON") def test_tokenizer_error_with_stdin(self): self.check_string(b"(1+2+3") - @unittest.expectedFailureIf(not support.is_android, "TODO: RUSTPYTHON") def test_decoding_error_at_the_end_of_the_line(self): self.check_string(br"'\u1f'") diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 0fb7f73da11..55b9673ef6c 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -20,9 +20,11 @@ requires_subprocess, verbose, ) +from test import support from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script +from test.test_importlib.util import temporary_pycache_prefix import runpy @@ -55,7 +57,6 @@ def f(): implicit_namespace = { "__name__": None, "__file__": None, - "__cached__": None, "__package__": None, "__doc__": None, "__spec__": None @@ -303,7 +304,6 @@ def _del_pkg(self, top): def _fix_ns_for_legacy_pyc(self, ns, alter_sys): char_to_add = "c" ns["__file__"] += char_to_add - ns["__cached__"] = ns["__file__"] spec = ns["__spec__"] new_spec = importlib.util.spec_from_file_location(spec.name, ns["__file__"]) @@ -323,7 +323,6 @@ def _check_module(self, depth, alter_sys=False, expected_ns.update({ "__name__": mod_name, "__file__": mod_fname, - "__cached__": mod_spec.cached, "__package__": mod_name.rpartition(".")[0], "__spec__": mod_spec, }) @@ -340,14 +339,16 @@ def create_ns(init_globals): self.check_code_execution(create_ns, expected_ns) importlib.invalidate_caches() __import__(mod_name) - os.remove(mod_fname) if not sys.dont_write_bytecode: - make_legacy_pyc(mod_fname) + make_legacy_pyc(mod_fname, allow_compile=True) unload(mod_name) # In case loader caches paths + os.remove(mod_fname) importlib.invalidate_caches() if verbose > 1: print("Running from compiled:", mod_name) self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) self.check_code_execution(create_ns, expected_ns) + else: + os.remove(mod_fname) finally: self._del_pkg(pkg_dir) if verbose > 1: print("Module executed successfully") @@ -364,7 +365,6 @@ def _check_package(self, depth, alter_sys=False, expected_ns.update({ "__name__": mod_name, "__file__": mod_fname, - "__cached__": importlib.util.cache_from_source(mod_fname), "__package__": pkg_name, "__spec__": mod_spec, }) @@ -381,14 +381,16 @@ def create_ns(init_globals): self.check_code_execution(create_ns, expected_ns) importlib.invalidate_caches() __import__(mod_name) - os.remove(mod_fname) if not sys.dont_write_bytecode: - make_legacy_pyc(mod_fname) + make_legacy_pyc(mod_fname, allow_compile=True) unload(mod_name) # In case loader caches paths + os.remove(mod_fname) if verbose > 1: print("Running from compiled:", pkg_name) importlib.invalidate_caches() self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) self.check_code_execution(create_ns, expected_ns) + else: + os.remove(mod_fname) finally: self._del_pkg(pkg_dir) if verbose > 1: print("Package executed successfully") @@ -441,7 +443,7 @@ def _check_relative_imports(self, depth, run_name=None): importlib.invalidate_caches() __import__(mod_name) os.remove(mod_fname) - if not sys.dont_write_bytecode: + if not sys.dont_write_bytecode and sys.implementation.cache_tag: make_legacy_pyc(mod_fname) unload(mod_name) # In case the loader caches paths if verbose > 1: print("Running from compiled:", mod_name) @@ -569,7 +571,6 @@ def test_run_name(self): expected_ns.update({ "__name__": run_name, "__file__": mod_fname, - "__cached__": importlib.util.cache_from_source(mod_fname), "__package__": mod_name.rpartition(".")[0], "__spec__": mod_spec, }) @@ -649,7 +650,6 @@ def create_ns(init_globals): expected_ns.update({ "__name__": expected_name, "__file__": expected_file, - "__cached__": mod_cached, "__package__": "", "__spec__": mod_spec, "run_argv0": expected_argv0, @@ -699,6 +699,8 @@ def test_basic_script_no_suffix(self): self._check_script(script_name, "", script_name, script_name, expect_spec=False) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag') def test_script_compiled(self): with temp_dir() as script_dir: mod_name = 'script' @@ -719,12 +721,10 @@ def test_directory_compiled(self): with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = py_compile.compile(script_name, doraise=True) + legacy_pyc = make_legacy_pyc(script_name, allow_compile=True) os.remove(script_name) - if not sys.dont_write_bytecode: - legacy_pyc = make_legacy_pyc(script_name) - self._check_script(script_dir, "", legacy_pyc, - script_dir, mod_name=mod_name) + self._check_script(script_dir, "", legacy_pyc, + script_dir, mod_name=mod_name) def test_directory_error(self): with temp_dir() as script_dir: @@ -756,7 +756,8 @@ def test_zipfile_compiled(self): with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = py_compile.compile(script_name, doraise=True) + compiled_name = script_name + 'c' + py_compile.compile(script_name, compiled_name, doraise=True) zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name) self._check_script(zip_name, "", fname, zip_name, @@ -793,6 +794,47 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + def test_run_module_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with (temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + ns = run_module('test.test_import.data.syntax_warnings') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = ns['__file__'] + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + def test_run_path_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.script\z') + warnings.filterwarnings('error', module='') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename, run_name='package.script') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + @force_not_colorized_test_class class TestExit(unittest.TestCase): @@ -867,7 +909,6 @@ def test_pymain_run_command_run_module(self): def test_pymain_run_command(self): self.assertSigInt(["-c", "import ham"], cwd=self.ham.parent) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pymain_run_stdin(self): self.assertSigInt([], input="import ham", cwd=self.ham.parent) From 19cf87837ba2fcf90d556d1d8c12d93ac2689306 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 16 Oct 2026 16:50:08 +0900 Subject: [PATCH 09/19] Synchronize runpy warning attribution and document remaining capabilities Migrate whole canonical runpy.py to pass module names to compile. Use update_lib to annotate only reproduced missing-capability cases: absent _testcapi import-event tests, unsupported startup/import-time features, and the deferred frozen-loader module-attribution caller. Assisted-by: OpenAI Codex:model version unavailable --- Lib/runpy.py | 8 +++----- Lib/test/test_audit.py | 3 +++ Lib/test/test_cmd_line.py | 2 ++ Lib/test/test_runpy.py | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/runpy.py b/Lib/runpy.py index 1d5ecf0cf15..a535b4f651a 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -80,7 +80,6 @@ def _run_code(code, run_globals, init_globals=None, pkg_name = mod_spec.parent run_globals.update(__name__ = mod_name, __file__ = fname, - __cached__ = cached, __doc__ = None, __loader__ = loader, __package__ = pkg_name, @@ -185,7 +184,6 @@ def _run_module_as_main(mod_name, alter_argv=True): At the very least, these variables in __main__ will be overwritten: __name__ __file__ - __cached__ __loader__ __package__ """ @@ -254,7 +252,7 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(fname): +def _get_code_from_file(fname, module): # Check for a compiled file first from pkgutil import read_code code_path = os.path.abspath(fname) @@ -263,7 +261,7 @@ def _get_code_from_file(fname): if code is None: # That didn't work, so try it as normal source code with io.open_code(code_path) as f: - code = compile(f.read(), fname, 'exec') + code = compile(f.read(), fname, 'exec', module=module) return code def run_path(path_name, init_globals=None, run_name=None): @@ -290,7 +288,7 @@ def run_path(path_name, init_globals=None, run_name=None): if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code = _get_code_from_file(path_name, run_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=path_name) else: diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 2a43c8f2663..8c1658ca604 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -344,12 +344,15 @@ def test_sys_remote_exec(self): if returncode: self.fail(stderr) + @unittest.expectedFailureIf(__import__("importlib.util", fromlist=["find_spec"]).find_spec("_testcapi") is None, "TODO: RUSTPYTHON; requires the absent _testcapi module for extension import audit events; recheck when discoverable") def test_import_module(self): self.do_test("test_import_module") + @unittest.expectedFailureIf(__import__("importlib.util", fromlist=["find_spec"]).find_spec("_testcapi") is None, "TODO: RUSTPYTHON; requires the absent _testcapi module for extension import audit events; recheck when discoverable") def test_builtin__import__(self): self.do_test("test_builtin__import__") + @unittest.expectedFailureIf(__import__("importlib.util", fromlist=["find_spec"]).find_spec("_testcapi") is None, "TODO: RUSTPYTHON; requires the absent _testcapi module for extension import audit events; recheck when discoverable") def test_import_statement(self): self.do_test("test_import_statement") diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index c7716fb7191..248d6047f45 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1285,6 +1285,7 @@ def test_import_time(self): assert_python_failure('-X', 'importtime=-1', '-c', code) assert_python_failure('-X', 'importtime=3', '-c', code) + @unittest.expectedFailure # TODO: RUSTPYTHON; -X importtime=2 timing output and surrogate module import exception semantics are not implemented correctly def test_import_time_unencodable_module_name(self): code = textwrap.dedent(""" import sys, types @@ -1370,6 +1371,7 @@ def test_presite(self): proc = assert_python_ok("-X", f"presite={entrypoint}", "-c", "pass") self.assertEqual(proc.out.rstrip(), b"presite func") + @unittest.expectedFailure # TODO: RUSTPYTHON; explicit PYTHONHOME startup path configuration and fatal encodings failure reporting are not implemented def test_dump_path_config(self): # gh-151253: At the first import (import encodings) during Python # startup, if the import fails, dump the Python path configuration. diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 55b9673ef6c..8da23dae886 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -794,6 +794,7 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + @unittest.expectedFailure # TODO: RUSTPYTHON; current frozen SourceLoader.get_code/source_to_code omit fullname when calling compile, so module-specific SyntaxWarning filters lack module attribution; recheck after coherent importlib synchronization def test_run_module_filter_syntax_warnings_by_module(self): module_re = r'test\.test_import\.data\.syntax_warnings\z' with (temp_dir() as tmpdir, From 08e4c220587293741d364a98c6487a7b72c92056 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 17 Oct 2026 01:31:12 +0900 Subject: [PATCH 10/19] Propagate interrupted native stdin reads to signal dispatch Preserve the shared input buffer and partial bytes while returning EINTR to the VM before retrying. Exercise the transport deterministically and verify a returning SIGINT handler while pipe stdin stays open. Assisted-by: OpenAI Codex:model version unavailable --- crates/host_env/src/readline.rs | 119 +++++++++++++++++++++++- crates/vm/src/vm/python_run.rs | 2 +- extra_tests/snippets/stdlib_signal.py | 128 ++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 3 deletions(-) diff --git a/crates/host_env/src/readline.rs b/crates/host_env/src/readline.rs index a7de2f50880..c37d3714de0 100644 --- a/crates/host_env/src/readline.rs +++ b/crates/host_env/src/readline.rs @@ -200,10 +200,125 @@ impl Readline { /// Borrow the shared buffered stdin for one physical read. The caller retains /// partial bytes across EINTR and checks signals after this lock is released. pub fn stdio_readline_bytes(prompt: &str, line: &mut Vec) -> io::Result { - use std::io::{BufRead, Write}; + use std::io::Write; let mut stderr = io::stderr().lock(); stderr.write_all(prompt.as_bytes())?; stderr.flush()?; drop(stderr); - io::stdin().lock().read_until(b'\n', line) + read_line_bytes(&mut io::stdin().lock(), line) +} + +fn read_line_bytes(reader: &mut impl io::BufRead, line: &mut Vec) -> io::Result { + let mut read = 0; + loop { + // Unlike read_until, return Interrupted to the VM before retrying. + let available = reader.fill_buf()?; + if available.is_empty() { + return Ok(read); + } + let newline = available.iter().position(|&byte| byte == b'\n'); + let used = newline.map_or(available.len(), |index| index + 1); + line.extend_from_slice(&available[..used]); + reader.consume(used); + read += used; + if newline.is_some() { + return Ok(read); + } + } +} + +#[cfg(test)] +mod tests { + use super::read_line_bytes; + use std::io::{self, BufRead, Read}; + + struct ScriptedReader { + chunks: alloc::collections::VecDeque>>, + consumed: usize, + } + + impl ScriptedReader { + fn new(chunks: impl IntoIterator>>) -> Self { + Self { + chunks: chunks.into_iter().collect(), + consumed: 0, + } + } + } + + impl Read for ScriptedReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + unreachable!("the transport uses BufRead directly") + } + } + + impl BufRead for ScriptedReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + if self.chunks.front().is_some_and(Result::is_err) { + return Err(self.chunks.pop_front().unwrap().unwrap_err()); + } + Ok(self + .chunks + .front() + .map_or(&[], |chunk| &chunk.as_ref().unwrap()[self.consumed..])) + } + + fn consume(&mut self, amount: usize) { + self.consumed += amount; + if self.consumed == self.chunks.front().unwrap().as_ref().unwrap().len() { + self.chunks.pop_front(); + self.consumed = 0; + } + } + } + + #[test] + fn physical_lines_preserve_buffered_remainder_and_count_only_new_bytes() { + let mut reader = ScriptedReader::new([Ok(b"one\ntwo\ntail".to_vec())]); + let mut line = b"prefix".to_vec(); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"prefixone\n"); + line.clear(); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"two\n"); + line.clear(); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"tail"); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 0); + assert_eq!(line, b"tail"); + } + + #[test] + fn errors_return_immediately_and_leave_partial_bytes_for_the_caller() { + for kind in [io::ErrorKind::Interrupted, io::ErrorKind::Other] { + let mut reader = ScriptedReader::new([ + Ok(b"part".to_vec()), + Err(io::Error::from(kind)), + Ok(b"ial\nnext".to_vec()), + ]); + let mut line = Vec::new(); + assert_eq!( + read_line_bytes(&mut reader, &mut line).unwrap_err().kind(), + kind + ); + assert_eq!(line, b"part"); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 4); + assert_eq!(line, b"partial\n"); + } + } + + #[test] + fn eof_after_interruption_keeps_partial_bytes() { + let mut reader = ScriptedReader::new([ + Ok(b"partial".to_vec()), + Err(io::Error::from(io::ErrorKind::Interrupted)), + ]); + let mut line = Vec::new(); + assert_eq!( + read_line_bytes(&mut reader, &mut line).unwrap_err().kind(), + io::ErrorKind::Interrupted + ); + assert_eq!(read_line_bytes(&mut reader, &mut line).unwrap(), 0); + assert_eq!(line, b"partial"); + } } diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index 9d7e0270421..7702678f49a 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -468,7 +468,7 @@ impl VirtualMachine { let result = crate::readline::stdio_readline_bytes(prompt, &mut line); prompt = ""; match result { - Ok(0) => { + Ok(0) if line.is_empty() => { eprintln!(); return Ok(None); } diff --git a/extra_tests/snippets/stdlib_signal.py b/extra_tests/snippets/stdlib_signal.py index c03bc70ab30..ba333f90032 100644 --- a/extra_tests/snippets/stdlib_signal.py +++ b/extra_tests/snippets/stdlib_signal.py @@ -218,3 +218,131 @@ def raise_wakeup_signal(): check_windows_wakeup_reporting() + + +def check_native_stdin_eintr(): + if sys.platform in ("win32", "wasi", "emscripten"): + return + if not sys.executable or not hasattr(sys, "_baserepl"): + return + + import os + import select + import subprocess + + if os.name != "posix": + return + + prompt = b"__NATIVE_STDIN_PROMPT__" + handled = b"__NATIVE_STDIN_HANDLED__" + executed = b"__NATIVE_STDIN_EXECUTED__" + + # This explicitly exercises the native adapter rather than an editor REPL. + # Lib/test/test_repl.py covers ordinary input, not a blocked native read + # interrupted while its pipe remains open and no complete line is supplied. + setup = ( + "import signal, sys\n" + f"sys.ps1 = {prompt.decode()!r}\n" + "sys.ps2 = '__NATIVE_STDIN_CONTINUATION__'\n" + ) + setup += ( + "def handler(signum, frame):\n" + " print('__NATIVE_' + 'STDIN_HANDLED__', flush=True)\n" + "signal.signal(signal.SIGINT, handler)\n" + ) + setup += "sys._baserepl()\n" + process = subprocess.Popen( + [sys.executable, "-i", "-S", "-u", "-c", setup], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + ) + output = bytearray() + output_eof = False + deadline = time.monotonic() + 10.0 + + def remaining(): + timeout = deadline - time.monotonic() + assert timeout > 0, ("native stdin child timed out", output) + return timeout + + def read_output(timeout): + nonlocal output_eof + ready, _, _ = select.select([process.stdout], [], [], min(timeout, remaining())) + if ready: + try: + chunk = os.read(process.stdout.fileno(), 4096) + except BlockingIOError: + return + if chunk: + output.extend(chunk) + assert len(output) <= 65536, ("child output limit", output) + else: + output_eof = True + + def expect(marker, start=0, interrupt=False): + observed = len(output) + retry_at = time.monotonic() + attempts = 0 + while marker not in output[start:]: + assert not output_eof, ("child exited before marker", marker, output) + timeout = remaining() + if interrupt and len(output) == observed and attempts < 10: + # Drain bytes already available before deciding to retry. + read_output(0) + if len(output) != observed or output_eof: + continue + now = time.monotonic() + if now >= retry_at: + # Prompt emission precedes read(). Retry only this child + # to cover that race; never send input to wake the read. + assert process.poll() is None, ("child exited", output) + process.send_signal(signal.SIGINT) + attempts += 1 + retry_at = now + 0.5 + timeout = min(timeout, max(0, retry_at - time.monotonic())) + # Any response stops retries, including a partial traceback or + # marker, so another SIGINT does not interrupt its reporting. + read_output(timeout) + + def write_input(data): + offset = 0 + while offset < len(data): + _, ready, _ = select.select([], [process.stdin], [], remaining()) + if ready: + try: + written = os.write(process.stdin.fileno(), data[offset:]) + except BlockingIOError: + continue + assert written > 0 + offset += written + + try: + os.set_blocking(process.stdin.fileno(), False) + os.set_blocking(process.stdout.fileno(), False) + expect(prompt) + after_prompt = len(output) + source = b"print('__NATIVE_' + 'STDIN_EXECUTED__', flush=True)" + # Keep the pipe empty until the native reader dispatches the signal. + # Only then provide a complete line and EOF; this does not assert + # CPython's behavior for a partial libc fgets interrupted by a signal. + expect(handled, after_prompt, interrupt=True) + write_input(source + b"\n") + process.stdin.close() + expect(executed, after_prompt) + while not output_eof: + read_output(remaining()) + returncode = process.wait(timeout=remaining()) + assert returncode == 0, (returncode, output) + assert output.count(executed) == 1, output + assert b"KeyboardInterrupt" not in output, output + finally: + process.stdin.close() + process.stdout.close() + if process.poll() is None: + process.kill() + process.wait(timeout=5.0) + + +check_native_stdin_eintr() From daf4ce4ddaac8a93adc4511b40523e64f9552861 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 00:56:28 +0900 Subject: [PATCH 11/19] Reject statements after a compound statement in single mode single_mode_multiple_statements_error now reports "invalid syntax" at the statement that follows a compound first statement, and interactive_input runs it right after parsing as compile() does. The lazy `from __future__` import check moved from lazy_future_import_error into unsupported_grammar_error, so it applies to every compile mode. Assisted-by: Claude Code:claude-opus-5-5 --- crates/compiler/src/lib.rs | 114 ++++++++++--------------------------- 1 file changed, 31 insertions(+), 83 deletions(-) diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index c6f8a4e103d..fd257a1d1b8 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -950,55 +950,6 @@ pub fn too_deeply_nested_error( })) } -/// Source-only rejection performed by CPython's checked_from_import parser action. -#[doc(hidden)] -#[must_use] -pub fn lazy_future_import_error(ast: &ast::Mod, source_file: &SourceFile) -> Option { - use ast::statement_visitor::StatementVisitor; - - struct Checker<'a> { - pending: Vec<&'a ast::Stmt>, - } - - impl<'a> StatementVisitor<'a> for Checker<'a> { - fn visit_stmt(&mut self, stmt: &'a ast::Stmt) { - self.pending.push(stmt); - } - } - - let ast::Mod::Module(module) = ast else { - return None; - }; - let mut checker = Checker { - pending: module.body.iter().collect(), - }; - let mut first = None; - while let Some(stmt) = checker.pending.pop() { - if let ast::Stmt::ImportFrom(import) = stmt - && import.is_lazy - && import.level == 0 - && import - .module - .as_ref() - .is_some_and(|name| name.as_str() == "__future__") - { - let start = import.range.start().to_usize(); - first = Some(first.map_or(start, |previous: usize| previous.min(start))); - } - ast::statement_visitor::walk_stmt(&mut checker, stmt); - } - first.map(|start| { - CompileError::from_source_error( - source_file, - CpythonDiagnostic::new( - "lazy from __future__ import is not allowed".to_owned(), - start, - start + 4, - ), - ) - }) -} - /// Syntax the reference grammar has no rule for, but that this parser accepts. /// /// A bare generator expression is one: `f(x for x in y)` is the `primary @@ -1180,8 +1131,30 @@ pub fn single_mode_multiple_statements_error( let ast::Mod::Module(module) = parsed.syntax() else { return None; }; - if is_compound_stmt(module.body.first()?) { - return None; + let first = module.body.first()?; + if is_compound_stmt(first) { + // The interactive grammar ends a compound statement with a NEWLINE, + // so the first token of a following statement is invalid syntax. + let next = module.body.get(1)?; + if parsed + .errors() + .iter() + .any(|error| error.location.start() <= next.start()) + { + return None; + } + let token = parsed + .tokens() + .iter() + .find(|token| token.start() >= next.start() && !token.range().is_empty())?; + return Some(CompileError::from_source_error( + source_file, + CpythonDiagnostic::new( + "invalid syntax".to_owned(), + token.start().to_usize(), + token.end().to_usize(), + ), + )); } let tokens = parsed.tokens(); let newline_index = tokens @@ -1678,6 +1651,9 @@ pub fn interactive_input( future_features.contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), ); let parsed = parser::parse_unchecked(prepared.source(), options); + if let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) { + return Err(error); + } if let Some(error) = prepared.diagnostic(parsed.errors().first(), &source_file) { return Err(error); } @@ -1685,39 +1661,11 @@ pub fn interactive_input( let ast::Mod::Module(module) = parsed.syntax() else { unreachable!() }; - if let Some(first) = module.body.first() - && is_compound_stmt(first) - { - if let Some(next) = module.body.get(1) { - let range = parsed - .tokens() - .iter() - .find(|token| { - token.start() >= next.start() - && token.start() < next.end() - && !token.range().is_empty() - }) - .map_or_else( - || ruff_text_size::TextRange::new(next.start(), next.start()), - Ranged::range, - ); - return Err(CompileError::from_source_error( - &source_file, - CpythonDiagnostic::new( - "invalid syntax".to_owned(), - range.start().to_usize(), - range.end().to_usize(), - ), - )); - } - if !terminated { - return Ok(None); - } - } - if let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) { - return Err(error); + // A compound statement ends with an empty line or EOF. + if !terminated && module.body.first().is_some_and(is_compound_stmt) { + return Ok(None); } - if let Some(error) = lazy_future_import_error(parsed.syntax(), &source_file) { + if let Some(error) = unsupported_grammar_error(parsed.syntax(), &source_file) { return Err(error); } let future_features = codegen::preprocess::checked_future_features_in_body(&module.body) From 6a2ecdd76746e949508c0dd74b0b1810c45b418a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 00:56:30 +0900 Subject: [PATCH 12/19] Remove passing expected failure from test_runpy test_run_module_filter_syntax_warnings_by_module passes. Assisted-by: Claude Code:claude-opus-5-5 --- Lib/test/test_runpy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 8da23dae886..55b9673ef6c 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -794,7 +794,6 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") - @unittest.expectedFailure # TODO: RUSTPYTHON; current frozen SourceLoader.get_code/source_to_code omit fullname when calling compile, so module-specific SyntaxWarning filters lack module attribution; recheck after coherent importlib synchronization def test_run_module_filter_syntax_warnings_by_module(self): module_re = r'test\.test_import\.data\.syntax_warnings\z' with (temp_dir() as tmpdir, From a4d5d4a1cf01002dcb5feb4bca816b7e1fc95beb Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 00:56:52 +0900 Subject: [PATCH 13/19] Restructure stdin execution after Modules/main.c and pythonrun.c - src/lib.rs follows pymain_run_python: pymain_run_stdin, pymain_repl, pymain_run_startup, pymain_run_interactive_hook and pymain_run_pyrepl return exit codes instead of raising SystemExit. - VirtualMachine gains run_any_stdin (_PyRun_AnyFile), run_interactive_loop (_PyRun_InteractiveLoop) and run_simple_stdin; run_shell feeds rustyline lines into run_interactive_loop, and sys._baserepl calls run_any_stdin. - print_uncaught_exception (PyErr_Print) replaces report_uncaught_exception, and handle_system_exit_and_keyboard_interrupt replaces handle_reported_exception. - An unhandled KeyboardInterrupt is recorded and the process exits through SIGINT after finalization. - Interpreter::run_main replaces run_with_exit_status and InterpreterExit; rustpython_capi::run_main_interpreter takes an exit-code callback. - config_set_inspect audits cpython.PyConfig_Set and rebuilds sys.flags through set_flag_obj (_PySys_SetFlagObj); the inspect state is an AtomicBool and dict replace_by_identity is removed. - add_main_module (PyImport_AddModuleRef("__main__")) is used by the interactive and simple-file paths. - The interactive parser warning helpers take their state directly. Assisted-by: Claude Code:claude-opus-5-5 --- .cspell.dict/cpython.txt | 1 + crates/capi/src/lib.rs | 20 +- crates/vm/src/dict_inner.rs | 18 - crates/vm/src/exceptions.rs | 86 ++-- crates/vm/src/signal.rs | 27 + crates/vm/src/stdlib/_testinternalcapi.rs | 6 +- crates/vm/src/stdlib/posix.rs | 2 - crates/vm/src/stdlib/sys.rs | 87 ++-- crates/vm/src/vm/compile.rs | 34 +- crates/vm/src/vm/interpreter.rs | 70 ++- crates/vm/src/vm/mod.rs | 149 +++--- crates/vm/src/vm/python_run.rs | 586 ++++++++++++---------- crates/vm/src/vm/vm_new.rs | 13 + src/lib.rs | 341 ++++++++----- src/shell.rs | 11 +- 15 files changed, 778 insertions(+), 673 deletions(-) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index a00cd9e2584..7fc991670d2 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -222,6 +222,7 @@ pylifecycle pymain pymem pyrepl +pyrun pystate pystrcmp PYTHONTRACEMALLOC diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index 4db968dbb9e..aeeff3537d7 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -4,7 +4,7 @@ use crate::pyerrors::init_exception_statics; use crate::pylifecycle::{MAIN_INTERP, MAIN_INTERP_PTR}; use core::sync::atomic::Ordering; pub use rustpython_vm::PyObject; -use rustpython_vm::{Context, Interpreter, PyResult, VirtualMachine}; +use rustpython_vm::{Context, Interpreter, VirtualMachine}; use std::sync::MutexGuard; extern crate alloc; @@ -72,20 +72,12 @@ pub fn init_main_interpreter(interpreter: Interpreter) { ); } -/// Run and finalize the main VM while leaving C API thread attachment available. +/// Py_RunMain for an interpreter that C API threads can attach to. +/// +/// `f` reports its own errors and returns the exit code. pub fn run_main_interpreter( - interpreter: Interpreter, - f: impl FnOnce(&VirtualMachine) -> PyResult<()>, -) -> u32 { - run_main_interpreter_with_exit_status(interpreter, |vm| { - f(vm).map(|()| rustpython_vm::vm::InterpreterExit::Code(0)) - }) -} - -/// Preserve the main VM and an already-reported result through C API shutdown. -pub fn run_main_interpreter_with_exit_status( mut interpreter: Interpreter, - f: impl FnOnce(&VirtualMachine) -> PyResult, + f: impl FnOnce(&VirtualMachine) -> u32, ) -> u32 { let main_thread = interpreter.take_main_thread(); init_main_interpreter(interpreter); @@ -107,5 +99,5 @@ pub fn run_main_interpreter_with_exit_status( // The registered interpreter is an inactive worker-VM template. Running // the original VM locally preserves main-thread state without keeping the // global mutex locked across Python execution or finalization callbacks. - main_thread.run_with_exit_status(f) + main_thread.run_main(f) } diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 0c496a36f45..8b4a8eb6def 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -424,24 +424,6 @@ impl DictInner { type PopInnerResult = ControlFlow>>; impl Dict { - /// Replace a retained entry without hashing, equality calls, or releasing owners. - /// The caller must keep key, expected, and replacement alive until its own - /// outer locks are released, and drop the returned owner only afterwards. - pub(crate) fn replace_by_identity( - &self, - key: &PyObject, - matches: impl FnOnce(&T) -> bool, - replacement: &T, - ) -> Option { - let mut inner = self.write(); - let entry = inner - .entries - .iter_mut() - .flatten() - .find(|entry| entry.key.is(key))?; - matches(&entry.value).then(|| core::mem::replace(&mut entry.value, replacement.clone())) - } - /// Current keys-version stamp, or 0 if none has been assigned since the /// last key-set change. Equal nonzero stamps guarantee an unchanged key /// set (values may differ). diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 856ff75d104..c283026d136 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -101,43 +101,32 @@ impl VirtualMachine { /// Print exception chain by calling sys.excepthook pub fn print_exception(&self, exc: &Py) { let vm = self; - let write_fallback = |exc, errstr| { - if let Ok(stderr) = sys::get_stderr(vm) { - let mut stderr = py_io::PyWriter(stderr, vm); - // if this fails stderr might be closed -- ignore it - let _ = writeln!(stderr, "{errstr}"); - let _ = self.write_exception(&mut stderr, exc); - } else { - eprintln!("{errstr}\nlost sys.stderr"); - let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), exc); - } - }; if let Ok(excepthook) = vm.sys_module.get_attr("excepthook", vm) { let (exc_type, exc_val, exc_tb) = vm.split_exception(exc.to_owned()); if let Err(eh_exc) = excepthook.call((exc_type, exc_val, exc_tb), vm) { - write_fallback(&eh_exc, "Error in sys.excepthook:"); - write_fallback(exc, "Original exception was:"); + self.write_exception_fallback(&eh_exc, "Error in sys.excepthook:"); + self.write_exception_fallback(exc, "Original exception was:"); } } else { - write_fallback(exc, "missing sys.excepthook"); + self.write_exception_fallback(exc, "missing sys.excepthook"); } } - /// Report an uncaught exception, returning an unconsumed SystemExit to the host. - /// Unlike `print_exception`, this updates the interactive last-exception state - /// and performs the sys.excepthook audit. - pub fn report_uncaught_exception(&self, exc: &Py) -> PyResult<()> { - if exc.fast_isinstance(self.ctx.exceptions.system_exit) && !self.inspect_requested() { - return Err(exc.to_owned()); - } + /// PyErr_Print + /// + /// Record the exception as `sys.last_exc` and display it through + /// `sys.excepthook`. A `SystemExit` that should end the process is returned + /// instead, and the caller exits with it. + pub fn print_uncaught_exception(&self, exc: &Py) -> PyResult<()> { + self.handle_system_exit(exc)?; let (exc_type, exc_val, exc_tb) = self.split_exception(exc.to_owned()); for (name, value) in [ - ("last_exc", exc_val.clone()), - ("last_type", exc_type.clone()), - ("last_value", exc_val.clone()), - ("last_traceback", exc_tb.clone()), + ("last_exc", &exc_val), + ("last_type", &exc_type), + ("last_value", &exc_val), + ("last_traceback", &exc_tb), ] { - let _ = self.sys_module.set_attr(name, value, self); + let _ = self.sys_module.set_attr(name, value.clone(), self); } let hook = self.sys_module.get_attr("excepthook", self).ok(); if let Err(audit_exc) = self.audit("sys.excepthook", || { @@ -157,32 +146,41 @@ impl VirtualMachine { self.ctx.none(), ); } - let write_fallback = |error, heading| { - if let Ok(stderr) = sys::get_stderr(self) { - let mut stderr = py_io::PyWriter(stderr, self); - let _ = writeln!(stderr, "{heading}"); - let _ = self.write_exception(&mut stderr, error); - } else { - eprintln!("{heading}\nlost sys.stderr"); - let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), error); - } - }; if let Some(hook) = hook { if let Err(hook_exc) = hook.call((exc_type, exc_val, exc_tb), self) { - if hook_exc.fast_isinstance(self.ctx.exceptions.system_exit) - && !self.inspect_requested() - { - return Err(hook_exc); - } - write_fallback(&hook_exc, "Error in sys.excepthook:"); - write_fallback(exc, "Original exception was:"); + self.handle_system_exit(&hook_exc)?; + self.write_exception_fallback(&hook_exc, "Error in sys.excepthook:"); + self.write_exception_fallback(exc, "\nOriginal exception was:"); } } else { - write_fallback(exc, "sys.excepthook is missing"); + self.write_exception_fallback(exc, "sys.excepthook is missing"); + } + Ok(()) + } + + /// handle_system_exit, except that the caller performs `Py_Exit()` with + /// the returned `SystemExit`. + fn handle_system_exit(&self, exc: &Py) -> PyResult<()> { + if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { + crate::signal::set_unhandled_keyboard_interrupt(); + } else if !self.config_inspect() && exc.fast_isinstance(self.ctx.exceptions.system_exit) { + return Err(exc.to_owned()); } Ok(()) } + fn write_exception_fallback(&self, exc: &Py, heading: &str) { + if let Ok(stderr) = sys::get_stderr(self) { + let mut stderr = py_io::PyWriter(stderr, self); + // if this fails stderr might be closed -- ignore it + let _ = writeln!(stderr, "{heading}"); + let _ = self.write_exception(&mut stderr, exc); + } else { + eprintln!("{heading}\nlost sys.stderr"); + let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), exc); + } + } + pub fn write_exception( &self, output: &mut W, diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index 8bef84d33da..7a414726559 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -58,6 +58,33 @@ pub(crate) static TRIGGERS: [AtomicBool; NSIG] = [ATOMIC_FALSE; NSIG]; #[cfg(windows)] static SIGINT_EVENT: AtomicIsize = AtomicIsize::new(0); +/// `_PyRuntime.signals.unhandled_keyboard_interrupt` +static UNHANDLED_KEYBOARD_INTERRUPT: AtomicBool = AtomicBool::new(false); + +pub(crate) fn set_unhandled_keyboard_interrupt() { + UNHANDLED_KEYBOARD_INTERRUPT.store(true, Ordering::Relaxed); +} + +pub(crate) fn unhandled_keyboard_interrupt() -> bool { + UNHANDLED_KEYBOARD_INTERRUPT.load(Ordering::Relaxed) +} + +/// exit_sigint: exit through the default SIGINT handler so that the parent +/// process sees that a KeyboardInterrupt went unhandled. +pub(crate) fn exit_sigint() -> u32 { + cfg_select! { + unix => { + if crate::host_env::signal::set_sigint_default_onstack().is_ok() { + let _ = crate::host_env::signal::send_sigint_to_self(); + } + libc::SIGINT as u32 + 128 + } + // STATUS_CONTROL_C_EXIT + windows => 0xC000013A, + _ => 2 + 128, + } +} + thread_local! { /// Prevent recursive signal handler invocation. When a Python signal /// handler is running, new signals are deferred until it completes. diff --git a/crates/vm/src/stdlib/_testinternalcapi.rs b/crates/vm/src/stdlib/_testinternalcapi.rs index 7581229b74a..9e368d4462b 100644 --- a/crates/vm/src/stdlib/_testinternalcapi.rs +++ b/crates/vm/src/stdlib/_testinternalcapi.rs @@ -423,11 +423,7 @@ mod _testinternalcapi { global.set_item("dev_mode", vm.ctx.new_bool(settings.dev_mode).into(), vm)?; global.set_item("verbose", vm.ctx.new_int(settings.verbose).into(), vm)?; global.set_item("quiet", vm.ctx.new_bool(settings.quiet).into(), vm)?; - global.set_item( - "inspect", - vm.ctx.new_bool(vm.inspect_requested()).into(), - vm, - )?; + global.set_item("inspect", vm.ctx.new_bool(vm.config_inspect()).into(), vm)?; global.set_item( "interactive", vm.ctx.new_bool(settings.interactive).into(), diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index b6abf4246e3..ae70b7001bb 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -788,7 +788,6 @@ pub mod module { unsafe { // PyGlobalState PyMutex locks - reinit_mutex_after_fork(&vm.state.inspect_requested); reinit_mutex_after_fork(&vm.state.before_forkers); reinit_mutex_after_fork(&vm.state.after_forkers_child); reinit_mutex_after_fork(&vm.state.after_forkers_parent); @@ -839,7 +838,6 @@ pub mod module { } unsafe { - reinit_mutex_after_fork(&state.inspect_requested); reinit_mutex_after_fork(&state.before_forkers); reinit_mutex_after_fork(&state.after_forkers_child); reinit_mutex_after_fork(&state.after_forkers_parent); diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 4efddb8a9c7..d8b99a31436 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -761,7 +761,11 @@ pub mod sys { #[cfg(feature = "rustpython-compiler")] #[pyfunction] fn _baserepl(vm: &VirtualMachine) -> PyResult<()> { - vm.run_basic_interactive() + // PyRun_AnyFileExFlags(stdin, "", 0, &cf) + if let Err(exc) = vm.run_any_stdin() { + vm.print_uncaught_exception(&exc)?; + } + Ok(()) } #[pyfunction] @@ -1030,59 +1034,34 @@ pub mod sys { PyFlags::from_data(FlagsData::from_settings(&vm.state.config.settings), vm) } - pub(crate) fn set_inspect_requested(vm: &VirtualMachine, requested: bool) -> PyResult<()> { - use crate::class::StaticType; - use crate::types::PyStructSequenceData; - - let dict = vm.sys_module.dict(); - loop { - // Retain every encountered key/value before taking the publication - // lock: a concurrent deletion must never finalize one under it. - let entries = dict.entries.items(); - let (key, current) = entries - .iter() - .find(|(key, _)| { - key.class().is(vm.ctx.types.str_type) - && key - .downcast_ref::() - .is_some_and(|name| name.as_bytes() == b"flags") - }) - .ok_or_else(|| vm.new_runtime_error("sys.flags is missing"))?; - if !current.class().is(PyFlags::static_type()) { - return Err(vm.new_type_error("sys.flags must be a sys.flags instance")); - } - let tuple = current - .downcast_ref::() - .ok_or_else(|| vm.new_type_error("invalid sys.flags payload"))?; - let fields = FlagsData::REQUIRED_FIELD_NAMES; - let field_count = fields.len() + FlagsData::OPTIONAL_FIELD_NAMES.len(); - if tuple.as_slice().len() != field_count { - return Err(vm.new_type_error("invalid sys.flags field count")); - } - let inspect_index = fields - .iter() - .position(|name| *name == "inspect") - .expect("sys.flags has an inspect field"); - let mut values = tuple.as_slice().to_vec(); - values[inspect_index] = vm.ctx.new_int(u8::from(requested)).into(); - let replacement: PyObjectRef = PyTuple::new_unchecked(values.into_boxed_slice()) - .into_ref_with_type(vm, PyFlags::static_type().to_owned())? - .into(); - - // No Python, hashing, equality, or owner destruction in this section. - // A racing publisher or direct sys.flags assignment causes a retry. - let mut effective = vm.state.inspect_requested.lock(); - let removed = - dict.entries - .replace_by_identity(key, |value| value.is(current), &replacement); - if removed.is_some() { - *effective = requested; - } - drop(effective); - if removed.is_some() { - return Ok(()); - } - } + /// _PySys_SetFlagObj + pub(crate) fn set_flag_obj( + vm: &VirtualMachine, + name: &str, + value: PyObjectRef, + ) -> PyResult<()> { + use crate::{class::StaticType, types::PyStructSequenceData}; + + let old_flags = vm + .sys_module + .get_attr("flags", vm) + .map_err(|_| vm.new_runtime_error("lost sys.flags"))?; + let old_flags = old_flags + .downcast_ref::() + .filter(|flags| { + flags.len() + == FlagsData::REQUIRED_FIELD_NAMES.len() + FlagsData::OPTIONAL_FIELD_NAMES.len() + }) + .ok_or_else(|| vm.new_type_error("sys.flags must be a sys.flags instance"))?; + let pos = FlagsData::REQUIRED_FIELD_NAMES + .iter() + .position(|field| *field == name) + .expect("unknown sys.flags field"); + let mut values = old_flags.as_slice().to_vec(); + values[pos] = value; + let new_flags = PyTuple::new_unchecked(values.into_boxed_slice()) + .into_ref_with_type(vm, PyFlags::static_type().to_owned())?; + vm.sys_module.set_attr("flags", new_flags, vm) } #[pyattr] diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 2c05f4e37c4..95dbcac5b8f 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -11,19 +11,16 @@ use crate::{ vm::compile_mode::{CompileStart, CompilerFlags, compile_future_features_from_flags}, }; -/// Warnings delivered while collecting one physical interactive record. +/// Warnings already emitted while reading one interactive statement. +/// +/// The source is warned about again after each line is read, but the +/// tokenizer warns about each token only once. #[cfg(feature = "parser")] #[derive(Default)] pub(super) struct InteractiveWarnings { emitted: alloc::collections::BTreeSet<(usize, String)>, } -#[cfg(feature = "parser")] -pub(super) struct ParserWarningState<'a> { - pub emitted_escapes: &'a [usize], - pub interactive: Option<&'a mut InteractiveWarnings>, -} - #[derive(Debug)] pub enum VmCompileError { Compile(CompileError), @@ -1766,7 +1763,7 @@ mod escape_warnings { enum WarningEmission<'a> { Standard { emitted_escapes: &'a [usize] }, Ast { tokenizer_recovery: bool }, - Interactive(super::ParserWarningState<'a>), + Interactive(&'a mut super::InteractiveWarnings), } impl VirtualMachine { @@ -1868,24 +1865,29 @@ mod escape_warnings { ) } + /// Emit the tokenizer warnings for the interactive input read so far. pub(in crate::vm) fn emit_interactive_parser_warnings( &self, source: &str, filename: &str, module: Option<&Py>, - mode: Option, inherited_barry_as_flufl: bool, - state: super::ParserWarningState<'_>, - ) -> Result, CompileWarningError> { - let (warnings, _) = - collect_parser_warnings(source, filename, mode, inherited_barry_as_flufl); + emitted: &mut super::InteractiveWarnings, + ) -> Result<(), CompileWarningError> { + let (warnings, _) = collect_parser_warnings( + source, + filename, + Some(ruff_python_parser::Mode::Module), + inherited_barry_as_flufl, + ); self.emit_parser_warning_batch( source, filename, module, warnings, - WarningEmission::Interactive(state), + WarningEmission::Interactive(emitted), ) + .map(|_| ()) } fn emit_parser_warning_batch( @@ -1899,9 +1901,7 @@ mod escape_warnings { let (emitted_escapes, ast_tokenizer_recovery, mut interactive) = match emission { WarningEmission::Standard { emitted_escapes } => (emitted_escapes, false, None), WarningEmission::Ast { tokenizer_recovery } => (&[][..], tokenizer_recovery, None), - WarningEmission::Interactive(state) => { - (state.emitted_escapes, false, state.interactive) - } + WarningEmission::Interactive(emitted) => (&[][..], false, Some(emitted)), }; let mut considered_escapes = Vec::new(); let mut parser_error = None; diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index f36b8780e7d..562346492d1 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -19,13 +19,6 @@ type InitFunc = Box; /// Matches CPython's behavior (see cpython/Python/pylifecycle.c). const EXITCODE_FLUSH_FAILURE: u32 = 120; -/// An execution result whose Python exception has already been reported. -/// Unreported exceptions continue to travel separately through `PyResult`. -pub enum InterpreterExit { - Code(u32), - ReportedException(PyBaseExceptionRef), -} - /// Configuration builder for constructing an Interpreter. /// /// This is the preferred way to configure and create an interpreter with custom modules. @@ -81,7 +74,7 @@ where use crate::codecs::CodecsRegistry; use crate::common::lock::PyMutex; use crate::warn::WarningsState; - use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64}; + use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize}; use crossbeam_utils::atomic::AtomicCell; // Before any lock this interpreter's threads can contend on exists. @@ -92,7 +85,7 @@ where // Subinterpreter: clone config and module tables from parent, fresh runtime state. let int_max_str_digits = AtomicCell::new(parent.int_max_str_digits.load()); let mut config = parent.config.clone(); - config.settings.inspect = *parent.inspect_requested.lock(); + config.settings.inspect = parent.inspect.load(Ordering::Relaxed); ( config, parent.module_defs.clone(), @@ -163,8 +156,8 @@ where runtime_root_id, whence, is_main, - inspect_requested: PyMutex::new(config.settings.inspect), - interactive_code_counter: AtomicU64::new(0), + inspect: AtomicBool::new(config.settings.inspect), + interactive_src_count: AtomicUsize::new(0), config, module_defs: all_module_defs, frozen, @@ -660,20 +653,20 @@ impl Interpreter { where F: FnOnce(&VirtualMachine) -> PyResult<()>, { - self.run_with_exit_status(|vm| f(vm).map(|()| InterpreterExit::Code(0))) + let res = self.enter(|vm| f(vm)); + self.finalize(res.err()) } - /// Run and finalize while preserving a host status for already-reported errors. - /// Errors returned by the callback are still reported during finalization. - pub fn run_with_exit_status(self, f: F) -> u32 + /// Py_RunMain + /// + /// Run a function that reports its own errors and returns the exit code, + /// then finalize with that exit code. + pub fn run_main(self, f: F) -> u32 where - F: FnOnce(&VirtualMachine) -> PyResult, + F: FnOnce(&VirtualMachine) -> u32, { - let (exc, exit_code) = match self.enter(f) { - Ok(exit_code) => (None, exit_code), - Err(exc) => (Some(exc), InterpreterExit::Code(0)), - }; - self.finalize_with_exit_status(exc, exit_code) + let exit_code = self.enter(f); + self.finalize_with_exit_code(exit_code) } /// Finalize vm and turns an exception to exit code. @@ -691,28 +684,22 @@ impl Interpreter { /// /// Note that calling `finalize` is not necessary by purpose though. pub fn finalize(self, exc: Option) -> u32 { - self.finalize_with_exit_status(exc, InterpreterExit::Code(0)) + self.finalize_inner(|vm| exc.map_or(0, |exc| vm.handle_exit_exception(exc))) } - /// Finalize with an explicit status when no unreported exception remains. - /// A pending exception takes precedence over the supplied status. - pub fn finalize_with_exit_status( - self, - exc: Option, - status: InterpreterExit, - ) -> u32 { - self.enter(|vm| { + /// Finalize vm with an exit code the caller has already determined. + /// + /// See [`Interpreter::finalize`] for the finalization steps. + pub fn finalize_with_exit_code(self, exit_code: u32) -> u32 { + self.finalize_inner(|_| exit_code) + } + + fn finalize_inner(self, exit_code: impl FnOnce(&VirtualMachine) -> u32) -> u32 { + let exit_code = self.enter(|vm| { let mut flush_status = vm.flush_std(); // See if any exception leaked out: - let exit_code = if let Some(exc) = exc { - vm.handle_exit_exception(exc) - } else { - match status { - InterpreterExit::Code(code) => code, - InterpreterExit::ReportedException(exc) => vm.handle_reported_exception(&exc), - } - }; + let exit_code = exit_code(vm); // Wait for thread shutdown - call threading._shutdown() if available. // This waits for all non-daemon threads to complete. @@ -790,7 +777,12 @@ impl Interpreter { crate::object::qsbr::QSBR.process(); exit_code - }) + }); + if crate::signal::unhandled_keyboard_interrupt() { + crate::signal::exit_sigint() + } else { + exit_code + } } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 7321f4bcadf..3ff1d44e6ce 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -54,7 +54,7 @@ use alloc::{borrow::Cow, collections::BTreeMap}; use core::ptr::NonNull; use core::{ cell::{Cell, OnceCell, RefCell}, - sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering}, }; use crossbeam_utils::atomic::AtomicCell; use std::{ @@ -63,7 +63,7 @@ use std::{ }; pub use context::Context; -pub use interpreter::{Interpreter, InterpreterBuilder, InterpreterExit}; +pub use interpreter::{Interpreter, InterpreterBuilder}; pub(crate) use method::PyMethod; pub use runtime::{ InterpFeatureFlags, InterpreterConfig, InterpreterGil, InterpreterInfo, InterpreterWhence, @@ -860,10 +860,10 @@ pub struct PyGlobalState { /// becomes *the* process main — see [`runtime::main_interpreter_id`]. pub is_main: bool, pub config: PyConfig, - /// Effective runtime inspect state, serialized with sys.flags publication. - pub(crate) inspect_requested: PyMutex, - /// Unique source identities shared by all interactive entries in this interpreter. - pub(crate) interactive_code_counter: AtomicU64, + /// `config.inspect`, which `PyConfig_Set` can change at runtime. + pub(crate) inspect: AtomicBool, + /// `_interactive_src_count`: numbers the code filenames of interactive inputs. + pub(crate) interactive_src_count: AtomicUsize, pub module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef>, pub frozen: HashMap<&'static str, FrozenModule, rapidhash::quality::RandomState>, pub stacksize: AtomicCell, @@ -2347,9 +2347,7 @@ impl VirtualMachine { path: &str, run: impl FnOnce(&Py) -> PyResult<()>, ) -> PyResult<()> { - let sys_modules = self.sys_module.get_attr(identifier!(self, modules), self)?; - let main_module = sys_modules.get_item(identifier!(self, __main__), self)?; - let module_dict = main_module.dict().expect("main module must have __dict__"); + let module_dict = self.add_main_module()?.dict(); // Track whether we set __file__ (for cleanup) let set_file_name = !module_dict.contains_key(identifier!(self, __file__), self); @@ -3913,88 +3911,81 @@ impl VirtualMachine { excs.stack.iter().rev().find_map(|e| e.clone()) } - pub fn inspect_requested(&self) -> bool { - *self.state.inspect_requested.lock() + /// The current `config.inspect`. + pub fn config_inspect(&self) -> bool { + self.state.inspect.load(Ordering::Relaxed) } - /// Publish a fresh immutable sys.flags snapshot and the effective inspect bit. - pub fn set_inspect_requested(&self, requested: bool) -> PyResult<()> { - stdlib::sys::sys::set_inspect_requested(self, requested) - } - - pub fn next_interactive_code_filename(&self) -> PyResult { - let serial = self - .state - .interactive_code_counter - .try_update(Ordering::Relaxed, Ordering::Relaxed, |serial| { - serial.checked_add(1) - }) - .map_err(|_| self.new_overflow_error("interactive code filename counter exhausted"))?; - Ok(format!("")) + /// `PyConfig_Set("inspect", value)`: update the config and `sys.flags.inspect`. + pub fn config_set_inspect(&self, inspect: bool) -> PyResult<()> { + let value: PyObjectRef = self.ctx.new_int(u8::from(inspect)).into(); + self.audit("cpython.PyConfig_Set", || ("inspect", value.clone()))?; + self.state.inspect.store(inspect, Ordering::Relaxed); + stdlib::sys::sys::set_flag_obj(self, "inspect", value) } + /// pymain_exit_err_print + /// + /// Report an exception that reached the top level and return the exit code. pub fn handle_exit_exception(&self, exc: PyBaseExceptionRef) -> u32 { - let exc = match self.report_uncaught_exception(&exc) { - Err(exit) => exit, - Ok(()) => return self.handle_reported_exception(&exc), - }; - if exc.fast_isinstance(self.ctx.exceptions.system_exit) { - let code = exc - .as_object() - .get_attr("code", self) - .unwrap_or_else(|_| exc.as_object().to_owned()); - let msg = match_class!(match code { - ref i @ PyInt => { - use num_traits::cast::ToPrimitive; - // Try u32 first, then i32 (for negative values), else -1 for overflow - let code = i - .as_bigint() - .to_u32() - .or_else(|| i.as_bigint().to_i32().map(|v| v as u32)) - .unwrap_or(-1i32 as u32); - return code; - } - code => { - if self.is_none(&code) { - return 0; - } - code.str(self).ok() - } - }); - if let Some(msg) = msg { - // Write using Python's write() to use stderr's error handler (backslashreplace) - if let Ok(stderr) = stdlib::sys::get_stderr(self) { - let _ = self.call_method(&stderr, "write", (msg,)); - let _ = self.call_method(&stderr, "write", ("\n",)); - } - } - 1 - } else { - self.handle_reported_exception(&exc) + if let Some(exit_code) = self.handle_system_exit_and_keyboard_interrupt(&exc) { + return exit_code; + } + match self.print_uncaught_exception(&exc) { + Ok(()) => 1, + Err(exc) => self + .handle_system_exit_and_keyboard_interrupt(&exc) + .unwrap_or(1), } } - /// Apply the host exit policy after an exception has already been reported. - /// Call only after deciding not to continue into an interactive session. - pub fn handle_reported_exception(&self, exc: &Py) -> u32 { + /// _Py_HandleSystemExitAndKeyboardInterrupt + /// + /// Return the exit code of a `SystemExit` that should end the process. + /// A `KeyboardInterrupt` is remembered so that the process exits through SIGINT. + pub fn handle_system_exit_and_keyboard_interrupt( + &self, + exc: &Py, + ) -> Option { if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { - cfg_select! { - unix => { - if crate::host_env::signal::set_sigint_default_onstack().is_ok() { - self.flush_std(); - crate::host_env::signal::send_sigint_to_self() - .expect("Expect to be killed."); - } - - (libc::SIGINT as u32) + 128 + crate::signal::set_unhandled_keyboard_interrupt(); + return None; + } + // Don't exit if -i flag was given. This flag is set to 0 + // when entering interactive mode for inspecting. + if self.config_inspect() || !exc.fast_isinstance(self.ctx.exceptions.system_exit) { + return None; + } + let code = exc + .as_object() + .get_attr("code", self) + .unwrap_or_else(|_| exc.as_object().to_owned()); + let msg = match_class!(match code { + ref i @ PyInt => { + use num_traits::cast::ToPrimitive; + // Try u32 first, then i32 (for negative values), else -1 for overflow + let code = i + .as_bigint() + .to_u32() + .or_else(|| i.as_bigint().to_i32().map(|v| v as u32)) + .unwrap_or(-1i32 as u32); + return Some(code); + } + code => { + if self.is_none(&code) { + return Some(0); } - // STATUS_CONTROL_C_EXIT - same as CPython - windows => 0xC000013A, - _ => 1, + code.str(self).ok() + } + }); + if let Some(msg) = msg { + // Write using Python's write() to use stderr's error handler (backslashreplace) + if let Ok(stderr) = stdlib::sys::get_stderr(self) { + let _ = self.call_method(&stderr, "write", (msg,)); + let _ = self.call_method(&stderr, "write", ("\n",)); } - } else { - 1 } + Some(1) } #[doc(hidden)] diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index 7702678f49a..dd5876a2a6a 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -1,11 +1,14 @@ //! Python code execution functions. use crate::{ - Py, PyObjectRef, PyResult, VirtualMachine, - builtins::PyCode, + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyBaseException, PyBaseExceptionRef, PyCode, PyModule, PyStr}, + bytecode::CodeFlags, compiler::{self}, scope::Scope, + stdlib, }; +use core::sync::atomic::Ordering; impl VirtualMachine { /// PyRun_SimpleString @@ -58,8 +61,8 @@ impl VirtualMachine { #[cfg(feature = "host_env")] mod file_run { use crate::{ - AsObject, Py, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyCode, PyDict}, + Py, PyResult, VirtualMachine, + builtins::{PyCode, PyDict}, compiler::{self}, scope::Scope, }; @@ -67,8 +70,8 @@ mod file_run { impl VirtualMachine { /// _PyRun_AnyFileObject (internal) /// - /// Execute a Python file. Currently always delegates to run_simple_file - /// (interactive mode is handled separately in shell.rs). + /// Execute a Python file. A named file is not interactive, so this + /// always delegates to run_simple_file; see `run_any_stdin` for stdin. /// /// Note: This is an internal function. Use `run_file` for the public interface. #[doc(hidden)] @@ -111,7 +114,12 @@ mod file_run { match crate::host_env::fs::read(path) { Ok(source_bytes) => { if let Some(null_at) = source_bytes.iter().position(|&b| b == 0) { - return Err(null_byte_syntax_error(self, path, &source_bytes, null_at)); + return Err(super::null_byte_syntax_error( + self, + path, + &source_bytes, + null_at, + )); } #[cfg(feature = "parser")] // Match compile() by honoring BOMs and encoding cookies in files. @@ -138,31 +146,6 @@ mod file_run { } } - fn null_byte_syntax_error( - vm: &VirtualMachine, - path: &str, - source_bytes: &[u8], - null_at: usize, - ) -> PyBaseExceptionRef { - let before = &source_bytes[..null_at]; - let lineno = before.iter().filter(|&&b| b == b'\n').count() + 1; - let line_start = before - .iter() - .rposition(|&b| b == b'\n') - .map_or(0, |i| i + 1); - let line = String::from_utf8_lossy(&source_bytes[line_start..null_at]); - let syntax_error = vm.new_exception_msg( - vm.ctx.exceptions.syntax_error.to_owned(), - "source code cannot contain null bytes".into(), - ); - let obj = syntax_error.as_object(); - obj.set_attr("filename", vm.ctx.new_str(path), vm).unwrap(); - obj.set_attr("lineno", vm.ctx.new_int(lineno), vm).unwrap(); - obj.set_attr("text", vm.ctx.new_str(format!("{line}\n")), vm) - .unwrap(); - syntax_error - } - fn set_main_loader( module_dict: &Py, filename: &str, @@ -216,236 +199,283 @@ mod file_run { } } -/// A physical stdin line or an already decoded editor line. +/// One line of interactive input: raw bytes read from stdin, or text from a +/// line editor. pub enum InteractiveLine { Bytes(Vec), Text(String), } +/// Source of one interactive statement, ready to be compiled. +struct InteractiveSource { + source: String, + parsed: compiler::InteractiveInput, +} + impl VirtualMachine { - /// Resolve the current main module without rewriting an existing loader. - pub fn stdin_main_module(&self) -> PyResult> { - let modules = self.sys_module.get_attr("modules", self)?; - if let Ok(module) = modules.get_item("__main__", self) - && let Ok(module) = module.downcast::() - { - return Ok(module); + /// _Py_FdIsInteractive(stdin, "") + pub fn stdin_is_interactive(&self) -> bool { + use std::io::IsTerminal; + std::io::stdin().is_terminal() || self.state.config.settings.interactive + } + + /// _PyRun_AnyFile(stdin, "") + /// + /// Run stdin as an interactive session or as a script. + pub fn run_any_stdin(&self) -> PyResult<()> { + if self.stdin_is_interactive() { + self.run_interactive_loop(|prompt| self.stdio_readline(prompt)) + } else { + self.run_simple_stdin() } - self.ensure_main_module() } - /// Evaluate native interactive records. Input policy belongs to the caller; - /// compilation, exception handling and namespace ownership stay in the VM. - pub fn run_interactive( + /// _PyRun_InteractiveLoop + /// + /// `readline` plays the role of `PyOS_Readline`: it returns `Ok(None)` at + /// the end of input. + pub fn run_interactive_loop( &self, mut readline: impl FnMut(&str) -> PyResult>, ) -> PyResult<()> { - use crate::{AsObject, bytecode::CodeFlags}; for (name, default) in [("ps1", ">>> "), ("ps2", "... ")] { if self.sys_module.dict().get_item_opt(name, self)?.is_none() { - self.sys_module - .set_attr(name, self.ctx.new_str(default), self)?; + let _ = self + .sys_module + .set_attr(name, self.ctx.new_str(default), self); } } - let mut future_features = CodeFlags::empty(); - let mut memory_errors = 0; + let mut flags = CodeFlags::empty(); + let mut nomem_count = 0; loop { - let mut eof = false; - let mut empty_eof = false; - let result = (|| { - let encoding = self - .sys_module - .get_attr("stdin", self) - .and_then(|stdin| stdin.get_attr("encoding", self)) - .ok() - .and_then(|encoding| encoding.downcast::().ok()) - .and_then(|encoding| encoding.to_str().map(str::to_owned)) - .unwrap_or_else(|| "utf-8".to_owned()); - let prompt = |name| { - self.sys_module - .get_attr(name, self) - .and_then(|prompt| prompt.str(self)) - .ok() - .and_then(|prompt| prompt.to_str().map(str::to_owned)) - .unwrap_or_default() - }; - let ps1 = prompt("ps1"); - let ps2 = prompt("ps2"); - self.audit("compile", || (self.ctx.none(), self.ctx.new_str("")))?; - let mut source = String::new(); - let mut raw_source = Vec::new(); - #[cfg(feature = "parser")] - let mut warnings = super::compile::InteractiveWarnings::default(); - loop { - let line = readline(if source.is_empty() { &ps1 } else { &ps2 })?; - let mut terminated = line.is_none(); - let mut unterminated_fragment = false; - match line { - None => { - eof = true; - if source.is_empty() { - empty_eof = true; - return Ok(()); - } - if !source.ends_with('\n') { - source.push('\n'); - } - } - Some(InteractiveLine::Text(mut line)) => { - if !line.ends_with('\n') { - line.push('\n'); - } - let line = line.replace("\r\n", "\n"); - terminated = line == "\n"; - source.push_str(&line); - } - Some(InteractiveLine::Bytes(line)) => { - unterminated_fragment = !line.ends_with(b"\n"); - terminated = line == b"\n" || line == b"\r\n"; - raw_source.extend_from_slice(&line); - source = self - .state - .codec_registry - .decode_text( - self.ctx.new_bytes(raw_source.clone()).into(), - &encoding, - None, - self, - )? - .to_str() - .ok_or_else(|| { - self.new_exception_msg( - self.ctx.exceptions.unicode_error.to_owned(), - "stdin source contains surrogates".into(), - ) - })? - .replace("\r\n", "\n"); + match self.run_interactive_one(&mut readline, &mut flags) { + Ok(true) => nomem_count = 0, + // E_EOF + Ok(false) => return Ok(()), + Err(exc) => { + // Prevent an endless loop after multiple consecutive + // MemoryErrors while still allowing an interactive + // command to fail with a MemoryError. + if exc.fast_isinstance(self.ctx.exceptions.memory_error) { + nomem_count += 1; + if nomem_count > 16 { + return Err(exc); } + } else { + nomem_count = 0; } - #[cfg(feature = "parser")] - self.emit_interactive_parser_warnings( - &source, - "", - Some(identifier!(self, __main__)), - Some(compiler::parser::Mode::Module), - future_features.contains(CodeFlags::FUTURE_BARRY_AS_BDFL), - super::compile::ParserWarningState { - emitted_escapes: &[], - interactive: Some(&mut warnings), - }, - ) - .map_err(|error| { - super::compile::VmCompileError::Warning(error) - .into_pyexception(self, Some(&source)) - })?; - let parsed = match compiler::interactive_input( - &source, - "", - future_features, - terminated, - ) { - Ok(Some(parsed)) => parsed, - Ok(None) => continue, - Err(error) => { - let unclosed = matches!(&error, compiler::CompileError::Parse(error) - if error.is_unclosed_bracket || error.is_unclosed_string); - let error = super::compile::VmCompileError::Compile(error) - .into_pyexception_maybe_incomplete( - self, - Some(&source), - !terminated, - ); - if !eof - && (unclosed - || (!terminated - && error.fast_isinstance( - self.ctx.exceptions.incomplete_input_error, - ))) - { - continue; - } - return Err(error); - } - }; - // A raw final fragment is not a completed physical line. - // Warnings and definitive parser errors are observable now, - // but acceptance waits for the transport's EOF transaction. - if unterminated_fragment { - continue; + if !self.config_inspect() + && exc.fast_isinstance(self.ctx.exceptions.system_exit) + { + return Err(exc); } - let main = self.stdin_main_module()?; - future_features |= parsed.future_features; - return (|| { - let filename = self.next_interactive_code_filename()?; - let code = self - .compile_after_parser_warnings( - &source, - if parsed.empty { - compiler::Mode::Exec - } else { - compiler::Mode::Single - }, - filename, - compiler::CompileOpts { - future_features, - ..self.compile_opts() - }, - Some(identifier!(self, __main__)), - ) - .map_err(|error| error.into_pyexception(self, Some(&source)))?; - let linecache = self.import("linecache", 0)?; - linecache.get_attr("_register_code", self)?.call( - ( - code.clone(), - self.ctx.new_str(source.as_str()), - self.ctx.new_str(""), - ), + self.print_uncaught_exception(&exc)?; + } + } + } + } + + /// _PyRun_InteractiveOne + /// + /// Read, compile and run one interactive statement in `__main__`. + /// Return `Ok(false)` at the end of input. + fn run_interactive_one( + &self, + readline: &mut impl FnMut(&str) -> PyResult>, + flags: &mut CodeFlags, + ) -> PyResult { + let Some(InteractiveSource { source, parsed }) = + self.interactive_parse(readline, *flags)? + else { + return Ok(false); + }; + let main_module = self.add_main_module()?; + // _PyAST_Compile merges the future features of the parsed statement + // into the flags before code generation. + *flags |= parsed.future_features; + self.run_interactive_mod(&source, parsed.empty, *flags, &main_module) + .inspect_err(|exc| self.fix_syntax_error_text(exc, &source))?; + self.flush_io(); + Ok(true) + } + + /// pyrun_one_parse_ast + /// + /// Read lines with `sys.ps1` and `sys.ps2` prompts until they form one + /// complete interactive statement. Return `None` at the end of input. + fn interactive_parse( + &self, + readline: &mut impl FnMut(&str) -> PyResult>, + flags: CodeFlags, + ) -> PyResult> { + let encoding = self + .sys_module + .get_attr("stdin", self) + .and_then(|stdin| stdin.get_attr("encoding", self)) + .ok() + .and_then(|encoding| encoding.downcast::().ok()) + .and_then(|encoding| encoding.to_str().map(str::to_owned)) + .unwrap_or_else(|| "utf-8".to_owned()); + let prompt = |name| { + self.sys_module + .get_attr(name, self) + .and_then(|prompt| prompt.str(self)) + .ok() + .and_then(|prompt| prompt.to_str().map(str::to_owned)) + .unwrap_or_default() + }; + let ps1 = prompt("ps1"); + let ps2 = prompt("ps2"); + + // _PyParser_InteractiveASTFromFile + self.audit("compile", || (self.ctx.none(), self.ctx.new_str("")))?; + let mut source = String::new(); + let mut raw_source = Vec::new(); + #[cfg(feature = "parser")] + let mut warnings = super::compile::InteractiveWarnings::default(); + loop { + let line = readline(if source.is_empty() { &ps1 } else { &ps2 }); + // tok_underflow_interactive ends the prompt line at EOF or on an + // interrupted read. + if !matches!(line, Ok(Some(_))) { + writeln!(stdlib::sys::PyStderr(self)); + } + let (terminated, unterminated_fragment, eof) = match line? { + None if source.is_empty() => return Ok(None), + None => { + if !source.ends_with('\n') { + source.push('\n'); + } + (true, false, true) + } + Some(InteractiveLine::Text(mut line)) => { + if !line.ends_with('\n') { + line.push('\n'); + } + let line = line.replace("\r\n", "\n"); + let terminated = line == "\n"; + source.push_str(&line); + (terminated, false, false) + } + Some(InteractiveLine::Bytes(line)) => { + let unterminated_fragment = !line.ends_with(b"\n"); + let terminated = line == b"\n" || line == b"\r\n"; + raw_source.extend_from_slice(&line); + source = self + .state + .codec_registry + .decode_text( + self.ctx.new_bytes(raw_source.clone()).into(), + &encoding, + None, self, - )?; - self.audit("exec", || (code.clone(),))?; - self.run_code_obj(code, Scope::with_builtins(None, main.dict(), self))?; - Ok(()) - })() - .inspect_err(|error| { - self.repair_interactive_syntax_error_text(error, &source) - }); + )? + .to_str() + .ok_or_else(|| { + self.new_exception_msg( + self.ctx.exceptions.unicode_error.to_owned(), + "stdin source contains surrogates".into(), + ) + })? + .replace("\r\n", "\n"); + (terminated, unterminated_fragment, false) } - })(); - if let Err(error) = result { - if error.fast_isinstance(self.ctx.exceptions.memory_error) { - memory_errors += 1; - if memory_errors > 16 { - return Err(error); + }; + #[cfg(feature = "parser")] + self.emit_interactive_parser_warnings( + &source, + "", + Some(identifier!(self, __main__)), + flags.contains(CodeFlags::FUTURE_BARRY_AS_BDFL), + &mut warnings, + ) + .map_err(|error| { + super::compile::VmCompileError::Warning(error).into_pyexception(self, Some(&source)) + })?; + let parsed = match compiler::interactive_input(&source, "", flags, terminated) { + Ok(Some(parsed)) => parsed, + Ok(None) => continue, + Err(error) => { + let unclosed = matches!(&error, compiler::CompileError::Parse(error) + if error.is_unclosed_bracket || error.is_unclosed_string); + let error = super::compile::VmCompileError::Compile(error) + .into_pyexception_maybe_incomplete(self, Some(&source), !terminated); + if !eof + && (unclosed + || (!terminated + && error + .fast_isinstance(self.ctx.exceptions.incomplete_input_error))) + { + continue; } - } else { - memory_errors = 0; + return Err(error); } - self.report_uncaught_exception(&error)?; - } else { - memory_errors = 0; - } - self.flush_io(); - if empty_eof { - return Ok(()); + }; + // A final fragment without a newline is not a complete line yet. + // Its warnings and definitive syntax errors are reported now, but + // it is only accepted once the next read reports EOF. + if unterminated_fragment { + continue; } + return Ok(Some(InteractiveSource { source, parsed })); } } - fn repair_interactive_syntax_error_text( + /// run_mod for interactive input + fn run_interactive_mod( &self, - error: &crate::builtins::PyBaseExceptionRef, source: &str, - ) { - use crate::AsObject; + empty: bool, + flags: CodeFlags, + main_module: &Py, + ) -> PyResult<()> { + let filename = get_interactive_filename( + "", + self.state + .interactive_src_count + .fetch_add(1, Ordering::Relaxed), + ); + let mode = if empty { + compiler::Mode::Exec + } else { + compiler::Mode::Single + }; + let opts = compiler::CompileOpts { + future_features: flags, + ..self.compile_opts() + }; + let code = self + .compile_after_parser_warnings( + source, + mode, + filename, + opts, + Some(identifier!(self, __main__)), + ) + .map_err(|error| error.into_pyexception(self, Some(source)))?; + let linecache = self.import("linecache", 0)?; + linecache.get_attr("_register_code", self)?.call( + ( + code.clone(), + self.ctx.new_str(source), + self.ctx.new_str(""), + ), + self, + )?; + self.audit("exec", || (code.clone(),))?; + let globals = main_module.dict(); + self.run_code_obj(code, Scope::with_builtins(None, globals, self))?; + Ok(()) + } - if !error.fast_isinstance(self.ctx.exceptions.syntax_error) { + /// Fix the `text` attribute of a SyntaxError raised by interactive input. + fn fix_syntax_error_text(&self, exc: &Py, source: &str) { + if !exc.fast_isinstance(self.ctx.exceptions.syntax_error) { return; } - // Match the interactive run_mod boundary: repair only text and keep - // the original exception even if attribute access/conversion fails. let source = self.ctx.new_str(source); let lines = source.splitlines(crate::anystr::SplitLinesArgs { keepends: true }, self); - let Ok(lineno) = error + let Ok(lineno) = exc .as_object() .get_attr("lineno", self) .and_then(|lineno| lineno.try_into_value::(self)) @@ -455,50 +485,54 @@ impl VirtualMachine { if lineno > 0 && let Some(line) = lines.get((lineno - 1) as usize) { - let _ = error.as_object().set_attr("text", line.clone(), self); + let _ = exc.as_object().set_attr("text", line.clone(), self); } } - /// Native fd0 adapter used by sys._baserepl and non-terminal CLI inspection. - pub fn run_basic_interactive(&self) -> PyResult<()> { - self.run_interactive(|prompt| { - let mut line = Vec::new(); - let mut prompt = prompt; - loop { - let result = crate::readline::stdio_readline_bytes(prompt, &mut line); - prompt = ""; - match result { - Ok(0) if line.is_empty() => { - eprintln!(); - return Ok(None); - } - Ok(_) => return Ok(Some(InteractiveLine::Bytes(line))), - Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { - self.check_signals()?; - } - Err(_) if !line.is_empty() => return Ok(Some(InteractiveLine::Bytes(line))), - Err(_) => return Ok(None), + /// PyOS_StdioReadline + /// + /// Write the prompt to stderr and read one line from stdin. + fn stdio_readline(&self, prompt: &str) -> PyResult> { + let mut line = Vec::new(); + let mut prompt = prompt; + loop { + let result = crate::readline::stdio_readline_bytes(prompt, &mut line); + prompt = ""; + match result { + Ok(0) if line.is_empty() => return Ok(None), + Ok(_) => return Ok(Some(InteractiveLine::Bytes(line))), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { + self.check_signals()?; } + // A read error ends the input like EOF. + Err(_) if line.is_empty() => return Ok(None), + Err(_) => return Ok(Some(InteractiveLine::Bytes(line))), } - }) + } } - /// Evaluate ordinary piped stdin as one file, without prompts or startup. - pub fn run_stdin_file(&self) -> PyResult<()> { + /// _PyRun_SimpleFile(stdin, "") + fn run_simple_stdin(&self) -> PyResult<()> { use std::io::Read; - let main = self.stdin_main_module()?; - self.with_simple_run("", |_| { + self.with_simple_run("", |module_dict| { + // _PyParser_ASTFromFile self.audit("compile", || (self.ctx.none(), self.ctx.new_str("")))?; let mut bytes = Vec::new(); + let mut buf = [0; 8192]; loop { - let result = std::io::stdin().lock().read_to_end(&mut bytes); - match result { + match std::io::stdin().read(&mut buf) { + Ok(0) => break, + Ok(n) => bytes.extend_from_slice(&buf[..n]), Err(error) if error.kind() == std::io::ErrorKind::Interrupted => { - self.check_signals()? + self.check_signals()?; } - _ => break, + // A read error ends the input like EOF. + Err(_) => break, } } + if let Some(null_at) = bytes.iter().position(|&b| b == 0) { + return Err(null_byte_syntax_error(self, "", &bytes, null_at)); + } #[cfg(feature = "parser")] let source = self.decode_source_bytes(&bytes, "", false)?; #[cfg(not(feature = "parser"))] @@ -515,12 +549,48 @@ impl VirtualMachine { ) .map_err(|error| error.into_pyexception(self, Some(&source)))?; self.audit("exec", || (code.clone(),))?; - self.run_code_obj(code, Scope::with_builtins(None, main.dict(), self)) - .map(drop) + let scope = Scope::with_builtins(None, module_dict.to_owned(), self); + self.run_code_obj(code, scope).map(drop) }) } } +fn null_byte_syntax_error( + vm: &VirtualMachine, + path: &str, + source_bytes: &[u8], + null_at: usize, +) -> PyBaseExceptionRef { + let before = &source_bytes[..null_at]; + let lineno = before.iter().filter(|&&b| b == b'\n').count() + 1; + let line_start = before + .iter() + .rposition(|&b| b == b'\n') + .map_or(0, |i| i + 1); + let line = String::from_utf8_lossy(&source_bytes[line_start..null_at]); + let syntax_error = vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + "source code cannot contain null bytes".into(), + ); + let obj = syntax_error.as_object(); + obj.set_attr("filename", vm.ctx.new_str(path), vm).unwrap(); + obj.set_attr("lineno", vm.ctx.new_int(lineno), vm).unwrap(); + obj.set_attr("text", vm.ctx.new_str(format!("{line}\n")), vm) + .unwrap(); + syntax_error +} + +/// get_interactive_filename +fn get_interactive_filename(filename: &str, count: usize) -> String { + match filename + .strip_prefix('<') + .and_then(|name| name.strip_suffix('>')) + { + Some(middle) => format!("<{middle}-{count}>"), + None => format!("{filename}-{count}"), + } +} + #[cfg(test)] mod tests { use crate::object::AsObject; diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index d611cae2acc..7230e24e75f 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -114,6 +114,19 @@ impl VirtualMachine { Ok(module) } + /// PyImport_AddModuleRef("__main__") + pub fn add_main_module(&self) -> PyResult> { + let modules = self.sys_module.get_attr(identifier!(self, modules), self)?; + if let Ok(module) = modules.get_item(identifier!(self, __main__), self) + && let Ok(module) = module.downcast::() + { + return Ok(module); + } + let module = self.new_module("__main__", self.ctx.new_dict(), None); + modules.set_item(identifier!(self, __main__), module.clone().into(), self)?; + Ok(module) + } + /// Set `__main__.__loader__` to BuiltinImporter when it is missing or None. pub fn set_main_builtin_importer( &self, diff --git a/src/lib.rs b/src/lib.rs index 09df89f682a..9c4af960d63 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,7 +59,9 @@ mod shell; #[cfg(feature = "rustpython-pylib")] pub use rustpython_pylib as pylib; -use rustpython_vm::{AsObject, PyObjectRef, PyResult, VirtualMachine, scope::Scope}; +use rustpython_vm::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, scope::Scope, +}; use std::env; use std::io::IsTerminal; use std::process::ExitCode; @@ -121,8 +123,8 @@ pub fn run(mut builder: InterpreterBuilder) -> ExitCode { let interp = builder.interpreter(); let exitcode = cfg_select! { - feature = "capi" => rustpython_capi::run_main_interpreter_with_exit_status(interp, move |vm| run_rustpython(vm, run_mode)), - _ => interp.run_with_exit_status(move |vm| run_rustpython(vm, run_mode)), + feature = "capi" => rustpython_capi::run_main_interpreter(interp, move |vm| run_rustpython(vm, run_mode)), + _ => interp.run_main(move |vm| run_rustpython(vm, run_mode)), }; rustpython_vm::host_env::os::exit_code(exitcode) @@ -213,7 +215,7 @@ fn script_sys_path0(argv0: &str) -> String { } // pymain_run_file_obj in Modules/main.c -fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { +fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult { // config_run_filename_abspath: __file__ and co_filename are absolute even // when the script was named relative to the working directory. let path = &abs_path(argv0); @@ -224,7 +226,7 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { let runpy = vm.import("runpy", 0)?; let run_module_as_main = runpy.get_attr("_run_module_as_main", vm)?; run_module_as_main.call((vm::identifier!(vm, __main__).to_owned(), false), vm)?; - return Ok(()); + return Ok(0); } // Add script directory to sys.path[0] @@ -234,7 +236,7 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { cfg_select! { feature = "host_env" => match rustpython_vm::host_env::fs::metadata(path) { - Ok(_) => vm.run_any_file(scope, path), + Ok(_) => vm.run_any_file(scope, path).map(|()| 0), Err(err) => cant_open_file(vm, path, &err), }, _ => { @@ -242,19 +244,19 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> { // The VM itself has no filesystem access. let path = if path.is_empty() { "???" } else { path }; match std::fs::read_to_string(path) { - Ok(source) => vm.run_string(scope, &source, path).map(drop), + Ok(source) => vm.run_string(scope, &source, path).map(|_| 0), Err(err) => cant_open_file(vm, path, &err), } } } } -fn cant_open_file(vm: &VirtualMachine, path: &str, err: &std::io::Error) -> PyResult<()> { +fn cant_open_file(vm: &VirtualMachine, path: &str, err: &std::io::Error) -> PyResult { let program = &vm.state.config.paths.executable; let filename_repr = vm.ctx.new_str(path).as_object().repr(vm)?; let errno = err.raw_os_error().unwrap_or(2); eprintln!("{program}: can't open file {filename_repr}: [Errno {errno}] {err}"); - Err(vm.new_system_exit(vec![vm.ctx.new_int(2).into()].into())) + Ok(2) } fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult> { @@ -288,12 +290,26 @@ fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult }) } -// pymain_run_python -fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult { - use vm::vm::InterpreterExit; +fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> u32 { #[cfg(feature = "flame-it")] let main_guard = flame::start_guard("RustPython main"); + let exitcode = + pymain_run_python(vm, run_mode).unwrap_or_else(|exc| vm.handle_exit_exception(exc)); + + #[cfg(feature = "flame-it")] + { + main_guard.end(); + if let Err(e) = write_profile(&vm.state.as_ref().config.settings) { + error!("Error writing profile information: {}", e); + } + } + + exitcode +} + +// pymain_run_python +fn pymain_run_python(vm: &VirtualMachine, run_mode: RunMode) -> PyResult { let scope = vm.new_scope_with_main()?; // Initialize warnings module to process sys.warnoptions @@ -338,8 +354,9 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult 0 || (is_repl && std::io::stdin().is_terminal())) + && (vm.state.config.settings.verbose > 0 || (is_repl && vm.stdin_is_interactive())) { eprintln!( "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", @@ -354,163 +371,213 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult { debug!("Running command {command}"); - vm.run_string(scope.clone(), &command, "").map(drop) + vm.run_string(scope.clone(), &command, "") + .map(|_| 0) } RunMode::Module(module) => { debug!("Running module {module}"); - vm.run_module(&module) + vm.run_module(&module).map(|()| 0) } - RunMode::InstallPip(installer) => install_pip(installer, scope.clone(), vm), + RunMode::InstallPip(installer) => install_pip(installer, scope.clone(), vm).map(|()| 0), RunMode::Script(script_path) => { // pymain_run_file_obj debug!("Running script {script_path}"); run_file(vm, scope.clone(), &script_path) } - RunMode::Repl => run_interactive(vm, scope.clone(), true), - }; - let result = if is_repl { - res.map(|()| InterpreterExit::Code(0)) - } else { - let reported = if let Err(error) = res { - vm.report_uncaught_exception(&error)?; - Some(error) - } else { - None - }; - if !vm.inspect_requested() - && !vm.state.config.settings.ignore_environment - && env::var_os("PYTHONINSPECT").is_some_and(|value| !value.is_empty()) - { - vm.set_inspect_requested(true)?; - } - if vm.inspect_requested() - && (std::io::stdin().is_terminal() || vm.state.config.settings.interactive) - { - run_interactive(vm, scope, false).map(|()| InterpreterExit::Code(0)) - } else { - Ok(reported.map_or(InterpreterExit::Code(0), InterpreterExit::ReportedException)) - } + RunMode::Repl => Ok(pymain_run_stdin(vm, scope.clone())), }; + let mut exitcode = result.unwrap_or_else(|exc| vm.handle_exit_exception(exc)); + vm.check_signals()?; - #[cfg(feature = "flame-it")] - { - main_guard.end(); - if let Err(e) = write_profile(&vm.state.as_ref().config.settings) { - error!("Error writing profile information: {}", e); - } - } + pymain_repl(vm, scope, !is_repl, &mut exitcode); + Ok(exitcode) +} - result +/// pymain_err_print +/// +/// Report `exc` and return the exit code if it ends the process. +fn pymain_err_print(vm: &VirtualMachine, exc: PyBaseExceptionRef) -> Option { + if let Some(exitcode) = vm.handle_system_exit_and_keyboard_interrupt(&exc) { + return Some(exitcode); + } + vm.print_uncaught_exception(&exc) + .err() + .map(|exc| vm.handle_exit_exception(exc)) } -// Follow the CLI entry points in CPython's Modules/main.c: initial -// interactive startup runs before the hook; inspection runs startup in pyrepl. -fn run_interactive(vm: &VirtualMachine, scope: Scope, initial: bool) -> PyResult<()> { - let tty = std::io::stdin().is_terminal(); - let interactive = tty || vm.state.config.settings.interactive; - if interactive { - vm.set_inspect_requested(false)?; +// pymain_set_inspect +fn pymain_set_inspect(vm: &VirtualMachine, inspect: bool) { + if let Err(exc) = vm.config_set_inspect(inspect) { + eprintln!("Could not set the inspect flag"); + vm.print_exception(&exc); } - if interactive - && initial - && !vm.state.config.settings.ignore_environment - && let Err(exc) = run_interactive_startup(vm, scope.clone()) - { - if exc.fast_isinstance(vm.ctx.exceptions.system_exit) { - return Err(exc); - } - vm.report_uncaught_exception(&exc)?; +} + +// pymain_run_startup +fn pymain_run_startup(vm: &VirtualMachine, scope: Scope) -> Option { + let exc = run_startup(vm, scope).err()?; + pymain_err_print(vm, exc) +} + +fn run_startup(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { + if vm.state.config.settings.ignore_environment { + return Ok(()); } - if interactive && let Err(exc) = run_interactive_hook(vm) { + let Some(startup) = env::var_os("PYTHONSTARTUP").filter(|value| !value.is_empty()) else { + return Ok(()); + }; + let startup = vm.fsdecode(startup); + vm.audit("cpython.run_startup", || (startup.clone(),))?; + let could_not_open = |err: std::io::Error| { + use vm::convert::ToPyException; writeln!( vm::stdlib::sys::PyStderr(vm), - "Failed calling sys.__interactivehook__" + "Could not open PYTHONSTARTUP" ); - if exc.fast_isinstance(vm.ctx.exceptions.system_exit) { - return Err(exc); - } - vm.report_uncaught_exception(&exc)?; - } - vm.check_signals()?; - vm.audit("cpython.run_stdin", || ())?; - let basic = !vm.state.config.settings.ignore_environment - && env::var_os("PYTHON_BASIC_REPL").is_some_and(|value| !value.is_empty()); - if tty && !basic { - let available = match vm.import("_pyrepl", 0) { - Ok(_) => true, - Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => false, - Err(exc) => return Err(exc), - }; - if available { - let package = vm.import("_pyrepl.main", 0)?; - let pyrepl = package.get_attr("main", vm)?; - let console = pyrepl.get_attr("interactive_console", vm)?; - let kwargs: vm::function::KwArgs = [ - ("mainmodule".to_owned(), vm.stdin_main_module()?.into()), - ("pythonstartup".to_owned(), vm.ctx.new_bool(!initial).into()), - ] - .into_iter() - .collect(); - return console - .call(vm::function::FuncArgs::new(vec![], kwargs), vm) - .map(drop); - } - } - if !interactive { - return vm.run_stdin_file(); + let exc = err.to_pyexception(vm); + exc.as_object().set_attr("filename", startup.clone(), vm)?; + Err(exc) + }; + let path = startup.expect_str(); + cfg_select! { + feature = "host_env" => match vm::host_env::fs::open(path) { + Ok(_) => vm.run_any_file(scope, path), + Err(err) => could_not_open(err), + }, + _ => match std::fs::read_to_string(path) { + Ok(source) => vm.run_string(scope, &source, path).map(drop), + Err(err) => could_not_open(err), + }, } - shell::run_shell(vm, scope) +} + +// pymain_run_interactive_hook +fn pymain_run_interactive_hook(vm: &VirtualMachine) -> Option { + let exc = run_interactive_hook(vm).err()?; + writeln!( + vm::stdlib::sys::PyStderr(vm), + "Failed calling sys.__interactivehook__" + ); + pymain_err_print(vm, exc) } fn run_interactive_hook(vm: &VirtualMachine) -> PyResult<()> { - if let Some(hook) = vm + let Some(hook) = vm .sys_module .dict() .get_item_opt("__interactivehook__", vm)? + else { + return Ok(()); + }; + vm.audit("cpython.run_interactivehook", || (hook.clone(),))?; + hook.call((), vm)?; + Ok(()) +} + +// pymain_run_stdin +fn pymain_run_stdin(vm: &VirtualMachine, scope: Scope) -> u32 { + if vm.stdin_is_interactive() { + // do exit on SystemExit + pymain_set_inspect(vm, false); + + if let Some(exitcode) = pymain_run_startup(vm, scope.clone()) { + return exitcode; + } + if let Some(exitcode) = pymain_run_interactive_hook(vm) { + return exitcode; + } + } + pymain_run_repl(vm, scope, false) +} + +// pymain_repl +fn pymain_repl(vm: &VirtualMachine, scope: Scope, run_code: bool, exitcode: &mut u32) { + // Check this environment variable at the end, to give programs the + // opportunity to set it from Python. + if !vm.config_inspect() + && !vm.state.config.settings.ignore_environment + && env::var_os("PYTHONINSPECT").is_some_and(|value| !value.is_empty()) { - vm.audit("cpython.run_interactivehook", || (hook.clone(),))?; - hook.call((), vm)?; + pymain_set_inspect(vm, true); } - Ok(()) + + if !(vm.config_inspect() && vm.stdin_is_interactive() && run_code) { + return; + } + + pymain_set_inspect(vm, false); + if let Some(code) = pymain_run_interactive_hook(vm) { + *exitcode = code; + return; + } + + *exitcode = pymain_run_repl(vm, scope, true); } -fn run_interactive_startup(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { - let Some(path) = env::var_os("PYTHONSTARTUP").filter(|value| !value.is_empty()) else { - return Ok(()); - }; - let path = vm.fsdecode(path); - vm.audit("cpython.run_startup", || (path.clone(),))?; - #[cfg(feature = "host_env")] - if let Some(filename) = path.to_str() { - // Opening precedes __main__ setup in pymain_run_startup. In - // particular, a missing startup file must not replace its loader. - if let Err(err) = vm::host_env::fs::open(filename) { - use vm::convert::ToPyException; - writeln!( - vm::stdlib::sys::PyStderr(vm), - "Could not open PYTHONSTARTUP" - ); - let exc = err.to_pyexception(vm); - exc.as_object().set_attr("filename", path.clone(), vm)?; - return Err(exc); +// _pymain_run_repl +fn pymain_run_repl(vm: &VirtualMachine, scope: Scope, startup: bool) -> u32 { + run_repl(vm, scope, startup).unwrap_or_else(|exc| vm.handle_exit_exception(exc)) +} + +fn run_repl(vm: &VirtualMachine, scope: Scope, startup: bool) -> PyResult { + vm.check_signals()?; + vm.audit("cpython.run_stdin", || ())?; + + let tty = std::io::stdin().is_terminal(); + if tty + && (vm.state.config.settings.ignore_environment + || env::var_os("PYTHON_BASIC_REPL").is_none_or(|value| value.is_empty())) + { + match vm.import("_pyrepl", 0) { + Ok(_) => return Ok(pymain_run_pyrepl(vm, startup)), + Err(exc) if !exc.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => { + eprintln!("Could not import _pyrepl.main"); + return Err(exc); + } + Err(_) => {} } - return vm.run_any_file(scope, filename); - } - let tokenize = vm.import("tokenize", 0)?; - let file = tokenize.get_attr("open", vm)?.call((path.clone(),), vm)?; - let source = file.get_attr("read", vm)?.call((), vm); - file.get_attr("close", vm)?.call((), vm)?; - let code = vm - .builtins - .get_attr("compile", vm)? - .call((source?, path, "exec"), vm)?; - vm.builtins - .get_attr("exec", vm)? - .call((code, scope.globals), vm) - .map(drop) + } + + // _PyRun_AnyFile(stdin, "") + if tty { + shell::run_shell(vm, scope)?; + } else { + vm.run_any_stdin()?; + } + Ok(0) +} + +// pymain_run_pyrepl +fn pymain_run_pyrepl(vm: &VirtualMachine, pythonstartup: bool) -> u32 { + run_pyrepl(vm, pythonstartup).map_or_else(|exc| vm.handle_exit_exception(exc), |()| 0) +} + +fn run_pyrepl(vm: &VirtualMachine, pythonstartup: bool) -> PyResult<()> { + let pyrepl = vm.import("_pyrepl.main", 0).inspect_err(|_| { + eprintln!("Could not import _pyrepl.main"); + })?; + let console = pyrepl + .get_attr("main", vm) + .and_then(|main| main.get_attr("interactive_console", vm)) + .inspect_err(|_| { + eprintln!("Could not access _pyrepl.main.interactive_console"); + })?; + let kwargs: vm::function::KwArgs = [ + ("mainmodule".to_owned(), vm.add_main_module()?.into()), + ( + "pythonstartup".to_owned(), + vm.ctx.new_bool(pythonstartup).into(), + ), + ] + .into_iter() + .collect(); + console.call(vm::function::FuncArgs::new(vec![], kwargs), vm)?; + Ok(()) } #[cfg(feature = "flame-it")] diff --git a/src/shell.rs b/src/shell.rs index 9413a2e05bf..7d206f28e9e 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -7,12 +7,11 @@ use rustpython_vm::{ vm::InteractiveLine, }; -/// Root-only editor/history policy, adapting into the VM's shared evaluator. +/// Enter a repl loop +/// +/// rustyline takes the role of `PyOS_Readline` with the readline module, +/// adding line editing, completion and history. pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { - use std::io::IsTerminal; - if !std::io::stdin().is_terminal() { - return vm.run_basic_interactive(); - } let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals)); let history = dirs::config_dir().map_or_else( || ".repl_history.txt".into(), @@ -23,7 +22,7 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { }, ); let _ = repl.load_history(&history); - let result = vm.run_interactive(|prompt| match repl.readline(prompt) { + let result = vm.run_interactive_loop(|prompt| match repl.readline(prompt) { ReadlineResult::Line(line) => { let _ = repl.add_history_entry(line.trim_end()); Ok(Some(InteractiveLine::Text(line))) From 8a20dd6f81d403f586c61529d579872dbcf12be7 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 01:54:09 +0900 Subject: [PATCH 14/19] Exit through Py_Exit from PyErr_Print and run the basic REPL via stdin - print_uncaught_exception (PyErr_Print) calls exit_process (Py_Exit) for a SystemExit when inspect mode is off, instead of returning the exception. run_interactive_loop no longer checks SystemExit itself and flushes after printing an error; sys._baserepl returns None. - exit_process finalizes through VirtualMachine::finalize_ex, which holds the Py_FinalizeEx steps split out of Interpreter::finalize, and exits through host_env::os::exit_with_cleanup (C exit()). - The basic REPL on a terminal (PYTHON_BASIC_REPL, or _pyrepl missing) runs run_any_stdin instead of the rustyline shell. - run_shell saves the history after every line. Assisted-by: Claude Code:claude-opus-5-5 --- crates/host_env/src/os.rs | 6 ++ crates/vm/src/exceptions.rs | 25 ++--- crates/vm/src/stdlib/sys.rs | 5 +- crates/vm/src/vm/interpreter.rs | 165 ++++++++++++++++++-------------- crates/vm/src/vm/mod.rs | 8 +- crates/vm/src/vm/python_run.rs | 8 +- src/lib.rs | 36 +++---- src/shell.rs | 9 +- 8 files changed, 134 insertions(+), 128 deletions(-) diff --git a/crates/host_env/src/os.rs b/crates/host_env/src/os.rs index f716d2770c1..7839965d2fd 100644 --- a/crates/host_env/src/os.rs +++ b/crates/host_env/src/os.rs @@ -519,6 +519,12 @@ pub fn device_encoding(_fd: i32) -> Option { } } +/// Terminate the process through the C `exit()`, which runs the C atexit +/// handlers and flushes C stdio. +pub fn exit_with_cleanup(code: i32) -> ! { + std::process::exit(code) +} + pub fn exit(code: i32) -> ! { // These callers require immediate termination without C exit handlers. cfg_select! { diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index c283026d136..e3ed5f022e3 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -115,10 +115,10 @@ impl VirtualMachine { /// PyErr_Print /// /// Record the exception as `sys.last_exc` and display it through - /// `sys.excepthook`. A `SystemExit` that should end the process is returned - /// instead, and the caller exits with it. - pub fn print_uncaught_exception(&self, exc: &Py) -> PyResult<()> { - self.handle_system_exit(exc)?; + /// `sys.excepthook`. A `SystemExit` exits the process unless inspect mode + /// is enabled. + pub fn print_uncaught_exception(&self, exc: &Py) { + self.handle_system_exit(exc); let (exc_type, exc_val, exc_tb) = self.split_exception(exc.to_owned()); for (name, value) in [ ("last_exc", &exc_val), @@ -138,7 +138,7 @@ impl VirtualMachine { ) }) { if audit_exc.fast_isinstance(self.ctx.exceptions.runtime_error) { - return Ok(()); + return; } self.run_unraisable( audit_exc, @@ -148,25 +148,20 @@ impl VirtualMachine { } if let Some(hook) = hook { if let Err(hook_exc) = hook.call((exc_type, exc_val, exc_tb), self) { - self.handle_system_exit(&hook_exc)?; + self.handle_system_exit(&hook_exc); self.write_exception_fallback(&hook_exc, "Error in sys.excepthook:"); self.write_exception_fallback(exc, "\nOriginal exception was:"); } } else { self.write_exception_fallback(exc, "sys.excepthook is missing"); } - Ok(()) } - /// handle_system_exit, except that the caller performs `Py_Exit()` with - /// the returned `SystemExit`. - fn handle_system_exit(&self, exc: &Py) -> PyResult<()> { - if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { - crate::signal::set_unhandled_keyboard_interrupt(); - } else if !self.config_inspect() && exc.fast_isinstance(self.ctx.exceptions.system_exit) { - return Err(exc.to_owned()); + /// handle_system_exit + fn handle_system_exit(&self, exc: &Py) { + if let Some(exit_code) = self.handle_system_exit_and_keyboard_interrupt(exc) { + self.exit_process(exit_code); } - Ok(()) } fn write_exception_fallback(&self, exc: &Py, heading: &str) { diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index d8b99a31436..3878dc52357 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -760,12 +760,11 @@ pub mod sys { #[cfg(feature = "rustpython-compiler")] #[pyfunction] - fn _baserepl(vm: &VirtualMachine) -> PyResult<()> { + fn _baserepl(vm: &VirtualMachine) { // PyRun_AnyFileExFlags(stdin, "", 0, &cf) if let Err(exc) = vm.run_any_stdin() { - vm.print_uncaught_exception(&exc)?; + vm.print_uncaught_exception(&exc); } - Ok(()) } #[pyfunction] diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 562346492d1..42eed81ae1c 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -696,87 +696,17 @@ impl Interpreter { fn finalize_inner(self, exit_code: impl FnOnce(&VirtualMachine) -> u32) -> u32 { let exit_code = self.enter(|vm| { - let mut flush_status = vm.flush_std(); + let flush_status = vm.flush_std(); // See if any exception leaked out: let exit_code = exit_code(vm); - // Wait for thread shutdown - call threading._shutdown() if available. - // This waits for all non-daemon threads to complete. - // threading module may not be imported, so ignore import errors. - #[cfg(feature = "threading")] - if let Ok(threading) = vm.import("threading", 0) - && let Ok(shutdown) = threading.get_attr("_shutdown", vm) - && let Err(e) = shutdown.call((), vm) - { - vm.run_unraisable( - e, - Some("Exception ignored in threading shutdown".to_owned()), - threading, - ); - } - - // Run atexit handlers before setting finalizing flag. - // This allows unraisable exceptions from atexit handlers to be reported. - atexit::_run_exitfuncs(vm); - - // Clean up any lingering subinterpreters. This has to happen before - // the finalizing flag is set, or else threads might get prematurely - // blocked. - #[cfg(feature = "threading")] - finalize_subinterpreters(vm); - - // Remaining daemon threads hang in `_PyThreadState_HangThread` when - // they next attach (or at the eval-breaker if they are already - // running). Their `_ThreadHandle` stays not-done so `is_alive()` - // is still true for `join()` during the GC that follows. - #[cfg(feature = "threading")] - { - vm.state.stop_the_world.stop_the_world(&vm.state); - vm.state - .finalizing_thread_ident - .store(crate::stdlib::_thread::get_ident()); - } - vm.state.finalizing.store(true, Ordering::Release); - #[cfg(feature = "threading")] - { - thread::set_other_threads_shutting_down(&vm.state); - vm.state.stop_the_world.start_the_world(&vm.state); - crate::signal::set_finalizing_bit(); - } - - // GC pass - collect cycles before module cleanup - vm.state.gc.collect_force(2); - - // Module finalization: remove modules from sys.modules, GC collect - // (while builtins is still available for __del__), then clear module dicts. - vm.finalize_modules(); - - // CPython clears low-level cross-interpreter container data from - // _PyAtExit_Fini(), after Python atexit callbacks and module - // finalization. In particular, values sent by an atexit callback - // must also become unbound when this interpreter goes away. - let interpreter_id = vm.state.interpreter_id; - crate::stdlib::_interpchannels::clear_interpreter(interpreter_id); - crate::stdlib::_interpqueues::clear_interpreter(interpreter_id); - - if vm.flush_std() < 0 && flush_status == 0 { - flush_status = -1; - } - // Match CPython: if exit_code is 0 and stdout flush failed, exit 120 - let exit_code = if exit_code == 0 && flush_status < 0 { + if vm.finalize_ex(flush_status) < 0 && exit_code == 0 { EXITCODE_FLUSH_FAILURE } else { exit_code - }; - - // Daemon threads may still exist, so use the safe `process()`, - // not `drain_all()`. - #[cfg(feature = "threading")] - crate::object::qsbr::QSBR.process(); - - exit_code + } }); if crate::signal::unhandled_keyboard_interrupt() { crate::signal::exit_sigint() @@ -786,6 +716,95 @@ impl Interpreter { } } +impl VirtualMachine { + /// Py_Exit + /// + /// Finalize the interpreter and exit the process with `status`. + pub fn exit_process(&self, status: u32) -> ! { + let status = if self.finalize_ex(self.flush_std()) < 0 { + EXITCODE_FLUSH_FAILURE + } else { + status + }; + crate::host_env::os::exit_with_cleanup(status as i32) + } + + /// The steps of Py_FinalizeEx after the first flush of stdout and stderr. + /// Return a negative value if any flush failed. + fn finalize_ex(&self, mut flush_status: i32) -> i32 { + let vm = self; + // Wait for thread shutdown - call threading._shutdown() if available. + // This waits for all non-daemon threads to complete. + // threading module may not be imported, so ignore import errors. + #[cfg(feature = "threading")] + if let Ok(threading) = vm.import("threading", 0) + && let Ok(shutdown) = threading.get_attr("_shutdown", vm) + && let Err(e) = shutdown.call((), vm) + { + vm.run_unraisable( + e, + Some("Exception ignored in threading shutdown".to_owned()), + threading, + ); + } + + // Run atexit handlers before setting finalizing flag. + // This allows unraisable exceptions from atexit handlers to be reported. + atexit::_run_exitfuncs(vm); + + // Clean up any lingering subinterpreters. This has to happen before + // the finalizing flag is set, or else threads might get prematurely + // blocked. + #[cfg(feature = "threading")] + finalize_subinterpreters(vm); + + // Remaining daemon threads hang in `_PyThreadState_HangThread` when + // they next attach (or at the eval-breaker if they are already + // running). Their `_ThreadHandle` stays not-done so `is_alive()` + // is still true for `join()` during the GC that follows. + #[cfg(feature = "threading")] + { + vm.state.stop_the_world.stop_the_world(&vm.state); + vm.state + .finalizing_thread_ident + .store(crate::stdlib::_thread::get_ident()); + } + vm.state.finalizing.store(true, Ordering::Release); + #[cfg(feature = "threading")] + { + thread::set_other_threads_shutting_down(&vm.state); + vm.state.stop_the_world.start_the_world(&vm.state); + crate::signal::set_finalizing_bit(); + } + + // GC pass - collect cycles before module cleanup + vm.state.gc.collect_force(2); + + // Module finalization: remove modules from sys.modules, GC collect + // (while builtins is still available for __del__), then clear module dicts. + vm.finalize_modules(); + + // CPython clears low-level cross-interpreter container data from + // _PyAtExit_Fini(), after Python atexit callbacks and module + // finalization. In particular, values sent by an atexit callback + // must also become unbound when this interpreter goes away. + let interpreter_id = vm.state.interpreter_id; + crate::stdlib::_interpchannels::clear_interpreter(interpreter_id); + crate::stdlib::_interpqueues::clear_interpreter(interpreter_id); + + if vm.flush_std() < 0 && flush_status == 0 { + flush_status = -1; + } + + // Daemon threads may still exist, so use the safe `process()`, + // not `drain_all()`. + #[cfg(feature = "threading")] + crate::object::qsbr::QSBR.process(); + + flush_status + } +} + /// `finalize_subinterpreters`: destroy the subinterpreters the program left /// behind, after telling the user they are still around. #[cfg(feature = "threading")] diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 3ff1d44e6ce..65b8866fc41 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -3931,12 +3931,8 @@ impl VirtualMachine { if let Some(exit_code) = self.handle_system_exit_and_keyboard_interrupt(&exc) { return exit_code; } - match self.print_uncaught_exception(&exc) { - Ok(()) => 1, - Err(exc) => self - .handle_system_exit_and_keyboard_interrupt(&exc) - .unwrap_or(1), - } + self.print_uncaught_exception(&exc); + 1 } /// _Py_HandleSystemExitAndKeyboardInterrupt diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index dd5876a2a6a..b8bf6c184dd 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -264,12 +264,8 @@ impl VirtualMachine { } else { nomem_count = 0; } - if !self.config_inspect() - && exc.fast_isinstance(self.ctx.exceptions.system_exit) - { - return Err(exc); - } - self.print_uncaught_exception(&exc)?; + self.print_uncaught_exception(&exc); + self.flush_io(); } } } diff --git a/src/lib.rs b/src/lib.rs index 9c4af960d63..c74568cccf1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -376,25 +376,24 @@ fn pymain_run_python(vm: &VirtualMachine, run_mode: RunMode) -> PyResult { let result = match run_mode { RunMode::Command(command) => { debug!("Running command {command}"); - vm.run_string(scope.clone(), &command, "") - .map(|_| 0) + vm.run_string(scope, &command, "").map(|_| 0) } RunMode::Module(module) => { debug!("Running module {module}"); vm.run_module(&module).map(|()| 0) } - RunMode::InstallPip(installer) => install_pip(installer, scope.clone(), vm).map(|()| 0), + RunMode::InstallPip(installer) => install_pip(installer, scope, vm).map(|()| 0), RunMode::Script(script_path) => { // pymain_run_file_obj debug!("Running script {script_path}"); - run_file(vm, scope.clone(), &script_path) + run_file(vm, scope, &script_path) } - RunMode::Repl => Ok(pymain_run_stdin(vm, scope.clone())), + RunMode::Repl => Ok(pymain_run_stdin(vm, scope)), }; let mut exitcode = result.unwrap_or_else(|exc| vm.handle_exit_exception(exc)); vm.check_signals()?; - pymain_repl(vm, scope, !is_repl, &mut exitcode); + pymain_repl(vm, !is_repl, &mut exitcode); Ok(exitcode) } @@ -405,9 +404,8 @@ fn pymain_err_print(vm: &VirtualMachine, exc: PyBaseExceptionRef) -> Option if let Some(exitcode) = vm.handle_system_exit_and_keyboard_interrupt(&exc) { return Some(exitcode); } - vm.print_uncaught_exception(&exc) - .err() - .map(|exc| vm.handle_exit_exception(exc)) + vm.print_uncaught_exception(&exc); + None } // pymain_set_inspect @@ -485,18 +483,18 @@ fn pymain_run_stdin(vm: &VirtualMachine, scope: Scope) -> u32 { // do exit on SystemExit pymain_set_inspect(vm, false); - if let Some(exitcode) = pymain_run_startup(vm, scope.clone()) { + if let Some(exitcode) = pymain_run_startup(vm, scope) { return exitcode; } if let Some(exitcode) = pymain_run_interactive_hook(vm) { return exitcode; } } - pymain_run_repl(vm, scope, false) + pymain_run_repl(vm, false) } // pymain_repl -fn pymain_repl(vm: &VirtualMachine, scope: Scope, run_code: bool, exitcode: &mut u32) { +fn pymain_repl(vm: &VirtualMachine, run_code: bool, exitcode: &mut u32) { // Check this environment variable at the end, to give programs the // opportunity to set it from Python. if !vm.config_inspect() @@ -516,15 +514,15 @@ fn pymain_repl(vm: &VirtualMachine, scope: Scope, run_code: bool, exitcode: &mut return; } - *exitcode = pymain_run_repl(vm, scope, true); + *exitcode = pymain_run_repl(vm, true); } // _pymain_run_repl -fn pymain_run_repl(vm: &VirtualMachine, scope: Scope, startup: bool) -> u32 { - run_repl(vm, scope, startup).unwrap_or_else(|exc| vm.handle_exit_exception(exc)) +fn pymain_run_repl(vm: &VirtualMachine, startup: bool) -> u32 { + run_repl(vm, startup).unwrap_or_else(|exc| vm.handle_exit_exception(exc)) } -fn run_repl(vm: &VirtualMachine, scope: Scope, startup: bool) -> PyResult { +fn run_repl(vm: &VirtualMachine, startup: bool) -> PyResult { vm.check_signals()?; vm.audit("cpython.run_stdin", || ())?; @@ -544,11 +542,7 @@ fn run_repl(vm: &VirtualMachine, scope: Scope, startup: bool) -> PyResult { } // _PyRun_AnyFile(stdin, "") - if tty { - shell::run_shell(vm, scope)?; - } else { - vm.run_any_stdin()?; - } + vm.run_any_stdin()?; Ok(0) } diff --git a/src/shell.rs b/src/shell.rs index 7d206f28e9e..6f8836eed29 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -22,9 +22,12 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { }, ); let _ = repl.load_history(&history); - let result = vm.run_interactive_loop(|prompt| match repl.readline(prompt) { + // A SystemExit exits the process from inside the loop, so the history + // is saved after every line. + vm.run_interactive_loop(|prompt| match repl.readline(prompt) { ReadlineResult::Line(line) => { let _ = repl.add_history_entry(line.trim_end()); + let _ = repl.save_history(&history); Ok(Some(InteractiveLine::Text(line))) } ReadlineResult::Eof => Ok(None), @@ -35,7 +38,5 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { #[cfg(unix)] ReadlineResult::OsError(_) => Ok(None), ReadlineResult::Other(_) => Ok(None), - }); - let _ = repl.save_history(&history); - result + }) } From e2ac3c01f5bf4a5db2c880800217cb9fe9218f35 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 09:13:21 +0900 Subject: [PATCH 15/19] Display exceptions through PyErr_Display and track the main thread running __main__ - display_exception (PyErr_Display) and display_exception_to (_PyErr_Display) try traceback._print_exception_bltin before the native printer and flush the file. PyErr_Print's excepthook fallback, print_exception and sys.excepthook use them. A missing sys.stderr is reported with dump_object (PyObject_Dump); a None sys.stderr prints nothing. - Paths gains program_name (argv[0]) from getpath. The "can't open file" message uses it with strerror(errno) and goes to sys.stderr. - running_main holds the ident of the thread running __main__. pymain_run_python sets it around running the main code, Py_Exit clears it, the fork child reinitializes it, and _interpreters.is_running reads it instead of treating the main interpreter as always running. - Interpreter::run_main clears the unhandled KeyboardInterrupt flag before running and applies exit_sigint after finalizing; finalize no longer does. - run_interactive_loop returns a SystemExit raised outside inspect mode again and no longer flushes after printing an error; run_shell saves the history once after the loop. - pymain_set_inspect reports its error through print_uncaught_exception. Assisted-by: Claude Code:claude-opus-5-5 --- crates/vm/src/exceptions.rs | 66 +++++++++++++++++++++++++-------- crates/vm/src/frame.rs | 2 +- crates/vm/src/getpath.rs | 23 +++++++++++- crates/vm/src/signal.rs | 4 ++ crates/vm/src/stdlib/posix.rs | 2 + crates/vm/src/stdlib/sys.rs | 16 ++------ crates/vm/src/vm/crossinterp.rs | 18 ++------- crates/vm/src/vm/interpreter.rs | 25 ++++++++----- crates/vm/src/vm/mod.rs | 42 ++++++++++++++++++++- crates/vm/src/vm/python_run.rs | 6 ++- crates/vm/src/vm/setting.rs | 2 + crates/vm/src/vm/vm_object.rs | 17 +++++++++ src/lib.rs | 23 ++++++++++-- src/shell.rs | 9 ++--- 14 files changed, 190 insertions(+), 65 deletions(-) diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index e3ed5f022e3..b6f0cad9303 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -104,11 +104,14 @@ impl VirtualMachine { if let Ok(excepthook) = vm.sys_module.get_attr("excepthook", vm) { let (exc_type, exc_val, exc_tb) = vm.split_exception(exc.to_owned()); if let Err(eh_exc) = excepthook.call((exc_type, exc_val, exc_tb), vm) { - self.write_exception_fallback(&eh_exc, "Error in sys.excepthook:"); - self.write_exception_fallback(exc, "Original exception was:"); + writeln!(sys::PyStderr(vm), "Error in sys.excepthook:"); + vm.display_exception(&eh_exc); + writeln!(sys::PyStderr(vm), "Original exception was:"); + vm.display_exception(exc); } } else { - self.write_exception_fallback(exc, "missing sys.excepthook"); + writeln!(sys::PyStderr(vm), "missing sys.excepthook"); + vm.display_exception(exc); } } @@ -149,11 +152,15 @@ impl VirtualMachine { if let Some(hook) = hook { if let Err(hook_exc) = hook.call((exc_type, exc_val, exc_tb), self) { self.handle_system_exit(&hook_exc); - self.write_exception_fallback(&hook_exc, "Error in sys.excepthook:"); - self.write_exception_fallback(exc, "\nOriginal exception was:"); + let _ = io::Write::flush(&mut io::stdout()); + writeln!(sys::PyStderr(self), "Error in sys.excepthook:"); + self.display_exception(&hook_exc); + writeln!(sys::PyStderr(self), "\nOriginal exception was:"); + self.display_exception(exc); } } else { - self.write_exception_fallback(exc, "sys.excepthook is missing"); + writeln!(sys::PyStderr(self), "sys.excepthook is missing"); + self.display_exception(exc); } } @@ -164,18 +171,47 @@ impl VirtualMachine { } } - fn write_exception_fallback(&self, exc: &Py, heading: &str) { - if let Ok(stderr) = sys::get_stderr(self) { - let mut stderr = py_io::PyWriter(stderr, self); - // if this fails stderr might be closed -- ignore it - let _ = writeln!(stderr, "{heading}"); - let _ = self.write_exception(&mut stderr, exc); - } else { - eprintln!("{heading}\nlost sys.stderr"); - let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), exc); + /// PyErr_Display + /// + /// Display `exc` on `sys.stderr`; nothing is written when it is `None`. + pub fn display_exception(&self, exc: &Py) { + let file = self.sys_module.dict().get_item_opt("stderr", self); + if let Ok(Some(file)) = &file { + if !self.is_none(file) { + self.display_exception_to(file, exc); + } + return; + } + self.dump_object(exc.as_object()); + eprintln!("lost sys.stderr"); + if let Err(err) = file { + self.dump_object(err.as_object()); } } + /// _PyErr_Display + /// + /// Display `exc` through `traceback._print_exception_bltin`, falling back + /// to the native printer writing to `file`. + pub(crate) fn display_exception_to(&self, file: &PyObject, exc: &Py) { + if let Ok(print_exception) = self + .import("traceback", 0) + .and_then(|traceback| traceback.get_attr("_print_exception_bltin", self)) + && print_exception.is_callable() + && print_exception.call((exc.to_owned(),), self).is_ok() + { + return; + } + if self + .write_exception(&mut py_io::PyWriter(file.to_owned(), self), exc) + .is_err() + { + self.dump_object(exc.as_object()); + eprintln!("lost sys.stderr"); + } + let _ = self.call_method(file, "flush", ()); + } + pub fn write_exception( &self, output: &mut W, diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 07d5a020983..20759fb5f8b 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -2729,7 +2729,7 @@ impl Py { /// 0 doubles as "no thread", which is what `attached_tid` wants for a build /// that cannot have a frame running anywhere else. #[inline] -fn current_thread_ident() -> u64 { +pub(crate) fn current_thread_ident() -> u64 { #[cfg(feature = "threading")] { crate::stdlib::_thread::get_ident() diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs index 02357f12660..68e95583a13 100644 --- a/crates/vm/src/getpath.rs +++ b/crates/vm/src/getpath.rs @@ -19,6 +19,7 @@ mod platform { pub(super) const BUILD_LANDMARK: &str = "Modules/Setup.local"; pub(super) const VENV_LANDMARK: &str = "pyvenv.cfg"; pub(super) const BUILDSTDLIB_LANDMARK: &str = "Lib/os.py"; + pub(super) const DEFAULT_PROGRAM_NAME: &str = "python3"; pub(super) fn stdlib_subdir() -> String { format!("lib/python{}.{}", version::MAJOR, version::MINOR) @@ -51,6 +52,7 @@ mod platform { pub(super) const VENV_LANDMARK: &str = "pyvenv.cfg"; pub(super) const BUILDSTDLIB_LANDMARK: &str = "Lib\\os.py"; pub(super) const STDLIB_SUBDIR: &str = "Lib"; + pub(super) const DEFAULT_PROGRAM_NAME: &str = "python"; pub(super) fn stdlib_landmarks() -> [String; 2] { ["Lib\\os.py".into(), "Lib\\os.pyc".into()] @@ -106,7 +108,10 @@ fn search_up_dir>(start: P, landmarks: &[&str]) -> Option Paths { - let mut paths = Paths::default(); + let mut paths = Paths { + program_name: program_name(), + ..Paths::default() + }; // Step 0: Get executable path let executable = get_executable_path(); @@ -488,6 +493,22 @@ fn env_executable_override() -> Option { None } +/// CALCULATE program_name: `argv[0]`, with the executable suffix added when +/// only the suffixed file exists. +fn program_name() -> String { + let program_name = env::args() + .next() + .filter(|arg| !arg.is_empty()) + .unwrap_or_else(|| platform::DEFAULT_PROGRAM_NAME.to_owned()); + if cfg!(windows) && !program_name.to_lowercase().ends_with(".exe") { + let suffixed = format!("{program_name}.exe"); + if Path::new(&suffixed).is_file() { + return suffixed; + } + } + program_name +} + /// Get the current executable path fn get_executable_path() -> Option { #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index 7a414726559..4ea023505a8 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -65,6 +65,10 @@ pub(crate) fn set_unhandled_keyboard_interrupt() { UNHANDLED_KEYBOARD_INTERRUPT.store(true, Ordering::Relaxed); } +pub(crate) fn clear_unhandled_keyboard_interrupt() { + UNHANDLED_KEYBOARD_INTERRUPT.store(false, Ordering::Relaxed); +} + pub(crate) fn unhandled_keyboard_interrupt() -> bool { UNHANDLED_KEYBOARD_INTERRUPT.load(Ordering::Relaxed) } diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index ae70b7001bb..8e142c41793 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -755,6 +755,8 @@ pub mod module { crate::object::qsbr::QSBR.reset_after_fork() }; + vm.state.reinit_running_main(); + // Phase 3: Clean up thread state. Locks are now reinit'd so we can // acquire them normally instead of using try_lock(). #[cfg(feature = "threading")] diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 3878dc52357..6051723b19a 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -861,7 +861,6 @@ pub mod sys { traceback: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - let stderr = super::get_stderr(vm)?; // Keep runtime SyntaxErrors on the normal traceback path. let has_traceback = !vm.is_none(&traceback); match vm.normalize_exception(exctype, value.clone(), traceback) { @@ -881,19 +880,11 @@ pub mod sys { .and_then(|text| text.downcast::().ok()) .is_some_and(|text| text.to_string_lossy().trim_end() == "\\"); if native_syntax_error_display { + let stderr = super::get_stderr(vm)?; return vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc); } - // PyErr_Display: try traceback._print_exception_bltin first - if let Ok(tb_mod) = vm.import("traceback", 0) - && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) - && print_exc_builtin - .call((exc.as_object().to_owned(),), vm) - .is_ok() - { - return Ok(()); - } - // Fallback to Rust-level exception printing - vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc) + vm.display_exception(&exc); + Ok(()) } Err(_) => { let type_name = value.class().name(); @@ -901,6 +892,7 @@ pub mod sys { "TypeError: print_exception(): Exception expected for value, {type_name} found\n" ); use crate::py_io::Write; + let stderr = super::get_stderr(vm)?; write!(&mut crate::py_io::PyWriter(stderr, vm), "{msg}")?; Ok(()) } diff --git a/crates/vm/src/vm/crossinterp.rs b/crates/vm/src/vm/crossinterp.rs index 70ea5104827..1fb46a64356 100644 --- a/crates/vm/src/vm/crossinterp.rs +++ b/crates/vm/src/vm/crossinterp.rs @@ -932,16 +932,7 @@ where )); } - if state - .running_main - .compare_exchange( - false, - true, - core::sync::atomic::Ordering::AcqRel, - core::sync::atomic::Ordering::Acquire, - ) - .is_err() - { + if !state.set_running_main() { return Err(crate::stdlib::_interpreters::interpreter_error( caller, "interpreter already running", @@ -951,9 +942,7 @@ where struct RunningGuard<'a>(&'a crate::vm::PyGlobalState); impl Drop for RunningGuard<'_> { fn drop(&mut self) { - self.0 - .running_main - .store(false, core::sync::atomic::Ordering::Release); + self.0.set_not_running_main(); } } let _guard = RunningGuard(&state); @@ -965,6 +954,5 @@ where #[must_use] pub fn is_running(id: i64) -> bool { - crate::vm::runtime::lookup_interpreter(id) - .is_some_and(|s| s.is_main || s.running_main.load(core::sync::atomic::Ordering::Acquire)) + crate::vm::runtime::lookup_interpreter(id).is_some_and(|s| s.is_running_main()) } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 42eed81ae1c..b7c4c345133 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -197,7 +197,7 @@ where stop_the_world: StopTheWorldState::new(), feature_flags, own_gil, - running_main: AtomicBool::new(false), + running_main: AtomicCell::new(None), ready: AtomicBool::new(false), id_refcount: AtomicI64::new(0), require_idref: AtomicBool::new(false), @@ -660,13 +660,20 @@ impl Interpreter { /// Py_RunMain /// /// Run a function that reports its own errors and returns the exit code, - /// then finalize with that exit code. + /// then finalize with that exit code. An unhandled `KeyboardInterrupt` + /// exits through the default SIGINT handler instead. pub fn run_main(self, f: F) -> u32 where F: FnOnce(&VirtualMachine) -> u32, { + crate::signal::clear_unhandled_keyboard_interrupt(); let exit_code = self.enter(f); - self.finalize_with_exit_code(exit_code) + let exit_code = self.finalize_with_exit_code(exit_code); + if crate::signal::unhandled_keyboard_interrupt() { + crate::signal::exit_sigint() + } else { + exit_code + } } /// Finalize vm and turns an exception to exit code. @@ -695,7 +702,7 @@ impl Interpreter { } fn finalize_inner(self, exit_code: impl FnOnce(&VirtualMachine) -> u32) -> u32 { - let exit_code = self.enter(|vm| { + self.enter(|vm| { let flush_status = vm.flush_std(); // See if any exception leaked out: @@ -707,12 +714,7 @@ impl Interpreter { } else { exit_code } - }); - if crate::signal::unhandled_keyboard_interrupt() { - crate::signal::exit_sigint() - } else { - exit_code - } + }) } } @@ -721,6 +723,9 @@ impl VirtualMachine { /// /// Finalize the interpreter and exit the process with `status`. pub fn exit_process(&self, status: u32) -> ! { + if self.state.thread_is_running_main() { + self.state.set_not_running_main(); + } let status = if self.finalize_ex(self.flush_std()) < 0 { EXITCODE_FLUSH_FAILURE } else { diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 65b8866fc41..0bbb420ff57 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -924,8 +924,8 @@ pub struct PyGlobalState { pub feature_flags: runtime::InterpFeatureFlags, /// Whether the interpreter was configured with `gil="own"`. pub own_gil: bool, - /// Whether `__main__` is currently executing via `_interpreters.exec` / `run_*`. - pub running_main: AtomicBool, + /// `threads.main`: the ident of the thread running `__main__`, if any. + pub running_main: AtomicCell>, /// True after `initialize()` has finished (CPython "ready"). pub ready: AtomicBool, /// Optional ID refcount used by `_interpreters.create(reqrefs=True)`. @@ -965,6 +965,44 @@ impl PyGlobalState { self.feature_flags.allow_daemon_threads } + /// _PyInterpreterState_SetRunningMain + /// + /// Record the calling thread as the one running `__main__`. Return `false` + /// when a thread already does. + pub fn set_running_main(&self) -> bool { + self.running_main + .compare_exchange(None, Some(crate::frame::current_thread_ident())) + .is_ok() + } + + /// _PyInterpreterState_SetNotRunningMain + pub fn set_not_running_main(&self) { + debug_assert!(self.thread_is_running_main()); + self.running_main.store(None); + } + + /// _PyInterpreterState_IsRunningMain + #[must_use] + pub fn is_running_main(&self) -> bool { + self.running_main.load().is_some() + } + + /// _PyThreadState_IsRunningMain + #[must_use] + pub fn thread_is_running_main(&self) -> bool { + self.running_main.load() == Some(crate::frame::current_thread_ident()) + } + + /// _PyInterpreterState_ReinitRunningMain + /// + /// After a fork only the forking thread survives, so `__main__` is no + /// longer running unless that thread was running it. + pub(crate) fn reinit_running_main(&self) { + if !self.thread_is_running_main() { + self.running_main.store(None); + } + } + /// The config this interpreter was created with, rebuilt from the flags it /// kept (`_PyInterpreterConfig_InitFromState`). #[must_use] diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index b8bf6c184dd..a9fcb56ca63 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -264,8 +264,12 @@ impl VirtualMachine { } else { nomem_count = 0; } + if !self.config_inspect() + && exc.fast_isinstance(self.ctx.exceptions.system_exit) + { + return Err(exc); + } self.print_uncaught_exception(&exc); - self.flush_io(); } } } diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index df07adf0fd1..9f3ef85f7d4 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -5,6 +5,8 @@ use std::ffi::OsString; /// Path configuration computed at runtime (like PyConfig path outputs) #[derive(Debug, Clone, Default)] pub struct Paths { + /// program_name: the name the interpreter was invoked as + pub program_name: String, /// sys.executable pub executable: String, /// sys._base_executable (original interpreter in venv) diff --git a/crates/vm/src/vm/vm_object.rs b/crates/vm/src/vm/vm_object.rs index baa0c8095c6..db9459b9b75 100644 --- a/crates/vm/src/vm/vm_object.rs +++ b/crates/vm/src/vm/vm_object.rs @@ -9,6 +9,23 @@ use crate::{ /// PyObject support impl VirtualMachine { + /// PyObject_Dump + /// + /// Write a description of `obj` to the process stderr, bypassing + /// `sys.stderr`. + pub fn dump_object(&self, obj: &PyObject) { + let class = obj.class(); + eprintln!("object address : {obj:p}"); + eprintln!("object refcount : {}", obj.strong_count()); + eprintln!("object type : {class:p}"); + eprintln!("object type name: {}", class.slot_name()); + eprint!("object repr : "); + if let Ok(repr) = obj.repr(self) { + eprint!("{}", repr.as_wtf8()); + } + eprintln!(); + } + #[track_caller] #[cold] fn _py_panic_failed(&self, exc: PyBaseExceptionRef, msg: &str) -> ! { diff --git a/src/lib.rs b/src/lib.rs index c74568cccf1..2f6ee077f53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -252,10 +252,14 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult { } fn cant_open_file(vm: &VirtualMachine, path: &str, err: &std::io::Error) -> PyResult { - let program = &vm.state.config.paths.executable; + let program_name = &vm.state.config.paths.program_name; let filename_repr = vm.ctx.new_str(path).as_object().repr(vm)?; let errno = err.raw_os_error().unwrap_or(2); - eprintln!("{program}: can't open file {filename_repr}: [Errno {errno}] {err}"); + let strerror = vm::host_env::errno::strerror_string(errno).unwrap_or_default(); + writeln!( + vm::stdlib::sys::PyStderr(vm), + "{program_name}: can't open file {filename_repr}: [Errno {errno}] {strerror}" + ); Ok(2) } @@ -372,6 +376,19 @@ fn pymain_run_python(vm: &VirtualMachine, run_mode: RunMode) -> PyResult { eprintln!("Type \"help\", \"copyright\", \"credits\" or \"license\" for more information."); } + let set_running_main = vm.state.set_running_main(); + let exitcode = + pymain_run_main(vm, scope, run_mode).unwrap_or_else(|exc| vm.handle_exit_exception(exc)); + if set_running_main { + vm.state.set_not_running_main(); + } + Ok(exitcode) +} + +/// The part of pymain_run_python that runs while the main thread owns +/// `__main__`. +fn pymain_run_main(vm: &VirtualMachine, scope: Scope, run_mode: RunMode) -> PyResult { + let is_repl = matches!(run_mode, RunMode::Repl); vm.check_signals()?; let result = match run_mode { RunMode::Command(command) => { @@ -412,7 +429,7 @@ fn pymain_err_print(vm: &VirtualMachine, exc: PyBaseExceptionRef) -> Option fn pymain_set_inspect(vm: &VirtualMachine, inspect: bool) { if let Err(exc) = vm.config_set_inspect(inspect) { eprintln!("Could not set the inspect flag"); - vm.print_exception(&exc); + vm.print_uncaught_exception(&exc); } } diff --git a/src/shell.rs b/src/shell.rs index 6f8836eed29..7d206f28e9e 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -22,12 +22,9 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { }, ); let _ = repl.load_history(&history); - // A SystemExit exits the process from inside the loop, so the history - // is saved after every line. - vm.run_interactive_loop(|prompt| match repl.readline(prompt) { + let result = vm.run_interactive_loop(|prompt| match repl.readline(prompt) { ReadlineResult::Line(line) => { let _ = repl.add_history_entry(line.trim_end()); - let _ = repl.save_history(&history); Ok(Some(InteractiveLine::Text(line))) } ReadlineResult::Eof => Ok(None), @@ -38,5 +35,7 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { #[cfg(unix)] ReadlineResult::OsError(_) => Ok(None), ReadlineResult::Other(_) => Ok(None), - }) + }); + let _ = repl.save_history(&history); + result } From 2a8fa66dea07111ff9997a367f3bacf2ddf5e9ab Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 11:12:29 +0900 Subject: [PATCH 16/19] Treat a parse error at the end of the source as incomplete input A parser error that starts at the end of the source is reported as _IncompleteInputError when incomplete input is allowed, so the interactive loop and codeop keep reading after `try:` and its block or after a decorator line. Assisted-by: Claude Code:claude-opus-5-5 --- crates/vm/src/vm/vm_new.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 7230e24e75f..529bb9edafe 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -560,6 +560,19 @@ impl VirtualMachine { self.ctx.exceptions.indentation_error } } + // _is_end_of_source: the parser failed at the end of the input, + // where more input can still follow. + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error, + raw_location, + .. + }) if !matches!(error, ParseErrorType::Lexical(_)) + && source + .is_some_and(|source| raw_location.start().to_usize() >= source.len()) => + { + incomplete_or_syntax(allow_incomplete) + } #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::OtherError(s), From 648022225b70360982024e194d165c83468be1ed Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 13:26:28 +0900 Subject: [PATCH 17/19] Place syntax errors on tokens made up at the end of the input A parse error on the NEWLINE, DEDENT or ENDMARKER the tokenizer makes up at the end of the input now carries an EofToken: the number of lines read and the last line. SyntaxError takes its lineno and text from it, and its offset is that line's length with end_offset -1, or 0 for both when the error is raised at the token itself (the generic syntax error and a missing forced `:` or `(`). Exec input that does not end with a newline is checked with one appended. Assisted-by: Claude Code:claude-opus-5-5 --- crates/compiler/src/lib.rs | 159 ++++++++++++++++++++++++++++++++++- crates/vm/src/stdlib/_ast.rs | 8 ++ crates/vm/src/vm/vm_new.rs | 15 +++- 3 files changed, 178 insertions(+), 4 deletions(-) diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index fd257a1d1b8..f7833a82a34 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -37,6 +37,60 @@ pub struct ParseError { pub is_unclosed_bracket: bool, /// Set when a string is still open at EOF and more input could close it. pub is_unclosed_string: bool, + /// Set when the parser failed on a token made up at the end of the input. + pub eof_token: Option>, +} + +/// The tokenizer state at the end of the input, where it makes up the +/// NEWLINE, DEDENT and ENDMARKER tokens that close it. Those tokens have no +/// column, so an error on one of them is placed where the tokenizer stopped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EofToken { + /// `tok->lineno`: the number of lines read, 0 for empty input. + pub lineno: usize, + /// The last line read, which the tokenizer stopped at the end of. + pub line: String, + /// The error is raised at the token itself, as the generic syntax error + /// and a missing forced token are, so its columns are 0. + pub known_location: bool, +} + +impl EofToken { + fn new(source: &str, mode: Mode, known_location: bool) -> Self { + let lineno = source.split_inclusive('\n').count(); + let mut line = source + .split_inclusive('\n') + .next_back() + .unwrap_or_default() + .to_owned(); + // translate_newlines: exec input that does not end with a newline + // gets one. + if matches!(mode, Mode::Exec | Mode::BlockExpr) && !line.is_empty() && !line.ends_with('\n') + { + line.push('\n'); + } + Self { + lineno, + line, + known_location, + } + } + + /// `SyntaxError.offset` + #[must_use] + pub fn offset(&self) -> usize { + if self.known_location { + 0 + } else { + self.line.chars().count() + } + } + + /// `SyntaxError.end_offset` + #[must_use] + pub const fn end_offset(&self) -> isize { + if self.known_location { 0 } else { -1 } + } } impl ::core::fmt::Display for ParseError { @@ -61,6 +115,7 @@ impl CompileError { mode: Mode, ) -> Self { let raw_location = error.location; + let at_eof_token = fails_at_eof_token(&error, source_file.source_text(), mode); let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) { Some(diagnostic) => diagnostic, None => { @@ -80,6 +135,11 @@ impl CompileError { } }; + let eof_token = at_eof_token.then(|| { + let source = source_file.source_text(); + let known_location = is_raised_at_token(&diagnostic.error, source, raw_location); + Box::new(EofToken::new(source, mode, known_location)) + }); Self::Parse(ParseError { error: diagnostic.error, raw_location, @@ -88,6 +148,7 @@ impl CompileError { source_path: source_file.name().to_owned(), is_unclosed_bracket: diagnostic.is_unclosed_bracket, is_unclosed_string: diagnostic.is_unclosed_string, + eof_token, }) } @@ -105,6 +166,7 @@ impl CompileError { source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) } @@ -117,8 +179,14 @@ impl CompileError { } #[must_use] - pub const fn python_location(&self) -> (usize, usize) { - if let Some(location) = self.location() { + pub fn python_location(&self) -> (usize, usize) { + if let Self::Parse(ParseError { + eof_token: Some(eof_token), + .. + }) = self + { + (eof_token.lineno, eof_token.offset()) + } else if let Some(location) = self.location() { (location.line.get(), location.character_offset.get()) } else { (0, 0) @@ -316,6 +384,92 @@ fn cpython_parse_diagnostic_override( None } +/// Whether the parser fails on a token made up at the end of the input. +fn fails_at_eof_token(error: &parser::ParseError, source: &str, mode: Mode) -> bool { + let start = error.location.start().to_usize(); + match &error.error { + parser::ParseErrorType::Lexical(_) => false, + // `NEWLINE !INDENT` fails on the token after the newline, and the + // error is reported at the newline when only blank lines and comments + // follow it. + parser::ParseErrorType::ExpectedIndentedBlock { .. } => { + let end = error.location.end().to_usize(); + source[start..end].chars().all(|c| c.is_ascii_whitespace()) + && is_blank_python_source(&source[end..]) + } + _ if start < source.len() => false, + // Exec input gets a newline appended, which is a real NEWLINE token, + // so the error is at the end only when it stays there once the + // newline is added. + _ if matches!(mode, Mode::Exec | Mode::BlockExpr) + && !source.is_empty() + && !source.ends_with('\n') => + { + let source = format!("{source}\n"); + let parsed = + parser::parse_unchecked(&source, parser::ParseOptions::from(parser::Mode::Module)); + parsed + .errors() + .first() + .is_none_or(|error| error.location.start().to_usize() >= source.len()) + } + _ => true, + } +} + +/// Whether the error is raised at the token the parser failed on, as the +/// generic syntax error and a missing forced token are, rather than where the +/// tokenizer stopped. +fn is_raised_at_token( + error: &parser::ParseErrorType, + source: &str, + range: ruff_text_size::TextRange, +) -> bool { + match error { + // Only `else`, `try`, `finally` and `def` force their `:`; the other + // clause headers report a missing one from an `invalid_*` rule. + parser::ParseErrorType::ExpectedToken { + expected: TokenKind::Colon, + found: TokenKind::Newline, + } => matches!( + clause_keyword(source, range.start()), + Some(TokenKind::Else | TokenKind::Try | TokenKind::Finally | TokenKind::Def) + ), + parser::ParseErrorType::ExpectedToken { .. } + | parser::ParseErrorType::ExpectedExpression + | parser::ParseErrorType::ExpectedIdentifier + | parser::ParseErrorType::UnexpectedExpressionToken + | parser::ParseErrorType::SimpleStatementsOnSameLine + | parser::ParseErrorType::SimpleAndCompoundStatementOnSameLine + | parser::ParseErrorType::InvalidStarredExpressionUsage => true, + parser::ParseErrorType::OtherError(message) => message == "invalid syntax", + _ => false, + } +} + +/// The keyword that starts the logical line `offset` is in, after `async`. +fn clause_keyword(source: &str, offset: TextSize) -> Option { + let parsed = parser::parse_unchecked(source, parser::ParseOptions::from(parser::Mode::Module)); + let mut keyword = None; + let mut at_line_start = true; + for token in parsed + .tokens() + .iter() + .take_while(|token| token.start() < offset) + { + match token.kind() { + TokenKind::Newline | TokenKind::Indent | TokenKind::Dedent => at_line_start = true, + TokenKind::NonLogicalNewline | TokenKind::Comment | TokenKind::Async => {} + kind if at_line_start => { + keyword = Some(kind); + at_line_start = false; + } + _ => {} + } + } + keyword +} + fn eof_parse_diagnostic( error: &parser::ParseError, source_file: &SourceFile, @@ -775,6 +929,7 @@ fn single_mode_blank_source_error(source_file: &SourceFile) -> Option>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) .into()); } @@ -1840,6 +1843,7 @@ pub(crate) fn parse>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) .into()); } @@ -1974,6 +1978,7 @@ pub(crate) fn parse_func_type>( source_path: filename.to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, } .into() }; @@ -2003,6 +2008,7 @@ pub(crate) fn parse_func_type>( source_path: filename.to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) .into()); }; @@ -2024,6 +2030,7 @@ pub(crate) fn parse_func_type>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) })?; let ast::Mod::Expression(expression) = parsed.into_syntax() else { @@ -2059,6 +2066,7 @@ pub(crate) fn parse_func_type>( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, + eof_token: None, }) })?; let ast::Mod::Expression(expression) = parsed.into_syntax() else { diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 529bb9edafe..217d63ab267 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -614,7 +614,16 @@ impl VirtualMachine { // Tokenizer/parser errors attach the current source line. Compiler // errors load that line with ProgramText, which is None when the // filename cannot be opened (compile of a string). - let mut statement = if matches!(error, crate::compiler::CompileError::Codegen(_)) { + let eof_token = match error { + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + eof_token: Some(eof_token), + .. + }) => Some(eof_token), + _ => None, + }; + let mut statement = if let Some(eof_token) = eof_token { + Some(eof_token.line.clone()) + } else if matches!(error, crate::compiler::CompileError::Codegen(_)) { self.program_text(error.source_path(), error.python_location().0) } else { source.and_then(|src| get_statement(src, error.location())) @@ -727,7 +736,9 @@ impl VirtualMachine { .as_deref() .and_then(|line| line.chars().next()) .is_some_and(|ch| ch.is_ascii_whitespace())); - let (end_lineno, end_offset) = if no_end_offset { + let (end_lineno, end_offset) = if let Some(eof_token) = eof_token { + (eof_token.lineno, eof_token.end_offset()) + } else if no_end_offset { (end_lineno, -1) } else if let Some(end_offset) = tokenizer_end_offset { (end_lineno, end_offset) From 934004fe2ea88304a146c9c3f6dba2b008583aab Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 14:44:26 +0900 Subject: [PATCH 18/19] Slice the source with get when checking an indented block error at EOF Assisted-by: Claude Code:claude-opus-5-5 --- crates/compiler/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index f7833a82a34..636182c7708 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -394,8 +394,10 @@ fn fails_at_eof_token(error: &parser::ParseError, source: &str, mode: Mode) -> b // follow it. parser::ParseErrorType::ExpectedIndentedBlock { .. } => { let end = error.location.end().to_usize(); - source[start..end].chars().all(|c| c.is_ascii_whitespace()) - && is_blank_python_source(&source[end..]) + source + .get(start..end) + .is_some_and(|text| text.chars().all(|c| c.is_ascii_whitespace())) + && source.get(end..).is_some_and(is_blank_python_source) } _ if start < source.len() => false, // Exec input gets a newline appended, which is a real NEWLINE token, From 05881b022c08c755d0cce9cac66e2e7a596e689d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 18 Oct 2026 16:20:45 +0900 Subject: [PATCH 19/19] Rename the host exits after C exit() and fix wasi and Windows builds - host_env::os::exit now calls the C exit() on every platform; on Windows std::process::exit ends in ExitProcess, so it flushes the Rust stdout and calls libc::exit there. The former exit, which calls _exit(), is now exit_immediately, used by os._exit, faulthandler and posixsubprocess. - errno::strerror_string is available on wasi. - PyGlobalState::reinit_running_main is built only where its fork caller is. Assisted-by: Claude Code:claude-opus-5-5 --- crates/host_env/src/errno.rs | 2 +- crates/host_env/src/os.rs | 18 ++++++++++++++---- crates/stdlib/src/faulthandler.rs | 4 ++-- crates/stdlib/src/posixsubprocess.rs | 2 +- crates/vm/src/stdlib/os.rs | 2 +- crates/vm/src/vm/interpreter.rs | 2 +- crates/vm/src/vm/mod.rs | 1 + 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/crates/host_env/src/errno.rs b/crates/host_env/src/errno.rs index e816a04a55f..c0a122ddbcc 100644 --- a/crates/host_env/src/errno.rs +++ b/crates/host_env/src/errno.rs @@ -2,7 +2,7 @@ /// Return the platform `strerror(errno)` message as an owned `String`. /// Returns `None` when the runtime gives no description for `errno`. -#[cfg(any(unix, windows))] +#[cfg(any(unix, windows, target_os = "wasi"))] #[must_use] pub fn strerror_string(errno: i32) -> Option { let ptr = unsafe { libc::strerror(errno) }; diff --git a/crates/host_env/src/os.rs b/crates/host_env/src/os.rs index 7839965d2fd..71469890587 100644 --- a/crates/host_env/src/os.rs +++ b/crates/host_env/src/os.rs @@ -521,12 +521,22 @@ pub fn device_encoding(_fd: i32) -> Option { /// Terminate the process through the C `exit()`, which runs the C atexit /// handlers and flushes C stdio. -pub fn exit_with_cleanup(code: i32) -> ! { - std::process::exit(code) +pub fn exit(code: i32) -> ! { + cfg_select! { + // std::process::exit ends in ExitProcess here, which skips the CRT + // atexit handlers and the stdio flush. + windows => { + use std::io::Write; + let _ = std::io::stdout().flush(); + unsafe { libc::exit(code) } + } + _ => std::process::exit(code), + } } -pub fn exit(code: i32) -> ! { - // These callers require immediate termination without C exit handlers. +/// Terminate the process through `_exit()`, without running the C atexit +/// handlers or flushing C stdio. +pub fn exit_immediately(code: i32) -> ! { cfg_select! { any(unix, windows, target_os = "wasi") => unsafe { libc::_exit(code) }, _ => std::process::exit(code), diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 05229ddd304..c017f0bcd92 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -408,7 +408,7 @@ mod decl { host_faulthandler::signal_default_and_raise(signum); // Fallback - rustpython_host_env::os::exit(1); + rustpython_host_env::os::exit_immediately(1); } // Windows vectored exception handler (faulthandler.c:417-480) @@ -687,7 +687,7 @@ mod decl { } if exit { - rustpython_host_env::os::exit(1); + rustpython_host_env::os::exit_immediately(1); } } } diff --git a/crates/stdlib/src/posixsubprocess.rs b/crates/stdlib/src/posixsubprocess.rs index f2ee7bbc2fe..268131e28bc 100644 --- a/crates/stdlib/src/posixsubprocess.rs +++ b/crates/stdlib/src/posixsubprocess.rs @@ -283,7 +283,7 @@ fn exec(args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>, vm: &VirtualMachine) -> let errno = e.raw_os_error().unwrap_or(0); let _ = write!(pipe, "OSError:{errno:x}:{}", ctx.as_msg()); } - rustpython_host_env::os::exit(255) + rustpython_host_env::os::exit_immediately(255) } } } diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 7a51f81ae0f..3a69305d2d0 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -1746,7 +1746,7 @@ pub(super) mod _os { #[pyfunction] fn _exit(status: ExitStatus) { - crate::host_env::os::exit(status.status) + crate::host_env::os::exit_immediately(status.status) } #[pyfunction] diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index b7c4c345133..b703da7d3ae 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -731,7 +731,7 @@ impl VirtualMachine { } else { status }; - crate::host_env::os::exit_with_cleanup(status as i32) + crate::host_env::os::exit(status as i32) } /// The steps of Py_FinalizeEx after the first flush of stdout and stderr. diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 0bbb420ff57..0ef153b7248 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -997,6 +997,7 @@ impl PyGlobalState { /// /// After a fork only the forking thread survives, so `__main__` is no /// longer running unless that thread was running it. + #[cfg(all(feature = "host_env", unix))] pub(crate) fn reinit_running_main(&self) { if !self.thread_is_running_main() { self.running_main.store(None);