diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index af359f11190..95d80d7fa03 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -24,6 +24,7 @@ use crate::{ }; use core::sync::atomic::{AtomicU32, Ordering::Relaxed}; use itertools::Itertools; +use rustpython_common::atomic::Radium; #[cfg(feature = "jit")] use rustpython_jit::CompiledCode; @@ -783,6 +784,170 @@ impl Py { } self.invoke_prepared_exact_args(taken, vm) } + + /// Fast path for unobserved Python-to-Python calls. + /// Allocates a LightFrame on the DataStack instead of a full Frame PyObject. + /// The frame is materialized lazily only if observed (traceback, sys._getframe). + /// + /// Falls back to `invoke_exact_args_slots` for generators/coroutines or when + /// tracing is active. + pub(crate) fn invoke_light_slots( + &self, + args: &mut [Option], + vm: &VirtualMachine, + ) -> PyResult { + use crate::frame::{ExecutionResult, FrameLocals, FrameSource, LightFrame, LocalsPlus}; + + let code: &Py = &self.code; + + // Generator/coroutine code and tracing must use the heavy path + // Fall back to the heavy path for generators/coroutines, tracing, + // and non-optimized/non-NEWLOCALS code (e.g. types.FunctionType + // with a non-standard code object). + if code.flags.intersects( + bytecode::CodeFlags::GENERATOR + | bytecode::CodeFlags::COROUTINE + | bytecode::CodeFlags::ASYNC_GENERATOR, + ) || vm.use_tracing.get() + || !code + .flags + .contains(bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED) + { + return self.invoke_exact_args_slots(args, vm); + } + + let nlocalsplus = code.localspluskinds.len(); + let max_stackdepth = code.max_stackdepth as usize; + let alloc_size = LightFrame::alloc_size(nlocalsplus, max_stackdepth); + + let base = vm.datastack_push(alloc_size); + let light = base as *mut LightFrame; + + unsafe { + // Initialize the LightFrame header with borrowed pointers (NO refcount bumps) + core::ptr::write( + light, + LightFrame { + code: code as *const _, + globals: &*self.globals as *const _, + builtins: &*self.builtins as *const _, + func_obj: self.as_object() as *const _, + lasti: Radium::new(0), + prev_line: 0, + previous_light: crate::vm::thread::get_current_light_frame(), + saved_current_frame: crate::vm::thread::get_current_frame(), + materialized: core::cell::UnsafeCell::new(core::ptr::null_mut()), + nlocalsplus: nlocalsplus as u32, + max_stackdepth: max_stackdepth as u32, + }, + ); + + // Zero-init the localsplus area + let lp_ptr = (*light).localsplus_ptr(); + let capacity = nlocalsplus + max_stackdepth; + core::ptr::write_bytes(lp_ptr, 0, capacity); + + // Move args into localsplus + let slots = + core::slice::from_raw_parts_mut(lp_ptr as *mut Option, nlocalsplus); + for (slot, arg_slot) in slots.iter_mut().zip(args.iter_mut()) { + *slot = arg_slot.take(); + } + + // Copy closure cells if present + if let Some(ref closure) = self.closure { + let nfrees = code.freevars.len(); + if nfrees > 0 { + let freevar_start = nlocalsplus - nfrees; + for (i, cell) in closure.iter().enumerate() { + slots[freevar_start + i] = Some(cell.clone().into()); + } + } + } + + // Push light frame onto TLS chain + let prev_light = crate::vm::thread::set_current_light_frame(light); + + // Recursion depth and C stack overflow check + let depth = vm.current_recursion_depth(); + if depth >= vm.recursion_limit.get() || (depth & 63 == 0 && vm.check_c_stack_overflow()) + { + // Clean up the light frame TLS before erroring + crate::vm::thread::set_current_light_frame(prev_light); + // Drop values we moved into localsplus + let slots = + core::slice::from_raw_parts_mut(lp_ptr as *mut Option, capacity); + for slot in slots.iter_mut() { + *slot = None; + } + vm.datastack_pop(base); + return Err(vm.new_recursion_error("maximum recursion depth exceeded".to_string())); + } + vm.recursion_depth_increment(); + + // Cache the materialized-field pointer so the panic guard doesn't + // need to borrow `light` (which is also used mutably for prev_line). + let materialized_cell: *const core::cell::UnsafeCell<*mut Py> = + &(*light).materialized; + + // Panic guard: restore TLS and recursion depth, reclaim materialized + // frame, pop DataStack. Localsplus values are dropped by + // LocalsPlus::drop (run_light_frame takes ownership). + // TLS restored first so destructors can re-enter the VM safely. + scopeguard::defer! { + vm.recursion_depth_decrement(); + crate::vm::thread::set_current_light_frame(prev_light); + let materialized_ptr = *(*materialized_cell).get(); + if !materialized_ptr.is_null() { + let reclaim = FrameRef::from_raw(materialized_ptr as *const _); + drop(reclaim); + } + vm.datastack_pop(base); + } + + // Build LocalsPlus view over the light frame data + let localsplus = LocalsPlus::from_datastack_raw(lp_ptr, capacity, nlocalsplus); + + // Create lazy FrameLocals (NEWLOCALS fast path) + let locals = FrameLocals::lazy(); + + // Build the builtins_dict cache + let builtins_dict = if self.globals.class().is(vm.ctx.types.dict_type) { + self.builtins + .downcast_ref_if_exact::(vm) + .map(|d| crate::PyExact::ref_unchecked(d)) + } else { + None + }; + + // Run the bytecode + let code_ref = (*(*light).code).to_owned(); + let result = crate::frame::run_light_frame( + &code_ref, + localsplus, + &locals, + &self.globals, + &self.builtins, + builtins_dict, + FrameSource::Light(light), + self.as_object(), + &(*light).lasti, + &mut (*light).prev_line, + vm, + ); + + // Normal exit: the scopeguard handles all cleanup. + + // Convert result + match result { + Ok(ExecutionResult::Return(value)) => Ok(value), + Ok(ExecutionResult::Yield(_)) => { + unreachable!("light frame should never yield") + } + Err(e) => Err(e), + } + } + } } pub(crate) fn datastack_frame_size_bytes_for_code(code: &Py) -> Option { diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 2743bf3d542..879e95c85fa 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -52,6 +52,237 @@ use rustpython_compiler_core::SourceLocation; pub type FrameRef = PyRef; +// LightFrame — stack-allocated frame header for unobserved Python→Python calls + +/// Lightweight interpreter frame allocated on the DataStack for unobserved calls. +/// Contains only what bytecode execution needs. A full Frame PyObject is +/// materialized lazily if the frame is observed (sys._getframe, traceback, tracing). +#[repr(C)] +pub struct LightFrame { + // Borrowed from PyFunction — alive for the call's duration because the + // caller holds the function on its evaluation stack. + pub code: *const Py, + pub globals: *const Py, + pub builtins: *const PyObject, + pub func_obj: *const PyObject, + + // Execution state (written during bytecode execution) + pub lasti: PyAtomic, + pub prev_line: u32, + + // Frame chain + pub previous_light: *const Self, + /// The heavy frame that was `CURRENT_FRAME` when this light frame was entered. + /// Used to interleave light and heavy frames during stack walks. + pub saved_current_frame: *const Frame, + + // Lazy materialization: NULL until first observation + pub materialized: core::cell::UnsafeCell<*mut Py>, + + // Layout info for localsplus array that follows this header + pub nlocalsplus: u32, + pub max_stackdepth: u32, +} + +// LightFrame is per-thread and never shared across threads. Raw pointers +// make it !Send + !Sync by default, which is the correct bound. + +impl LightFrame { + /// Pointer to the localsplus data that follows immediately after this header. + #[inline(always)] + pub(crate) fn localsplus_ptr(&self) -> *mut usize { + let base = self as *const Self as *mut u8; + let header_size = core::mem::size_of::(); + // Align to 16 bytes (same as DataStack ALIGN) + let aligned = (header_size + 15) & !15; + unsafe { base.add(aligned) as *mut usize } + } + + /// Total allocation size for a LightFrame with the given localsplus capacity. + #[inline] + pub(crate) fn alloc_size(nlocalsplus: usize, max_stackdepth: usize) -> usize { + let header_size = core::mem::size_of::(); + let aligned_header = (header_size + 15) & !15; + let capacity = nlocalsplus + max_stackdepth; + aligned_header + capacity * core::mem::size_of::() + } +} + +/// Source of frame data for `ExecutingFrame`. +/// +/// Heavy frames have a full `Py` object; light frames only have a +/// `LightFrame` header on the DataStack. Cold paths that need a real frame +/// object call `ensure_heavy` which materializes on demand. +pub(crate) enum FrameSource<'a> { + /// Normal (heavy) frame backed by a PyObject. + Heavy(&'a Py), + /// Light frame on the DataStack. `*mut LightFrame` is valid for the + /// duration of the call. + Light(*mut LightFrame), +} + +impl<'a> FrameSource<'a> { + /// Get a reference to the heavy frame, materializing if necessary. + /// This is a cold path — only called for tracing, traceback, sys._getframe, etc. + #[cold] + pub(crate) fn ensure_heavy(&self, vm: &VirtualMachine) -> FrameRef { + match self { + FrameSource::Heavy(frame) => (*frame).to_owned(), + FrameSource::Light(light) => unsafe { materialize_light_frame(*light, vm) }, + } + } + + /// Get the heavy frame reference if available (without materializing). + #[inline] + pub(crate) fn heavy_ref(&self) -> Option<&'a Py> { + match self { + FrameSource::Heavy(frame) => Some(frame), + FrameSource::Light(_) => None, + } + } +} + +/// Materialize a LightFrame into a full Frame PyObject. +/// +/// # Safety +/// `light` must point to a valid LightFrame whose borrowed pointers are still alive. +#[cold] +unsafe fn materialize_light_frame(light: *mut LightFrame, vm: &VirtualMachine) -> FrameRef { + unsafe { + // Check if already materialized — synchronize execution state + let existing = *(*light).materialized.get(); + if !existing.is_null() { + let frame_ref: FrameRef = (&*existing).to_owned(); + sync_light_to_materialized(light, &frame_ref); + return frame_ref; + } + + // Create owned references from borrowed pointers + let code: PyRef = (*(*light).code).to_owned(); + let globals: PyDictRef = (*(*light).globals).to_owned(); + let builtins: PyObjectRef = (*(*light).builtins).to_owned(); + let func_obj: Option = if (*light).func_obj.is_null() { + None + } else { + Some((*(*light).func_obj).to_owned()) + }; + + let nlocalsplus = (*light).nlocalsplus as usize; + + // Build the frame with heap-backed localsplus + let frame = Frame::new( + code, + Scope::new(None, globals), + builtins, + &[], // closure cells are already in localsplus + func_obj, + false, // heap-backed, not datastack + vm, + ) + .into_ref(&vm.ctx); + + // Copy localsplus data from LightFrame into the materialized frame + let src_ptr = (*light).localsplus_ptr(); + let iframe = (&mut *frame.iframe.get()).as_mut().unwrap(); + let dst = iframe.localsplus.fastlocals_mut(); + // Copy fastlocals (not the stack -- materialization happens on cold paths + // where we just need frame identity, not evaluation state) + for (i, slot) in dst.iter_mut().enumerate().take(nlocalsplus) { + let src_val = core::ptr::read(src_ptr.add(i) as *const Option); + if let Some(ref obj) = src_val { + *slot = Some(obj.clone()); + } + // Don't drop the source -- it's still owned by the light frame + core::mem::forget(src_val); + } + + // Copy lasti and prev_line + let lasti_val = (*light).lasti.load(Relaxed); + iframe.lasti.store(lasti_val, Relaxed); + iframe.prev_line = (*light).prev_line; + + // Set up the `previous` frame pointer for f_back: + // Link f_back to the correct predecessor: + // - If previous_light has the same saved_current_frame, it is the + // direct caller → materialize and link to it. + // - Otherwise, the direct caller is the heavy frame recorded at entry + // time (saved_current_frame). + let prev_light = (*light).previous_light; + let saved = (*light).saved_current_frame; + let prev_frame_ptr = + if !prev_light.is_null() && core::ptr::eq((*prev_light).saved_current_frame, saved) { + let prev_materialized = materialize_light_frame(prev_light as *mut _, vm); + // Store a strong reference in retained_back so the predecessor + // stays alive even after the predecessor's light frame is cleaned + // up (f_back can be accessed after the caller returns). + *iframe.retained_back.lock() = Some(prev_materialized.clone()); + // Use payload pointer (not Py pointer) — the chain stores + // &**frame (Frame payload) consistently. + let ptr = (&**prev_materialized) as *const Frame as *mut Frame; + // Drop the temporary; retained_back keeps the predecessor alive. + drop(prev_materialized); + ptr + } else { + saved as *mut Frame + }; + iframe + .previous + .store(prev_frame_ptr, atomic::Ordering::Relaxed); + + // Store the materialized frame pointer back + *(*light).materialized.get() = &*frame as *const Py as *mut Py; + + // Keep the frame alive: increment the refcount because the light frame's + // `materialized` pointer is a raw pointer (not ref-counted). + // We return one owned ref and leak another to keep it alive. + // The leaked ref will be reclaimed during light frame cleanup. + let leaked = frame.clone(); + core::mem::forget(leaked); + + frame + } +} + +/// Synchronize the live light frame's execution state into its materialized +/// heavy frame. Called on every re-observation so introspection APIs +/// (`f_lineno`, `f_locals`, `inspect.currentframe()`) see current values. +/// +/// # Safety +/// Both `light` and `frame` must be valid and the light frame must still be +/// executing (its localsplus on the DataStack is live). +#[cold] +unsafe fn sync_light_to_materialized(light: *const LightFrame, frame: &Py) { + unsafe { + let iframe = (&mut *frame.iframe.get()).as_mut().unwrap(); + // Sync lasti and prev_line + let lasti_val = (*light).lasti.load(Relaxed); + iframe.lasti.store(lasti_val, Relaxed); + iframe.prev_line = (*light).prev_line; + // Sync fastlocals (clone current values from light frame) + let nlocalsplus = (*light).nlocalsplus as usize; + let src_ptr = (*light).localsplus_ptr(); + let dst = iframe.localsplus.fastlocals_mut(); + for (i, slot) in dst.iter_mut().enumerate().take(nlocalsplus) { + let src_val = core::ptr::read(src_ptr.add(i) as *const Option); + if let Some(ref obj) = src_val { + *slot = Some(obj.clone()); + } else { + *slot = None; + } + // Don't drop the source — it's still owned by the light frame + core::mem::forget(src_val); + } + } +} + +/// Public wrapper for `materialize_light_frame`. +/// +/// # Safety +/// `light` must point to a valid LightFrame whose borrowed pointers are still alive. +pub unsafe fn materialize_light_frame_pub(light: *mut LightFrame, vm: &VirtualMachine) -> FrameRef { + unsafe { materialize_light_frame(light, vm) } +} + /// Recover an owned reference to a live chain frame, or `None` for null. /// /// # Safety @@ -68,14 +299,41 @@ unsafe fn owned_chain_frame(frame: *const Frame) -> Option { } /// The current thread's topmost frame object, if any. +/// If light frames are active, they are on top of the heavy chain. #[must_use] pub fn current_thread_frame() -> Option { // SAFETY: the chain top executes on this thread, hence is alive. unsafe { owned_chain_frame(crate::vm::thread::get_current_frame()) } } +/// The current thread's topmost frame, materializing any light frame if needed. +/// +/// The top frame is determined by comparing the topmost light frame's +/// `saved_current_frame` against the current heavy frame pointer. If a heavy +/// frame has been pushed after the light frame, the heavy frame is on top. +#[must_use] +pub fn current_thread_frame_vm(vm: &VirtualMachine) -> Option { + let heavy = crate::vm::thread::get_current_frame(); + let light = crate::vm::thread::get_current_light_frame(); + if !light.is_null() { + // The light frame recorded which heavy frame was current when it started. + // If the current heavy frame is DIFFERENT, a heavy frame was pushed after + // the light frame (e.g. native code calling back into Python). In that case, + // the heavy frame is on top. + let saved = unsafe { (*light).saved_current_frame }; + if core::ptr::eq(heavy, saved) { + // No heavy frame has been pushed since this light frame — it's the topmost. + return Some(unsafe { materialize_light_frame(light as *mut _, vm) }); + } + // A heavy frame was pushed after the light frame — use the heavy chain. + } + // SAFETY: the chain top executes on this thread, hence is alive. + unsafe { owned_chain_frame(heavy) } +} + /// The frame `offset` positions below the current thread's top frame (offset 0 /// is the top), or `None` if the stack is not that deep. +/// Does not account for light frames. #[must_use] pub fn frame_at_offset(offset: usize) -> Option { let mut cur = crate::vm::thread::get_current_frame(); @@ -90,6 +348,70 @@ pub fn frame_at_offset(offset: usize) -> Option { unsafe { owned_chain_frame(cur) } } +/// The frame `offset` positions below the top of the unified frame stack, +/// materializing light frames as needed. +/// +/// The unified stack interleaves light and heavy frames in call order. +/// Heavy frames pushed after the topmost light frame are walked first. +#[must_use] +pub fn frame_at_offset_vm(offset: usize, vm: &VirtualMachine) -> Option { + let mut remaining = offset; + let mut light = crate::vm::thread::get_current_light_frame(); + let mut heavy = crate::vm::thread::get_current_frame(); + + // Walk heavy frames that are ABOVE the topmost light frame. + // These are heavy frames pushed by native callbacks from within a light frame. + if !light.is_null() { + let saved = unsafe { (*light).saved_current_frame }; + while !core::ptr::eq(heavy, saved) && !heavy.is_null() { + if remaining == 0 { + return unsafe { owned_chain_frame(heavy) }; + } + remaining -= 1; + heavy = unsafe { (*heavy).previous_frame() }; + } + } + + while !light.is_null() { + // This light frame is the next in the unified stack + if remaining == 0 { + return Some(unsafe { materialize_light_frame(light as *mut _, vm) }); + } + remaining -= 1; + + // Walk any heavy frames between this light frame and the next + let next_light = unsafe { (*light).previous_light }; + let stop_at = if !next_light.is_null() { + unsafe { (*next_light).saved_current_frame } + } else { + core::ptr::null() + }; + let saved = unsafe { (*light).saved_current_frame }; + // Walk heavy frames from saved_current_frame down to stop_at (exclusive) + let mut h = saved; + while !h.is_null() && !core::ptr::eq(h, stop_at) { + if remaining == 0 { + return unsafe { owned_chain_frame(h) }; + } + remaining -= 1; + h = unsafe { (*h).previous_frame() }; + } + heavy = stop_at; + light = next_light; + } + + // Walk remaining heavy frames + let mut cur = heavy; + while !cur.is_null() { + if remaining == 0 { + return unsafe { owned_chain_frame(cur) }; + } + remaining -= 1; + cur = unsafe { (*cur).previous_frame() }; + } + None +} + /// If `target` is a frame on the current thread's chain, return an owned /// reference to it; otherwise `None`. Presence on the chain proves liveness. #[must_use] @@ -107,7 +429,7 @@ pub fn find_owned_chain_frame(target: *const Frame) -> Option { } /// Invoke `f` for each frame on the current thread's chain, from the topmost -/// frame down to the bottom. +/// frame down to the bottom. Does not visit light frames. pub fn for_each_current_frame(mut f: impl FnMut(&Py)) { let mut cur = crate::vm::thread::get_current_frame(); while !cur.is_null() { @@ -265,6 +587,27 @@ impl LocalsPlus { } } + /// Create a LocalsPlus backed by an already-initialized raw pointer. + /// Used by light frame path where the LightFrame header + localsplus + /// are allocated together in a single DataStack push. + /// + /// # Safety + /// - `ptr` must point to `capacity` valid `usize` slots, zero-initialized. + /// - The data must remain valid for the lifetime of this LocalsPlus. + /// - Caller is responsible for cleanup. + pub(crate) unsafe fn from_datastack_raw( + ptr: *mut usize, + capacity: usize, + nlocalsplus: usize, + ) -> Self { + let nlocalsplus_u32 = u32::try_from(nlocalsplus).expect("nlocalsplus exceeds u32"); + Self { + data: LocalsPlusData::DataStack { ptr, capacity }, + nlocalsplus: nlocalsplus_u32, + stack_top: 0, + } + } + /// Migrate data-stack-backed storage to the heap, preserving all values. /// Returns the data stack base pointer for `DataStack::pop()`. /// Returns `None` if already heap-backed. @@ -601,7 +944,7 @@ impl FrameLocals { /// Create an empty lazy locals (for NEWLOCALS frames). /// The dict will be created on first access. - fn lazy() -> Self { + pub(crate) fn lazy() -> Self { Self { inner: OnceCell::new(), } @@ -1575,7 +1918,8 @@ impl Py { None }, lasti: &iframe.lasti, - object: self, + frame_source: FrameSource::Heavy(self), + func_obj: iframe.func_obj.as_deref(), prev_line: &mut iframe.prev_line, monitoring_mask: 0, }; @@ -1627,7 +1971,8 @@ impl Py { builtins: &iframe.builtins, builtins_dict: None, lasti: &iframe.lasti, - object: self, + frame_source: FrameSource::Heavy(self), + func_obj: iframe.func_obj.as_deref(), prev_line: &mut iframe.prev_line, monitoring_mask: 0, }; @@ -1655,7 +2000,7 @@ impl Py { /// An executing frame; borrows mutable frame-internal data for the duration /// of bytecode execution. -struct ExecutingFrame<'a> { +pub(crate) struct ExecutingFrame<'a> { code: &'a PyRef, localsplus: &'a mut LocalsPlus, locals: &'a FrameLocals, @@ -1666,7 +2011,10 @@ struct ExecutingFrame<'a> { /// subclasses), so that `__missing__` / `__getitem__` overrides are /// not bypassed. builtins_dict: Option<&'a PyExact>, - object: &'a Py, + /// Source of frame data: either a heavy Py or a light DataStack frame. + frame_source: FrameSource<'a>, + /// Borrowed function object that created this frame (if any). + func_obj: Option<&'a PyObject>, lasti: &'a PyAtomic, prev_line: &'a mut u32, /// Cached monitoring events mask. Reloaded at Resume instruction only, @@ -2017,7 +2365,95 @@ impl fmt::Debug for ExecutingFrame<'_> { } } +/// Run bytecode in a light frame context. +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_light_frame<'a>( + code: &'a PyRef, + mut localsplus: LocalsPlus, + locals: &'a FrameLocals, + globals: &'a PyDictRef, + builtins: &'a PyObjectRef, + builtins_dict: Option<&'a PyExact>, + frame_source: FrameSource<'a>, + func_obj: &'a PyObject, + lasti: &'a PyAtomic, + prev_line: &'a mut u32, + vm: &VirtualMachine, +) -> PyResult { + let mut exec = ExecutingFrame { + code, + localsplus: &mut localsplus, + locals, + globals, + builtins, + builtins_dict, + frame_source, + func_obj: Some(func_obj), + lasti, + prev_line, + monitoring_mask: 0, + }; + exec.run(vm) +} + impl ExecutingFrame<'_> { + /// Get the heavy frame object, materializing from a light frame if needed. + /// This is a cold path used only for tracing, traceback creation, etc. + #[cold] + #[inline(never)] + fn frame_object(&self, vm: &VirtualMachine) -> FrameRef { + self.frame_source.ensure_heavy(vm) + } + + /// Get a reference to the heavy `Py` if this is a heavy frame. + /// Returns None for light frames (without materializing). + #[inline] + fn heavy_frame_ref(&self) -> Option<&Py> { + self.frame_source.heavy_ref() + } + + /// Access the heavy frame's trace lock. Only valid when `vm.use_tracing` + /// is set — light frames are never created under tracing. + #[inline] + fn trace_is_set(&self, vm: &VirtualMachine) -> bool { + if let Some(frame) = self.heavy_frame_ref() { + !vm.is_none(&frame.trace.lock()) + } else { + false + } + } + + /// Access the heavy frame's trace_opcodes lock. Only valid on heavy frames. + #[inline] + fn trace_opcodes_is_set(&self) -> bool { + if let Some(frame) = self.heavy_frame_ref() { + *frame.trace_opcodes.lock() + } else { + false + } + } + + /// Get pending_stack_pops from the heavy frame. + #[inline] + fn pending_stack_pops(&self) -> u32 { + self.heavy_frame_ref().map_or(0, |f| f.pending_stack_pops()) + } + + /// Get pending_unwind_from_stack from the heavy frame. + #[inline] + fn pending_unwind_from_stack(&self) -> i64 { + self.heavy_frame_ref() + .map_or(0, |f| f.pending_unwind_from_stack()) + } + + /// Set pending_stack_pops on the heavy frame. + #[inline] + fn set_pending_stack_pops(&self, val: u32) { + if let Some(frame) = self.heavy_frame_ref() { + frame.set_pending_stack_pops(val); + } + } + /// Run `__init__` for the tp_new specialization. `args` holds the /// `__init__` args with slot 0 left empty; it is filled with `new_obj` /// here. Enforces the `__init__() should return None` contract and @@ -2108,7 +2544,7 @@ impl ExecutingFrame<'_> { /// Matches `_PyEval_MonitorRaise` → `PY_MONITORING_EVENT_RAISE` → /// `sys_trace_exception_func` in legacy_tracing.c. fn fire_exception_trace(&self, exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<()> { - if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + if vm.use_tracing.get() && self.trace_is_set(vm) { let exc_type: PyObjectRef = exc.class().to_owned().into(); let exc_value: PyObjectRef = exc.clone().into(); let exc_tb: PyObjectRef = exc @@ -2140,7 +2576,7 @@ impl ExecutingFrame<'_> { // (frames entered before sys.settrace() have trace=None). // Skip RESUME – it should not generate user-visible line events. if vm.use_tracing.get() - && !vm.is_none(&self.object.trace.lock()) + && self.trace_is_set(vm) && !matches!( self.code.instructions.read_op(idx), Instruction::Resume { .. } | Instruction::InstrumentedResume @@ -2155,11 +2591,11 @@ impl ExecutingFrame<'_> { if self.lasti() != (idx as u32 + 1) { // set_f_lineno defers stack unwinding because we hold // the state mutex. Perform it now. - let pops = self.object.pending_stack_pops(); + let pops = self.pending_stack_pops(); if pops > 0 { - let from_stack = self.object.pending_unwind_from_stack(); + let from_stack = self.pending_unwind_from_stack(); self.unwind_stack_for_lineno(pops as usize, from_stack, vm); - self.object.set_pending_stack_pops(0); + self.set_pending_stack_pops(0); } arg_state.reset(); continue; @@ -2184,8 +2620,8 @@ impl ExecutingFrame<'_> { // Fire 'opcode' trace event for sys.settrace when f_trace_opcodes // is set. Skip RESUME and ExtendedArg // (_Py_call_instrumentation_instruction). - if !vm.is_none(&self.object.trace.lock()) - && *self.object.trace_opcodes.lock() + if self.trace_is_set(vm) + && self.trace_opcodes_is_set() && !matches!( op.into(), Opcode::Resume | Opcode::InstrumentedResume | Opcode::ExtendedArg @@ -2208,7 +2644,7 @@ impl ExecutingFrame<'_> { let next = exception.__traceback__(); let new_traceback = PyTraceback::new( next, - frame.object.to_owned(), + frame.frame_object(vm), idx as u32 * 2, loc.line, ); @@ -2277,7 +2713,7 @@ impl ExecutingFrame<'_> { let new_traceback = PyTraceback::new( next, - frame.object.to_owned(), + frame.frame_object(vm), idx as u32 * 2, loc.line, ); @@ -2388,7 +2824,9 @@ impl ExecutingFrame<'_> { // The traceback was created with the correct lasti when exception // was first raised, but frame.lasti may have changed during cleanup if let Some(tb) = exception.__traceback__() - && core::ptr::eq::>(&*tb.frame, self.object) + && self + .heavy_frame_ref() + .is_some_and(|obj| core::ptr::eq::>(&*tb.frame, obj)) { // This traceback entry is for this frame - restore its lasti // tb.lasti is in bytes (idx * 2), convert back to instruction index @@ -2477,12 +2915,8 @@ impl ExecutingFrame<'_> { if idx < self.code.locations.len() { let (loc, _end_loc) = self.code.locations[idx]; let next = err.__traceback__(); - let new_traceback = PyTraceback::new( - next, - self.object.to_owned(), - idx as u32 * 2, - loc.line, - ); + let new_traceback = + PyTraceback::new(next, self.frame_object(vm), idx as u32 * 2, loc.line); err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); } @@ -2524,7 +2958,7 @@ impl ExecutingFrame<'_> { let next = err.__traceback__(); let new_traceback = PyTraceback::new( next, - self.object.to_owned(), + self.frame_object(vm), idx as u32 * 2, loc.line, ); @@ -2566,7 +3000,7 @@ impl ExecutingFrame<'_> { let (loc, _end_loc) = self.code.locations[idx]; let next = exception.__traceback__(); let new_traceback = - PyTraceback::new(next, self.object.to_owned(), idx as u32 * 2, loc.line); + PyTraceback::new(next, self.frame_object(vm), idx as u32 * 2, loc.line); exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); } @@ -2930,9 +3364,7 @@ impl ExecutingFrame<'_> { let n = n.get(arg) as usize; if n > 0 { let closure = self - .object .func_obj - .as_ref() .and_then(|f| f.downcast_ref::()) .and_then(|f| f.closure.as_ref()); let nlocalsplus = self.code.localspluskinds.len(); @@ -4240,7 +4672,7 @@ impl ExecutingFrame<'_> { Ok(None) } PyIterReturn::StopIteration(value) => { - if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + if vm.use_tracing.get() && self.trace_is_set(vm) { let stop_exc = vm.new_stop_iteration(value.clone()); self.fire_exception_trace(&stop_exc, vm)?; } @@ -4277,7 +4709,7 @@ impl ExecutingFrame<'_> { return Ok(None); } PyIterReturn::StopIteration(value) => { - if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + if vm.use_tracing.get() && self.trace_is_set(vm) { let stop_exc = vm.new_stop_iteration(value.clone()); self.fire_exception_trace(&stop_exc, vm)?; } @@ -4295,7 +4727,7 @@ impl ExecutingFrame<'_> { Ok(None) } PyIterReturn::StopIteration(value) => { - if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + if vm.use_tracing.get() && self.trace_is_set(vm) { let stop_exc = vm.new_stop_iteration(value.clone()); self.fire_exception_trace(&stop_exc, vm)?; } @@ -4662,7 +5094,7 @@ impl ExecutingFrame<'_> { let owner = self.pop_value(); let attr_name = self.code.names[oparg.name_idx() as usize].to_owned().into(); let result = - func.invoke_exact_args_slots(&mut [Some(owner), Some(attr_name)], vm)?; + func.invoke_light_slots(&mut [Some(owner), Some(attr_name)], vm)?; self.push_value(result); return Ok(None); } @@ -4709,7 +5141,7 @@ impl ExecutingFrame<'_> { && self.specialization_has_datastack_space_for_func(vm, func) { let owner = self.pop_value(); - let result = func.invoke_exact_args_slots(&mut [Some(owner)], vm)?; + let result = func.invoke_light_slots(&mut [Some(owner)], vm)?; self.push_value(result); return Ok(None); } @@ -4857,7 +5289,7 @@ impl ExecutingFrame<'_> { debug_assert!(func.has_exact_argcount(2)); let sub = self.pop_value(); let owner = self.pop_value(); - let result = func.invoke_exact_args_slots(&mut [Some(owner), Some(sub)], vm)?; + let result = func.invoke_light_slots(&mut [Some(owner), Some(sub)], vm)?; self.push_value(result); return Ok(None); } @@ -5021,7 +5453,7 @@ impl ExecutingFrame<'_> { } let callable = self.pop_value(); let func = callable.downcast_ref_if_exact::(vm).unwrap(); - let result = func.invoke_exact_args_slots(args, vm)?; + let result = func.invoke_light_slots(args, vm)?; self.push_value(result); Ok(None) } else { @@ -5073,7 +5505,7 @@ impl ExecutingFrame<'_> { self.pop_value_opt(); // null (self_or_null) self.pop_value(); // callable (bound method) args[0] = Some(bound_self); - let result = func.invoke_exact_args_slots(args, vm)?; + let result = func.invoke_light_slots(args, vm)?; self.push_value(result); return Ok(None); } @@ -6299,7 +6731,7 @@ impl ExecutingFrame<'_> { self.push_value(value); } Ok(PyIterReturn::StopIteration(value)) => { - if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + if vm.use_tracing.get() && self.trace_is_set(vm) { let stop_exc = vm.new_stop_iteration(value); self.fire_exception_trace(&stop_exc, vm)?; } @@ -7578,7 +8010,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_int(value).into()); return Ok(true); } - if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + if vm.use_tracing.get() && self.trace_is_set(vm) { let stop_exc = vm.new_stop_iteration(None); self.fire_exception_trace(&stop_exc, vm)?; } @@ -7597,7 +8029,7 @@ impl ExecutingFrame<'_> { Ok(PyIterReturn::StopIteration(value)) => { // Fire 'exception' trace event for StopIteration, matching // FOR_ITER's inline call to _PyEval_MonitorRaise. - if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + if vm.use_tracing.get() && self.trace_is_set(vm) { let stop_exc = vm.new_stop_iteration(value); self.fire_exception_trace(&stop_exc, vm)?; } diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 65917865d07..405f4abdee3 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -970,7 +970,7 @@ pub mod sys { #[pyfunction] fn _getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult { let offset = offset.into_option().unwrap_or(0); - let frame_ref = crate::frame::frame_at_offset(offset) + let frame_ref = crate::frame::frame_at_offset_vm(offset, vm) .ok_or_else(|| vm.new_value_error("call stack is not deep enough"))?; frame_ref.mark_escaped(); @@ -992,7 +992,7 @@ pub mod sys { } // Get the frame at the specified depth - let func_obj = match crate::frame::frame_at_offset(depth) { + let func_obj = match crate::frame::frame_at_offset_vm(depth, vm) { Some(frame) => frame.func_obj.clone(), None => return Ok(vm.ctx.none()), }; diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 3062ea07f98..bbba1b5c11e 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1576,6 +1576,18 @@ impl VirtualMachine { self.recursion_depth.get() } + /// Increment recursion depth. Used by light frame path. + #[inline] + pub(crate) fn recursion_depth_increment(&self) { + self.recursion_depth.update(|d| d + 1); + } + + /// Decrement recursion depth. Used by light frame path. + #[inline] + pub(crate) fn recursion_depth_decrement(&self) { + self.recursion_depth.update(|d| d - 1); + } + /// Stack margin bytes (like _PyOS_STACK_MARGIN_BYTES). /// The margin is doubled for debug/sanitized builds because frame /// evaluation consumes more native stack in those configurations. @@ -1667,7 +1679,7 @@ impl VirtualMachine { /// single native frame can exceed the margin and step past it. #[cfg(all(not(miri), not(target_env = "musl")))] #[inline(always)] - fn check_c_stack_overflow(&self) -> bool { + pub(crate) fn check_c_stack_overflow(&self) -> bool { let current_sp = psm::stack_pointer() as usize; let soft_limit = self.c_stack_soft_limit.get(); current_sp < soft_limit @@ -1677,7 +1689,7 @@ impl VirtualMachine { /// the probe during stdlib bootstrap. #[cfg(any(miri, target_env = "musl"))] #[inline(always)] - fn check_c_stack_overflow(&self) -> bool { + pub(crate) fn check_c_stack_overflow(&self) -> bool { false } @@ -1702,55 +1714,87 @@ impl VirtualMachine { frame: FrameRef, f: F, ) -> PyResult { - self.with_recursion("", || { - // SAFETY: `frame` (FrameRef) stays alive for the entire closure scope, - // keeping the FramePtr valid. We pass a clone to `f` so that `f` - // consuming its FrameRef doesn't invalidate our pointer. - // Publish the frame for sys._current_frames() and faulthandler. - // On unix, set_current_frame below publishes the top frame into the - // thread slot; only non-unix builds maintain the mutex-guarded Vec. - #[cfg(all(not(unix), feature = "threading"))] - crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&*frame))); - // Link frame into the signal-safe frame chain (previous pointer). - // This chain is the single source for the current thread's frame - // stack (current_frame, sys._getframe, f_back, monitoring). - let old_frame = crate::vm::thread::set_current_frame((&**frame) as *const Frame); - frame.previous.store( - old_frame as *mut Frame, - core::sync::atomic::Ordering::Relaxed, - ); - // Normal frame calls share the caller's exc_info slot so that - // callees can see the caller's handled exception via sys.exc_info(). - // Save the current value to restore on exit — this prevents - // exc_info pollution from frames with unbalanced - // PUSH_EXC_INFO/POP_EXCEPT (e.g., exception escaping an except block - // whose cleanup entry is missing from the exception table). - // A callee whose bytecode never mutates the slot cannot pollute it, - // so the save/restore is skipped for it. - let save_exc = frame.code.has_exc_handling; - let saved_exc = if save_exc { - self.current_exception() + // Inline recursion check (avoids with_recursion closure overhead) + self.check_recursive_call("")?; + + // C stack overflow check: amortized over every 64th call to reduce + // the cost of psm::stack_pointer() on the hot path. + let depth = self.recursion_depth.get(); + if depth & 63 == 0 && self.check_c_stack_overflow() { + return Err(self.new_recursion_error(String::new())); + } + + self.recursion_depth.update(|d| d + 1); + + // Publish the frame for sys._current_frames() and faulthandler. + #[cfg(all(not(unix), feature = "threading"))] + crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&*frame))); + // Link frame into the signal-safe frame chain. + // If a light frame is currently on top, materialize it so that + // f_back and inspect.stack() see the correct interleaved order + // (H_new → L_materialized → H_old instead of H_new → H_old). + let light = crate::vm::thread::get_current_light_frame(); + let old_frame = crate::vm::thread::set_current_frame((&**frame) as *const Frame); + let _materialized_back = if !light.is_null() { + let saved = unsafe { (*light).saved_current_frame }; + if core::ptr::eq(old_frame, saved) { + // The top light frame belongs to this heavy call level — + // materialize and use as the predecessor. + let mat = + unsafe { crate::frame::materialize_light_frame_pub(light as *mut _, self) }; + let payload = (&**mat) as *const crate::frame::Frame as *mut crate::frame::Frame; + frame + .previous + .store(payload, core::sync::atomic::Ordering::Relaxed); + // Keep the materialized frame alive via retained_back so + // f_back can find it even after the light frame's cleanup. + *frame.retained_back.lock() = Some(mat.clone()); + Some(mat) } else { + frame.previous.store( + old_frame as *mut crate::frame::Frame, + core::sync::atomic::Ordering::Relaxed, + ); None - }; - let old_owner = frame.owner.swap( - crate::frame::FrameOwner::Thread as i8, - core::sync::atomic::Ordering::AcqRel, + } + } else { + frame.previous.store( + old_frame as *mut crate::frame::Frame, + core::sync::atomic::Ordering::Relaxed, ); + None + }; + // Save exc_info if this frame does exception handling. + let save_exc = frame.code.has_exc_handling; + let saved_exc = if save_exc { + self.current_exception() + } else { + None + }; + let old_owner = frame.owner.swap( + crate::frame::FrameOwner::Thread as i8, + core::sync::atomic::Ordering::AcqRel, + ); - // Ensure cleanup on panic: restore owner, exc_info, and frame chain. - scopeguard::defer! { - frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); - if save_exc { - self.restore_exception(saved_exc); - } - crate::vm::thread::set_current_frame(old_frame); - #[cfg(all(not(unix), feature = "threading"))] - crate::vm::thread::pop_thread_frame(); + // Ensure all state is restored on panic or normal exit. Installed + // immediately after all setup so no intermediate panic can leak. + scopeguard::defer! { + frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); + if save_exc { + self.restore_exception(saved_exc); } + crate::vm::thread::set_current_frame(old_frame); + #[cfg(all(not(unix), feature = "threading"))] + crate::vm::thread::pop_thread_frame(); + self.recursion_depth.update(|d| d - 1); + } + // Execute: skip dispatch_traced_frame when tracing is off (hot path). + if self.use_tracing.get() { self.dispatch_traced_frame(&frame, |frame| f(frame.to_owned())) - }) + } else { + f(frame.to_owned()) + } } /// Frame execution for generator/coroutine resume. @@ -1881,7 +1925,7 @@ impl VirtualMachine { } pub fn current_frame(&self) -> Option { - crate::frame::current_thread_frame() + crate::frame::current_thread_frame_vm(self) } pub fn current_locals(&self) -> PyResult { diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 9948b936f9d..4a91b94c734 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -89,6 +89,12 @@ thread_local! { pub(crate) static CURRENT_FRAME: AtomicPtr = const { AtomicPtr::new(core::ptr::null_mut()) }; + /// Current top light frame for unobserved Python-to-Python calls. + /// NULL when no light frame is active. Light frames form their own + /// chain via `LightFrame::previous_light`. + pub(crate) static CURRENT_LIGHT_FRAME: Cell<*const crate::frame::LightFrame> = + const { Cell::new(core::ptr::null()) }; + /// Cached pointer to this thread's `ThreadSlot::top_frame`, so the hot /// push/pop path can publish the top frame with a single relaxed store and /// no `CURRENT_THREAD_SLOT` RefCell borrow. Null until the slot is @@ -694,6 +700,20 @@ pub fn get_current_frame() -> *const Frame { CURRENT_FRAME.with(|c| c.load(Ordering::Relaxed) as *const Frame) } +/// Set the current thread's top light frame pointer. +/// Returns the previous pointer so it can be restored on pop. +pub fn set_current_light_frame( + frame: *const crate::frame::LightFrame, +) -> *const crate::frame::LightFrame { + CURRENT_LIGHT_FRAME.with(|c| c.replace(frame)) +} + +/// Get the current thread's top light frame pointer. +#[must_use] +pub fn get_current_light_frame() -> *const crate::frame::LightFrame { + CURRENT_LIGHT_FRAME.with(|c| c.get()) +} + /// Update the current thread's exception slot atomically (no locks). /// Called from push_exception/pop_exception/set_exception. #[cfg(feature = "threading")]