diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index a27eb7399dc..edb7825eef6 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1865,7 +1865,6 @@ class D(C): self.assertEqual(b.foo, 3) self.assertEqual(b.__class__, D) - @unittest.expectedSuccess # TODO: RUSTPYTHON; The `expectedFailure` here is from CPython, so this test must fail @unittest.expectedFailure def test_bad_new(self): self.assertRaises(TypeError, object.__new__) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 3ed2aabafcb..20230067875 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -236,7 +236,6 @@ def test_function(self): # is 3 because it includes f's code object. self.assertIn(gc.collect(), (2, 3)) - @unittest.expectedFailure # TODO: RUSTPYTHON; - weakref clear ordering differs from 3.15+ def test_function_tp_clear_leaves_consistent_state(self): # https://github.com/python/cpython/issues/91636 code = """if 1: diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 5b5b50effc1..493bb428cd4 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -751,7 +751,6 @@ class D: del c1, c2, C, D gc.collect() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_in_cycle_resurrection(self): import gc @@ -798,7 +797,6 @@ def C_went_away(ignore): gc.collect() self.assertEqual(alist, []) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_callbacks_on_callback(self): import gc diff --git a/benches/execution.rs b/benches/execution.rs index 56d4fc4fc19..6525e53912f 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -39,17 +39,23 @@ fn bench_rustpython_code(b: &mut Bencher<'_>, name: &str, source: &str) { settings.write_bytecode = false; settings.user_site_directory = false; let builder = Interpreter::builder(settings); - let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); - builder.add_native_modules(&defs).build().enter(|vm| { - // Note: bench_cpython is both compiling and executing the code. - // As such we compile the code in the benchmark loop as well. - b.iter(|| { - let code = vm.compile(source, Mode::Exec, name).unwrap(); - let scope = vm.new_scope_with_builtins(); - let res: PyResult = vm.run_code_obj(code, scope); - vm.unwrap_pyresult(res); - }) - }) + let defs = rustpython_stdlib::stdlib_module_defs(unsafe { builder.context() }); + // SAFETY: compilation and execution retain no objects beyond this entry. + unsafe { + builder + .add_native_modules(&defs) + .build() + .enter_unchecked(|vm| { + // Note: bench_cpython is both compiling and executing the code. + // As such we compile the code in the benchmark loop as well. + b.iter(|| { + let code = vm.compile(source, Mode::Exec, name).unwrap(); + let scope = vm.new_scope_with_builtins(); + let res: PyResult = vm.run_code_obj(code, scope); + vm.unwrap_pyresult(res); + }) + }) + } } pub fn benchmark_file_execution( diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 4d48278e447..46eb9aa2ee2 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -111,58 +111,61 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &Micro settings.user_site_directory = false; let builder = Interpreter::builder(settings); - let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); - let interp = builder.add_native_modules(&defs).build(); - interp.enter(|vm| { - let setup_code = vm - .compile(&bench.setup, Mode::Exec, &bench.name) - .expect("Error compiling setup code"); - let bench_code = vm - .compile(&bench.code, Mode::Exec, &bench.name) - .expect("Error compiling bench code"); - - let bench_func = |scope| { - let res: PyResult = vm.run_code_obj(bench_code.clone(), scope); - vm.unwrap_pyresult(res); - }; + let defs = rustpython_stdlib::stdlib_module_defs(unsafe { builder.context() }); + let interp = unsafe { builder.add_native_modules(&defs) }.build(); + // SAFETY: benchmark scopes and code objects are released before leaving. + unsafe { + interp.enter_unchecked(|vm| { + let setup_code = vm + .compile(&bench.setup, Mode::Exec, &bench.name) + .expect("Error compiling setup code"); + let bench_code = vm + .compile(&bench.code, Mode::Exec, &bench.name) + .expect("Error compiling bench code"); + + let bench_func = |scope| { + let res: PyResult = vm.run_code_obj(bench_code.clone(), scope); + vm.unwrap_pyresult(res); + }; - let bench_setup = |iterations| { - let scope = vm.new_scope_with_builtins(); - if let Some(idx) = iterations { + let bench_setup = |iterations| { + let scope = vm.new_scope_with_builtins(); + if let Some(idx) = iterations { + scope + .locals + .as_ref() + .expect("new_scope_with_builtins always provides locals") + .as_object() + .set_item("ITERATIONS", vm.new_pyobj(idx), vm) + .expect("Error adding ITERATIONS local variable"); + } + let setup_result = vm.run_code_obj(setup_code.clone(), scope.clone()); + vm.unwrap_pyresult(setup_result); scope - .locals - .as_ref() - .expect("new_scope_with_builtins always provides locals") - .as_object() - .set_item("ITERATIONS", vm.new_pyobj(idx), vm) - .expect("Error adding ITERATIONS local variable"); - } - let setup_result = vm.run_code_obj(setup_code.clone(), scope.clone()); - vm.unwrap_pyresult(setup_result); - scope - }; + }; - if bench.iterate { - for idx in iteration_counts() { - group.throughput(Throughput::Elements(idx as u64)); - group.bench_with_input( - BenchmarkId::new("rustpython", &bench.name), - &idx, - |b, idx| { - b.iter_batched( - || bench_setup(Some(*idx)), - bench_func, - BatchSize::LargeInput, - ); - }, - ); + if bench.iterate { + for idx in iteration_counts() { + group.throughput(Throughput::Elements(idx as u64)); + group.bench_with_input( + BenchmarkId::new("rustpython", &bench.name), + &idx, + |b, idx| { + b.iter_batched( + || bench_setup(Some(*idx)), + bench_func, + BatchSize::LargeInput, + ); + }, + ); + } + } else { + group.bench_function(BenchmarkId::new("rustpython", &bench.name), move |b| { + b.iter_batched(|| bench_setup(None), bench_func, BatchSize::LargeInput); + }); } - } else { - group.bench_function(BenchmarkId::new("rustpython", &bench.name), move |b| { - b.iter_batched(|| bench_setup(None), bench_func, BatchSize::LargeInput); - }); - } - }) + }) + } } /// `true` when the benchmarks are executed by the CodSpeed runner. diff --git a/crates/capi/src/bytearrayobject.rs b/crates/capi/src/bytearrayobject.rs index d6b5dedde6c..6c94ad841e2 100644 --- a/crates/capi/src/bytearrayobject.rs +++ b/crates/capi/src/bytearrayobject.rs @@ -59,7 +59,9 @@ pub unsafe extern "C" fn PyByteArray_Size(bytearray: *mut PyObject) -> isize { pub unsafe extern "C" fn PyByteArray_AsString(bytearray: *mut PyObject) -> *mut c_char { with_vm(|vm| { let bytearray = unsafe { bytearray.assume_borrowed_and_cast::(vm) }?; - Ok(bytearray.borrow_buf_mut().as_mut_ptr()) + // SAFETY: the C caller owns the PyByteArray_AsString lifetime and + // synchronization contract; no resizable Rust Vec escapes this API. + Ok(unsafe { bytearray.as_mut_ptr_unchecked() }) }) } diff --git a/crates/capi/src/ceval.rs b/crates/capi/src/ceval.rs index 4064b289caf..a4b0b4dfee0 100644 --- a/crates/capi/src/ceval.rs +++ b/crates/capi/src/ceval.rs @@ -180,7 +180,7 @@ mod tests { #[pyfunction] fn legacy_locals_is_frame_mapping() -> bool { let locals = super::PyEval_GetLocals(); - rustpython_vm::vm::thread::with_current_vm(|vm| { + crate::pystate::with_current_vm(|vm| { let frame = vm.current_frame().unwrap(); core::ptr::eq( locals.cast_const(), diff --git a/crates/capi/src/descrobject.rs b/crates/capi/src/descrobject.rs index 1e53b72e0ce..5fbea6d09e1 100644 --- a/crates/capi/src/descrobject.rs +++ b/crates/capi/src/descrobject.rs @@ -5,7 +5,7 @@ use crate::pystate::with_vm; use crate::util::{CStrExt, FfiPtrExt}; use core::ffi::{c_char, c_int, c_void}; use rustpython_vm::builtins::{ - DescriptorMemberDef, MemberAccess, MemberKind, PyDescriptorOwned, PyGetSet, PyMappingProxy, + DescriptorMemberDef, MemberAccess, MemberKind, PyDescriptor, PyGetSet, PyMappingProxy, PyMemberDescriptor, PyMemberFlags, PyType, }; use rustpython_vm::common::lock::PyRwLock; @@ -40,11 +40,7 @@ impl PyGetSetDef { } }) } - pub(crate) fn build( - &self, - ty: &'static Py, - vm: &VirtualMachine, - ) -> PyResult> { + pub(crate) fn build(&self, ty: &Py, vm: &VirtualMachine) -> PyResult> { let name = unsafe { self.name.try_as_str(vm) }?; let closure = self.closure as usize; @@ -167,7 +163,7 @@ impl PyMemberDef { }); let descriptor = PyMemberDescriptor { - common: PyDescriptorOwned { + common: PyDescriptor { typ: ty.to_owned(), name: vm.ctx.intern_str(name), qualname: PyRwLock::new(None), diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index 90ef71da7a8..9fe851658e9 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -63,7 +63,7 @@ pub fn init_main_interpreter(interpreter: Interpreter) { let mut interp = get_main_interpreter(); assert!(interp.is_none(), "Main interpreter is already set"); // Safety: Interpreter was not initialized before, so we can safely assume the statics are not used - unsafe { init_exception_statics(&Context::genesis().exceptions) }; + unsafe { init_exception_statics(&Context::genesis_unchecked().exceptions) }; *interp = Some(interpreter); MAIN_INTERP_PTR.store( interp.as_ref().unwrap() as *const _ as *mut _, diff --git a/crates/capi/src/object/pytype.rs b/crates/capi/src/object/pytype.rs index 27254dc83cc..bf14e4487c7 100644 --- a/crates/capi/src/object/pytype.rs +++ b/crates/capi/src/object/pytype.rs @@ -58,7 +58,7 @@ pub unsafe extern "C" fn Py_IS_TYPE(op: *mut PyObject, ty: *mut PyTypeObject) -> #[unsafe(no_mangle)] pub unsafe extern "C" fn PyType_GetFlags(ptr: *mut PyTypeObject) -> c_ulong { let ty = unsafe { ptr.assume_borrowed() }; - ty.slots.flags.load().bits() as u32 as c_ulong + ty.effective_flags().bits() as u32 as c_ulong } #[unsafe(no_mangle)] @@ -227,27 +227,20 @@ pub extern "C" fn PyType_FromSlots(slots: *const PySlot) -> *mut PyObject { |msg| vm.new_system_error(format!("Failed to create type from slots: {msg}")), )?; - let attrs = &class.attributes; - let class_static = unsafe { &*((&*class) as *const _) }; + let attrs = class.attributes(); for (name, method) in methods { attrs.insert( vm.ctx.intern_str(name), - method.build_method(class_static, vm).into(), + method.build_method(&class, vm).into(), ); } for getset in getsets { let name = unsafe { getset.name.try_as_str(vm)? }; - attrs.insert( - vm.ctx.intern_str(name), - getset.build(class_static, vm)?.into(), - ); + attrs.insert(vm.ctx.intern_str(name), getset.build(&class, vm)?.into()); } for member in members { let name = unsafe { member.name.try_as_str(vm)? }; - attrs.insert( - vm.ctx.intern_str(name), - member.build(class_static, vm)?.into(), - ); + attrs.insert(vm.ctx.intern_str(name), member.build(&class, vm)?.into()); } Ok(class) diff --git a/crates/capi/src/objimpl.rs b/crates/capi/src/objimpl.rs index e79adb92baf..bb38faeba59 100644 --- a/crates/capi/src/objimpl.rs +++ b/crates/capi/src/objimpl.rs @@ -10,7 +10,10 @@ pub unsafe extern "C" fn PyObject_GC_Track(op: *mut PyObject) { with_vm(|_vm| { let obj = unsafe { op.assume_borrowed() }; if !obj.is_gc_tracked() { - unsafe { gc_state::gc_state().track_object(obj.into(), gc_state::current_owner()) }; + unsafe { + gc_state::gc_state_unchecked() + .track_object(obj.into(), gc_state::current_owner_unchecked()) + }; } }) } @@ -20,7 +23,7 @@ pub unsafe extern "C" fn PyObject_GC_UnTrack(op: *mut PyObject) { with_vm(|_vm| { let obj = unsafe { op.assume_borrowed() }; if obj.is_gc_tracked() { - unsafe { gc_state::gc_state().untrack_object(obj.into()) }; + unsafe { gc_state::gc_state_unchecked().untrack_object(obj.into()) }; } }) } diff --git a/crates/capi/src/pyframe.rs b/crates/capi/src/pyframe.rs index 2214ee55820..04aeba4cc84 100644 --- a/crates/capi/src/pyframe.rs +++ b/crates/capi/src/pyframe.rs @@ -41,7 +41,7 @@ mod tests { use rustpython_vm::AsObject; fn current_frame_ptr() -> *mut super::PyFrameObject { - rustpython_vm::vm::thread::with_current_vm(|vm| { + crate::pystate::with_current_vm(|vm| { vm.current_frame() .unwrap() .as_object() @@ -83,7 +83,7 @@ mod tests { #[pyfunction] fn frame_globals_is_globals() -> bool { - rustpython_vm::vm::thread::with_current_vm(|vm| { + crate::pystate::with_current_vm(|vm| { let frame = vm.current_frame().unwrap(); let got = unsafe { super::PyFrame_GetGlobals(current_frame_ptr()) }; let expected = frame.iframe().globals().as_object().as_raw(); @@ -95,7 +95,7 @@ mod tests { #[pyfunction] fn frame_builtins_is_builtins() -> bool { - rustpython_vm::vm::thread::with_current_vm(|vm| { + crate::pystate::with_current_vm(|vm| { let frame = vm.current_frame().unwrap(); let got = unsafe { super::PyFrame_GetBuiltins(current_frame_ptr()) }; let expected = frame.iframe().builtins().as_object().as_raw(); diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index 7255a0e34a5..fa0e2899d87 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -4,7 +4,6 @@ use crate::pystate::ensure_thread_has_vm_attached; use alloc::ffi::CString; use core::ffi::{c_char, c_int, c_ulong}; use core::sync::atomic::{AtomicPtr, Ordering}; -use rustpython_vm::common::rc::PyRc; use rustpython_vm::stdlib::sys; use rustpython_vm::version::{MAJOR, MICRO, MINOR, RUSTPYTHON_BUILD_INFO, VERSION_HEX}; use rustpython_vm::vm::thread::ThreadedVirtualMachine; @@ -19,7 +18,7 @@ pub(crate) fn request_vm_from_interpreter() -> ThreadedVirtualMachine { get_main_interpreter() .as_ref() .expect("Interpreter not initialized") - .enter(|vm| vm.new_thread()) + .new_thread() } #[unsafe(no_mangle)] @@ -40,20 +39,21 @@ pub extern "C" fn Py_InitializeEx(_initsigs: c_int) { let mut interp = get_main_interpreter(); if interp.is_none() { // Safety: Interpreter was not initialized before, so we can safely assume the statics are not used - unsafe { init_exception_statics(&Context::genesis().exceptions) }; + unsafe { init_exception_statics(&Context::genesis_unchecked().exceptions) }; let builder = Interpreter::builder(Default::default()); - let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); - *interp = builder - .add_native_modules(&defs) - .init_hook(|vm| { - let state = PyRc::get_mut(&mut vm.state).unwrap(); - let path = rustpython_pylib::LIB_PATH.to_owned(); - - state.config.paths.stdlib_dir = Some(path.clone()); - state.config.paths.module_search_paths.insert(0, path); - }) - .build() - .into(); + let defs = rustpython_stdlib::stdlib_module_defs(unsafe { builder.context() }); + *interp = unsafe { + builder + .add_native_modules(&defs) + .configure(|config| { + let path = rustpython_pylib::LIB_PATH.to_owned(); + + config.paths.stdlib_dir = Some(path.clone()); + config.paths.module_search_paths.insert(0, path); + }) + .build() + .into() + }; MAIN_INTERP_PTR.store( interp.as_ref().unwrap() as *const _ as *mut _, Ordering::Release, diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs index cec3bc240b1..7946a972de1 100644 --- a/crates/capi/src/pystate.rs +++ b/crates/capi/src/pystate.rs @@ -4,10 +4,16 @@ use core::ffi::c_int; use core::sync::atomic::Ordering; use rustpython_vm::vm::thread::{ CurrentVmAttachState, SavedThreadState, attach_current_thread, release_current_thread, - restore_current_thread, save_current_thread, with_current_vm, + restore_current_thread_unchecked, save_current_thread_unchecked, }; use rustpython_vm::{Interpreter, VirtualMachine}; +// Native C API callbacks already require an attached interpreter and follow +// CPython's ownership rules. Keep that boundary inside this crate. +pub(crate) fn with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { + unsafe { rustpython_vm::vm::thread::with_current_vm_unchecked(f) } +} + pub(crate) fn with_vm, O>(f: impl FnOnce(&VirtualMachine) -> R) -> O { with_current_vm(|vm| f(vm).into_output(vm)) } @@ -28,7 +34,8 @@ pub struct PyThreadState { /// Make sure this thread has a running vm attached. This only creates a new vm if we don't already /// have one. So this will only create a new vm when we are in a new thread created outside RustPython. pub(crate) fn ensure_thread_has_vm_attached() -> CurrentVmAttachState { - attach_current_thread(request_vm_from_interpreter) + // SAFETY: the C API pairs Ensure/Release on the calling native thread. + unsafe { attach_current_thread(request_vm_from_interpreter) } } #[unsafe(no_mangle)] @@ -42,7 +49,7 @@ pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { #[unsafe(no_mangle)] pub extern "C" fn PyGILState_Release(state: PyGILState_STATE) { if state == PYGILSTATE_UNLOCKED { - release_current_thread(CurrentVmAttachState::Attached); + unsafe { release_current_thread(CurrentVmAttachState::Attached) }; } } @@ -51,7 +58,7 @@ pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState { let interp = PyInterpreterState_Get(); let state = Box::new(PyThreadState { interp, - vm: save_current_thread(), + vm: unsafe { save_current_thread_unchecked() }, }); Box::into_raw(state) } @@ -62,7 +69,7 @@ pub unsafe extern "C" fn PyEval_RestoreThread(state: *mut PyThreadState) { // SAFETY: PyEval_SaveThread returns this allocation and CPython's API // requires callers to restore exactly that thread state once. let state = unsafe { Box::from_raw(state) }; - restore_current_thread(state.vm); + unsafe { restore_current_thread_unchecked(state.vm) }; } #[unsafe(no_mangle)] @@ -87,10 +94,11 @@ pub extern "C" fn PyInterpreterState_GetID(interp: *mut PyInterpreterState) -> i #[cfg(test)] mod tests { + use super::with_current_vm; use crate::get_main_interpreter; use crate::pystate::{PyGILState_Ensure, PyGILState_Release}; use pyo3::prelude::*; - use rustpython_vm::vm::thread::{current_vm_is_set, with_current_vm}; + use rustpython_vm::vm::thread::current_vm_is_set; #[test] fn new_thread() { @@ -128,21 +136,20 @@ mod tests { Python::initialize(); // let RustPython create a vm for this thread. - let vm = get_main_interpreter() - .as_ref() - .unwrap() - .enter(|vm| vm.new_thread()); + let vm = get_main_interpreter().as_ref().unwrap().new_thread(); // Attach the vm using RustPython - vm.run(|_vm| { - assert!(current_vm_is_set(), "This thread should have a vm attached"); + unsafe { + vm.run_unchecked(|_vm| { + assert!(current_vm_is_set(), "This thread should have a vm attached"); - Python::attach(|_py| { - with_current_vm(|_vm| { - assert!(current_vm_is_set()); + Python::attach(|_py| { + with_current_vm(|_vm| { + assert!(current_vm_is_set()); + }) }) }) - }); + }; } #[test] diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index f7d117cb76b..8a24e286f8a 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -6,7 +6,14 @@ use crate::{ varint::{read_varint, read_varint_with_start, write_varint_be, write_varint_with_start}, {OneIndexed, SourceLocation}, }; -use alloc::{borrow::ToOwned, boxed::Box, collections::BTreeSet, fmt, string::String, vec::Vec}; +use alloc::{ + borrow::{Cow, ToOwned}, + boxed::Box, + collections::BTreeSet, + fmt, + string::String, + vec::Vec, +}; use bitflags::bitflags; use core::{ cell::UnsafeCell, @@ -303,8 +310,12 @@ impl Constant for ConstantData { Self::Bytes { value } => BorrowedConstant::Bytes { value }, Self::Code { code } => BorrowedConstant::Code { code }, Self::Tuple { elements } => BorrowedConstant::Tuple { elements }, - Self::Slice { elements } => BorrowedConstant::Slice { elements }, - Self::Frozenset { elements } => BorrowedConstant::Frozenset { elements }, + Self::Slice { elements } => BorrowedConstant::Slice { + elements: elements.each_ref(), + }, + Self::Frozenset { elements } => BorrowedConstant::Frozenset { + elements: Cow::Borrowed(elements), + }, Self::None => BorrowedConstant::None, Self::Ellipsis => BorrowedConstant::Ellipsis, } @@ -1112,7 +1123,9 @@ impl hash::Hash for ConstantData { } } -/// A borrowed Constant +/// A constant view. Containers may own a temporary snapshot when their native +/// representation cannot expose a contiguous slice of constants. +#[derive(Clone)] pub enum BorrowedConstant<'a, C: Constant> { Integer { value: &'a BigInt }, Float { value: f64 }, @@ -1122,20 +1135,12 @@ pub enum BorrowedConstant<'a, C: Constant> { Bytes { value: &'a [u8] }, Code { code: &'a CodeObject }, Tuple { elements: &'a [C] }, - Slice { elements: &'a [C; 3] }, - Frozenset { elements: &'a [C] }, + Slice { elements: [&'a C; 3] }, + Frozenset { elements: Cow<'a, [C]> }, None, Ellipsis, } -impl Copy for BorrowedConstant<'_, C> {} - -impl Clone for BorrowedConstant<'_, C> { - fn clone(&self) -> Self { - *self - } -} - impl BorrowedConstant<'_, C> { pub fn fmt_display(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -1173,7 +1178,7 @@ impl BorrowedConstant<'_, C> { BorrowedConstant::Frozenset { elements } => { write!(f, "frozenset({{")?; let mut first = true; - for c in *elements { + for c in elements.iter() { if first { first = false } else { @@ -1213,7 +1218,7 @@ impl BorrowedConstant<'_, C> { .collect(), }, BorrowedConstant::Slice { elements } => ConstantData::Slice { - elements: Box::new(elements.each_ref().map(|c| c.borrow_constant().to_owned())), + elements: Box::new(elements.map(|c| c.borrow_constant().to_owned())), }, BorrowedConstant::Frozenset { elements } => ConstantData::Frozenset { elements: elements diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 1615c1d6a78..65a78ae6159 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -778,7 +778,7 @@ impl MarshalBag for Bag { let elements = [start, stop, step]; Ok( self.make_constant::(BorrowedConstant::Slice { - elements: &elements, + elements: elements.each_ref(), }), ) } @@ -806,7 +806,7 @@ impl MarshalBag for Bag { let elements: Vec = it.collect(); Ok( self.make_constant::(BorrowedConstant::Frozenset { - elements: &elements, + elements: alloc::borrow::Cow::Borrowed(&elements), }), ) } @@ -1271,19 +1271,19 @@ pub enum DumpableValue<'a, D: Dumpable> { Slice(&'a D, &'a D, &'a D), } -impl<'a, C: Constant> From> for DumpableValue<'a, C> { - fn from(c: BorrowedConstant<'a, C>) -> Self { +impl<'a, C: Constant> From<&'a BorrowedConstant<'_, C>> for DumpableValue<'a, C> { + fn from(c: &'a BorrowedConstant<'_, C>) -> Self { match c { BorrowedConstant::Integer { value } => Self::Integer(value), - BorrowedConstant::Float { value } => Self::Float(value), - BorrowedConstant::Complex { value } => Self::Complex(value), - BorrowedConstant::Boolean { value } => Self::Boolean(value), + BorrowedConstant::Float { value } => Self::Float(*value), + BorrowedConstant::Complex { value } => Self::Complex(*value), + BorrowedConstant::Boolean { value } => Self::Boolean(*value), BorrowedConstant::Str { value } => Self::Str(value), BorrowedConstant::Bytes { value } => Self::Bytes(value), BorrowedConstant::Code { code } => Self::Code(code), BorrowedConstant::Tuple { elements } => Self::Tuple(elements), BorrowedConstant::Slice { elements } => { - Self::Slice(&elements[0], &elements[1], &elements[2]) + Self::Slice(elements[0], elements[1], elements[2]) } BorrowedConstant::Frozenset { elements } => Self::Frozenset(elements), BorrowedConstant::None => Self::None, @@ -1298,7 +1298,7 @@ impl Dumpable for C { #[inline(always)] fn with_dump(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result { - Ok(f(self.borrow_constant().into())) + Ok(f((&self.borrow_constant()).into())) } } @@ -1463,7 +1463,7 @@ pub fn serialize_value( /// co_localsplusnames/co_localspluskinds. pub fn serialize_code(buf: &mut W, code: &CodeObject) { serialize_code_with(buf, code, |buf, constant| { - serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {}); + serialize_value(buf, (&constant.borrow_constant()).into()).unwrap_or_else(|x| match x {}); Ok::<(), core::convert::Infallible>(()) }) .unwrap_or_else(|x| match x {}) diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index be59d7a339d..be41e0bf190 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -199,8 +199,8 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul }, parse_quote! { fn __extend_py_class( - ctx: &'static ::rustpython_vm::Context, - class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + ctx: &::rustpython_vm::Context, + class: &::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, ) { #getset_impl #extend_impl @@ -262,8 +262,8 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul const INTERNAL_DOC: ::rustpython_vm::function::ItemDoc = #internal_doc; fn impl_extend_class( - ctx: &'static ::rustpython_vm::Context, - class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + ctx: &::rustpython_vm::Context, + class: &::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, ) { #holder::__extend_py_class(ctx, class); #with_impl @@ -309,8 +309,9 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul let slots_impl = &context.extend_slots_items.validate()?; let class_extensions = &context.class_extensions; let call_extend_slots = if has_extend_slots { + let trait_name = &trai.ident; quote! { - Self::extend_slots(slots); + ::extend_slots(slots); } } else { quote! {} @@ -321,8 +322,8 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul }, parse_quote! { fn __extend_py_class( - ctx: &'static ::rustpython_vm::Context, - class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + ctx: &::rustpython_vm::Context, + class: &::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, ) { #getset_impl #extend_impl @@ -542,6 +543,7 @@ struct MemberTableTokens<'a> { struct ClassDefExtras<'a> { attrs: &'a [Attribute], member_table: MemberTableTokens<'a>, + interpreter_local: bool, } fn generate_class_def( @@ -554,6 +556,7 @@ fn generate_class_def( extras: ClassDefExtras<'_>, ) -> Result { let attrs = extras.attrs; + let interpreter_local = extras.interpreter_local; let module_key = module_name.unwrap_or("builtins"); let (attr_docs, attr_names) = crate::class_docs::attr_docs_tokens(module_name, name); let rust_doc = attrs.doc().filter(|doc| !doc.is_empty()); @@ -621,9 +624,13 @@ fn generate_class_def( } .map(|typ| { quote! { - fn static_baseclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + unsafe fn static_baseclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { use rustpython_vm::class::StaticType; - #typ::static_type() + unsafe { #typ::static_type() } + } + + fn baseclass(ctx: &::rustpython_vm::Context) -> ::rustpython_vm::builtins::PyTypeRef { + <#typ as ::rustpython_vm::class::PyClassImpl>::make_class(ctx) } } }); @@ -631,9 +638,9 @@ fn generate_class_def( let meta_class = metaclass.map(|typ| { let typ = Ident::new(&typ, ident.span()); quote! { - fn static_metaclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + unsafe fn static_metaclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { use rustpython_vm::class::StaticType; - #typ::static_type() + unsafe { #typ::static_type() } } } }); @@ -669,9 +676,22 @@ fn generate_class_def( checks: member_checks, } = extras.member_table; + let native_layout = if is_repr_transparent { + base.as_ref().map(|base| { + quote! { + const NATIVE_LAYOUT_ID: ::core::any::TypeId = + <#base as ::rustpython_vm::class::PyClassDef>::NATIVE_LAYOUT_ID; + } + }) + } else { + None + }; + let tokens = quote! { impl ::rustpython_vm::class::PyClassDef for #ident { const NAME: &'static str = #name; + const INTERPRETER_LOCAL: bool = #interpreter_local; + #native_layout const MODULE_NAME: Option<&'static str> = #module_name; const TP_NAME: &'static str = #module_class_name; const DOC: ::rustpython_vm::function::ItemDoc = #doc; @@ -694,7 +714,7 @@ fn generate_class_def( #member_parts_impl impl ::rustpython_vm::class::StaticType for #ident { - fn static_cell() -> &'static ::rustpython_vm::common::static_cell::StaticCell<::rustpython_vm::builtins::PyTypeRef> { + unsafe fn static_cell() -> &'static ::rustpython_vm::common::static_cell::StaticCell<::rustpython_vm::builtins::PyTypeRef> { ::rustpython_vm::common::static_cell! { static CELL: ::rustpython_vm::builtins::PyTypeRef; } @@ -800,6 +820,7 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result Result::static_type() } + quote! { ::make_class(ctx) } }; quote! { @@ -903,12 +924,22 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result::PAYLOAD_TYPE_ID; + fn supports_native_layout(layout: ::core::any::TypeId) -> bool { + layout == ::NATIVE_LAYOUT_ID + || <#base_type as ::rustpython_vm::PyPayload>::supports_native_layout(layout) + } + #[inline] unsafe fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { - ::BASICSIZE <= obj.class().payload().slots.basicsize && obj.class().fast_issubclass(::static_type()) + obj.supports_native_layout(::NATIVE_LAYOUT_ID) + && if ::INTERPRETER_LOCAL { + obj.class().is_native_subclass::() + } else { + obj.class().fast_issubclass(unsafe { ::static_type() }) + } } - fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + fn class(ctx: &::rustpython_vm::vm::Context) -> ::rustpython_vm::builtins::PyTypeRef { #class_fn } } @@ -918,8 +949,8 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { - ctx.types.#ctx_type_ident + fn class(ctx: &::rustpython_vm::vm::Context) -> ::rustpython_vm::builtins::PyTypeRef { + ctx.types.#ctx_type_ident.to_owned() } } } @@ -2578,9 +2609,6 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result - let new_class = <#ident as ::rustpython_vm::class::PyClassImpl>::make_static_type(); + let new_class = <#ident as ::rustpython_vm::class::PyClassImpl>::make_class(ctx); // Only set __module__ string if the class doesn't already have a // getset descriptor for __module__ (which provides instance-level // module resolution, e.g. TypeAliasType) @@ -880,7 +880,7 @@ impl ModuleItem for StructSequenceItem { // Generate the class creation code let class_new = quote_spanned!(pytype_ident.span() => - let new_class = <#pytype_ident as ::rustpython_vm::class::PyClassImpl>::make_static_type(); + let new_class = <#pytype_ident as ::rustpython_vm::class::PyClassImpl>::make_class(ctx); { let module_key = rustpython_vm::identifier!(ctx, __module__); let has_module_getset = new_class.payload().attributes @@ -950,11 +950,18 @@ impl ModuleItem for AttributeItem { let (ident, py_name, let_obj) = match args.item { Item::Fn(syn::ItemFn { sig, block, .. }) => { let ident = &sig.ident; - // If `once` keyword is in #[pyattr], - // wrapping it with static_cell for preventing it from using it as function + // `once` values belong to the interpreter, even when their + // native factory is shared by every module instance. let attr_meta = AttrItemMeta::from_attr(ident.clone(), &attr)?; if attr_meta.inner()._bool("once")? { let stmts = &block.stmts; + let Some(syn::FnArg::Typed(arg)) = sig.inputs.first() else { + bail_span!(sig, "#[pyattr(once)] requires a VM argument"); + }; + let syn::Pat::Ident(vm_arg) = arg.pat.as_ref() else { + bail_span!(arg, "#[pyattr(once)] requires a named VM argument"); + }; + let vm_arg = &vm_arg.ident; let return_type = match &sig.output { syn::ReturnType::Default => { unreachable!("#[pyattr] attached function must have return type.") @@ -963,14 +970,10 @@ impl ModuleItem for AttributeItem { }; let stmt: syn::Stmt = parse_quote! { { - rustpython_common::static_cell! { - static ERROR: #return_type; - } - ERROR - .get_or_init(|| { - #(#stmts)* - }) - .clone() + struct __PyAttrCacheKey; + #vm_arg.__cached_native::<__PyAttrCacheKey, #return_type>(|| { + #(#stmts)* + }) } }; block.stmts = vec![stmt]; diff --git a/crates/derive-impl/src/pypayload.rs b/crates/derive-impl/src/pypayload.rs index e1c3bcb6979..c7b90a9573f 100644 --- a/crates/derive-impl/src/pypayload.rs +++ b/crates/derive-impl/src/pypayload.rs @@ -8,8 +8,8 @@ pub(crate) fn impl_pypayload(input: DeriveInput) -> TokenStream { quote! { impl ::rustpython_vm::PyPayload for #ty { #[inline] - fn class(_ctx: &::rustpython_vm::vm::Context) -> &'static rustpython_vm::Py<::rustpython_vm::builtins::PyType> { - ::static_type() + fn class(ctx: &::rustpython_vm::vm::Context) -> ::rustpython_vm::builtins::PyTypeRef { + ::make_class(ctx) } } } diff --git a/crates/derive-impl/src/pystructseq.rs b/crates/derive-impl/src/pystructseq.rs index aa89b3717bb..3b6c6a6776d 100644 --- a/crates/derive-impl/src/pystructseq.rs +++ b/crates/derive-impl/src/pystructseq.rs @@ -390,7 +390,8 @@ pub(crate) struct PyStructSequenceMeta { } impl ItemMeta for PyStructSequenceMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "module", "data", "no_attr"]; + const ALLOWED_NAMES: &'static [&'static str] = + &["name", "module", "data", "no_attr", "interpreter_local"]; fn from_inner(inner: ItemMetaInner) -> Self { Self { inner } @@ -500,6 +501,7 @@ pub(crate) fn impl_pystruct_sequence( let ident = struct_item.ident.clone(); let fake_ident = Ident::new("pystruct_sequence", ident.span()); let meta = PyStructSequenceMeta::from_nested(ident, fake_ident, attr.into_iter())?; + let interpreter_local = meta.inner()._bool("interpreter_local")?; let pytype_ident = struct_item.ident.clone(); let pytype_vis = struct_item.vis.clone(); @@ -553,6 +555,7 @@ pub(crate) fn impl_pystruct_sequence( const ATTR_DOCS: &'static [&'static str] = #attr_names; }} const BASICSIZE: usize = 0; + const INTERPRETER_LOCAL: bool = #interpreter_local; const UNHASHABLE: bool = false; type Base = ::rustpython_vm::builtins::PyTuple; @@ -560,16 +563,16 @@ pub(crate) fn impl_pystruct_sequence( // StaticType for Python type impl ::rustpython_vm::class::StaticType for #pytype_ident { - fn static_cell() -> &'static ::rustpython_vm::common::static_cell::StaticCell<::rustpython_vm::builtins::PyTypeRef> { + unsafe fn static_cell() -> &'static ::rustpython_vm::common::static_cell::StaticCell<::rustpython_vm::builtins::PyTypeRef> { ::rustpython_vm::common::static_cell! { static CELL: ::rustpython_vm::builtins::PyTypeRef; } &CELL } - fn static_baseclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + unsafe fn static_baseclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { use ::rustpython_vm::class::StaticType; - ::rustpython_vm::builtins::PyTuple::static_type() + unsafe { ::rustpython_vm::builtins::PyTuple::static_type() } } } @@ -577,13 +580,22 @@ pub(crate) fn impl_pystruct_sequence( impl ::rustpython_vm::PyPayload for #pytype_ident { const PAYLOAD_TYPE_ID: ::core::any::TypeId = <::rustpython_vm::builtins::PyTuple as ::rustpython_vm::PyPayload>::PAYLOAD_TYPE_ID; + fn supports_native_layout(layout: ::core::any::TypeId) -> bool { + <::rustpython_vm::builtins::PyTuple as ::rustpython_vm::PyPayload>::supports_native_layout(layout) + } + #[inline] unsafe fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { - obj.class().fast_issubclass(::static_type()) + obj.supports_native_layout(::core::any::TypeId::of::<::rustpython_vm::builtins::PyTuple>()) + && if ::INTERPRETER_LOCAL { + obj.class().is_native_subclass::() + } else { + obj.class().fast_issubclass(unsafe { ::static_type() }) + } } - fn class(_ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { - ::static_type() + fn class(ctx: &::rustpython_vm::vm::Context) -> ::rustpython_vm::builtins::PyTypeRef { + ::make_class(ctx) } } diff --git a/crates/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs index 985a8312301..e9d1dc27e31 100644 --- a/crates/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -367,6 +367,7 @@ impl ItemMeta for ClassItemMeta { "traverse", "clear", // tp_clear "payload", + "interpreter_local", ]; fn from_inner(inner: ItemMetaInner) -> Self { @@ -379,6 +380,9 @@ impl ItemMeta for ClassItemMeta { } impl ClassItemMeta { + pub(crate) fn interpreter_local(&self) -> Result { + self.inner()._bool("interpreter_local") + } pub(crate) fn class_name(&self) -> Result { const KEY: &str = "name"; let inner = self.inner(); diff --git a/crates/derive/src/lib.rs b/crates/derive/src/lib.rs index d6cd7bc2ba3..a2b212285e4 100644 --- a/crates/derive/src/lib.rs +++ b/crates/derive/src/lib.rs @@ -205,7 +205,7 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { /// #### Examples /// ```rust, ignore /// #[extend_class] -/// fn extend_class_with_fields(ctx: &Context, class: &'static Py) { +/// fn extend_class_with_fields(ctx: &Context, class: &Py) { /// class.set_attr( /// identifier!(ctx, _fields), /// ctx.new_tuple(vec![ diff --git a/crates/host_env/src/signal.rs b/crates/host_env/src/signal.rs index 090269123fc..8deba58c0dc 100644 --- a/crates/host_env/src/signal.rs +++ b/crates/host_env/src/signal.rs @@ -104,12 +104,26 @@ mod ffi { /// The caller must ensure `signalnum` is a valid platform signal number. #[cfg(any(unix, windows))] pub unsafe fn probe_handler(signalnum: i32) -> Option { - let handler = unsafe { libc::signal(signalnum, libc::SIG_IGN) }; - if handler == libc::SIG_ERR as sighandler_t { - None - } else { - unsafe { libc::signal(signalnum, handler) }; - Some(handler) + #[cfg(unix)] + { + // Query without changing disposition: even temporarily ignoring SIGCHLD + // can discard a child's exit status, and signal() also replaces masks. + let mut action = core::mem::MaybeUninit::::uninit(); + if unsafe { libc::sigaction(signalnum, core::ptr::null(), action.as_mut_ptr()) } == 0 { + Some(unsafe { action.assume_init() }.sa_sigaction) + } else { + None + } + } + #[cfg(windows)] + { + let handler = unsafe { libc::signal(signalnum, libc::SIG_IGN) }; + if handler == libc::SIG_ERR as sighandler_t { + None + } else { + unsafe { libc::signal(signalnum, handler) }; + Some(handler) + } } } @@ -379,3 +393,49 @@ pub fn sigset_contains(mask: libc::sigset_t, signum: i32) -> bool { pub fn valid_signals(_max_signum: usize) -> io::Result> { Ok(VALID_SIGNALS.to_vec()) } + +#[cfg(test)] +#[cfg(unix)] +mod tests { + #[test] + fn probing_preserves_signal_flags_and_mask() { + unsafe extern "C" fn handler(_: libc::c_int) {} + + struct Restore(libc::sigaction); + impl Drop for Restore { + fn drop(&mut self) { + unsafe { libc::sigaction(libc::SIGUSR2, &self.0, core::ptr::null_mut()) }; + } + } + + // No other host_env test uses SIGUSR2; retain the host disposition even + // when an assertion fails. Do not deliver any signal to the process. + unsafe { + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = handler as *const () as libc::sighandler_t; + action.sa_flags = libc::SA_NODEFER; + assert_eq!(libc::sigemptyset(&mut action.sa_mask), 0); + assert_eq!(libc::sigaddset(&mut action.sa_mask, libc::SIGUSR1), 0); + let mut original = core::mem::MaybeUninit::uninit(); + assert_eq!( + libc::sigaction(libc::SIGUSR2, &action, original.as_mut_ptr()), + 0 + ); + let _restore = Restore(original.assume_init()); + + assert_eq!( + super::probe_handler(libc::SIGUSR2), + Some(action.sa_sigaction) + ); + let mut observed = core::mem::MaybeUninit::::uninit(); + assert_eq!( + libc::sigaction(libc::SIGUSR2, core::ptr::null(), observed.as_mut_ptr()), + 0 + ); + let observed = observed.assume_init(); + assert_eq!(observed.sa_sigaction, action.sa_sigaction); + assert_ne!(observed.sa_flags & libc::SA_NODEFER, 0); + assert_eq!(libc::sigismember(&observed.sa_mask, libc::SIGUSR1), 1); + } + } +} diff --git a/crates/stdlib/src/_asyncio.rs b/crates/stdlib/src/_asyncio.rs index f56943c19d7..8e1fffb23c7 100644 --- a/crates/stdlib/src/_asyncio.rs +++ b/crates/stdlib/src/_asyncio.rs @@ -8,6 +8,7 @@ pub(crate) use _asyncio::module_def; pub(crate) mod _asyncio { use crate::common::wtf8::{Wtf8Buf, wtf8_concat}; use crate::contextvars::PyContext; + use crate::vm::common::rc::PyRc; use crate::{ common::lock::PyRwLock, vm::{ @@ -2458,16 +2459,8 @@ pub(crate) mod _asyncio { copy_context: PyObjectRef, } - // `PyObjectRef`/`PyRef` are not `Sync` (they are not meant to be - // shared across threads via a plain global), so the caches live in - // thread-locals, mirroring the `CONTEXTS` thread-local already used by - // `contextvars.rs`. Each thread pays the resolution cost once (or again - // after a module reload, detected via the identity check below) rather - // than on every task step. - thread_local! { - static ASYNCIO_CACHE: core::cell::RefCell> = const { core::cell::RefCell::new(None) }; - static CONTEXTVARS_CACHE: core::cell::RefCell> = const { core::cell::RefCell::new(None) }; - } + // Cache snapshots belong to the interpreter and are released while its + // owner is attached. Callers run outside the native-cache directory lock. /// Access the cached `_asyncio` module state, refreshing it first if the /// module it was built from is no longer the currently-imported one @@ -2479,10 +2472,10 @@ pub(crate) mod _asyncio { ) -> PyResult { let asyncio_module = vm.import("_asyncio", 0)?; - let hit = - ASYNCIO_CACHE.with_borrow(|c| c.as_ref().is_some_and(|c| c.module.is(&asyncio_module))); - if hit { - return Ok(ASYNCIO_CACHE.with_borrow(|c| f(c.as_ref().unwrap()))); + if let Some(cache) = vm.__get_native::>() + && cache.module.is(&asyncio_module) + { + return Ok(f(&cache)); } // Slow path: (re)resolve everything and populate the cache. @@ -2507,7 +2500,7 @@ pub(crate) mod _asyncio { current_tasks, }; let result = f(&cache); - ASYNCIO_CACHE.with_borrow_mut(|c| *c = Some(cache)); + vm.__replace_native::(PyRc::new(cache)); Ok(result) } @@ -2519,10 +2512,10 @@ pub(crate) mod _asyncio { ) -> PyResult { let contextvars_module = vm.import("contextvars", 0)?; - let hit = CONTEXTVARS_CACHE - .with_borrow(|c| c.as_ref().is_some_and(|c| c.module.is(&contextvars_module))); - if hit { - return Ok(CONTEXTVARS_CACHE.with_borrow(|c| f(c.as_ref().unwrap()))); + if let Some(cache) = vm.__get_native::>() + && cache.module.is(&contextvars_module) + { + return Ok(f(&cache)); } let copy_context = vm @@ -2534,7 +2527,7 @@ pub(crate) mod _asyncio { copy_context, }; let result = f(&cache); - CONTEXTVARS_CACHE.with_borrow_mut(|c| *c = Some(cache)); + vm.__replace_native::(PyRc::new(cache)); Ok(result) } diff --git a/crates/stdlib/src/_datetime.rs b/crates/stdlib/src/_datetime.rs index e424d903b1d..5734e03e7ec 100644 --- a/crates/stdlib/src/_datetime.rs +++ b/crates/stdlib/src/_datetime.rs @@ -937,11 +937,11 @@ mod _datetime { // Type checks fn date_type() -> &'static Py { - PyDate::static_type() + unsafe { PyDate::static_type() } } pub(crate) fn datetime_type() -> &'static Py { - PyDateTime::static_type() + unsafe { PyDateTime::static_type() } } pub(crate) fn timedelta_from_seconds( @@ -953,7 +953,7 @@ mod _datetime { } fn time_type() -> &'static Py { - PyTime::static_type() + unsafe { PyTime::static_type() } } fn as_date(obj: &PyObject) -> Option<&Py> { @@ -969,7 +969,7 @@ mod _datetime { } fn is_tzinfo(obj: &PyObject) -> bool { - obj.fast_isinstance(PyTzInfo::static_type()) + obj.fast_isinstance(unsafe { PyTzInfo::static_type() }) } /// The `tzinfo` a `time` or `datetime` holds; `None` for naive values and dates. @@ -1575,7 +1575,7 @@ mod _datetime { return Ok(None); } if tzoffset == 0 && tz_useconds == 0 { - return Ok(Some(utc().to_owned().into())); + return Ok(Some(utc(vm).into())); } let delta = PyDelta::new_ref(0, i64::from(tzoffset), i64::from(tz_useconds), true, vm)?; Ok(Some(new_timezone(delta, None, vm)?.into())) @@ -2053,7 +2053,7 @@ mod _datetime { vm.ctx.new_int(day + 1).into(), ]; Ok(PyTuple::new_unchecked(items.into_boxed_slice()) - .into_ref_with_type(vm, PyIsoCalendarDate::make_static_type())? + .into_ref_with_type(vm, unsafe { PyIsoCalendarDate::make_static_type() })? .into()) } @@ -2111,7 +2111,7 @@ mod _datetime { class.set_attr(ctx.intern_str("max"), make(MAXYEAR, 12, 31)); let resolution = PyRef::new_ref( PyDelta::new_unchecked(1, 0, 0), - PyDelta::static_type().to_owned(), + unsafe { PyDelta::static_type() }.to_owned(), None, ); class.set_attr(ctx.intern_str("resolution"), resolution.into()); @@ -2430,12 +2430,16 @@ mod _datetime { name: Option, } - fn utc() -> &'static Py { - UTC.get().expect("the timezone type creates UTC") - } - - rustpython_common::static_cell! { - static UTC: PyRef; + fn utc(vm: &VirtualMachine) -> PyRef { + struct Utc; + vm.__cached_native::(|| { + PyTimeZone { + base: PyTzInfo::default(), + offset: PyDelta::new_unchecked(0, 0, 0).into_ref(&vm.ctx), + name: None, + } + .into_ref(&vm.ctx) + }) } /// `new_timezone`: UTC for a zero, unnamed offset; the offset must be within a day. @@ -2445,7 +2449,7 @@ mod _datetime { vm: &VirtualMachine, ) -> PyResult> { if name.is_none() && !offset.is_nonzero() { - return Ok(utc().to_owned()); + return Ok(utc(vm)); } if offset_out_of_range(&offset) { return Err(offset_range_error(offset.as_object(), vm)?); @@ -2510,7 +2514,7 @@ mod _datetime { if let Some(name) = &zelf.name { return name.clone(); } - if zelf.is(utc()) || !zelf.offset.is_nonzero() { + if zelf.is(&utc(vm)) || !zelf.offset.is_nonzero() { return vm.ctx.new_str("UTC"); } vm.ctx @@ -2546,7 +2550,7 @@ mod _datetime { impl Representable for PyTimeZone { fn repr_str(zelf: &Py, vm: &VirtualMachine) -> PyResult { let type_name = type_name(zelf.as_object()); - if zelf.is(utc()) { + if zelf.is(&utc(vm)) { return Ok(format!("{type_name}.utc")); } let offset = zelf.offset.as_object().repr(vm)?; @@ -2612,7 +2616,7 @@ mod _datetime { #[extend_class] fn extend_class(ctx: &crate::vm::Context, class: &Py) { - let delta_type = PyDelta::static_type(); + let delta_type = unsafe { PyDelta::static_type() }; let make = |seconds: i32| { let (days, seconds) = if seconds < 0 { (-1, seconds + 24 * 3600) @@ -2634,8 +2638,9 @@ mod _datetime { None, ) }; - let utc = UTC.get_or_init(|| make(0)); - class.set_attr(ctx.intern_str("utc"), utc.clone().into()); + // SAFETY: this native constructor runs in the currently attached owner. + let utc = unsafe { crate::vm::vm::thread::with_current_vm_unchecked(utc) }; + class.set_attr(ctx.intern_str("utc"), utc.into()); class.set_attr(ctx.intern_str("min"), make(-(23 * 3600 + 59 * 60)).into()); class.set_attr(ctx.intern_str("max"), make(23 * 3600 + 59 * 60).into()); } @@ -3074,7 +3079,7 @@ mod _datetime { class.set_attr(ctx.intern_str("max"), make(23, 59, 59, 999_999)); let resolution = PyRef::new_ref( PyDelta::new_unchecked(0, 0, 1), - PyDelta::static_type().to_owned(), + unsafe { PyDelta::static_type() }.to_owned(), None, ); class.set_attr(ctx.intern_str("resolution"), resolution.into()); @@ -4018,7 +4023,7 @@ mod _datetime { 0, 0, 0, - Some(utc().to_owned().into()), + Some(utc(vm).into()), 0, datetime_type(), vm, @@ -4162,7 +4167,7 @@ mod _datetime { let utc_result = Self::add_delta(zelf, &offset, -1, vm)? .downcast::() .map_err(|_| vm.new_type_error("datetime arithmetic must return a datetime"))?; - utc_result.set_tzinfo(utc().to_owned().into()); + utc_result.set_tzinfo(utc(vm).into()); let tzinfo: PyObjectRef = match target { Some(tz) => tz, None => { @@ -4174,7 +4179,7 @@ mod _datetime { 0, 0, 0, - Some(utc().to_owned().into()), + Some(utc(vm).into()), 0, datetime_type(), vm, @@ -4221,7 +4226,7 @@ mod _datetime { ); let resolution = PyRef::new_ref( PyDelta::new_unchecked(0, 0, 1), - PyDelta::static_type().to_owned(), + unsafe { PyDelta::static_type() }.to_owned(), None, ); class.set_attr(ctx.intern_str("resolution"), resolution.into()); @@ -4372,7 +4377,7 @@ mod _datetime { } #[pyattr(name = "UTC")] - fn utc_attr(_vm: &VirtualMachine) -> PyRef { - utc().to_owned() + fn utc_attr(vm: &VirtualMachine) -> PyRef { + utc(vm) } } diff --git a/crates/stdlib/src/_opcode.rs b/crates/stdlib/src/_opcode.rs index 6a3c500c537..35eb6e88d47 100644 --- a/crates/stdlib/src/_opcode.rs +++ b/crates/stdlib/src/_opcode.rs @@ -210,21 +210,21 @@ mod tests { const FNAME: &str = ""; let builder = vm::Interpreter::builder(Default::default()); - let stdlib_defs = crate::stdlib_module_defs(&builder.ctx); - let interp = builder - .add_native_modules(&stdlib_defs) + let stdlib_defs = crate::stdlib_module_defs(unsafe { builder.context() }); + let interp = unsafe { builder.add_native_modules(&stdlib_defs) } .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) .build(); - interp.enter(|vm| { - let scope = vm.new_scope_with_builtins(); - let code_obj = vm - .compile(source.trim(), Mode::Exec, FNAME) - .map_err(|err| err.into_pyexception(vm, Some(source))) - .unwrap(); - scope.globals.set_item("code", code_obj.into(), vm).unwrap(); + unsafe { + interp.enter_unchecked(|vm| { + let scope = vm.new_scope_with_builtins(); + let code_obj = vm + .compile(source.trim(), Mode::Exec, FNAME) + .map_err(|err| err.into_pyexception(vm, Some(source))) + .unwrap(); + scope.globals.set_item("code", code_obj.into(), vm).unwrap(); - let py_source = r#" + let py_source = r#" import dis import io import re @@ -241,15 +241,16 @@ tmp_output = buf.getvalue() output = re.sub(r'(0xdeadbeef', tmp_output) "#; - let py_code_obj = vm - .compile(py_source, Mode::Exec, FNAME) - .map_err(|err| err.into_pyexception(vm, Some(py_source))) - .unwrap(); + let py_code_obj = vm + .compile(py_source, Mode::Exec, FNAME) + .map_err(|err| err.into_pyexception(vm, Some(py_source))) + .unwrap(); - vm.run_code_obj(py_code_obj, scope.clone()).unwrap(); - let py_output = scope.globals.get_item("output", vm).unwrap(); - py_output.str(vm).unwrap().to_string() - }) + vm.run_code_obj(py_code_obj, scope.clone()).unwrap(); + let py_output = scope.globals.get_item("output", vm).unwrap(); + py_output.str(vm).unwrap().to_string() + }) + } } #[test] diff --git a/crates/stdlib/src/_queue.rs b/crates/stdlib/src/_queue.rs index 27155b7c3f3..064dde9b58f 100644 --- a/crates/stdlib/src/_queue.rs +++ b/crates/stdlib/src/_queue.rs @@ -54,7 +54,7 @@ mod _queue { /// /// [`empty_error`](https://github.com/python/cpython/blob/v3.14.5/Modules/_queuemodule.c#L347-L355). fn empty_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception_empty(PyEmptyError::class(&vm.ctx).to_owned()) + vm.new_exception_empty(PyEmptyError::class(&vm.ctx)) } #[cfg(feature = "threading")] diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs index 75c228fa3e6..b1c66c7f263 100644 --- a/crates/stdlib/src/_sqlite3.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -49,7 +49,7 @@ mod _sqlite3 { atomic::{Ordering, PyAtomic, Radium}, hash::PyHash, lock::{PyMappedMutexGuard, PyMutex, PyMutexGuard}, - static_cell, + rc::PyRc, }; use rustpython_vm::{ __exports::paste, @@ -82,28 +82,23 @@ mod _sqlite3 { macro_rules! exceptions { ($(($x:ident, $base:expr)),*) => { paste::paste! { - static_cell! { - $( - static [<$x:snake:upper>]: PyTypeRef; - )* - } $( #[allow(dead_code)] fn [](vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { - vm.new_exception_msg([<$x:snake _type>]().to_owned(), msg.into()) + vm.new_exception_msg([<$x:snake _type>](vm), msg.into()) } - fn [<$x:snake _type>]() -> &'static Py { - [<$x:snake:upper>].get().expect("exception type not initialize") + fn [<$x:snake _type>](vm: &VirtualMachine) -> PyTypeRef { + struct $x; + vm.__cached_native::<$x, _>(|| { + #[allow(clippy::redundant_closure_call)] + let base = $base(vm); + vm.ctx.new_exception_type("_sqlite3", stringify!($x), Some(vec![base.to_owned()])) + }) } )* fn setup_module_exceptions(module: &PyObject, vm: &VirtualMachine) { $( - #[allow(clippy::redundant_closure_call)] - let exception = [<$x:snake:upper>].get_or_init(|| { - let base = $base(vm); - vm.ctx.new_exception_type("_sqlite3", stringify!($x), Some(vec![base.to_owned()])) - }); - module.set_attr(stringify!($x), exception.clone().into_object(), vm).unwrap(); + module.set_attr(stringify!($x), [<$x:snake _type>](vm).into_object(), vm).unwrap(); )* } } @@ -119,14 +114,23 @@ mod _sqlite3 { .ctx .exceptions .exception_type), - (InterfaceError, |_| error_type()), - (DatabaseError, |_| error_type()), - (DataError, |_| database_error_type()), - (OperationalError, |_| database_error_type()), - (IntegrityError, |_| database_error_type()), - (InternalError, |_| database_error_type()), - (ProgrammingError, |_| database_error_type()), - (NotSupportedError, |_| database_error_type()) + (InterfaceError, |vm: &VirtualMachine| error_type(vm)), + (DatabaseError, |vm: &VirtualMachine| error_type(vm)), + (DataError, |vm: &VirtualMachine| database_error_type(vm)), + (OperationalError, |vm: &VirtualMachine| database_error_type( + vm + )), + (IntegrityError, |vm: &VirtualMachine| database_error_type( + vm + )), + (InternalError, |vm: &VirtualMachine| database_error_type(vm)), + (ProgrammingError, |vm: &VirtualMachine| database_error_type( + vm + )), + ( + NotSupportedError, + |vm: &VirtualMachine| database_error_type(vm) + ) ); #[pyattr] @@ -371,7 +375,7 @@ mod _sqlite3 { isolation_level: IsolationLevelArg, #[pyarg(any, default = true)] check_same_thread: bool, - #[pyarg(any, default = Connection::class(&vm.ctx).to_owned(), py_default = "ConnectionType")] + #[pyarg(any, default = Connection::class(&vm.ctx), py_default = "ConnectionType")] factory: PyTypeRef, // TODO: cache statements #[allow(dead_code)] @@ -460,20 +464,32 @@ mod _sqlite3 { } struct CallbackData { - obj: NonNull, - vm: *const VirtualMachine, + obj: PyObjectRef, } impl CallbackData { fn new(obj: PyObjectRef, vm: &VirtualMachine) -> Option { - (!vm.is_none(&obj)).then_some(Self { - obj: obj.into_raw(), - vm, - }) + (!vm.is_none(&obj)).then_some(Self { obj }) } - fn retrieve(&self) -> (&PyObject, &VirtualMachine) { - unsafe { (self.obj.as_ref(), &*self.vm) } + fn with_vm(&self, f: impl FnOnce(&PyObject, &VirtualMachine) -> R) -> Option { + // Connections can outlive the VM thread that registered a function. + // Resolve the currently attached owner instead of retaining its VM. + rustpython_vm::vm::thread::with_vm(&self.obj, |vm| f(&self.obj, vm)) + } + + fn with_context(context: SqliteContext, f: impl FnOnce(&PyObject, &VirtualMachine)) { + // SAFETY: SQLite keeps user data alive throughout its callback. + let data = unsafe { &*context.user_data::() }; + if data.with_vm(f).is_none() { + unsafe { + sqlite3_result_error( + context.ctx, + c"Python interpreter is unavailable".as_ptr(), + -1, + ) + }; + } } unsafe extern "C" fn destructor(data: *mut c_void) { @@ -486,25 +502,26 @@ mod _sqlite3 { argv: *mut *mut sqlite3_value, ) { let context = SqliteContext::from(context); - let (func, vm) = unsafe { (*context.user_data::()).retrieve() }; - let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; + Self::with_context(context, |func, vm| { + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; - let f = || -> PyResult<()> { - let db = context.db_handle(); - let args = args - .iter() - .copied() - .map(|val| value_to_object(val, db, vm)) - .collect::>>()?; + let f = || -> PyResult<()> { + let db = context.db_handle(); + let args = args + .iter() + .copied() + .map(|val| value_to_object(val, db, vm)) + .collect::>>()?; - let val = func.call(args, vm)?; + let val = func.call(args, vm)?; - context.result_from_object(&val, vm) - }; + context.result_from_object(&val, vm) + }; - if let Err(exc) = f() { - context.result_exception(vm, exc, c"user-defined function raised exception") - } + if let Err(exc) = f() { + context.result_exception(vm, exc, c"user-defined function raised exception") + } + }); } unsafe extern "C" fn step_callback( @@ -513,35 +530,47 @@ mod _sqlite3 { argv: *mut *mut sqlite3_value, ) { let context = SqliteContext::from(context); - let (cls, vm) = unsafe { (*context.user_data::()).retrieve() }; - let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; - let instance = context.aggregate_context::<*const PyObject>(); - if unsafe { (*instance).is_null() } { - match cls.call((), vm) { - Ok(obj) => unsafe { *instance = obj.into_raw().as_ptr() }, - Err(exc) => { - return context.result_exception( - vm, - exc, - c"user-defined aggregate's '__init__' method raised error", - ); + Self::with_context(context, |cls, vm| { + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; + let instance = context.aggregate_context::<*const PyObject>(); + if instance.is_null() { + unsafe { sqlite3_result_error_nomem(context.ctx) }; + return; + } + if unsafe { (*instance).is_null() } { + match cls.call((), vm) { + Ok(obj) => unsafe { *instance = obj.into_raw().as_ptr() }, + Err(exc) => { + return context.result_exception( + vm, + exc, + c"user-defined aggregate's '__init__' method raised error", + ); + } } } - } - let instance = unsafe { &**instance }; + let instance = unsafe { &**instance }; - Self::call_method_with_args(context, instance, "step", args, vm); + Self::call_method_with_args(context, instance, "step", args, vm); + }); } unsafe extern "C" fn finalize_callback(context: *mut sqlite3_context) { let context = SqliteContext::from(context); - let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; let instance = context.aggregate_context::<*const PyObject>(); - let Some(instance) = (unsafe { (*instance).as_ref() }) else { + if instance.is_null() { + unsafe { sqlite3_result_error_nomem(context.ctx) }; return; - }; - - Self::callback_result_from_method(context, instance, "finalize", vm); + } + // xFinal owns the reference acquired by xStep, including when + // finalization runs from sqlite3_finalize during native teardown. + let object = unsafe { core::mem::replace(&mut *instance, core::ptr::null()) }; + if let Some(object) = NonNull::new(object.cast_mut()) { + let instance = unsafe { PyObjectRef::from_raw(object) }; + Self::with_context(context, move |_, vm| { + Self::callback_result_from_method(context, &instance, "finalize", vm); + }); + } } unsafe extern "C" fn collation_callback( @@ -551,38 +580,45 @@ mod _sqlite3 { b_len: c_int, b_ptr: *const c_void, ) -> c_int { - let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; - - let f = || -> PyResult { - let text1 = ptr_to_string(a_ptr.cast(), a_len, null_mut(), vm)?; - let text1 = vm.ctx.new_str(text1); - let text2 = ptr_to_string(b_ptr.cast(), b_len, null_mut(), vm)?; - let text2 = vm.ctx.new_str(text2); + let data = unsafe { &*data.cast::() }; + data.with_vm(|callable, vm| { + let f = || -> PyResult { + let text1 = ptr_to_string(a_ptr.cast(), a_len, null_mut(), vm)?; + let text1 = vm.ctx.new_str(text1); + let text2 = ptr_to_string(b_ptr.cast(), b_len, null_mut(), vm)?; + let text2 = vm.ctx.new_str(text2); + + let val = callable.call((text1, text2), vm)?; + let Some(val) = val.number().index(vm) else { + return Ok(0); + }; - let val = callable.call((text1, text2), vm)?; - let Some(val) = val.number().index(vm) else { - return Ok(0); - }; + let val = match val?.as_bigint().sign() { + Sign::Plus => 1, + Sign::Minus => -1, + Sign::NoSign => 0, + }; - let val = match val?.as_bigint().sign() { - Sign::Plus => 1, - Sign::Minus => -1, - Sign::NoSign => 0, + Ok(val) }; - Ok(val) - }; - - f().unwrap_or(0) + f().unwrap_or(0) + }) + .unwrap_or(0) } unsafe extern "C" fn value_callback(context: *mut sqlite3_context) { let context = SqliteContext::from(context); - let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; - let instance = context.aggregate_context::<*const PyObject>(); - let instance = unsafe { &**instance }; - - Self::callback_result_from_method(context, instance, "value", vm); + Self::with_context(context, |_, vm| { + let instance = context.aggregate_context::<*const PyObject>(); + if instance.is_null() { + unsafe { sqlite3_result_error_nomem(context.ctx) }; + return; + } + if let Some(instance) = unsafe { (*instance).as_ref() } { + Self::callback_result_from_method(context, instance, "value", vm); + } + }); } unsafe extern "C" fn inverse_callback( @@ -591,12 +627,17 @@ mod _sqlite3 { argv: *mut *mut sqlite3_value, ) { let context = SqliteContext::from(context); - let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; - let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; - let instance = context.aggregate_context::<*const PyObject>(); - let instance = unsafe { &**instance }; - - Self::call_method_with_args(context, instance, "inverse", args, vm); + Self::with_context(context, |_, vm| { + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; + let instance = context.aggregate_context::<*const PyObject>(); + if instance.is_null() { + unsafe { sqlite3_result_error_nomem(context.ctx) }; + return; + } + if let Some(instance) = unsafe { (*instance).as_ref() } { + Self::call_method_with_args(context, instance, "inverse", args, vm); + } + }); } unsafe extern "C" fn authorizer_callback( @@ -607,21 +648,24 @@ mod _sqlite3 { db_name: *const libc::c_char, access: *const libc::c_char, ) -> c_int { - let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; - let f = || -> PyResult { - let arg1 = ptr_to_str_or_none(arg1, vm)?; - let arg2 = ptr_to_str_or_none(arg2, vm)?; - let db_name = ptr_to_str_or_none(db_name, vm)?; - let access = ptr_to_str_or_none(access, vm)?; - - let val = callable.call((action, arg1, arg2, db_name, access), vm)?; - let Some(val) = val.downcast_ref::() else { - return Ok(SQLITE_DENY); + let data = unsafe { &*data.cast::() }; + data.with_vm(|callable, vm| { + let f = || -> PyResult { + let arg1 = ptr_to_str_or_none(arg1, vm)?; + let arg2 = ptr_to_str_or_none(arg2, vm)?; + let db_name = ptr_to_str_or_none(db_name, vm)?; + let access = ptr_to_str_or_none(access, vm)?; + + let val = callable.call((action, arg1, arg2, db_name, access), vm)?; + let Some(val) = val.downcast_ref::() else { + return Ok(SQLITE_DENY); + }; + val.try_to_primitive::(vm) }; - val.try_to_primitive::(vm) - }; - f().unwrap_or(SQLITE_DENY) + f().unwrap_or(SQLITE_DENY) + }) + .unwrap_or(SQLITE_DENY) } unsafe extern "C" fn trace_callback( @@ -630,25 +674,31 @@ mod _sqlite3 { stmt: *mut c_void, sql: *mut c_void, ) -> c_int { - let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; - let expanded = unsafe { sqlite3_expanded_sql(stmt.cast()) }; - let f = || -> PyResult<()> { - let stmt = ptr_to_str(expanded, vm).or_else(|_| ptr_to_str(sql.cast(), vm))?; - callable.call((stmt,), vm)?; - Ok(()) - }; - let _ = f(); + let data = unsafe { &*data.cast::() }; + data.with_vm(|callable, vm| { + let expanded = unsafe { sqlite3_expanded_sql(stmt.cast()) }; + scopeguard::defer! { unsafe { libsqlite3_sys::sqlite3_free(expanded.cast()) }; } + let f = || -> PyResult<()> { + let stmt = ptr_to_str(expanded, vm).or_else(|_| ptr_to_str(sql.cast(), vm))?; + callable.call((stmt,), vm)?; + Ok(()) + }; + let _ = f(); + }); 0 } unsafe extern "C" fn progress_callback(data: *mut c_void) -> c_int { - let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; - if let Ok(val) = callable.call((), vm) - && let Ok(val) = val.is_true(vm) - { - return val as c_int; - } - -1 + let data = unsafe { &*data.cast::() }; + data.with_vm(|callable, vm| { + if let Ok(val) = callable.call((), vm) + && let Ok(val) = val.is_true(vm) + { + return val as c_int; + } + -1 + }) + .unwrap_or(-1) } fn callback_result_from_method( @@ -730,14 +780,6 @@ mod _sqlite3 { } } - impl Drop for CallbackData { - fn drop(&mut self) { - unsafe { - let _ = PyObjectRef::from_raw(self.obj); - }; - } - } - #[pyfunction] fn connect(args: ConnectArgs, vm: &VirtualMachine) -> PyResult { let factory = args.factory.clone(); @@ -753,22 +795,28 @@ mod _sqlite3 { } #[pyfunction] - fn enable_callback_tracebacks(flag: bool) { - enable_traceback().store(flag, Ordering::Relaxed); + fn enable_callback_tracebacks(flag: bool, vm: &VirtualMachine) { + module_state(vm) + .enable_traceback + .store(flag, Ordering::Relaxed); } #[pyfunction] fn register_adapter(typ: PyTypeRef, adapter: ArgCallable, vm: &VirtualMachine) -> PyResult<()> { - if typ.is(PyInt::class(&vm.ctx)) - || typ.is(PyFloat::class(&vm.ctx)) - || typ.is(PyStr::class(&vm.ctx)) - || typ.is(PyByteArray::class(&vm.ctx)) + if typ.is(&PyInt::class(&vm.ctx)) + || typ.is(&PyFloat::class(&vm.ctx)) + || typ.is(&PyStr::class(&vm.ctx)) + || typ.is(&PyByteArray::class(&vm.ctx)) { - let _ = BASE_TYPE_ADAPTED.set(()); + module_state(vm) + .base_type_adapted + .store(true, Ordering::Relaxed); } - let protocol = PrepareProtocol::class(&vm.ctx).to_owned(); + let protocol = PrepareProtocol::class(&vm.ctx); let key = vm.ctx.new_tuple(vec![typ.into(), protocol.into()]); - adapters().set_item(key.as_object(), adapter.into(), vm) + module_state(vm) + .adapters + .set_item(key.as_object(), adapter.into(), vm) } #[pyfunction] @@ -778,7 +826,9 @@ mod _sqlite3 { vm: &VirtualMachine, ) -> PyResult<()> { let name = typename.expect_str().to_uppercase(); - converters().set_item(&name, converter.into(), vm) + module_state(vm) + .converters + .set_item(&name, converter.into(), vm) } fn _adapt(obj: &PyObject, proto: PyTypeRef, alt: F, vm: &VirtualMachine) -> PyResult @@ -790,7 +840,10 @@ mod _sqlite3 { .ctx .new_tuple(vec![obj.class().to_owned().into(), proto.clone()]); - if let Some(adapter) = adapters().get_item_opt(key.as_object(), vm)? { + if let Some(adapter) = module_state(vm) + .adapters + .get_item_opt(key.as_object(), vm)? + { return adapter.call((obj,), vm); } if let Ok(adapter) = proto.get_attr("__adapt__", vm) { @@ -842,7 +895,7 @@ mod _sqlite3 { let proto = proto .flatten() - .unwrap_or_else(|| PrepareProtocol::class(&vm.ctx).to_owned()); + .unwrap_or_else(|| PrepareProtocol::class(&vm.ctx)); _adapt( &obj, @@ -859,7 +912,7 @@ mod _sqlite3 { } fn need_adapt(obj: &PyObject, vm: &VirtualMachine) -> bool { - if BASE_TYPE_ADAPTED.get().is_some() { + if module_state(vm).base_type_adapted.load(Ordering::Relaxed) { true } else { let cls = obj.class(); @@ -870,32 +923,24 @@ mod _sqlite3 { } } - static_cell! { - static CONVERTERS: PyDictRef; - static ADAPTERS: PyDictRef; - static BASE_TYPE_ADAPTED: (); - static USER_FUNCTION_EXCEPTION: PyAtomicRef>; - static ENABLE_TRACEBACK: PyAtomic; - } - - fn converters() -> &'static Py { - CONVERTERS.get().expect("converters not initialize") + struct ModuleState { + converters: PyDictRef, + adapters: PyDictRef, + base_type_adapted: PyAtomic, + user_function_exception: PyAtomicRef>, + enable_traceback: PyAtomic, } - fn adapters() -> &'static Py { - ADAPTERS.get().expect("adapters not initialize") - } - - fn user_function_exception() -> &'static PyAtomicRef> { - USER_FUNCTION_EXCEPTION - .get() - .expect("user function exception not initialize") - } - - fn enable_traceback() -> &'static PyAtomic { - ENABLE_TRACEBACK - .get() - .expect("enable traceback not initialize") + fn module_state(vm: &VirtualMachine) -> PyRc { + vm.__cached_native::(|| { + PyRc::new(ModuleState { + converters: vm.ctx.new_dict(), + adapters: vm.ctx.new_dict(), + base_type_adapted: Radium::new(false), + user_function_exception: PyAtomicRef::from(None), + enable_traceback: Radium::new(false), + }) + }) } pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { @@ -909,13 +954,8 @@ mod _sqlite3 { setup_module_exceptions(module.as_object(), vm); - let _ = CONVERTERS.set(vm.ctx.new_dict()); - let _ = ADAPTERS.set(vm.ctx.new_dict()); - let _ = USER_FUNCTION_EXCEPTION.set(PyAtomicRef::from(None)); - let _ = ENABLE_TRACEBACK.set(Radium::new(false)); - - module.set_attr("converters", converters().to_owned(), vm)?; - module.set_attr("adapters", adapters().to_owned(), vm)?; + module.set_attr("converters", module_state(vm).converters.to_owned(), vm)?; + module.set_attr("adapters", module_state(vm).adapters.to_owned(), vm)?; Ok(()) } @@ -945,11 +985,11 @@ mod _sqlite3 { type Args = ConnectArgs; fn py_new(cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let text_factory = PyStr::class(&vm.ctx).to_owned().into_object(); + let text_factory = PyStr::class(&vm.ctx).into_object(); // For non-subclassed Connection, initialize in __new__ // For subclassed Connection, leave db as None and require __init__ to be called - let is_base_class = cls.is(Self::class(&vm.ctx)); + let is_base_class = cls.is(&Self::class(&vm.ctx)); let db = if is_base_class { // Initialize immediately for base class @@ -1049,7 +1089,7 @@ mod _sqlite3 { } fn reset_factories(&self, vm: &VirtualMachine) { - let default_text_factory = PyStr::class(&vm.ctx).to_owned().into_object(); + let default_text_factory = PyStr::class(&vm.ctx).into_object(); let _ = unsafe { self.row_factory.swap(None) }; let _ = unsafe { self.text_factory.swap(default_text_factory) }; } @@ -1102,12 +1142,12 @@ mod _sqlite3 { let factory = match args.factory { OptionalArg::Present(f) => f, - OptionalArg::Missing => Cursor::class(&vm.ctx).to_owned().into(), + OptionalArg::Missing => Cursor::class(&vm.ctx).into(), }; let cursor = factory.call((zelf.clone(),), vm)?; - if !cursor.class().fast_issubclass(Cursor::class(&vm.ctx)) { + if !cursor.class().fast_issubclass(&Cursor::class(&vm.ctx)) { return Err(vm.new_type_error(format!( "factory must return a cursor, not {}", cursor.class() @@ -1765,44 +1805,44 @@ mod _sqlite3 { } #[pygetset(name = "Warning")] - fn exc_warning(_zelf: &Py) -> PyTypeRef { - warning_type().to_owned() + fn exc_warning(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + warning_type(vm) } #[pygetset(name = "Error")] - fn exc_error(_zelf: &Py) -> PyTypeRef { - error_type().to_owned() + fn exc_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + error_type(vm) } #[pygetset(name = "InterfaceError")] - fn exc_interface_error(_zelf: &Py) -> PyTypeRef { - interface_error_type().to_owned() + fn exc_interface_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + interface_error_type(vm) } #[pygetset(name = "DatabaseError")] - fn exc_database_error(_zelf: &Py) -> PyTypeRef { - database_error_type().to_owned() + fn exc_database_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + database_error_type(vm) } #[pygetset(name = "DataError")] - fn exc_data_error(_zelf: &Py) -> PyTypeRef { - data_error_type().to_owned() + fn exc_data_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + data_error_type(vm) } #[pygetset(name = "OperationalError")] - fn exc_operational_error(_zelf: &Py) -> PyTypeRef { - operational_error_type().to_owned() + fn exc_operational_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + operational_error_type(vm) } #[pygetset(name = "IntegrityError")] - fn exc_integrity_error(_zelf: &Py) -> PyTypeRef { - integrity_error_type().to_owned() + fn exc_integrity_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + integrity_error_type(vm) } #[pygetset(name = "InternalError")] - fn exc_internal_error(_zelf: &Py) -> PyTypeRef { - internal_error_type().to_owned() + fn exc_internal_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + internal_error_type(vm) } #[pygetset(name = "ProgrammingError")] - fn exc_programming_error(_zelf: &Py) -> PyTypeRef { - programming_error_type().to_owned() + fn exc_programming_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + programming_error_type(vm) } #[pygetset(name = "NotSupportedError")] - fn exc_not_supported_error(_zelf: &Py) -> PyTypeRef { - not_supported_error_type().to_owned() + fn exc_not_supported_error(_zelf: &Py, vm: &VirtualMachine) -> PyTypeRef { + not_supported_error_type(vm) } } @@ -1953,7 +1993,7 @@ mod _sqlite3 { let ret = st.step(); if ret != SQLITE_DONE && ret != SQLITE_ROW { - if let Some(exc) = unsafe { user_function_exception().swap(None) } { + if let Some(exc) = unsafe { module_state(vm).user_function_exception.swap(None) } { return Err(exc); } return Err(db.error_extended(vm)); @@ -2044,7 +2084,7 @@ mod _sqlite3 { st.reset(); } - // if let Some(exc) = unsafe { user_function_exception().swap(None) } { + // if let Some(exc) = unsafe { module_state(vm).user_function_exception.swap(None) } { // return Err(exc); // } } @@ -2234,7 +2274,9 @@ mod _sqlite3 { .take_while(|&x| x != ']') .flat_map(|x| x.to_uppercase()) .collect::>(); - if let Some(converter) = converters().get_item_opt(&*col_name, vm)? { + if let Some(converter) = + module_state(vm).converters.get_item_opt(&*col_name, vm)? + { cast_map.push(Some(converter)); continue; } @@ -2244,7 +2286,9 @@ mod _sqlite3 { let decltype = ptr_to_str(decltype, vm)?; if let Some(decltype) = decltype.split_terminator(&[' ', '(']).next() { let decltype = decltype.to_uppercase(); - if let Some(converter) = converters().get_item_opt(&decltype, vm)? { + if let Some(converter) = + module_state(vm).converters.get_item_opt(&decltype, vm)? + { cast_map.push(Some(converter)); continue; } @@ -2344,7 +2388,7 @@ mod _sqlite3 { let text_factory = zelf.connection.text_factory.to_owned(); - if text_factory.is(PyStr::class(&vm.ctx)) { + if text_factory.is(&PyStr::class(&vm.ctx)) { let text = String::from_utf8(text).map_err(|err| { let col_name = st.column_name(i); let col_name_str = ptr_to_str(col_name, vm).unwrap_or("?"); @@ -2356,9 +2400,9 @@ mod _sqlite3 { new_operational_error(vm, msg) })?; vm.ctx.new_str(text).into() - } else if text_factory.is(PyBytes::class(&vm.ctx)) { + } else if text_factory.is(&PyBytes::class(&vm.ctx)) { vm.ctx.new_bytes(text).into() - } else if text_factory.is(PyByteArray::class(&vm.ctx)) { + } else if text_factory.is(&PyByteArray::class(&vm.ctx)) { PyByteArray::from(text).into_ref(&vm.ctx).into() } else { let bytes = vm.ctx.new_bytes(text); @@ -3091,7 +3135,7 @@ mod _sqlite3 { let errmsg = unsafe { CStr::from_ptr(errmsg) }; let errmsg = errmsg.to_str().unwrap().to_owned(); - raise_exception(typ.to_owned(), extended_errcode, errmsg, vm) + raise_exception(typ, extended_errcode, errmsg, vm) } fn open(path: *const libc::c_char, uri: bool, vm: &VirtualMachine) -> PyResult { @@ -3288,7 +3332,7 @@ mod _sqlite3 { fn step_row_else_done(self, vm: &VirtualMachine) -> PyResult { let ret = self.step(); - if let Some(exc) = unsafe { user_function_exception().swap(None) } { + if let Some(exc) = unsafe { module_state(vm).user_function_exception.swap(None) } { Err(exc) } else if ret == SQLITE_ROW { Ok(true) @@ -3317,7 +3361,7 @@ mod _sqlite3 { let obj = if need_adapt(parameter, vm) { adapted = _adapt( parameter, - PrepareProtocol::class(&vm.ctx).to_owned(), + PrepareProtocol::class(&vm.ctx), |x| Ok(x.to_owned()), vm, )?; @@ -3599,7 +3643,7 @@ mod _sqlite3 { } else { unsafe { sqlite3_result_error(self.ctx, msg.as_ptr(), -1) } } - if enable_traceback().load(Ordering::Relaxed) { + if module_state(vm).enable_traceback.load(Ordering::Relaxed) { vm.print_exception(&exc); } } @@ -3772,18 +3816,18 @@ mod _sqlite3 { Ok((ptr, len)) } - fn exception_type_from_errcode(errcode: c_int, vm: &VirtualMachine) -> &'static Py { + fn exception_type_from_errcode(errcode: c_int, vm: &VirtualMachine) -> PyTypeRef { match errcode { - SQLITE_INTERNAL | SQLITE_NOTFOUND => internal_error_type(), - SQLITE_NOMEM => vm.ctx.exceptions.memory_error, + SQLITE_INTERNAL | SQLITE_NOTFOUND => internal_error_type(vm), + SQLITE_NOMEM => vm.ctx.exceptions.memory_error.to_owned(), SQLITE_ERROR | SQLITE_PERM | SQLITE_ABORT | SQLITE_BUSY | SQLITE_LOCKED | SQLITE_READONLY | SQLITE_INTERRUPT | SQLITE_IOERR | SQLITE_FULL | SQLITE_CANTOPEN - | SQLITE_PROTOCOL | SQLITE_EMPTY | SQLITE_SCHEMA => operational_error_type(), - SQLITE_CORRUPT => database_error_type(), - SQLITE_TOOBIG => data_error_type(), - SQLITE_CONSTRAINT | SQLITE_MISMATCH => integrity_error_type(), - SQLITE_MISUSE | SQLITE_RANGE => interface_error_type(), - _ => database_error_type(), + | SQLITE_PROTOCOL | SQLITE_EMPTY | SQLITE_SCHEMA => operational_error_type(vm), + SQLITE_CORRUPT => database_error_type(vm), + SQLITE_TOOBIG => data_error_type(vm), + SQLITE_CONSTRAINT | SQLITE_MISMATCH => integrity_error_type(vm), + SQLITE_MISUSE | SQLITE_RANGE => interface_error_type(vm), + _ => database_error_type(vm), } } diff --git a/crates/stdlib/src/_zoneinfo.rs b/crates/stdlib/src/_zoneinfo.rs index 9ab94cbefcd..cf997bb6518 100644 --- a/crates/stdlib/src/_zoneinfo.rs +++ b/crates/stdlib/src/_zoneinfo.rs @@ -218,7 +218,7 @@ mod _zoneinfo { } fn is_base(cls: &Py) -> bool { - cls.is(ZoneInfo::static_type()) + cls.is(unsafe { ZoneInfo::static_type() }) } fn strong_entries(vm: &VirtualMachine) -> PyResult> { diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index cb6e40d0d54..133e0d0efe6 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -7,10 +7,12 @@ pub mod array { use crate::{ common::{ atomic::{self, AtomicUsize}, + borrow::{BorrowedValue, BorrowedValueMut}, lock::{ - PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutex, PyRwLock, - PyRwLockReadGuard, PyRwLockWriteGuard, + PyDetachingRwLock, PyDetachingRwLockReadGuard, PyDetachingRwLockWriteGuard, + PyMappedDetachingRwLockReadGuard, PyMappedDetachingRwLockWriteGuard, PyMutex, }, + rc::PyRc, str::wchar_t, }, vm::{ @@ -745,17 +747,52 @@ pub mod array { #[pyclass(name = "array", unhashable = true)] #[derive(Debug, PyPayload)] pub struct PyArray { - array: PyRwLock, + storage: PyRc, + } + + #[doc(hidden)] + #[derive(Debug)] + pub struct ArrayStorage { + array: PyDetachingRwLock, exports: AtomicUsize, } + impl core::ops::Deref for PyArray { + type Target = ArrayStorage; + fn deref(&self) -> &Self::Target { + &self.storage + } + } + + #[derive(Debug)] + struct SharedArray(PyRc); + + impl Drop for SharedArray { + fn drop(&mut self) { + self.0.exports.fetch_sub(1, atomic::Ordering::Release); + } + } + + // SAFETY: the export prohibits resizing and owns the Rust-only allocation. + // All accesses, including the original array's, acquire the same lock. + unsafe impl crate::vm::protocol::SharedBufferStorage for SharedArray { + fn read(&self) -> BorrowedValue<'_, [u8]> { + PyDetachingRwLockReadGuard::map(self.0.array.read(), |x| x.get_bytes()).into() + } + fn write(&self) -> BorrowedValueMut<'_, [u8]> { + PyDetachingRwLockWriteGuard::map(self.0.array.write(), |x| x.get_bytes_mut()).into() + } + } + pub type PyArrayRef = PyRef; impl From for PyArray { fn from(array: ArrayContentType) -> Self { Self { - array: PyRwLock::new(array), - exports: AtomicUsize::new(0), + storage: PyRc::new(ArrayStorage { + array: PyDetachingRwLock::new(array), + exports: AtomicUsize::new(0), + }), } } } @@ -780,7 +817,7 @@ pub mod array { vm.new_type_error("array() argument 1 must be a unicode character, not str") })?; - if cls.is(Self::class(&vm.ctx)) && !kwargs.is_empty() { + if cls.is(&Self::class(&vm.ctx)) && !kwargs.is_empty() { return Err(vm.new_type_error("array.array() takes no keyword arguments")); } @@ -851,7 +888,7 @@ pub mod array { } #[pyclass( - flags(BASETYPE, HAS_WEAKREF), + flags(BASETYPE, HAS_WEAKREF, SEQUENCE), with( Comparable, AsBuffer, @@ -863,11 +900,11 @@ pub mod array { ) )] impl PyArray { - fn read(&self) -> PyRwLockReadGuard<'_, ArrayContentType> { + fn read(&self) -> PyDetachingRwLockReadGuard<'_, ArrayContentType> { self.array.read() } - fn write(&self) -> PyRwLockWriteGuard<'_, ArrayContentType> { + fn write(&self) -> PyDetachingRwLockWriteGuard<'_, ArrayContentType> { self.array.write() } @@ -1166,12 +1203,12 @@ pub mod array { Ok(()) } - pub(crate) fn get_bytes(&self) -> PyMappedRwLockReadGuard<'_, [u8]> { - PyRwLockReadGuard::map(self.read(), |a| a.get_bytes()) + pub(crate) fn get_bytes(&self) -> PyMappedDetachingRwLockReadGuard<'_, [u8]> { + PyDetachingRwLockReadGuard::map(self.read(), |a| a.get_bytes()) } - pub(crate) fn get_bytes_mut(&self) -> PyMappedRwLockWriteGuard<'_, [u8]> { - PyRwLockWriteGuard::map(self.write(), |a| a.get_bytes_mut()) + pub(crate) fn get_bytes_mut(&self) -> PyMappedDetachingRwLockWriteGuard<'_, [u8]> { + PyDetachingRwLockWriteGuard::map(self.write(), |a| a.get_bytes_mut()) } #[pymethod] @@ -1361,6 +1398,7 @@ pub mod array { let bytes = vm.ctx.new_bytes(array.get_bytes().to_vec()); let code = MachineFormatCode::from_typecode(array.typecode()).unwrap(); let code = PyInt::from(u8::from(code)).into_pyobject(vm); + drop(array); let module = vm.import("array", 0)?; let func = module.get_attr("_array_reconstructor", vm)?; Ok(( @@ -1460,8 +1498,7 @@ pub mod array { } impl PyArray { - fn buffer_desc(&self) -> BufferDescriptor { - let array = self.read(); + fn buffer_desc(array: &ArrayContentType) -> BufferDescriptor { BufferDescriptor::format( array.len() * array.itemsize(), false, @@ -1484,15 +1521,17 @@ pub mod array { let zelf = zelf .downcast_ref::() .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?; - let desc = zelf.buffer_desc().projected(flags); + let array = zelf.read(); + let desc = Self::buffer_desc(&array).projected(flags); flags.check_writable(desc.readonly, "Object is not writable.", vm)?; Ok(PyBuffer::new(zelf.to_owned().into(), desc, &BUFFER_METHODS)) } fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + let array = zelf.read(); Ok(PyBuffer::new( zelf.to_owned().into(), - zelf.buffer_desc(), + Self::buffer_desc(&array), &BUFFER_METHODS, )) } @@ -1517,6 +1556,11 @@ pub mod array { } static BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| { + let storage = buffer.obj_as::().storage.clone(); + storage.exports.fetch_add(1, atomic::Ordering::Release); + Some(PyRc::new(SharedArray(storage))) + }), obj_bytes: |buffer| buffer.obj_as::().get_bytes().into(), obj_bytes_mut: |buffer| buffer.obj_as::().get_bytes_mut().into(), release: |buffer| { @@ -1610,12 +1654,16 @@ pub mod array { } impl BufferResizeGuard for PyArray { - type Resizable<'a> = PyRwLockWriteGuard<'a, ArrayContentType>; + type Resizable<'a> = PyDetachingRwLockWriteGuard<'a, ArrayContentType>; fn try_resizable_opt(&self) -> Option> { // An export is a borrow someone else still holds, so it is // answered before the lock rather than by waiting on it. - (self.exports.load(atomic::Ordering::SeqCst) == 0).then(|| self.write()) + if self.exports.load(atomic::Ordering::Acquire) != 0 { + return None; + } + let array = self.write(); + (self.exports.load(atomic::Ordering::Acquire) == 0).then_some(array) } fn try_resizable(&self, vm: &VirtualMachine) -> PyResult> { @@ -1814,7 +1862,7 @@ pub mod array { } fn check_array_type(typ: PyTypeRef, vm: &VirtualMachine) -> PyResult { - if !typ.fast_issubclass(PyArray::class(&vm.ctx)) { + if !typ.fast_issubclass(&PyArray::class(&vm.ctx)) { return Err( vm.new_type_error(format!("{} is not a subtype of array.array", typ.name())) ); diff --git a/crates/stdlib/src/cjkcodecs/multibytecodec.rs b/crates/stdlib/src/cjkcodecs/multibytecodec.rs index ca1f85b0103..e3230e828c2 100644 --- a/crates/stdlib/src/cjkcodecs/multibytecodec.rs +++ b/crates/stdlib/src/cjkcodecs/multibytecodec.rs @@ -607,7 +607,7 @@ mod _multibytecodec { return Err(vm.new_lookup_error("no such codec is supported.")); }; // Keep the type alive whichever module was imported first. - let class = ::make_static_type(); + let class = unsafe { ::make_static_type() }; let payload = MultibyteCodec { codec: CodecRef { codec, encoding }, }; diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 3c8cf9fa169..955cd62ad63 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -630,7 +630,9 @@ mod _contextvars { fn old_value(zelf: &Py, _vm: &VirtualMachine) -> PyObjectRef { match &zelf.old_value { Some(value) => value.clone(), - None => ContextTokenMissing::static_type().to_owned().into(), + None => unsafe { ContextTokenMissing::static_type() } + .to_owned() + .into(), } } @@ -714,8 +716,15 @@ mod _contextvars { ) -> PyResult<()> { __module_exec(vm, module); - let token_type = module.get_attr("Token", vm)?; - token_type.set_attr("MISSING", ContextTokenMissing::static_type().to_owned(), vm)?; + let token_type = module + .get_attr("Token", vm)? + .downcast::() + .expect("Token is a native type"); + token_type.set_str_attr( + "MISSING", + unsafe { ContextTokenMissing::static_type() }.to_owned(), + &vm.ctx, + ); Ok(()) } diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index a0f75300eda..d5d6d067a5b 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -12,9 +12,9 @@ mod _csv { types::{Callable, Constructor, IterNext, Iterable, SelfIter}, }; use alloc::fmt; + use core::sync::atomic::{AtomicIsize, Ordering}; use itertools::Itertools; - use parking_lot::Mutex; - use rustpython_common::{lock::LazyLock, wtf8::Wtf8Buf}; + use rustpython_common::{rc::PyRc, wtf8::Wtf8Buf}; use rustpython_vm::match_class; use std::collections::HashMap; @@ -48,11 +48,19 @@ mod _csv { ) } - static GLOBAL_HASHMAP: LazyLock>> = LazyLock::new(|| { - let m = HashMap::new(); - Mutex::new(m) - }); - static GLOBAL_FIELD_LIMIT: LazyLock> = LazyLock::new(|| Mutex::new(131072)); + struct CsvState { + dialects: PyMutex>, + field_limit: AtomicIsize, + } + + fn state(vm: &VirtualMachine) -> PyRc { + vm.__cached_native::(|| { + PyRc::new(CsvState { + dialects: PyMutex::default(), + field_limit: AtomicIsize::new(131072), + }) + }) + } fn new_csv_error(vm: &VirtualMachine, msg: impl Into) -> PyBaseExceptionRef { vm.new_exception_msg(super::_csv::error(vm), msg.into()) @@ -333,7 +341,8 @@ mod _csv { let dialect = opts.result(vm)?; validate_dialect(vm, &dialect)?; - GLOBAL_HASHMAP + state(vm) + .dialects .lock() .insert(name.as_str().to_owned(), dialect); @@ -350,13 +359,8 @@ mod _csv { })?; let name: PyUtf8StrRef = name.try_into_utf8(vm)?; - let g = GLOBAL_HASHMAP.lock(); - - if let Some(dialect) = g.get(name.as_str()) { - return Ok(dialect.clone()); - } - - Err(new_csv_error(vm, "unknown dialect")) + let dialect = state(vm).dialects.lock().get(name.as_str()).cloned(); + dialect.ok_or_else(|| new_csv_error(vm, "unknown dialect")) } #[pyfunction] @@ -369,21 +373,17 @@ mod _csv { })?; let name: PyUtf8StrRef = name.try_into_utf8(vm)?; - let mut g = GLOBAL_HASHMAP.lock(); - - if let Some(_removed) = g.remove(name.as_str()) { - return Ok(()); - } - - Err(new_csv_error(vm, "unknown dialect")) + let removed = state(vm).dialects.lock().remove(name.as_str()); + removed + .map(|_| ()) + .ok_or_else(|| new_csv_error(vm, "unknown dialect")) } #[pyfunction] fn list_dialects(vm: &VirtualMachine) -> rustpython_vm::builtins::PyListRef { - let g = GLOBAL_HASHMAP.lock(); - let t = g - .keys() - .cloned() + let names = state(vm).dialects.lock().keys().cloned().collect_vec(); + let t = names + .into_iter() .map(|x| vm.ctx.new_str(x).into()) .collect_vec(); // .iter().map(|x| vm.ctx.new_str(x.clone()).into_pyobject(vm)).collect_vec(); @@ -398,12 +398,15 @@ mod _csv { #[pyfunction] fn field_size_limit(args: FieldSizeLimitArgs, vm: &VirtualMachine) -> PyResult { - let old_size = GLOBAL_FIELD_LIMIT.lock().to_owned(); + let state = state(vm); + let old_size = state.field_limit.load(Ordering::Relaxed); if let OptionalArg::Present(limit) = args.new_limit { let Ok(new_size) = limit.try_int(vm) else { return Err(vm.new_type_error("limit must be an integer")); }; - *GLOBAL_FIELD_LIMIT.lock() = new_size.try_to_primitive::(vm)?; + state + .field_limit + .store(new_size.try_to_primitive::(vm)?, Ordering::Relaxed); } Ok(old_size) } @@ -782,9 +785,9 @@ mod _csv { fn result(&self, vm: &VirtualMachine) -> PyResult { let dialect = match &self.dialect { DialectItem::Str(name) => { - let g = GLOBAL_HASHMAP.lock(); - if let Some(dialect) = g.get(name) { - Ok(self.update_py_dialect(dialect.clone())) + let dialect = state(vm).dialects.lock().get(name).cloned(); + if let Some(dialect) = dialect { + Ok(self.update_py_dialect(dialect)) } else { Err(new_csv_error(vm, format!("{name} is not registered."))) } @@ -1116,7 +1119,8 @@ mod _csv { impl IterNext for Reader { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let mut parser = CsvParser::new(*GLOBAL_FIELD_LIMIT.lock()); + let state = state(vm); + let mut parser = CsvParser::new(state.field_limit.load(Ordering::Relaxed)); loop { match next_input_item(zelf, vm)? { @@ -1135,7 +1139,7 @@ mod _csv { })?; zelf.state.lock().line_num += 1; - parser.field_limit = *GLOBAL_FIELD_LIMIT.lock(); + parser.field_limit = state.field_limit.load(Ordering::Relaxed); for &byte in string.as_bytes() { parser.process_parser_input( ParserInput::Byte(byte), diff --git a/crates/stdlib/src/elementtree.rs b/crates/stdlib/src/elementtree.rs index 57c793d5494..54ea074073e 100644 --- a/crates/stdlib/src/elementtree.rs +++ b/crates/stdlib/src/elementtree.rs @@ -208,7 +208,12 @@ pub(crate) mod _elementtree { let Some(mut inner) = self.inner.try_write() else { return; }; - let none = || -> PyObjectRef { crate::vm::Context::genesis().none.to_owned().into() }; + let none = || -> PyObjectRef { + unsafe { crate::vm::Context::genesis_unchecked() } + .none + .to_owned() + .into() + }; out.append(&mut inner.children); if let Some(attrib) = inner.attrib.take() { out.push(attrib.into()); @@ -1049,7 +1054,7 @@ pub(crate) mod _elementtree { return Ok(obj.to_owned()); } if let Some(elem) = obj.downcast_ref::() - && elem.class().is(PyElement::class(&vm.ctx)) + && elem.class().is(&PyElement::class(&vm.ctx)) { return deepcopy_element(elem, memo, vm); } @@ -1427,7 +1432,7 @@ pub(crate) mod _elementtree { child: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - if element.class().is(PyElement::class(&vm.ctx)) { + if element.class().is(&PyElement::class(&vm.ctx)) { check_element(&child, vm)?; element.downcast_ref::().unwrap().push(child); Ok(()) @@ -1444,7 +1449,7 @@ pub(crate) mod _elementtree { is_tail: bool, vm: &VirtualMachine, ) -> PyResult<()> { - if element.class().is(PyElement::class(&vm.ctx)) { + if element.class().is(&PyElement::class(&vm.ctx)) { let elem = element.downcast_ref::().unwrap(); enum Fast { Done, @@ -1994,7 +1999,7 @@ pub(crate) mod _elementtree { handle_doctype: optional_handler(&target, "doctype", vm)?, native_target: target .class() - .is(PyTreeBuilder::class(&vm.ctx)) + .is(&PyTreeBuilder::class(&vm.ctx)) .then(|| target.clone().downcast::().ok()) .flatten(), }; diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs index f28f27b1055..d2dc4ca0981 100644 --- a/crates/stdlib/src/hashlib.rs +++ b/crates/stdlib/src/hashlib.rs @@ -244,14 +244,14 @@ pub(crate) mod _hashlib { return Ok(algo.to_owned()); } Err(vm.new_exception_msg( - UnsupportedDigestmodError::static_type().to_owned(), + unsafe { UnsupportedDigestmodError::static_type() }.to_owned(), "unsupported digestmod".into(), )) } fn unsupported_hash(name: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { vm.new_exception_msg( - UnsupportedDigestmodError::static_type().to_owned(), + unsafe { UnsupportedDigestmodError::static_type() }.to_owned(), format!("unsupported hash type {name}").into(), ) } diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index 8b9da96cc8e..75ed9e3abfc 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -12,7 +12,6 @@ mod _json { protocol::PyIterReturn, types::{Callable, Constructor, PyComparisonOp}, }; - use core::cell::RefCell; use core::str::FromStr; use malachite_bigint::BigInt; use rustpython_common::{ @@ -44,7 +43,7 @@ mod _json { } #[pyattr(name = "make_scanner")] - #[pyclass(name = "Scanner", traverse)] + #[pyclass(name = "Scanner", traverse, interpreter_local)] #[derive(Debug, PyPayload)] struct JsonScanner { #[pytraverse(skip)] @@ -725,32 +724,24 @@ mod _json { basic_fn: PyObjectRef, } - thread_local! { - static BUILTIN_ESCAPERS: RefCell> = const { RefCell::new(None) }; - } - fn builtin_escaper_refs(vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { + use crate::vm::common::rc::PyRc; + struct EscaperCache; let module = vm.import("_json", 0)?; - - let hit = BUILTIN_ESCAPERS.with_borrow(|c| { - c.as_ref() - .filter(|c| c.module.is(&module)) - .map(|c| (c.ascii_fn.clone(), c.basic_fn.clone())) - }); - if let Some(r) = hit { - return Ok(r); + if let Some(cache) = vm.__get_native::>() + && cache.module.is(&module) + { + return Ok((cache.ascii_fn.clone(), cache.basic_fn.clone())); } let ascii_fn = module.get_attr("encode_basestring_ascii", vm)?; let basic_fn = module.get_attr("encode_basestring", vm)?; let result = (ascii_fn.clone(), basic_fn.clone()); - BUILTIN_ESCAPERS.with_borrow_mut(|c| { - *c = Some(BuiltinEscapers { - module, - ascii_fn, - basic_fn, - }) - }); + vm.__replace_native::(PyRc::new(BuiltinEscapers { + module, + ascii_fn, + basic_fn, + })); Ok(result) } @@ -761,7 +752,7 @@ mod _json { // (matching what `json/encoder.py`'s `encode()` does with the result: // `''.join(chunks)`). #[pyattr(name = "make_encoder")] - #[pyclass(name = "Encoder", traverse)] + #[pyclass(name = "Encoder", traverse, interpreter_local)] #[derive(Debug, PyPayload)] struct JsonEncoder { #[pytraverse(skip)] diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index 2a3ee0bc460..3e3e8b0c1e5 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -6,7 +6,8 @@ pub(crate) use mmap::module_def; mod mmap { use crate::common::{ borrow::{BorrowedValue, BorrowedValueMut}, - lock::{MapImmutable, PyMutex, PyMutexGuard}, + lock::{PyDetachingRwLock, PyDetachingRwLockReadGuard, PyDetachingRwLockWriteGuard}, + rc::PyRc, }; use crate::vm::{ AsObject, FromArgs, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, @@ -184,8 +185,13 @@ mod mmap { #[pyclass(name = "mmap")] #[derive(Debug, PyPayload)] struct PyMmap { + storage: PyRc, + } + + #[derive(Debug)] + struct MmapStorage { closed: AtomicCell, - mmap: PyMutex>, + mmap: PyDetachingRwLock>, #[cfg(unix)] fd: AtomicCell, #[cfg(windows)] @@ -199,7 +205,34 @@ mod mmap { trackfd: bool, } - impl PyMmap { + impl Deref for PyMmap { + type Target = MmapStorage; + fn deref(&self) -> &Self::Target { + &self.storage + } + } + + #[derive(Debug)] + struct SharedMmap(PyRc); + + impl Drop for SharedMmap { + fn drop(&mut self) { + self.0.exports.fetch_sub(1); + } + } + + // SAFETY: exported storage cannot be resized or closed, contains no Python + // references, and serializes all views using the mapping's original mutex. + unsafe impl crate::vm::protocol::SharedBufferStorage for SharedMmap { + fn read(&self) -> BorrowedValue<'_, [u8]> { + self.0.as_bytes() + } + fn write(&self) -> BorrowedValueMut<'_, [u8]> { + self.0.as_bytes_mut() + } + } + + impl MmapStorage { /// Close the underlying file handle/descriptor if open fn close_handle(&self) { #[cfg(unix)] @@ -217,7 +250,7 @@ mod mmap { } } - impl Drop for PyMmap { + impl Drop for MmapStorage { fn drop(&mut self) { self.close_handle(); } @@ -467,19 +500,21 @@ mod mmap { .map_err(|e| e.to_pyexception(vm))?; Ok(Self { - closed: AtomicCell::new(false), - mmap: PyMutex::new(Some(MmapObj::Mapped(mmap))), - fd: AtomicCell::new(if trackfd { - fd.map_or(-1, |owned| owned.into_raw()) - } else { - -1 + storage: PyRc::new(MmapStorage { + closed: AtomicCell::new(false), + mmap: PyDetachingRwLock::new(Some(MmapObj::Mapped(mmap))), + fd: AtomicCell::new(if trackfd { + fd.map_or(-1, |owned| owned.into_raw()) + } else { + -1 + }), + offset, + size: AtomicCell::new(map_size), + pos: AtomicCell::new(0), + exports: AtomicCell::new(0), + access, + trackfd, }), - offset, - size: AtomicCell::new(map_size), - pos: AtomicCell::new(0), - exports: AtomicCell::new(0), - access, - trackfd, }) } @@ -605,14 +640,16 @@ mod mmap { })?; return Ok(Self { - closed: AtomicCell::new(false), - mmap: PyMutex::new(Some(MmapObj::Named(named))), - handle: AtomicCell::new(host_mmap::INVALID_HANDLE as isize), - offset, - size: AtomicCell::new(map_size), - pos: AtomicCell::new(0), - exports: AtomicCell::new(0), - access, + storage: PyRc::new(MmapStorage { + closed: AtomicCell::new(false), + mmap: PyDetachingRwLock::new(Some(MmapObj::Named(named))), + handle: AtomicCell::new(host_mmap::INVALID_HANDLE as isize), + offset, + size: AtomicCell::new(map_size), + pos: AtomicCell::new(0), + exports: AtomicCell::new(0), + access, + }), }); } @@ -627,19 +664,26 @@ mod mmap { }; Ok(Self { - closed: AtomicCell::new(false), - mmap: PyMutex::new(Some(mmap)), - handle: AtomicCell::new(handle), - offset, - size: AtomicCell::new(map_size), - pos: AtomicCell::new(0), - exports: AtomicCell::new(0), - access, + storage: PyRc::new(MmapStorage { + closed: AtomicCell::new(false), + mmap: PyDetachingRwLock::new(Some(mmap)), + handle: AtomicCell::new(handle), + offset, + size: AtomicCell::new(map_size), + pos: AtomicCell::new(0), + exports: AtomicCell::new(0), + access, + }), }) } } static BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| { + let storage = buffer.obj_as::().storage.clone(); + storage.exports.fetch_add(1); + Some(PyRc::new(SharedMmap(storage))) + }), obj_bytes: |buffer| buffer.obj_as::().as_bytes(), obj_bytes_mut: |buffer| buffer.obj_as::().as_bytes_mut(), release: |buffer| { @@ -653,7 +697,12 @@ mod mmap { impl AsBuffer for PyMmap { const RELEASE_BUFFER: bool = true; - fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + fn as_buffer(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let mmap = zelf.mmap.read(); + if mmap.is_none() { + drop(mmap); + return Err(vm.new_value_error("mmap closed or invalid")); + } let readonly = matches!(zelf.access, AccessMode::Read); let buf = PyBuffer::new( zelf.to_owned().into(), @@ -712,13 +761,9 @@ mod mmap { } } - #[pyclass( - with(Constructor, AsMapping, AsSequence, AsBuffer, Representable), - flags(BASETYPE, HAS_WEAKREF) - )] - impl PyMmap { + impl MmapStorage { fn as_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { - PyMutexGuard::map(self.mmap.lock(), |m| { + PyDetachingRwLockWriteGuard::map(self.mmap.write(), |m| { match m.as_mut().expect("mmap closed or invalid") { MmapObj::Mapped(mmap) => mmap.as_mut_slice(), #[cfg(windows)] @@ -729,12 +774,18 @@ mod mmap { } fn as_bytes(&self) -> BorrowedValue<'_, [u8]> { - PyMutexGuard::map_immutable(self.mmap.lock(), |m| { + PyDetachingRwLockReadGuard::map(self.mmap.read(), |m| { m.as_ref().expect("mmap closed or invalid").as_slice() }) .into() } + } + #[pyclass( + with(Constructor, AsMapping, AsSequence, AsBuffer, Representable), + flags(BASETYPE, HAS_WEAKREF) + )] + impl PyMmap { fn __len__(&self) -> usize { self.size.load() } @@ -766,8 +817,11 @@ mod mmap { } } - fn check_valid(&self, vm: &VirtualMachine) -> PyResult>> { - let m = self.mmap.lock(); + fn check_valid( + &self, + vm: &VirtualMachine, + ) -> PyResult>> { + let m = self.mmap.write(); if m.is_none() { return Err(vm.new_value_error("mmap closed or invalid")); @@ -810,7 +864,11 @@ mod mmap { return Err(vm.new_buffer_error("cannot close exported pointers exist.")); } - let mut mmap = zelf.mmap.lock(); + let mut mmap = zelf.mmap.write(); + if zelf.exports.load() > 0 { + drop(mmap); + return Err(vm.new_buffer_error("cannot close exported pointers exist.")); + } zelf.closed.store(true); *mmap = None; @@ -1061,7 +1119,7 @@ mod mmap { let handle = zelf.handle.load(); // Get the lock on mmap - let mut mmap_guard = zelf.mmap.lock(); + let mut mmap_guard = zelf.mmap.write(); // Check if this is a Named mmap - these cannot be resized if let Some(MmapObj::Named(_)) = mmap_guard.as_ref() { @@ -1455,7 +1513,7 @@ mod mmap { impl Representable for PyMmap { #[inline] fn repr_str(zelf: &Py, _vm: &VirtualMachine) -> PyResult { - let mmap = zelf.mmap.lock(); + let mmap = zelf.mmap.write(); if mmap.is_none() { return Ok("".to_owned()); diff --git a/crates/stdlib/src/openssl.rs b/crates/stdlib/src/openssl.rs index 5470f788462..49b6adaec8b 100644 --- a/crates/stdlib/src/openssl.rs +++ b/crates/stdlib/src/openssl.rs @@ -1415,7 +1415,7 @@ mod _ssl { if zelf.protocol == SslVersion::TlsServer { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "Cannot add PSK client callback to a PROTOCOL_TLS_SERVER context" .to_owned(), @@ -1450,7 +1450,7 @@ mod _ssl { if zelf.protocol == SslVersion::TlsClient { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "Cannot add PSK server callback to a PROTOCOL_TLS_CLIENT context" .to_owned(), @@ -1516,11 +1516,7 @@ mod _ssl { "not enough data: cadata does not contain a certificate" }; return Err(vm - .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), - None, - msg.to_owned(), - ) + .new_os_subtype_error(PySSLError::class(&vm.ctx), None, msg.to_owned()) .upcast()); } @@ -2132,8 +2128,7 @@ mod _ssl { }; // Convert to PyRef (heap allocation) to avoid use-after-free - let py_ref = - py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx).to_owned())?; + let py_ref = py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx))?; // Check if SNI callback is configured (minimize lock time) let has_sni_callback = zelf.sni_callback.lock().is_some(); @@ -2201,8 +2196,7 @@ mod _ssl { }; // Convert to PyRef (heap allocation) to avoid use-after-free - let py_ref = - py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx).to_owned())?; + let py_ref = py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx))?; // Check if SNI callback is configured (minimize lock time) let has_sni_callback = zelf.sni_callback.lock().is_some(); @@ -2994,7 +2988,7 @@ mod _ssl { let result = vm.allow_threads(|| stream.do_handshake()).map_err(|e| { let exc = convert_ssl_error(vm, e); // If it's a cert verification error, set verify info - if exc.class().is(PySSLCertVerificationError::class(&vm.ctx)) { + if exc.class().is(&PySSLCertVerificationError::class(&vm.ctx)) { set_verify_error_info(&exc, ssl_ptr, vm); } exc @@ -3057,7 +3051,7 @@ mod _ssl { } let exc = convert_ssl_error(vm, err); // If it's a cert verification error, set verify info - if exc.class().is(PySSLCertVerificationError::class(&vm.ctx)) { + if exc.class().is(&PySSLCertVerificationError::class(&vm.ctx)) { set_verify_error_info(&exc, ssl_ptr, vm); } // Clean up SNI ex_data before returning error @@ -3790,7 +3784,7 @@ mod _ssl { /// Helper function to create SSL error with proper OSError subtype handling fn new_ssl_error(vm: &VirtualMachine, msg: impl ToString) -> PyBaseExceptionRef { - vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), None, msg.to_string()) + vm.new_os_subtype_error(PySSLError::class(&vm.ctx), None, msg.to_string()) .upcast() } @@ -3850,9 +3844,9 @@ mod _ssl { // Use SSLCertVerificationError for certificate verification failures let cls = if is_cert_verify_error { - PySSLCertVerificationError::class(&vm.ctx).to_owned() + PySSLCertVerificationError::class(&vm.ctx) } else { - PySSLError::class(&vm.ctx).to_owned() + PySSLError::class(&vm.ctx) }; // Build message @@ -3893,7 +3887,7 @@ mod _ssl { ) } None => { - let cls = PySSLError::class(&vm.ctx).to_owned(); + let cls = PySSLError::class(&vm.ctx); vm.new_os_subtype_error(cls, None, "unknown SSL error") .upcast() } @@ -3962,12 +3956,12 @@ mod _ssl { return convert_openssl_error(vm, ssl_err.clone()); } ( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), "A failure in the SSL library occurred", ) } _ => ( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), "A failure in the SSL library occurred", ), }; diff --git a/crates/stdlib/src/pickle.rs b/crates/stdlib/src/pickle.rs index 417a5721165..e54f81e23e8 100644 --- a/crates/stdlib/src/pickle.rs +++ b/crates/stdlib/src/pickle.rs @@ -742,7 +742,7 @@ mod _pickle { } fn has_override(zelf: &Py, name: &str, vm: &VirtualMachine) -> PyResult { - if !zelf.class().is(PyUnpickler::class(&vm.ctx)) { + if !zelf.class().is(&PyUnpickler::class(&vm.ctx)) { return Ok(true); } match zelf.as_object().dict() { @@ -3706,7 +3706,7 @@ mod _pickle { } let pers_func = resolve_hook( zelf.as_object(), - PyPickler::class(&vm.ctx), + &PyPickler::class(&vm.ctx), "persistent_id", vm, )?; diff --git a/crates/stdlib/src/pyexpat.rs b/crates/stdlib/src/pyexpat.rs index 5a516dd7821..426cd1a8291 100644 --- a/crates/stdlib/src/pyexpat.rs +++ b/crates/stdlib/src/pyexpat.rs @@ -856,7 +856,7 @@ mod _pyexpat { } #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + fn extend_class_with_fields(ctx: &Context, class: &Py) { let attributes = &class.attributes; create_property!(ctx, attributes, "StartElementHandler", class, start_element); @@ -1379,7 +1379,7 @@ mod _pyexpat { self.teardown_backend(); self.finished.store(true, Ordering::SeqCst); return Err(vm.new_exception_msg( - PyExpatError::class(&vm.ctx).to_owned(), + PyExpatError::class(&vm.ctx), "cannot continue parsing across a fork()".to_owned().into(), )); } diff --git a/crates/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs index 2a3c0dfe341..ac21aeda639 100644 --- a/crates/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -409,6 +409,6 @@ pub(crate) mod _struct { #[pyattr(name = "error")] fn error_type(vm: &VirtualMachine) -> PyTypeRef { - struct_error_type(vm).to_owned() + struct_error_type(vm) } } diff --git a/crates/stdlib/src/select.rs b/crates/stdlib/src/select.rs index 06cabef301f..68e599bc2b1 100644 --- a/crates/stdlib/src/select.rs +++ b/crates/stdlib/src/select.rs @@ -18,11 +18,10 @@ struct Selectable { impl TryFromObject for Selectable { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let fno = obj.try_to_value(vm).or_else(|_| { - let meth = vm.get_method_or_type_error( - obj.clone(), - vm.ctx.interned_str("fileno").unwrap(), - || "select arg must be an int or object with a fileno() method".to_owned(), - )?; + let meth = + vm.get_method_or_type_error(obj.clone(), vm.ctx.intern_str("fileno"), || { + "select arg must be an int or object with a fileno() method".to_owned() + })?; meth.call((), vm)?.try_into_value(vm) })?; Ok(Self { obj, fno }) @@ -48,7 +47,7 @@ mod decl { #[cfg(unix)] { use crate::vm::class::PyClassImpl; - let _ = poll::PyPoll::make_static_type(); + let _ = unsafe { poll::PyPoll::make_static_type() }; } __module_exec(vm, module); @@ -410,7 +409,7 @@ mod decl { #[pyattr(name = "epoll", once)] fn epoll(_vm: &VirtualMachine) -> PyTypeRef { use crate::vm::class::PyClassImpl; - epoll::PyEpoll::make_static_type() + unsafe { epoll::PyEpoll::make_static_type() } } #[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))] @@ -649,7 +648,7 @@ mod decl { #[pyattr(name = "kqueue", once)] fn kqueue_type(_vm: &VirtualMachine) -> PyTypeRef { use crate::vm::class::PyClassImpl; - kqueue::PyKqueue::make_static_type() + unsafe { kqueue::PyKqueue::make_static_type() } } #[cfg(any( @@ -663,7 +662,7 @@ mod decl { #[pyattr(name = "kevent", once)] fn kevent_type(_vm: &VirtualMachine) -> PyTypeRef { use crate::vm::class::PyClassImpl; - kqueue::PyKevent::make_static_type() + unsafe { kqueue::PyKevent::make_static_type() } } #[cfg(any( diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 74750cced43..dcbe683b665 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -838,12 +838,8 @@ mod _ssl { let msg = io_err.to_string(); if msg.contains("Failed to decrypt") || msg.contains("wrong password") { // Wrong password error - vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), - None, - msg, - ) - .upcast() + vm.new_os_subtype_error(PySSLError::class(&vm.ctx), None, msg) + .upcast() } else { // [SSL] PEM lib super::compat::SslError::create_ssl_error_with_reason( @@ -880,7 +876,7 @@ mod _ssl { } else { e }; - vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), Some(0), msg) + vm.new_os_subtype_error(PySSLError::class(&vm.ctx), Some(0), msg) .upcast() })?; @@ -903,7 +899,7 @@ mod _ssl { // Additional validation: Create CertifiedKey to ensure rustls accepts it let signing_key = crypto_ext.any_supported_key(&key).map_err(|_| { vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "[SSL: KEY_VALUES_MISMATCH] key values mismatch", ) @@ -914,7 +910,7 @@ mod _ssl { if certified_key.keys_match().is_err() { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "[SSL: KEY_VALUES_MISMATCH] key values mismatch", ) @@ -1227,7 +1223,7 @@ mod _ssl { .filter(|(suites, _)| suites.iter().any(|s| s.tls13().is_none())) .ok_or_else(|| { vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "No cipher can be selected.".to_owned(), ) @@ -1588,7 +1584,7 @@ mod _ssl { if args.server_side && zelf.protocol == PROTOCOL_TLS_CLIENT { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context", ) @@ -1597,7 +1593,7 @@ mod _ssl { if !args.server_side && zelf.protocol == PROTOCOL_TLS_SERVER { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "Cannot create a client socket with a PROTOCOL_TLS_SERVER context", ) @@ -1668,7 +1664,7 @@ mod _ssl { if server_side && zelf.protocol == PROTOCOL_TLS_CLIENT { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context", ) @@ -1677,7 +1673,7 @@ mod _ssl { if !server_side && zelf.protocol == PROTOCOL_TLS_SERVER { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "Cannot create a client socket with a PROTOCOL_TLS_SERVER context", ) @@ -1821,25 +1817,21 @@ mod _ssl { let err_msg = e.to_string(); if err_msg.contains("no start line") { vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "no start line: cadata does not contain a certificate", ) .upcast() } else if err_msg.contains("not enough data") { vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "not enough data: cadata does not contain a certificate", ) .upcast() } else { - vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), - None, - err_msg, - ) - .upcast() + vm.new_os_subtype_error(PySSLError::class(&vm.ctx), None, err_msg) + .upcast() } }) } @@ -3190,7 +3182,7 @@ mod _ssl { if matches!(*zelf.state.lock(), TlsState::ShutDown) { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "cannot read after shutdown", ) @@ -3344,7 +3336,7 @@ mod _ssl { ) { return Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "cannot write after shutdown", ) @@ -3740,9 +3732,9 @@ mod _ssl { .map_err(|e| e.into_py_err(vm))?; })(); if let Err(error) = &result - && error.fast_isinstance(PySSLError::class(&vm.ctx)) - && !error.fast_isinstance(PySSLWantReadError::class(&vm.ctx)) - && !error.fast_isinstance(PySSLWantWriteError::class(&vm.ctx)) + && error.fast_isinstance(&PySSLError::class(&vm.ctx)) + && !error.fast_isinstance(&PySSLWantReadError::class(&vm.ctx)) + && !error.fast_isinstance(&PySSLWantWriteError::class(&vm.ctx)) { // A protocol failure cannot resume as a normal shutdown. Keep // any fatal alert rustls queued and preserve the original error. @@ -3846,7 +3838,7 @@ mod _ssl { // Raise SSLError instead of NotImplementedError for compatibility Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, "Post-handshake authentication is not supported by the rustls backend. \ The rustls TLS library does not provide an API to request client certificates \ @@ -3969,12 +3961,8 @@ mod _ssl { let bytes_like = ArgBytesLike::try_from_object(vm, b)?; let data = bytes_like.borrow_buf(); zelf.inner.lock().write(&data).map_err(|err| { - vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), - None, - err.to_string(), - ) - .upcast() + vm.new_os_subtype_error(PySSLError::class(&vm.ctx), None, err.to_string()) + .upcast() }) } diff --git a/crates/stdlib/src/ssl/compat.rs b/crates/stdlib/src/ssl/compat.rs index 09c180bcaab..f8e59516686 100644 --- a/crates/stdlib/src/ssl/compat.rs +++ b/crates/stdlib/src/ssl/compat.rs @@ -64,11 +64,7 @@ pub(super) fn create_ssl_cert_verification_error( let msg = format!("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: {verify_message}",); - let exc = vm.new_os_subtype_error( - PySSLCertVerificationError::class(&vm.ctx).to_owned(), - None, - msg, - ); + let exc = vm.new_os_subtype_error(PySSLCertVerificationError::class(&vm.ctx), None, msg); // Set verify_code and verify_message attributes // Ignore errors as they're extremely rare (e.g., out of memory) @@ -181,7 +177,7 @@ impl SslError { let msg = message.into(); // SSLError args should be (errno, message) format // FIXME: Use 1 as generic SSL error code - let exc = vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), Some(1), msg); + let exc = vm.new_os_subtype_error(PySSLError::class(&vm.ctx), Some(1), msg); // Set library and reason attributes // Ignore errors as they're extremely rare (e.g., out of memory) @@ -249,7 +245,7 @@ impl SslError { } Self::Ssl(msg) => vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, format!("SSL error: {msg}"), ) @@ -274,7 +270,7 @@ impl SslError { } Self::Io(err) if err.raw_os_error().is_none() => vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), None, format!("SSL error: {err}"), ) @@ -358,7 +354,7 @@ pub(super) fn send_all_bytes( socket .flush_pending_tls_output(vm, deadline) .map_err(|error| { - if error.fast_isinstance(PySSLWantWriteError::class(&vm.ctx)) { + if error.fast_isinstance(&PySSLWantWriteError::class(&vm.ctx)) { SslError::WantWrite } else if error.fast_isinstance(vm.ctx.exceptions.timeout_error) { SslError::Timeout("The write operation timed out".to_owned()) diff --git a/crates/stdlib/src/ssl/error.rs b/crates/stdlib/src/ssl/error.rs index 4aeba986c58..8f58ad88092 100644 --- a/crates/stdlib/src/ssl/error.rs +++ b/crates/stdlib/src/ssl/error.rs @@ -89,7 +89,7 @@ pub(crate) mod ssl_error { #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub(crate) fn create_ssl_want_read_error(vm: &VirtualMachine) -> PyRef { vm.new_os_subtype_error( - PySSLWantReadError::class(&vm.ctx).to_owned(), + PySSLWantReadError::class(&vm.ctx), Some(SSL_ERROR_WANT_READ), "The operation did not complete (read)", ) @@ -98,7 +98,7 @@ pub(crate) mod ssl_error { #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub(crate) fn create_ssl_want_write_error(vm: &VirtualMachine) -> PyRef { vm.new_os_subtype_error( - PySSLWantWriteError::class(&vm.ctx).to_owned(), + PySSLWantWriteError::class(&vm.ctx), Some(SSL_ERROR_WANT_WRITE), "The operation did not complete (write)", ) @@ -107,7 +107,7 @@ pub(crate) mod ssl_error { #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub(crate) fn create_ssl_eof_error(vm: &VirtualMachine) -> PyRef { vm.new_os_subtype_error( - PySSLEOFError::class(&vm.ctx).to_owned(), + PySSLEOFError::class(&vm.ctx), Some(SSL_ERROR_EOF), "EOF occurred in violation of protocol", ) @@ -122,7 +122,7 @@ pub(crate) mod ssl_error { )] pub(crate) fn create_ssl_zero_return_error(vm: &VirtualMachine) -> PyRef { vm.new_os_subtype_error( - PySSLZeroReturnError::class(&vm.ctx).to_owned(), + PySSLZeroReturnError::class(&vm.ctx), Some(SSL_ERROR_ZERO_RETURN), "TLS/SSL connection has been closed (EOF)", ) @@ -140,7 +140,7 @@ pub(crate) mod ssl_error { msg: impl Into, ) -> PyRef { vm.new_os_subtype_error( - PySSLSyscallError::class(&vm.ctx).to_owned(), + PySSLSyscallError::class(&vm.ctx), Some(SSL_ERROR_SYSCALL), msg.into(), ) diff --git a/crates/stdlib/src/ssl_wasm.rs b/crates/stdlib/src/ssl_wasm.rs index 21f46586c18..805f1e335fe 100644 --- a/crates/stdlib/src/ssl_wasm.rs +++ b/crates/stdlib/src/ssl_wasm.rs @@ -157,7 +157,7 @@ mod _ssl { fn ssl_error(vm: &VirtualMachine, message: impl Into) -> PyResult { Err(vm .new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), + PySSLError::class(&vm.ctx), Some(SSL_ERROR_SSL), message.into(), ) @@ -730,12 +730,8 @@ mod _ssl { let bytes_like = ArgBytesLike::try_from_object(vm, buf)?; let data = bytes_like.borrow_buf(); zelf.inner.lock().write(&data).map_err(|err| { - vm.new_os_subtype_error( - PySSLError::class(&vm.ctx).to_owned(), - None, - err.to_string(), - ) - .upcast() + vm.new_os_subtype_error(PySSLError::class(&vm.ctx), None, err.to_string()) + .upcast() }) } diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs index 15c0050cb9c..e290be376e6 100644 --- a/crates/vm/src/buffer.rs +++ b/crates/vm/src/buffer.rs @@ -1,7 +1,7 @@ use crate::{ - AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytesRef, PyComplex, PyTuple, PyTupleRef, PyType, PyTypeRef}, - common::{lock::PyRwLock, rc::PyRc, static_cell, str::wchar_t}, + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef, PyComplex, PyTuple, PyTupleRef, PyTypeRef}, + common::{lock::PyRwLock, rc::PyRc, str::wchar_t}, convert::ToPyObject, exceptions, function::{ArgBytesLike, ArgIntoBool, ArgIntoComplex, ArgIntoFloat}, @@ -991,17 +991,15 @@ fn unpack_pascal(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef { } // XXX: are those functions expected to be placed here? -pub fn struct_error_type(vm: &VirtualMachine) -> &'static Py { - static_cell! { - static INSTANCE: PyTypeRef; - } - INSTANCE.get_or_init(|| vm.ctx.new_exception_type("struct", "error", None)) +pub fn struct_error_type(vm: &VirtualMachine) -> PyTypeRef { + struct StructError; + vm.__cached_native::(|| vm.ctx.new_exception_type("struct", "error", None)) } pub fn new_struct_error>(vm: &VirtualMachine, msg: T) -> PyBaseExceptionRef { // can't just STRUCT_ERROR.get().unwrap() cause this could be called before from buffer // machinery, independent of whether _struct was ever imported - vm.new_exception_msg(struct_error_type(vm).to_owned(), msg.into()) + vm.new_exception_msg(struct_error_type(vm), msg.into()) } #[cfg(test)] @@ -1011,7 +1009,7 @@ mod tests { #[test] fn format_cache_reuses_specs_and_releases_evicted_entries() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let cache = &vm.state.struct_format_cache; let spec = cache.get_or_parse(b" &'static Py { - ctx.types.async_generator + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.async_generator).to_owned() } } @@ -264,8 +264,8 @@ unsafe impl Traverse for PyAsyncGenWrappedValue { impl PyPayload for PyAsyncGenWrappedValue { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.async_generator_wrapped_value + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.async_generator_wrapped_value).to_owned() } } @@ -327,8 +327,8 @@ unsafe impl Traverse for PyAsyncGenASend { impl PyPayload for PyAsyncGenASend { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.async_generator_asend + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.async_generator_asend).to_owned() } } @@ -479,8 +479,8 @@ unsafe impl Traverse for PyAsyncGenAThrow { impl PyPayload for PyAsyncGenAThrow { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.async_generator_athrow + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.async_generator_athrow).to_owned() } } @@ -696,8 +696,8 @@ unsafe impl Traverse for PyAnextAwaitable { impl PyPayload for PyAnextAwaitable { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.anext_awaitable + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.anext_awaitable).to_owned() } } diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index 79e2c1ca721..933d37febb6 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -1,19 +1,23 @@ -use super::{PyStrInterned, PyStrRef, PyType, type_}; +use super::{PyStrInterned, PyStrRef, PyType, PyTypeRef, type_}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::wtf8::Wtf8, convert::TryFromObject, function::{Callee, FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags, PyNativeFn}, + object::{Traverse, TraverseFn}, types::{Callable, Comparable, PyComparisonOp, Representable}, }; use alloc::fmt; // PyCFunctionObject in CPython #[repr(C)] -#[pyclass(name = "builtin_function_or_method", module = false, traverse)] +#[pyclass( + name = "builtin_function_or_method", + module = false, + traverse = "manual" +)] pub struct PyNativeFunction { - #[pytraverse(skip)] pub(crate) value: &'static PyMethodDef, pub(crate) zelf: Option, // Module that owns this function. Not passed as a call argument. @@ -24,9 +28,26 @@ pub struct PyNativeFunction { pub(crate) _method_def_owner: Option, } +unsafe impl Traverse for PyNativeFunction { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.zelf.traverse(tracer_fn); + self.module_object.traverse(tracer_fn); + self.module.traverse(tracer_fn); + self._method_def_owner.traverse(tracer_fn); + } + + fn clear(&mut self, out: &mut Vec) { + out.extend(self.zelf.take()); + out.extend(self.module_object.take()); + // GC has exclusive access while clearing this unreachable object. + out.extend(unsafe { self.module.swap(None) }); + // Keep the definition owner until deallocation: `value` borrows it. + } +} + impl PyPayload for PyNativeFunction { - fn class(ctx: &Context) -> &'static Py { - ctx.types.builtin_function_or_method_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.builtin_function_or_method_type).to_owned() } } @@ -78,7 +99,7 @@ impl PyNativeFunction { self.zelf.as_deref().or(self.module_object.as_deref()) } - pub const fn as_func(&self) -> &'static dyn PyNativeFn { + pub const fn as_func(&self) -> &dyn PyNativeFn { self.value.func } } @@ -234,10 +255,40 @@ impl PyNativeFunction { } // PyCMethodObject in CPython -#[pyclass(name = "builtin_function_or_method", module = false, base = PyNativeFunction, ctx = "builtin_function_or_method_type")] +#[pyclass(name = "builtin_function_or_method", module = false, base = PyNativeFunction, ctx = "builtin_function_or_method_type", traverse = "manual", payload = "manual")] pub struct PyNativeMethod { pub(crate) func: PyNativeFunction, - pub(crate) class: &'static Py, // TODO: the actual life is &'self + pub(crate) class: PyRef, +} + +impl PyPayload for PyNativeMethod { + const PAYLOAD_TYPE_ID: core::any::TypeId = PyNativeFunction::PAYLOAD_TYPE_ID; + + fn supports_native_layout(layout: core::any::TypeId) -> bool { + layout == core::any::TypeId::of::() + || PyNativeFunction::supports_native_layout(layout) + } + + unsafe fn validate_downcastable_from(obj: &PyObject) -> bool { + // Both payloads use builtin_function_or_method as their Python class. + // The immutable allocation vtable distinguishes the larger CMethod. + obj.supports_native_layout(core::any::TypeId::of::()) + } + + fn class(ctx: &Context) -> PyTypeRef { + ctx.types.builtin_function_or_method_type.to_owned() + } +} + +unsafe impl Traverse for PyNativeMethod { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.func.traverse(tracer_fn); + self.class.traverse(tracer_fn); + } + + fn clear(&mut self, out: &mut Vec) { + self.func.clear(out); + } } // All Python-visible behavior (getters, slots) is registered by PyNativeFunction::extend_class. diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index 19e149d9df0..96d88526155 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -17,10 +17,12 @@ use crate::{ class::PyClassImpl, common::{ atomic::{AtomicUsize, Ordering}, + borrow::{BorrowedValue, BorrowedValueMut}, lock::{ PyDetachingRwLock, PyDetachingRwLockReadGuard, PyDetachingRwLockWriteGuard, PyMappedDetachingRwLockReadGuard, PyMappedDetachingRwLockWriteGuard, PyMutex, }, + rc::PyRc, }, convert::{ToPyObject, ToPyResult}, function::{ArgBytesLike, PyComparisonValue, PySsize}, @@ -41,10 +43,43 @@ use core::mem::size_of; #[pyclass(module = false, name = "bytearray", unhashable = true)] #[derive(Debug, Default)] pub struct PyByteArray { + storage: PyRc, +} + +#[doc(hidden)] +#[derive(Debug, Default)] +pub struct ByteArrayStorage { inner: PyDetachingRwLock, exports: AtomicUsize, } +impl core::ops::Deref for PyByteArray { + type Target = ByteArrayStorage; + fn deref(&self) -> &Self::Target { + &self.storage + } +} + +#[derive(Debug)] +struct SharedByteArray(PyRc); + +impl Drop for SharedByteArray { + fn drop(&mut self) { + self.0.exports.fetch_sub(1, Ordering::Release); + } +} + +// SAFETY: the export keeps resizing disabled and the byte-only allocation alive; +// both the exporter and all independent views use the same lock. +unsafe impl crate::protocol::SharedBufferStorage for SharedByteArray { + fn read(&self) -> BorrowedValue<'_, [u8]> { + PyDetachingRwLockReadGuard::map(self.0.inner.read(), |x| x.elements.as_slice()).into() + } + fn write(&self) -> BorrowedValueMut<'_, [u8]> { + PyDetachingRwLockWriteGuard::map(self.0.inner.write(), |x| x.elements.as_mut_slice()).into() + } +} + pub(crate) type PyByteArrayRef = PyRef; impl From for PyByteArray { @@ -60,8 +95,8 @@ impl From> for PyByteArray { } impl PyPayload for PyByteArray { - fn class(ctx: &Context) -> &'static Py { - ctx.types.bytearray_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.bytearray_type).to_owned() } } @@ -77,10 +112,12 @@ impl PyByteArray { Self::from(data).into_ref(ctx) } - const fn from_inner(inner: PyBytesInner) -> Self { + fn from_inner(inner: PyBytesInner) -> Self { Self { - inner: PyDetachingRwLock::new(inner), - exports: AtomicUsize::new(0), + storage: PyRc::new(ByteArrayStorage { + inner: PyDetachingRwLock::new(inner), + exports: AtomicUsize::new(0), + }), } } @@ -88,10 +125,20 @@ impl PyByteArray { PyDetachingRwLockReadGuard::map(self.inner.read(), |inner| &*inner.elements) } - pub fn borrow_buf_mut(&self) -> PyMappedDetachingRwLockWriteGuard<'_, Vec> { + fn borrow_buf_mut(&self) -> PyMappedDetachingRwLockWriteGuard<'_, Vec> { PyDetachingRwLockWriteGuard::map(self.inner.write(), |inner| &mut inner.elements) } + /// Return the native byte storage for C-API integration. + /// + /// # Safety + /// The caller must keep this bytearray alive, prevent resizing, and + /// synchronize all access for the entire lifetime of the returned pointer. + #[must_use] + pub unsafe fn as_mut_ptr_unchecked(&self) -> *mut u8 { + self.borrow_buf_mut().as_mut_ptr() + } + fn repeat(&self, value: isize, vm: &VirtualMachine) -> PyResult { self.inner().mul(value, vm).map(|x| x.into()) } @@ -173,11 +220,6 @@ impl PyByteArray { fn inner(&self) -> PyDetachingRwLockReadGuard<'_, PyBytesInner> { self.inner.read() } - #[inline] - fn inner_mut(&self) -> PyDetachingRwLockWriteGuard<'_, PyBytesInner> { - self.inner.write() - } - fn __len__(&self) -> usize { self.borrow_buf().len() } @@ -757,7 +799,7 @@ impl Initializer for PyByteArray { fn init(zelf: &Py, options: Self::Args, vm: &VirtualMachine) -> PyResult<()> { // First unpack bytearray and *then* get a lock to set it. let mut inner = options.get_inner(bytearray_from_object, vm)?; - core::mem::swap(&mut *zelf.inner_mut(), &mut inner); + core::mem::swap(&mut *zelf.try_resizable(vm)?, &mut inner); Ok(()) } } @@ -777,6 +819,11 @@ impl Comparable for PyByteArray { } static BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| { + let storage = buffer.obj_as::().storage.clone(); + storage.exports.fetch_add(1, Ordering::Release); + Some(PyRc::new(SharedByteArray(storage))) + }), obj_bytes: |buffer| buffer.obj_as::().borrow_buf().into(), obj_bytes_mut: |buffer| { PyMappedDetachingRwLockWriteGuard::map( @@ -815,9 +862,10 @@ impl AsBuffer for PyByteArray { } fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + let inner = zelf.inner.read(); Ok(PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor::simple(zelf.__len__(), false), + BufferDescriptor::simple(inner.elements.len(), false), &BUFFER_METHODS, )) } @@ -829,7 +877,11 @@ impl BufferResizeGuard for PyByteArray { fn try_resizable_opt(&self) -> Option> { // An export is a borrow someone else still holds, so it is answered // before the lock rather than by waiting on it. - (self.exports.load(Ordering::SeqCst) == 0).then(|| self.inner.write()) + if self.exports.load(Ordering::Acquire) != 0 { + return None; + } + let inner = self.inner.write(); + (self.exports.load(Ordering::Acquire) == 0).then_some(inner) } } @@ -881,7 +933,7 @@ impl AsSequence for PyByteArray { if let Some(value) = value { zelf._setitem_by_index(i, &value, vm) } else { - zelf.borrow_buf_mut().delitem_by_index(vm, i) + zelf.try_resizable(vm)?.elements.delitem_by_index(vm, i) } }), contains: atomic_func!(|seq, other, vm| { @@ -944,8 +996,8 @@ pub(crate) struct PyByteArrayIterator { impl PyPayload for PyByteArrayIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.bytearray_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.bytearray_iterator_type).to_owned() } } diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index 415cf4df4df..ed8ad3ca77f 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -86,9 +86,10 @@ impl AsRef<[u8]> for PyBytesRef { } impl PyPayload for PyBytes { + const OWNER_NEUTRAL: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.bytes_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.bytes_type).to_owned() } } @@ -627,6 +628,15 @@ impl PyRef { } static BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| { + // PyBytes owns its Vec inside a Python payload. An independent export + // copies it so a foreign interpreter never borrows that payload. + Some(crate::common::rc::PyRc::new( + crate::protocol::shared_buffer::ImmutableBuffer( + buffer.obj_as::().as_bytes().into(), + ), + )) + }), obj_bytes: |buffer| buffer.obj_as::().as_bytes().into(), obj_bytes_mut: |_| panic!(), release: |_| {}, @@ -775,8 +785,8 @@ pub(crate) struct PyBytesIterator { impl PyPayload for PyBytesIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.bytes_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.bytes_iterator_type).to_owned() } } diff --git a/crates/vm/src/builtins/capsule.rs b/crates/vm/src/builtins/capsule.rs index a432ae6a203..89493501339 100644 --- a/crates/vm/src/builtins/capsule.rs +++ b/crates/vm/src/builtins/capsule.rs @@ -1,4 +1,3 @@ -use super::PyType; use crate::{ AsObject, Context, Py, PyObject, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, @@ -19,8 +18,8 @@ pub struct PyCapsule { impl PyPayload for PyCapsule { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.capsule_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.capsule_type).to_owned() } } diff --git a/crates/vm/src/builtins/classmethod.rs b/crates/vm/src/builtins/classmethod.rs index 28e4b9ff32f..c47b206407e 100644 --- a/crates/vm/src/builtins/classmethod.rs +++ b/crates/vm/src/builtins/classmethod.rs @@ -27,8 +27,8 @@ impl From for PyClassMethod { impl PyPayload for PyClassMethod { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.classmethod_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.classmethod_type).to_owned() } } diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index 5e3174ea11f..991f848e445 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -207,7 +207,10 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> { match_class!(match obj { ref i @ super::int::PyInt => { let value = i.as_bigint(); - if obj.class().is(super::bool_::PyBool::static_type()) { + if obj + .class() + .is(unsafe { super::bool_::PyBool::static_type() }) + { BorrowedConstant::Boolean { value: !value.is_zero(), } @@ -236,21 +239,30 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> { super::singletons::PyNone => BorrowedConstant::None, super::slice::PyEllipsis => BorrowedConstant::Ellipsis, ref s @ super::slice::PySlice => { - // Constant pool slices always store Some() for start/step (even for None). - // Box::leak the array so it outlives the borrow. Leak is acceptable since - // constant pool objects live for the program's lifetime. - let start = s.start.clone().unwrap(); - let stop = s.stop.clone(); - let step = s.step.clone().unwrap(); - let arr = Box::leak(Box::new([Literal(start), Literal(stop), Literal(step)])); - BorrowedConstant::Slice { elements: arr } + let literal = |object: &PyObjectRef| { + // SAFETY: Literal is transparent over PyObjectRef. Slice + // constants contain only other constants. + unsafe { &*core::ptr::from_ref(object).cast::() } + }; + // SAFETY: PyRef and PyObjectRef are transparent, thin + // references to the same Python allocation; Literal wraps the + // latter. This immortal None reference outlives the slice view. + let none: &Literal = + unsafe { &*core::ptr::from_ref(&Context::genesis().none).cast::() }; + BorrowedConstant::Slice { + elements: [ + s.start.as_ref().map_or(none, literal), + literal(&s.stop), + s.step.as_ref().map_or(none, literal), + ], + } } ref fs @ super::set::PyFrozenSet => { - // Box::leak the elements so they outlive the borrow. Leak is acceptable since - // constant pool objects live for the program's lifetime. - let elems: Vec = fs.elements().into_iter().map(Literal).collect(); - let elements = Box::leak(elems.into_boxed_slice()); - BorrowedConstant::Frozenset { elements } + BorrowedConstant::Frozenset { + elements: alloc::borrow::Cow::Owned( + fs.elements().into_iter().map(Literal).collect(), + ), + } } _ => panic!("unexpected payload for constant python value"), }) @@ -310,8 +322,7 @@ impl ConstantBag for PyObjBag<'_> { let start_obj = self.make_constant(start.borrow_constant()).0; let stop_obj = self.make_constant(stop.borrow_constant()).0; let step_obj = self.make_constant(step.borrow_constant()).0; - // Store as PySlice with Some() for all fields (even None values) - // so borrow_obj_constant can reference them. + // Preserve all three constant fields, including explicit None. use crate::builtins::PySlice; PySlice { start: Some(start_obj), @@ -783,8 +794,8 @@ impl PyCode { impl PyPayload for PyCode { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.code_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.code_type).to_owned() } } @@ -1721,3 +1732,34 @@ impl<'a> LineTableReader<'a> { pub(crate) fn init(ctx: &'static Context) { PyCode::extend_class(ctx, ctx.types.code_type); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn constant_snapshots_release_their_temporary_references() { + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let element: PyObjectRef = vm.ctx.new_str("owned constant element").into(); + let slice: PyObjectRef = super::super::PySlice { + start: None, + stop: element.clone(), + step: None, + } + .into_ref(&vm.ctx) + .into(); + let frozen: PyObjectRef = PyFrozenSet::from_iter(vm, [element.clone()]) + .unwrap() + .into_ref(&vm.ctx) + .into(); + for object in [slice, frozen] { + let before = element.strong_count(); + let expected = borrow_obj_constant(&object).to_owned(); + for _ in 0..3 { + assert_eq!(borrow_obj_constant(&object).to_owned(), expected); + assert_eq!(element.strong_count(), before); + } + } + }); + } +} diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index f471c43c580..fb542bb439d 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -39,12 +39,13 @@ thread_local! { } impl PyPayload for PyComplex { + const OWNER_NEUTRAL: bool = true; const MAX_FREELIST: usize = 100; const HAS_FREELIST: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.complex_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.complex_type).to_owned() } #[inline] diff --git a/crates/vm/src/builtins/coroutine.rs b/crates/vm/src/builtins/coroutine.rs index 649481c8cc3..19333fb89e2 100644 --- a/crates/vm/src/builtins/coroutine.rs +++ b/crates/vm/src/builtins/coroutine.rs @@ -1,4 +1,4 @@ -use super::{PyCode, PyGenericAlias, PyStrRef, PyTupleRef, PyType, PyTypeRef}; +use super::{PyCode, PyGenericAlias, PyStrRef, PyTupleRef, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, @@ -36,8 +36,8 @@ impl PyPayload for PyCoroutine { const NEW_REF_UNTRACKED: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.coroutine_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.coroutine_type).to_owned() } } @@ -202,8 +202,8 @@ unsafe impl Traverse for PyCoroutineWrapper { impl PyPayload for PyCoroutineWrapper { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.coroutine_wrapper_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.coroutine_wrapper_type).to_owned() } } diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index b3189e592a8..93b68df7834 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -8,6 +8,7 @@ use crate::{ function::{ Callee, FuncArgs, ItemDoc, PyMethodDef, PyMethodFlags, PySetterValue, PySsize, plain_doc, }, + object::{Traverse, TraverseFn}, protocol::{PyNumberBinaryFunc, PyNumberTernaryFunc, PyNumberUnaryFunc}, types::{ Callable, Comparable, DelFunc, DescrGetFunc, DescrSetFunc, GenericMethod, GetDescriptor, @@ -22,34 +23,48 @@ use rustpython_common::lock::PyRwLock; #[derive(Debug)] pub struct PyDescriptor { - pub typ: &'static Py, + pub typ: PyRef, pub name: &'static PyStrInterned, pub qualname: PyRwLock>, } -#[derive(Debug)] -pub struct PyDescriptorOwned { - pub typ: PyRef, - pub name: &'static PyStrInterned, - pub qualname: PyRwLock>, +impl PyDescriptor { + fn bind( + &self, + method: &'static PyMethodDef, + owner: Option, + obj: PyObjectRef, + ctx: &Context, + ) -> PyRef { + let mut bound = method.to_bound_method(obj, &self.typ); + bound.func._method_def_owner = owner; + bound.into_ref(ctx) + } } -#[pyclass(name = "method_descriptor", module = false)] +#[pyclass(name = "method_descriptor", module = false, traverse = "manual")] pub struct PyMethodDescriptor { #[pymember(name = "__objclass__", path = "typ")] #[pymember(name = "__name__", path = "name")] pub common: PyDescriptor, - pub method: &'static PyMethodDef, + pub(crate) method: &'static PyMethodDef, // vectorcall: vector_call_func, /// Prevent HeapMethodDef from being freed while this descriptor references it pub(crate) _method_def_owner: Option, } +unsafe impl Traverse for PyMethodDescriptor { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.common.typ.traverse(tracer_fn); + self._method_def_owner.traverse(tracer_fn); + } +} + impl PyMethodDescriptor { - pub fn new(method: &'static PyMethodDef, typ: &'static Py, ctx: &Context) -> Self { + pub fn new(method: &'static PyMethodDef, typ: &Py, ctx: &Context) -> Self { Self { common: PyDescriptor { - typ, + typ: typ.to_owned(), name: ctx.intern_str(method.name), qualname: PyRwLock::new(None), }, @@ -60,8 +75,8 @@ impl PyMethodDescriptor { } impl PyPayload for PyMethodDescriptor { - fn class(ctx: &Context) -> &'static Py { - ctx.types.method_descriptor_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.method_descriptor_type).to_owned() } } @@ -125,7 +140,8 @@ impl Callable for PyMethodDescriptor { impl PyMethodDescriptor { pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef { - self.method.build_bound_method(ctx, obj, self.common.typ) + self.common + .bind(self.method, self._method_def_owner.clone(), obj, ctx) } } @@ -173,20 +189,27 @@ impl Representable for PyMethodDescriptor { } // METH_CLASS descriptors. Same layout as method_descriptor; a distinct type. -#[pyclass(name = "classmethod_descriptor", module = false)] +#[pyclass(name = "classmethod_descriptor", module = false, traverse = "manual")] pub struct PyClassMethodDescriptor { #[pymember(name = "__objclass__", path = "typ")] #[pymember(name = "__name__", path = "name")] pub common: PyDescriptor, - pub method: &'static PyMethodDef, + pub(crate) method: &'static PyMethodDef, pub(crate) _method_def_owner: Option, } +unsafe impl Traverse for PyClassMethodDescriptor { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.common.typ.traverse(tracer_fn); + self._method_def_owner.traverse(tracer_fn); + } +} + impl PyClassMethodDescriptor { - pub fn new(method: &'static PyMethodDef, typ: &'static Py, ctx: &Context) -> Self { + pub fn new(method: &'static PyMethodDef, typ: &Py, ctx: &Context) -> Self { Self { common: PyDescriptor { - typ, + typ: typ.to_owned(), name: ctx.intern_str(method.name), qualname: PyRwLock::new(None), }, @@ -196,13 +219,14 @@ impl PyClassMethodDescriptor { } pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef { - self.method.build_bound_method(ctx, obj, self.common.typ) + self.common + .bind(self.method, self._method_def_owner.clone(), obj, ctx) } } impl PyPayload for PyClassMethodDescriptor { - fn class(ctx: &Context) -> &'static Py { - ctx.types.classmethod_descriptor_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.classmethod_descriptor_type).to_owned() } } @@ -249,7 +273,7 @@ impl GetDescriptor for PyClassMethodDescriptor { obj.class().name() )) })?; - if !typ.fast_issubclass(descr.common.typ) { + if !typ.fast_issubclass(&descr.common.typ) { return Err(vm.new_type_error(format!( "descriptor '{}' requires a subtype of '{}' but received '{}'", descr.common.name, @@ -736,16 +760,22 @@ impl PyMemberSpec { } // = PyMemberDescrObject -#[pyclass(name = "member_descriptor", module = false)] +#[pyclass(name = "member_descriptor", module = false, traverse = "manual")] #[derive(Debug)] pub struct PyMemberDescriptor { #[pymember(name = "__objclass__", path = "typ")] #[pymember(name = "__name__", path = "name")] - pub common: PyDescriptorOwned, + pub common: PyDescriptor, pub member: PyMemberDef, pub access: MemberAccess, } +unsafe impl Traverse for PyMemberDescriptor { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.common.typ.traverse(tracer_fn); + } +} + impl PyMemberDescriptor { /// Byte offset of an object-pointer member. `None` for bool, float, and /// struct-sequence indexes, which slot specialization must not treat as cells. @@ -812,12 +842,12 @@ impl PyMemberDescriptor { } impl PyPayload for PyMemberDescriptor { - fn class(ctx: &Context) -> &'static Py { - ctx.types.member_descriptor_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.member_descriptor_type).to_owned() } } -fn calculate_qualname(descr: &PyDescriptorOwned, vm: &VirtualMachine) -> PyResult> { +fn calculate_qualname(descr: &PyDescriptor, vm: &VirtualMachine) -> PyResult> { if let Some(qualname) = vm.get_attribute_opt(descr.typ.as_object(), "__qualname__")? { let str = qualname.downcast::().map_err(|_| { vm.new_type_error(".__objclass__.__qualname__ is not a unicode object") @@ -1544,7 +1574,7 @@ fn method_descr_typecheck( ) -> PyResult<()> { if descr.method.flags.contains(PyMethodFlags::STATIC) || descr.method.flags.contains(PyMethodFlags::CLASS) - || obj.fast_isinstance(descr.common.typ) + || obj.fast_isinstance(&descr.common.typ) { return Ok(()); } @@ -1596,7 +1626,7 @@ fn vectorcall_wrapper( ))); } let obj = args.remove(0); - if !obj.fast_isinstance(zelf.typ) { + if !obj.fast_isinstance(&zelf.typ) { return Err(vm.new_type_error(format!( "descriptor '{}' requires a '{}' object but received a '{}'", zelf.name.as_str(), @@ -1951,11 +1981,11 @@ fn parse_buffer_flags( // wrapper_descriptor: wraps a slot function as a Python method // = PyWrapperDescrObject -#[pyclass(name = "wrapper_descriptor", module = false)] +#[pyclass(name = "wrapper_descriptor", module = false, traverse = "manual")] #[derive(Debug)] pub(crate) struct PyWrapper { #[pymember(name = "__objclass__")] - pub typ: &'static Py, + pub typ: PyRef, #[pymember(name = "__name__")] pub name: &'static PyStrInterned, pub wrapped: SlotFunc, @@ -1966,9 +1996,15 @@ pub(crate) struct PyWrapper { pub plain_len: u32, } +unsafe impl Traverse for PyWrapper { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.typ.traverse(tracer_fn); + } +} + impl PyPayload for PyWrapper { - fn class(ctx: &Context) -> &'static Py { - ctx.types.wrapper_descriptor_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.wrapper_descriptor_type).to_owned() } } @@ -2000,7 +2036,7 @@ impl Callable for PyWrapper { // list.__init__(l, [1,2,3]) form - first arg is self let (obj, rest): (PyObjectRef, FuncArgs) = args.bind(vm)?; - if !obj.fast_isinstance(zelf.typ) { + if !obj.fast_isinstance(&zelf.typ) { return Err(vm.new_type_error(format!( "descriptor '{}' requires a '{}' object but received a '{}'", zelf.name.as_str(), @@ -2061,13 +2097,12 @@ impl Representable for PyWrapper { pub(crate) struct PyMethodWrapper { pub wrapper: PyRef, #[pymember(name = "__self__")] - #[pytraverse(skip)] pub obj: PyObjectRef, } impl PyPayload for PyMethodWrapper { - fn class(ctx: &Context) -> &'static Py { - ctx.types.method_wrapper_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.method_wrapper_type).to_owned() } } @@ -2076,7 +2111,7 @@ impl Callable for PyMethodWrapper { fn call(zelf: &Py, args: FuncArgs, vm: &VirtualMachine) -> PyResult { // bpo-37619: Check type compatibility before calling wrapped slot - if !zelf.obj.fast_isinstance(zelf.wrapper.typ) { + if !zelf.obj.fast_isinstance(&zelf.wrapper.typ) { return Err(vm.new_type_error(format!( "descriptor '{}' requires a '{}' object but received a '{}'", zelf.wrapper.name.as_str(), @@ -2188,6 +2223,28 @@ mod tests { use crate::{protocol::PyNumberMethods, types::AsNumber}; use core::sync::atomic::{AtomicU8, Ordering}; + #[test] + fn bound_heap_method_keeps_its_definition_alive() { + use crate::function::HeapMethodDef; + + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let owner = HeapMethodDef::new(PyMethodDef::new_const( + "identity", + |zelf: PyObjectRef| zelf, + PyMethodFlags::METHOD, + ItemDoc::NONE, + )) + .into_ref(&vm.ctx); + let descriptor = owner.build_method(vm.ctx.types.object_type, vm); + let obj = vm.ctx.new_list(Vec::new()); + let method = descriptor.bind(obj.clone().into(), &vm.ctx); + drop(descriptor); + assert_eq!(owner.as_object().strong_count(), 2); + drop(owner); + assert!(method.as_object().call((), vm).unwrap().is(&obj)); + }); + } + #[pyclass(name = "UByteMembers", module = false)] #[derive(Debug, PyPayload)] #[repr(C)] @@ -2227,8 +2284,8 @@ mod tests { #[test] fn readonly_atomic_members_use_atomic_storage() { - crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { - let _class = ReadonlyAtomicMembers::make_static_type(); + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let _class = unsafe { ReadonlyAtomicMembers::make_static_type() }; let obj = ReadonlyAtomicMembers { byte: AtomicU8::new(128), object: None.into(), @@ -2269,8 +2326,8 @@ mod tests { #[test] fn unsigned_byte_members_preserve_adjacent_fields() { - crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { - let _class = UByteMembers::make_static_type(); + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let _class = unsafe { UByteMembers::make_static_type() }; let obj = UByteMembers { prefix: 17, readonly: 255, @@ -2312,7 +2369,7 @@ mod tests { .unwrap(); assert_eq!(obj.writable.load(Ordering::Relaxed), 42); - let filters = vm.state.warnings.filters.to_owned(); + let filters = vm.state.warnings.filters(); filters.borrow_vec_mut().insert( 0, vm.ctx diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index d709521ea6d..a6d6c51c8cf 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -71,8 +71,8 @@ impl PyPayload for PyDict { const HAS_FREELIST: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.dict_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.dict_type).to_owned() } #[inline] @@ -1256,8 +1256,8 @@ macro_rules! dict_view { } impl PyPayload for $name { - fn class(ctx: &Context) -> &'static Py { - ctx.types.$class + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.$class).to_owned() } } @@ -1302,8 +1302,8 @@ macro_rules! dict_view { impl PyPayload for $iter_name { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.$iter_class + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.$iter_class).to_owned() } } @@ -1403,8 +1403,8 @@ macro_rules! dict_view { impl PyPayload for $reverse_iter_name { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.$reverse_iter_class + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.$reverse_iter_class).to_owned() } } diff --git a/crates/vm/src/builtins/enumerate.rs b/crates/vm/src/builtins/enumerate.rs index 76ee3cca2b7..7748030a4dc 100644 --- a/crates/vm/src/builtins/enumerate.rs +++ b/crates/vm/src/builtins/enumerate.rs @@ -46,8 +46,8 @@ pub struct PyEnumerate { impl PyPayload for PyEnumerate { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.enumerate_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.enumerate_type).to_owned() } } @@ -140,8 +140,8 @@ pub(crate) struct PyReverseSequenceIterator { impl PyPayload for PyReverseSequenceIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.reverse_iter_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.reverse_iter_type).to_owned() } } diff --git a/crates/vm/src/builtins/filter.rs b/crates/vm/src/builtins/filter.rs index 11e11fc2795..3f459cda729 100644 --- a/crates/vm/src/builtins/filter.rs +++ b/crates/vm/src/builtins/filter.rs @@ -16,8 +16,8 @@ pub struct PyFilter { impl PyPayload for PyFilter { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.filter_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.filter_type).to_owned() } } diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 78d3bef7794..70182f80bfc 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -46,12 +46,13 @@ thread_local! { } impl PyPayload for PyFloat { + const OWNER_NEUTRAL: bool = true; const MAX_FREELIST: usize = 100; const HAS_FREELIST: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.float_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.float_type).to_owned() } #[inline] diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs index cbd21527da3..16f4b40b86f 100644 --- a/crates/vm/src/builtins/frame_locals_proxy.rs +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -40,8 +40,8 @@ unsafe impl Traverse for FrameLocalsProxy { impl PyPayload for FrameLocalsProxy { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.frame_locals_proxy_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.frame_locals_proxy_type).to_owned() } } @@ -165,15 +165,18 @@ impl FrameLocalsProxy { } } -#[pyclass(with( - Constructor, - AsMapping, - AsSequence, - AsNumber, - Iterable, - Comparable, - Representable -))] +#[pyclass( + flags(MAPPING), + with( + Constructor, + AsMapping, + AsSequence, + AsNumber, + Iterable, + Comparable, + Representable + ) +)] impl Py { #[pymethod] fn keys(&self, vm: &VirtualMachine) -> PyResult { diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index c6c4aaf2b79..2a220be92c5 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -989,8 +989,8 @@ pub(crate) fn datastack_frame_size_bytes_for_code(code: &Py) -> Option &'static Py { - ctx.types.function_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.function_type).to_owned() } } @@ -1384,9 +1384,9 @@ impl Hashable for PyBoundMethod { impl GetAttr for PyBoundMethod { fn getattro(zelf: &Py, name: &Py, vm: &VirtualMachine) -> PyResult { - let class_attr = vm - .ctx - .interned_str(name) + let class_attr = zelf + .class() + .interned_attr_name(name, vm) .and_then(|attr_name| zelf.get_class_attr(attr_name)); if let Some(obj) = class_attr { return vm.call_if_get_descriptor(&obj, zelf.to_owned().into()); @@ -1524,8 +1524,8 @@ impl Py { impl PyPayload for PyBoundMethod { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.bound_method_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.bound_method_type).to_owned() } } @@ -1564,8 +1564,8 @@ pub(crate) type PyCellRef = PyRef; impl PyPayload for PyCell { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.cell_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.cell_type).to_owned() } } diff --git a/crates/vm/src/builtins/generator.rs b/crates/vm/src/builtins/generator.rs index 8eb6013e703..8419f663cd7 100644 --- a/crates/vm/src/builtins/generator.rs +++ b/crates/vm/src/builtins/generator.rs @@ -2,7 +2,7 @@ * The mythical generator. */ -use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; +use super::{PyCode, PyGenericAlias, PyStrRef, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, @@ -33,8 +33,8 @@ impl PyPayload for PyGenerator { const NEW_REF_UNTRACKED: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.generator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.generator_type).to_owned() } } diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index 31e1b4ef1c7..17eb3eb535d 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -53,8 +53,8 @@ impl fmt::Debug for PyGenericAlias { impl PyPayload for PyGenericAlias { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.generic_alias_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.generic_alias_type).to_owned() } } diff --git a/crates/vm/src/builtins/getset.rs b/crates/vm/src/builtins/getset.rs index c8e37a7c42a..8e4c39b1410 100644 --- a/crates/vm/src/builtins/getset.rs +++ b/crates/vm/src/builtins/getset.rs @@ -54,8 +54,8 @@ unsafe impl Traverse for PyGetSet { impl PyPayload for PyGetSet { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.getset_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.getset_type).to_owned() } } diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index d6799474582..c761ca6e211 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -56,12 +56,13 @@ thread_local! { } impl PyPayload for PyInt { + const OWNER_NEUTRAL: bool = true; const MAX_FREELIST: usize = 100; const HAS_FREELIST: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.int_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.int_type).to_owned() } fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { diff --git a/crates/vm/src/builtins/interpolation.rs b/crates/vm/src/builtins/interpolation.rs index 73126102ea9..df40a2ee95c 100644 --- a/crates/vm/src/builtins/interpolation.rs +++ b/crates/vm/src/builtins/interpolation.rs @@ -28,8 +28,8 @@ pub struct PyInterpolation { impl PyPayload for PyInterpolation { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.interpolation_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.interpolation_type).to_owned() } } diff --git a/crates/vm/src/builtins/iter.rs b/crates/vm/src/builtins/iter.rs index 85f40c25977..e3a9e242808 100644 --- a/crates/vm/src/builtins/iter.rs +++ b/crates/vm/src/builtins/iter.rs @@ -2,7 +2,7 @@ * iterator types */ -use super::{PyInt, PyTupleRef, PyType}; +use super::{PyInt, PyTupleRef}; use crate::{ Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, @@ -232,8 +232,8 @@ pub struct PySequenceIterator { impl PyPayload for PySequenceIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.iter_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.iter_type).to_owned() } } @@ -308,8 +308,8 @@ pub struct PyCallableIterator { impl PyPayload for PyCallableIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.callable_iterator + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.callable_iterator).to_owned() } } diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index d6a2c1669a7..8d16ec4d195 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -92,8 +92,8 @@ impl PyPayload for PyList { const HAS_FREELIST: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.list_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.list_type).to_owned() } #[inline] @@ -1080,8 +1080,8 @@ pub(crate) struct PyListIterator { impl PyPayload for PyListIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.list_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.list_iterator_type).to_owned() } } @@ -1144,8 +1144,8 @@ pub(crate) struct PyListReverseIterator { impl PyPayload for PyListReverseIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.list_reverseiterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.list_reverseiterator_type).to_owned() } } diff --git a/crates/vm/src/builtins/map.rs b/crates/vm/src/builtins/map.rs index 0a62f2d792b..fd7511d8413 100644 --- a/crates/vm/src/builtins/map.rs +++ b/crates/vm/src/builtins/map.rs @@ -20,8 +20,8 @@ pub struct PyMap { impl PyPayload for PyMap { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.map_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.map_type).to_owned() } } diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs index 9ee9ff17228..280a7854353 100644 --- a/crates/vm/src/builtins/mappingproxy.rs +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -1,4 +1,4 @@ -use super::{PyDict, PyDictRef, PyGenericAlias, PyList, PyTuple, PyType, PyTypeRef}; +use super::{PyDict, PyDictRef, PyGenericAlias, PyList, PyStr, PyTuple, PyType, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, @@ -38,8 +38,8 @@ unsafe impl Traverse for MappingProxyInner { impl PyPayload for PyMappingProxy { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.mappingproxy_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.mappingproxy_type).to_owned() } } @@ -98,7 +98,8 @@ impl PyMappingProxy { match class.attributes.as_dict() { Some(dict) => dict.get_item_opt(key, vm), None => Ok(key - .as_interned_str(vm) + .downcast_ref::() + .and_then(|name| class.interned_attr_name(name, vm)) .and_then(|key| class.attributes.get(key))), } } @@ -121,7 +122,8 @@ impl PyMappingProxy { match class.attributes.as_dict() { Some(dict) => dict.contains_key(key, vm), None => key - .as_interned_str(vm) + .downcast_ref::() + .and_then(|name| class.interned_attr_name(name, vm)) .is_some_and(|key| class.attributes.contains(key)), } } @@ -170,16 +172,19 @@ impl PyMappingProxy { } } -#[pyclass(with( - AsMapping, - Iterable, - Constructor, - AsSequence, - Comparable, - Hashable, - AsNumber, - Representable -))] +#[pyclass( + flags(MAPPING), + with( + AsMapping, + Iterable, + Constructor, + AsSequence, + Comparable, + Hashable, + AsNumber, + Representable + ) +)] impl Py { #[pymethod] fn get( diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index 2c9e8c62809..92cbd020e74 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -522,10 +522,10 @@ impl PyMemoryView { if zelf.desc.ndim() == 0 { let a_pos = zelf.desc.offset as usize; let b_pos = other.desc.offset as usize; - let a_bytes = zelf.buffer.obj_bytes(); + let a_bytes = zelf.buffer.obj_bytes_unlocked(vm)?; let a_val = format_unpack(a_format_spec, &a_bytes[a_pos..a_pos + a_itemsize], vm)?; drop(a_bytes); - let b_bytes = other.obj_bytes(); + let b_bytes = other.obj_bytes_unlocked(vm)?; let b_val = format_unpack(b_format_spec, &b_bytes[b_pos..b_pos + b_itemsize], vm)?; drop(b_bytes); return vm.bool_eq(&a_val, &b_val); @@ -533,8 +533,8 @@ impl PyMemoryView { // TODO: optimize cmp by format let mut ret = Ok(true); - let a_bytes = zelf.buffer.obj_bytes(); - let b_bytes = other.obj_bytes(); + let a_bytes = zelf.buffer.obj_bytes_unlocked(vm)?; + let b_bytes = other.obj_bytes_unlocked(vm)?; zelf.desc.zip_eq(&other.desc, false, |a_range, b_range| { let a_range = a_range.start as usize..a_range.start as usize + a_itemsize; let b_range = b_range.start as usize..b_range.start as usize + b_itemsize; @@ -650,19 +650,10 @@ impl Py { )); } - // copy_buffer reads the source as it stood before the copy began, which an - // overlapping assignment depends on and which also keeps the two borrows - // below off the same storage. - let src = if root_exporter(&src).is(&root_exporter(&dest.buffer)) { - let owned = src.to_contiguous(vm); - drop(src); - owned - } else { - src - }; - + // Snapshot mutable sources before locking the destination: independent + // XI views can alias, and concurrent A->B/B->A copies must not deadlock. + let src_bytes = src.obj_bytes_unlocked(vm)?; let mut bytes_mut = dest.buffer.obj_bytes_mut(); - let src_bytes = src.obj_bytes(); dest.desc.zip_eq(&src.desc, true, |a_range, b_range| { let a_range = a_range.start as usize..a_range.end as usize; let b_range = b_range.start as usize..b_range.end as usize; @@ -1273,6 +1264,7 @@ impl TryFromObject for SubscriptNeedle { } static BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| buffer.obj_as::().buffer.shared_storage()), obj_bytes: |buffer| buffer.obj_as::().buffer.obj_bytes(), obj_bytes_mut: |buffer| buffer.obj_as::().buffer.obj_bytes_mut(), // memory_releasebuf / memory_getbuf: a consumer's export of this view is a @@ -1414,8 +1406,8 @@ impl Hashable for PyMemoryView { impl PyPayload for PyMemoryView { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.memoryview_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.memoryview_type).to_owned() } } @@ -1434,12 +1426,12 @@ impl Representable for PyMemoryView { pub(crate) fn init(ctx: &'static Context) { PyMemoryView::extend_class(ctx, ctx.types.memoryview_type); PyMemoryViewIterator::extend_class(ctx, ctx.types.memoryviewiterator_type); - let wrapper_type = PyBufferWrapper::init_builtin_type(); + let wrapper_type = unsafe { PyBufferWrapper::init_builtin_type() }; // bufferwrapper_as_buffer: bf_releasebuffer and no bf_getbuffer, so the type // has `__release_buffer__` but no `__buffer__`. wrapper_type.slots.has_release_buffer.store(true); PyBufferWrapper::extend_class(ctx, wrapper_type); - PyBufferWindow::extend_class(ctx, PyBufferWindow::init_builtin_type()); + PyBufferWindow::extend_class(ctx, unsafe { PyBufferWindow::init_builtin_type() }); } #[pyclass(module = false, name = "_buffer_wrapper", traverse)] @@ -1458,8 +1450,8 @@ struct PyBufferWrapper { } impl PyPayload for PyBufferWrapper { - fn class(_ctx: &Context) -> &'static Py { - Self::static_type() + fn class(_ctx: &Context) -> crate::builtins::PyTypeRef { + (unsafe { Self::static_type() }).to_owned() } } @@ -1467,6 +1459,7 @@ impl PyPayload for PyBufferWrapper { impl PyBufferWrapper {} static BUFFER_WRAPPER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| buffer.obj_as::().view.shared_storage()), obj_bytes: |buffer| buffer.obj_as::().view.obj_bytes(), obj_bytes_mut: |buffer| buffer.obj_as::().view.obj_bytes_mut(), retain: |buffer| { @@ -1511,8 +1504,8 @@ struct PyBufferWindow { } impl PyPayload for PyBufferWindow { - fn class(_ctx: &Context) -> &'static Py { - Self::static_type() + fn class(_ctx: &Context) -> crate::builtins::PyTypeRef { + (unsafe { Self::static_type() }).to_owned() } } @@ -1520,35 +1513,13 @@ impl PyPayload for PyBufferWindow { impl PyBufferWindow {} static BUFFER_WINDOW_METHODS: BufferMethods = BufferMethods { + shared_storage: None, obj_bytes: |buffer| buffer.obj_as::().source.obj_bytes(), obj_bytes_mut: |buffer| buffer.obj_as::().source.obj_bytes_mut(), retain: |_buffer| {}, release: |_buffer| {}, }; -/// The object that ultimately owns the bytes a buffer reads, seen through the -/// payloads that only forward to another export: a view, the wrapper holding what -/// a `__buffer__` returned, and the window handed to `__release_buffer__`. -/// -/// Two buffers that resolve to the same object address the same storage, so -/// borrowing one for writing while the other is borrowed for reading would -/// deadlock on it. -fn root_exporter(buffer: &PyBuffer) -> PyObjectRef { - let mut obj = buffer.obj.clone(); - loop { - let next = if let Some(view) = obj.downcast_ref::() { - view.buffer.obj.clone() - } else if let Some(wrapper) = obj.downcast_ref::() { - wrapper.view.obj.clone() - } else if let Some(window) = obj.downcast_ref::() { - window.source.obj.clone() - } else { - return obj; - }; - obj = next; - } -} - // slot_bf_getbuffer pub(crate) fn buffer_from_python_getbuffer( obj: &PyObject, @@ -1605,7 +1576,7 @@ pub(crate) fn release_buffer_from_python( // releasebuffer_call_python, for a buffer acquired from a native exporter pub(crate) fn release_buffer_call_python(buffer: &PyBuffer) { - crate::vm::thread::try_with_current_vm(|vm| { + crate::vm::thread::with_vm(&buffer.obj, |vm| { let exporter = buffer.obj.clone(); let window = PyBufferWindow { source: buffer.detached(), @@ -1633,7 +1604,7 @@ pub(crate) fn release_buffer_call_python(buffer: &PyBuffer) { } fn call_python_release_buffer(exporter: &PyObject, mv: PyRef) { - crate::vm::thread::try_with_current_vm(|vm| { + crate::vm::thread::with_vm(exporter, |vm| { let method = vm.get_special_method(exporter, identifier!(vm, __release_buffer__)); if let Ok(Some(method)) = method && let Err(exc) = method.invoke((mv,), vm) @@ -1709,8 +1680,8 @@ pub(crate) struct PyMemoryViewIterator { } impl PyPayload for PyMemoryViewIterator { - fn class(ctx: &Context) -> &'static Py { - ctx.types.memoryviewiterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.memoryviewiterator_type).to_owned() } } diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs index c7404fcd169..e8c2361dc88 100644 --- a/crates/vm/src/builtins/mod.rs +++ b/crates/vm/src/builtins/mod.rs @@ -100,8 +100,8 @@ pub(crate) mod union_; pub use union_::{PyUnion, make_union}; pub mod descriptor; pub use descriptor::{ - MemberAccess, MemberKind, PyDescriptorOwned, PyMemberDef as DescriptorMemberDef, - PyMemberDescriptor, PyMemberFlags, + MemberAccess, MemberKind, PyDescriptor, PyMemberDef as DescriptorMemberDef, PyMemberDescriptor, + PyMemberFlags, }; pub use float::float_from_string as parse_float_from_string; diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index aae75da816a..7ab2869184d 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -1,4 +1,4 @@ -use super::{PyDict, PyDictRef, PyStr, PyStrRef, PyType, PyTypeRef, PyUtf8Str}; +use super::{PyDict, PyDictRef, PyStr, PyStrRef, PyTypeRef, PyUtf8Str}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyStrInterned, pystr::AsPyStr}, @@ -98,8 +98,8 @@ pub struct PyModule { impl PyPayload for PyModule { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.module_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.module_type).to_owned() } } @@ -446,7 +446,7 @@ impl Representable for PyModule { #[inline] fn repr(zelf: &Py, vm: &VirtualMachine) -> PyResult { // Use cached importlib reference (like interp->importlib) - let module_repr = vm.importlib.get_attr("_module_repr", vm)?; + let module_repr = vm.importlib().get_attr("_module_repr", vm)?; let repr = module_repr.call((zelf.to_owned(),), vm)?; repr.downcast() .map_err(|_| vm.new_type_error("_module_repr did not return a string")) diff --git a/crates/vm/src/builtins/namespace.rs b/crates/vm/src/builtins/namespace.rs index a872fd6c92e..9e74ff980b4 100644 --- a/crates/vm/src/builtins/namespace.rs +++ b/crates/vm/src/builtins/namespace.rs @@ -1,4 +1,4 @@ -use super::{PyStr, PyTupleRef, PyType, tuple::IntoPyTuple}; +use super::{PyStr, PyTupleRef, tuple::IntoPyTuple}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyDict, @@ -20,8 +20,8 @@ pub struct PyNamespace {} impl PyPayload for PyNamespace { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.namespace_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.namespace_type).to_owned() } } diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index d7de843a7a0..d5a7bb53ba2 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -19,8 +19,8 @@ pub struct PyBaseObject; impl PyPayload for PyBaseObject { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.object_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.object_type).to_owned() } } diff --git a/crates/vm/src/builtins/property.rs b/crates/vm/src/builtins/property.rs index 23185df1a01..2c13a038fca 100644 --- a/crates/vm/src/builtins/property.rs +++ b/crates/vm/src/builtins/property.rs @@ -58,8 +58,8 @@ pub struct PyProperty { impl PyPayload for PyProperty { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.property_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.property_type).to_owned() } } diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index e8d2c30dbce..f2904a8adc1 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -1,5 +1,5 @@ use super::{ - PyGenericAlias, PyInt, PyIntRef, PySlice, PyTupleRef, PyType, PyTypeRef, builtins_iter, + PyGenericAlias, PyInt, PyIntRef, PySlice, PyTupleRef, PyTypeRef, builtins_iter, tuple::tuple_hash, }; use crate::common::lock::LazyLock; @@ -82,8 +82,8 @@ impl PyPayload for PyRange { const HAS_FREELIST: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.range_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.range_type).to_owned() } #[inline] @@ -585,8 +585,8 @@ pub(crate) struct PyLongRangeIterator { impl PyPayload for PyLongRangeIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.long_range_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.long_range_iterator_type).to_owned() } } @@ -650,8 +650,8 @@ pub(crate) struct PyRangeIterator { impl PyPayload for PyRangeIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.range_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.range_iterator_type).to_owned() } } diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index e3e8ade64c4..7e1fdaede81 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -162,15 +162,15 @@ impl fmt::Debug for PyFrozenSet { impl PyPayload for PySet { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.set_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.set_type).to_owned() } } impl PyPayload for PyFrozenSet { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.frozenset_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.frozenset_type).to_owned() } } @@ -1670,8 +1670,8 @@ impl fmt::Debug for PySetIterator { impl PyPayload for PySetIterator { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.set_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.set_iterator_type).to_owned() } } diff --git a/crates/vm/src/builtins/singletons.rs b/crates/vm/src/builtins/singletons.rs index c28be2dd932..c7ea0b4229e 100644 --- a/crates/vm/src/builtins/singletons.rs +++ b/crates/vm/src/builtins/singletons.rs @@ -14,9 +14,10 @@ use crate::{ pub struct PyNone; impl PyPayload for PyNone { + const OWNER_NEUTRAL: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.none_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.none_type).to_owned() } } @@ -100,9 +101,10 @@ impl Hashable for PyNone { pub struct PyNotImplemented; impl PyPayload for PyNotImplemented { + const OWNER_NEUTRAL: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.not_implemented_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.not_implemented_type).to_owned() } } diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index 8cc53fd16c1..a2f48bbc8d4 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -56,8 +56,8 @@ impl PyPayload for PySlice { const HAS_FREELIST: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.slice_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.slice_type).to_owned() } #[inline] @@ -370,8 +370,8 @@ pub struct PyEllipsis; impl PyPayload for PyEllipsis { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.ellipsis_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.ellipsis_type).to_owned() } } diff --git a/crates/vm/src/builtins/staticmethod.rs b/crates/vm/src/builtins/staticmethod.rs index 387e661a878..b0c4f34c1f5 100644 --- a/crates/vm/src/builtins/staticmethod.rs +++ b/crates/vm/src/builtins/staticmethod.rs @@ -22,8 +22,8 @@ pub struct PyStaticMethod { impl PyPayload for PyStaticMethod { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.staticmethod_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.staticmethod_type).to_owned() } } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index c66b87cfb5c..5a86db0eedb 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -321,8 +321,8 @@ unsafe impl Traverse for PyStrIterator { } impl PyPayload for PyStrIterator { - fn class(ctx: &Context) -> &'static Py { - ctx.types.str_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.str_iterator_type).to_owned() } } @@ -1777,9 +1777,10 @@ pub(crate) fn encode_string( } impl PyPayload for PyStr { + const OWNER_NEUTRAL: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.str_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.str_type).to_owned() } } @@ -2088,7 +2089,7 @@ impl fmt::Display for PyUtf8Str { } impl MaybeTraverse for PyUtf8Str { - const HAS_TRAVERSE: bool = true; + const HAS_TRAVERSE: bool = ::HAS_TRAVERSE; const HAS_CLEAR: bool = false; fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>) { @@ -2101,9 +2102,10 @@ impl MaybeTraverse for PyUtf8Str { } impl PyPayload for PyUtf8Str { + const OWNER_NEUTRAL: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.str_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.str_type).to_owned() } const PAYLOAD_TYPE_ID: core::any::TypeId = core::any::TypeId::of::(); @@ -2764,7 +2766,7 @@ mod tests { #[test] fn str_maketrans_and_translate() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let table = vm.ctx.new_dict(); table .set_item("a", vm.ctx.new_str("🎅").into(), vm) diff --git a/crates/vm/src/builtins/super.rs b/crates/vm/src/builtins/super.rs index 9eb76e9df0e..ad7a6a3277d 100644 --- a/crates/vm/src/builtins/super.rs +++ b/crates/vm/src/builtins/super.rs @@ -60,8 +60,8 @@ impl PySuper { impl PyPayload for PySuper { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.super_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.super_type).to_owned() } } @@ -188,7 +188,28 @@ fn super_init_without_args(vm: &VirtualMachine) -> PyResult<(PyTypeRef, PyObject with(GetAttr, GetDescriptor, Constructor, Initializer, Representable), flags(BASETYPE) )] -impl PySuper {} +impl PySuper { + #[extend_class] + fn extend_super(ctx: &crate::Context, class: &Py) { + const SUPER_DOC: &str = "\ +super() -> same as super(__class__, ) +super(type) -> unbound super object +super(type, obj) -> bound super object; requires isinstance(obj, type) +super(type, type2) -> bound super object; requires issubclass(type2, type) +Typical use to call a cooperative superclass method: +class C(B): + def meth(self, arg): + super().meth(arg) +This works for class methods too: +class C(B): + @classmethod + def cmeth(cls, arg): + super().cmeth(arg) +"; + + class.set_attr(ctx.names.__doc__, ctx.new_str(SUPER_DOC).into()); + } +} impl GetAttr for PySuper { fn getattro(zelf: &Py, name: &Py, vm: &VirtualMachine) -> PyResult { @@ -206,18 +227,15 @@ impl GetAttr for PySuper { return skip(zelf, name); } - if let Some(name) = vm.ctx.interned_str(name) { - // Walk start_type's MRO by reference (no Vec allocation, no - // per-class clone) up to and including zelf.typ, then look for - // the first class past it that declares `name` directly. - // Both locks are dropped before any arbitrary Python code - // (the descriptor call below) runs, so they can't be held - // across a call that might re-enter and want them again. + if let Some(name) = start_type.interned_attr_name(name, vm) { + // Walk a snapshot past zelf.typ to the first direct declaration. + // Release the MRO lock before reading namespaces, whose lazy + // initialization can re-enter type code, and calling descriptors. let Some(su_type) = zelf.typ.load_owned() else { return skip(zelf, name); }; let descr = { - let mro = start_type.mro.read(); + let mro = start_type.mro.read().clone(); mro.iter() .skip_while(|cls| !cls.is(&su_type)) .skip(1) // skip su->type (if any) @@ -319,26 +337,5 @@ fn super_check(ty: &Py, obj: &PyObject, vm: &VirtualMachine) -> PyResult } pub(crate) fn init(context: &'static Context) { - let super_type = &context.types.super_type; - PySuper::extend_class(context, super_type); - - const SUPER_DOC: &str = "\ -super() -> same as super(__class__, ) -super(type) -> unbound super object -super(type, obj) -> bound super object; requires isinstance(obj, type) -super(type, type2) -> bound super object; requires issubclass(type2, type) -Typical use to call a cooperative superclass method: -class C(B): - def meth(self, arg): - super().meth(arg) -This works for class methods too: -class C(B): - @classmethod - def cmeth(cls, arg): - super().cmeth(arg) -"; - - extend_class!(context, super_type, { - "__doc__" => context.new_str(SUPER_DOC), - }); + PySuper::extend_class(context, context.types.super_type); } diff --git a/crates/vm/src/builtins/template.rs b/crates/vm/src/builtins/template.rs index b072f4afc81..7757bd9c073 100644 --- a/crates/vm/src/builtins/template.rs +++ b/crates/vm/src/builtins/template.rs @@ -27,8 +27,8 @@ pub struct PyTemplate { impl PyPayload for PyTemplate { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.template_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.template_type).to_owned() } } @@ -272,8 +272,8 @@ pub struct PyTemplateIter { impl PyPayload for PyTemplateIter { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.template_iter_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.template_iter_type).to_owned() } } diff --git a/crates/vm/src/builtins/traceback.rs b/crates/vm/src/builtins/traceback.rs index a91ec13c86f..e76adebb116 100644 --- a/crates/vm/src/builtins/traceback.rs +++ b/crates/vm/src/builtins/traceback.rs @@ -23,8 +23,8 @@ pub(crate) type PyTracebackRef = PyRef; impl PyPayload for PyTraceback { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.traceback_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.traceback_type).to_owned() } } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index a1c06e49106..4382870d51c 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -8,7 +8,7 @@ use crate::object::{Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, - class::{PyClassDef, PyClassImpl}, + class::{PyClassDef, PyClassImpl, StaticType}, convert::{ToPyObject, TransmuteFromObject}, function::{FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue, PySsize}, iter::PyExactSizeIterator, @@ -119,9 +119,19 @@ impl PyPayload for PyTuple { const MAX_FREELIST: usize = 2000; const HAS_FREELIST: bool = true; + fn gc_is_acyclic(&self, class: &Py) -> bool { + unsafe { Self::static_cell() } + .get() + .is_some_and(|base| class.is(base)) + && self + .elements + .iter() + .all(|obj| obj.gc_is_atomic() || obj.gc_is_acyclic()) + } + #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.tuple_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.tuple_type).to_owned() } #[inline] @@ -728,8 +738,8 @@ pub(crate) struct PyTupleIterator { } impl PyPayload for PyTupleIterator { - fn class(ctx: &Context) -> &'static Py { - ctx.types.tuple_iterator_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.tuple_iterator_type).to_owned() } } diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 3eb9c94df52..c7575ed4dd0 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -8,8 +8,7 @@ use crate::{ builtins::{ PyBaseExceptionRef, descriptor::{ - MemberAccess, MemberKind, PyDescriptorOwned, PyMemberDef, PyMemberDescriptor, - PyMemberFlags, + MemberAccess, MemberKind, PyDescriptor, PyMemberDef, PyMemberDescriptor, PyMemberFlags, }, function::{PyCellRef, PyFunction}, tuple::{IntoPyTuple, PyTuple}, @@ -23,6 +22,7 @@ use crate::{ ArgumentError, FromArgs, FuncArgs, ItemDoc, KwArgs, Param, PyMethodDef, PySetterValue, db_doc, }, + intern::MaybeInternedString, object::{Traverse, TraverseFn}, protocol::{PyIterReturn, PyNumberMethods}, types::{ @@ -34,7 +34,7 @@ use crate::{ use core::{ any::Any, borrow::Borrow, - cell::Cell, + cell::RefCell, ops::Deref, pin::Pin, sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, Ordering}, @@ -58,7 +58,6 @@ pub struct PyType { pub attributes: TypeNamespace, #[pymember(name = "__itemsize__", path = "itemsize")] #[pymember(name = "__basicsize__", path = "basicsize")] - #[pymember(name = "__flags__", path = "flags")] pub slots: PyTypeSlots, pub heaptype_ext: Option>>, /// Type version tag for inline caching. 0 means unassigned/invalidated. @@ -104,7 +103,7 @@ struct TypeCacheEntry { name: AtomicPtr, /// Cached lookup result as raw pointer. null = empty. /// The cache holds a **borrowed** pointer (no refcount increment). - /// Safety: `type_cache_clear()` nullifies all entries during GC, + /// Safety: `TypeCache::clear()` nullifies all entries during GC, /// and `type_cache_clear_version()` nullifies entries when a type /// is modified — both before the source dict entry is removed. /// Types are always part of reference cycles (via `mro` self-reference) @@ -184,19 +183,25 @@ impl TypeCacheEntry { } } -// std::sync::LazyLock is used here (not crate::common::lock::LazyLock) -// because TYPE_CACHE is a global shared across test threads. The common -// LazyLock delegates to LazyCell in non-threading mode, which is !Sync. -static TYPE_CACHE: std::sync::LazyLock> = std::sync::LazyLock::new(|| { - (0..TYPE_CACHE_SIZE) - .map(|_| TypeCacheEntry::new()) - .collect::>() - .into_boxed_slice() -}); +/// Borrowed lookup results never leave the interpreter that owns their +/// namespace. Type version tags remain globally unique. +pub(crate) struct TypeCache { + entries: Box<[TypeCacheEntry]>, + clearing: AtomicBool, + closed: AtomicBool, +} -/// When true, find_name_in_mro skips populating the cache. -/// Set during GC's type_cache_clear to prevent re-population from drops. -static TYPE_CACHE_CLEARING: AtomicBool = AtomicBool::new(false); +impl Default for TypeCache { + fn default() -> Self { + Self { + entries: (0..TYPE_CACHE_SIZE) + .map(|_| TypeCacheEntry::new()) + .collect(), + clearing: AtomicBool::new(false), + closed: AtomicBool::new(false), + } + } +} /// MCACHE_HASH: XOR of version and name pointer hash, masked to cache size. #[inline] @@ -208,52 +213,57 @@ fn type_cache_hash(version: u32, name: &'static PyStrInterned) -> usize { /// Invalidate cache entries for a specific version tag. /// Called from modified() when a type is changed. fn type_cache_clear_version(version: u32) { - for entry in TYPE_CACHE.iter() { - if entry.version.load(Ordering::Relaxed) == version { - entry.begin_write(); + crate::vm::thread::try_with_current_vm(|vm| vm.state.type_cache.clear_version(version)); +} + +impl TypeCache { + pub(crate) fn close(&self) { + self.closed.store(true, Ordering::Release); + self.clear(); + } + + fn available(&self, vm: &VirtualMachine) -> bool { + !self.closed.load(Ordering::Acquire) + && !crate::vm::native_types::initializing(vm.state.interpreter_id) + } + + fn clear_version(&self, version: u32) { + for entry in &self.entries { if entry.version.load(Ordering::Relaxed) == version { - entry.version.store(0, Ordering::Release); - entry.clear_value(); + entry.begin_write(); + if entry.version.load(Ordering::Relaxed) == version { + entry.version.store(0, Ordering::Release); + entry.clear_value(); + } + entry.end_write(); } - entry.end_write(); } } -} -/// Clear all method cache entries (_PyType_ClearCache). -/// Called during GC collection to nullify borrowed pointers before -/// the collector breaks cycles. -/// -/// Sets TYPE_CACHE_CLEARING to suppress cache re-population during the -/// entire operation, preventing concurrent lookups from repopulating -/// entries while we're clearing them. -pub(crate) fn type_cache_clear() { - TYPE_CACHE_CLEARING.store(true, Ordering::Release); - for entry in TYPE_CACHE.iter() { - entry.begin_write(); - entry.version.store(0, Ordering::Release); - entry.clear_value(); - entry.end_write(); + pub(crate) fn clear(&self) { + self.clearing.store(true, Ordering::Release); + for entry in &self.entries { + entry.begin_write(); + entry.version.store(0, Ordering::Release); + entry.clear_value(); + entry.end_write(); + } + self.clearing.store(false, Ordering::Release); } - TYPE_CACHE_CLEARING.store(false, Ordering::Release); -} -/// Repair type-cache SeqLock state in the post-fork child. -/// -/// If fork happens while a writer holds an entry SeqLock, the child inherits -/// the odd sequence value with no surviving writer to release it. Clear only -/// those in-progress entries, matching `_PyTypes_AfterFork()`. -#[cfg(all(feature = "host_env", unix))] -pub(crate) unsafe fn type_cache_after_fork() { - for entry in TYPE_CACHE.iter() { - let seq = entry.sequence.load(Ordering::Relaxed); - if (seq & 1) == 0 { - continue; + /// A vanished fork writer may have left its sequence odd. + #[cfg(all(feature = "host_env", unix))] + pub(crate) unsafe fn after_fork(&self) { + for entry in &self.entries { + if entry.sequence.load(Ordering::Relaxed) & 1 == 0 { + continue; + } + entry.value.store(core::ptr::null_mut(), Ordering::Relaxed); + entry.name.store(core::ptr::null_mut(), Ordering::Relaxed); + entry.version.store(0, Ordering::Relaxed); + entry.sequence.store(0, Ordering::Relaxed); } - entry.value.store(core::ptr::null_mut(), Ordering::Relaxed); - entry.name.store(core::ptr::null_mut(), Ordering::Relaxed); - entry.version.store(0, Ordering::Relaxed); - entry.sequence.store(0, Ordering::Relaxed); + self.clearing.store(false, Ordering::Relaxed); } } @@ -283,7 +293,10 @@ unsafe impl crate::object::Traverse for PyType { // Clone the empty tuple before taking the write lock: `object`'s // `bases` is this same lock, and `.read()` while exclusive panics // on CellRwLock (and hangs on parking_lot). - let empty = object::PyBaseObject::static_type().bases.read().clone(); + let empty = unsafe { object::PyBaseObject::static_type() } + .bases + .read() + .clone(); if let Some(mut bases) = self.bases.try_write() { let old_bases = core::mem::replace(&mut *bases, empty); out.push(old_bases.into_untyped().into()); @@ -315,12 +328,14 @@ pub struct HeapTypeExt { /// The interpreter this type was created in, or `None` for the types the /// shared context builds before any interpreter exists. pub interpreter_id: Option, + /// Rust payload definition, independent of the Python identity of this VM's type. + native_definition: Option, } impl HeapTypeExt { /// The interpreter a type created right now belongs to. fn creating_interpreter_id() -> Option { - crate::vm::thread::try_with_current_vm(|vm| vm.state.interpreter_id) + crate::gc_state::current_interpreter_id() } } @@ -427,14 +442,17 @@ unsafe impl Traverse for PyAttributes { /// The types `Context::genesis` builds cannot have one: hashing a string needs /// a VM and none exists yet, so they keep an interned-key map instead. pub enum TypeNamespace { - Attributes(PyRwLock), + Attributes( + PyRwLock, + std::sync::OnceLock, + ), Dict(PyDictRef), } unsafe impl Traverse for TypeNamespace { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { match self { - Self::Attributes(attrs) => { + Self::Attributes(attrs, _) => { if let Some(attrs) = attrs.try_read_recursive() { attrs.traverse(tracer_fn); } @@ -446,21 +464,47 @@ unsafe impl Traverse for TypeNamespace { impl Default for TypeNamespace { fn default() -> Self { - Self::Attributes(PyRwLock::default()) + Self::Attributes(PyRwLock::default(), std::sync::OnceLock::new()) } } impl From for TypeNamespace { fn from(attrs: PyAttributes) -> Self { - Self::Attributes(PyRwLock::new(attrs)) + Self::Attributes(PyRwLock::new(attrs), std::sync::OnceLock::new()) } } impl TypeNamespace { + pub(crate) fn define_native( + &self, + definition: crate::vm::native_types::NativeNamespaceDefinition, + ) { + let Self::Attributes(_, slot) = self else { + panic!("native definition installed on a heap namespace"); + }; + assert!( + slot.set(definition).is_ok(), + "native namespace defined twice" + ); + } + + fn local(&self) -> Option> { + let Self::Attributes(_, definition) = self else { + return None; + }; + let definition = definition.get()?; + let owner = crate::gc_state::current_interpreter_id()?; + crate::vm::thread::try_with_current_vm(|vm| { + (vm.state.interpreter_id == owner) + .then(|| vm.state.native_types.namespace(definition, vm)) + }) + .flatten() + } + /// The namespace as a dict, for the types that have one. pub fn as_dict(&self) -> Option<&Py> { match self { - Self::Attributes(_) => None, + Self::Attributes(_, _) => None, Self::Dict(dict) => Some(dict), } } @@ -482,15 +526,21 @@ impl TypeNamespace { } pub fn get(&self, name: &'static PyStrInterned) -> Option { + if let Some(local) = self.local() { + return local.get(name); + } match self { - Self::Attributes(attrs) => attrs.read().get(name).cloned(), + Self::Attributes(attrs, _) => attrs.read().get(name).cloned(), Self::Dict(dict) => dict_get(dict, name), } } pub fn contains(&self, name: &'static PyStrInterned) -> bool { + if let Some(local) = self.local() { + return local.contains(name); + } match self { - Self::Attributes(attrs) => attrs.read().contains_key(name), + Self::Attributes(attrs, _) => attrs.read().contains_key(name), Self::Dict(dict) => { match crate::vm::thread::try_with_current_vm(|vm| dict.contains_key(name, vm)) { Some(found) => found, @@ -508,8 +558,11 @@ impl TypeNamespace { /// Bind `name` to `value` and hand back what it displaced, so the caller /// can decide where the old value is dropped. pub fn insert(&self, name: &'static PyStrInterned, value: PyObjectRef) -> Option { + if let Some(local) = self.local() { + return local.insert(name, value); + } match self { - Self::Attributes(attrs) => attrs.write().insert(name, value), + Self::Attributes(attrs, _) => attrs.write().insert(name, value), Self::Dict(dict) => { let previous = dict_get(dict, name); if let Some(Err(_)) | None = @@ -523,8 +576,11 @@ impl TypeNamespace { } pub fn remove(&self, name: &'static PyStrInterned) -> Option { + if let Some(local) = self.local() { + return local.remove(name); + } match self { - Self::Attributes(attrs) => attrs.write().shift_remove(name), + Self::Attributes(attrs, _) => attrs.write().shift_remove(name), Self::Dict(dict) => { let previous = dict_get(dict, name)?; crate::vm::thread::try_with_current_vm(|vm| dict.del_item(name, vm).ok()); @@ -535,8 +591,11 @@ impl TypeNamespace { /// The namespace contents in insertion order. pub fn entries(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + if let Some(local) = self.local() { + return local.entries(); + } match self { - Self::Attributes(attrs) => attrs + Self::Attributes(attrs, _) => attrs .read() .iter() .map(|(name, value)| ((*name).to_object(), value.clone())) @@ -548,8 +607,11 @@ impl TypeNamespace { /// The namespace keyed by interned name; dict keys that are not strings /// are left out, since `PyAttributes` has nowhere to put them. pub fn attributes(&self, ctx: &Context) -> PyAttributes { + if let Some(local) = self.local() { + return local.attributes(ctx); + } match self { - Self::Attributes(attrs) => attrs.read().clone(), + Self::Attributes(attrs, _) => attrs.read().clone(), Self::Dict(dict) => dict .into_iter() .filter_map(|(key, value)| { @@ -563,8 +625,11 @@ impl TypeNamespace { /// The interned names in the namespace; dict keys that are not strings are /// left out. pub fn interned_names(&self, ctx: &Context) -> Vec<&'static PyStrInterned> { + if let Some(local) = self.local() { + return local.interned_names(ctx); + } match self { - Self::Attributes(attrs) => attrs.read().keys().copied().collect(), + Self::Attributes(attrs, _) => attrs.read().keys().copied().collect(), Self::Dict(dict) => dict .into_iter() .filter_map(|(key, _)| { @@ -578,7 +643,7 @@ impl TypeNamespace { /// Empty the namespace, handing the values to the caller. Used by tp_clear. fn drain_into(&mut self, out: &mut Vec) { match self { - Self::Attributes(attrs) => { + Self::Attributes(attrs, _) => { if let Some(mut guard) = attrs.try_write() { out.extend(guard.drain(..).map(|(_, value)| value)); } @@ -620,8 +685,8 @@ impl core::fmt::Debug for PyType { impl PyPayload for PyType { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.type_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.type_type).to_owned() } } @@ -651,28 +716,38 @@ fn is_subtype_with_mro(a_mro: &[PyTypeRef], a: &Py, b: &Py) -> b a_mro.iter().any(|item| item.is(b)) } +thread_local! { + static HELD_TYPE_LOCKS: RefCell> = const { RefCell::new(Vec::new()) }; +} + impl PyType { + #[cfg(all(unix, feature = "threading"))] + pub(crate) unsafe fn repair_type_lock_after_fork(state: &crate::vm::PyGlobalState) { + if !HELD_TYPE_LOCKS.with(|held| held.borrow().contains(&state.interpreter_id)) { + unsafe { crate::common::lock::zero_reinit_after_fork(state.type_mutex.raw()) }; + } + } + #[inline] - fn with_type_lock(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + pub(crate) fn with_type_lock(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { // Drops deferred via try_defer_drop inside the critical section run // after the guard is released, outside the lock. // // The lock is reentrant on the same thread so a custom mro() can // assign __bases__ (and re-enter here) without deadlocking. - thread_local! { - static HELD: Cell = const { Cell::new(false) }; - } rustpython_common::refcount::with_deferred_drops(|| { - HELD.with(|held| { - if held.get() { - f() - } else { - let _guard = vm.state.type_mutex.lock(); - held.set(true); - let result = f(); - held.set(false); - result + HELD_TYPE_LOCKS.with(|held| { + let owner = vm.state.interpreter_id; + if held.borrow().contains(&owner) { + return f(); } + let _lock = vm.state.type_mutex.write(); + held.borrow_mut().push(owner); + let _recursion = scopeguard::guard((), |()| { + let previous = held.borrow_mut().pop(); + debug_assert_eq!(previous, Some(owner)); + }); + f() }) }) } @@ -708,6 +783,9 @@ impl PyType { } pub(crate) fn version_for_specialization(&self, vm: &VirtualMachine) -> u32 { + if !vm.state.type_cache.available(vm) { + return 0; + } let version = self.tp_version_tag.load(Ordering::Acquire); if version != 0 { return version; @@ -728,11 +806,8 @@ impl PyType { if old_version == 0 { return; } - let subclasses = self.subclasses.read(); - for weak_ref in subclasses.iter() { - if let Some(sub) = weak_ref.upgrade() { - sub.downcast_ref::().unwrap().modified_inner(); - } + for sub in self.subclass_snapshot() { + sub.modified_inner(); } self.tp_version_tag.store(0, Ordering::SeqCst); // Nullify borrowed pointers in cache entries for this version @@ -776,6 +851,65 @@ impl PyType { } } + fn local_subclasses(&self) -> Option>>>> { + if self.heaptype_ext.is_some() { + return None; + } + let owner = crate::gc_state::current_interpreter_id()?; + crate::vm::thread::try_with_current_vm(|vm| { + (vm.state.interpreter_id == owner).then(|| vm.state.native_types.subclasses(self)) + }) + .flatten() + } + + fn with_subclass_registry(&self, f: impl FnOnce(&PyRwLock>>) -> R) -> R { + if let Some(registry) = self.local_subclasses() { + f(®istry) + } else { + f(&self.subclasses) + } + } + + fn add_subclass(&self, subclass: &Py) { + let weak = subclass + .as_object() + .downgrade_with_weakref_typ_opt( + None, + unsafe { super::PyWeak::static_type() }.to_owned(), + ) + .unwrap(); + self.with_subclass_registry(|registry| registry.write().push(weak)); + } + + pub(crate) fn subclass_snapshot(&self) -> Vec { + fn snapshot(registry: &PyRwLock>>) -> Vec { + let mut retired = Vec::new(); + let mut subclasses = registry.write(); + let mut live = Vec::with_capacity(subclasses.len()); + subclasses.retain(|weak| { + if let Some(subclass) = weak.upgrade_visible() { + live.push(subclass.downcast::().unwrap()); + true + } else if weak.is_dead() { + retired.push(weak.clone()); + false + } else { + true + } + }); + drop(subclasses); + drop(retired); + live + } + // Shared native parents retain only other shared native classes. + // Their private descendants are rooted in the current interpreter. + let mut live = snapshot(&self.subclasses); + if let Some(registry) = self.local_subclasses() { + live.extend(snapshot(®istry)); + } + live + } + pub fn new_simple_heap( name: &str, base: &Py, @@ -786,7 +920,7 @@ impl PyType { vec![base.to_owned()], Default::default(), Default::default(), - Self::static_type().to_owned(), + unsafe { Self::static_type() }.to_owned(), ctx, ) } @@ -813,6 +947,7 @@ impl PyType { type_data: PyRwLock::new(None), specialization_cache: TypeSpecializationCache::new(), interpreter_id: HeapTypeExt::creating_interpreter_id(), + native_definition: None, }; let bases = PyTuple::new_ref_typed(bases, ctx); let base = bases.as_slice()[0].clone(); @@ -829,6 +964,48 @@ impl PyType { ) } + pub(crate) fn new_native_heap( + name: &str, + base: PyTypeRef, + slots: PyTypeSlots, + metaclass: PyTypeRef, + ctx: &Context, + ) -> Result { + slots.flags.insert(PyTypeFlags::HEAPTYPE); + let name = ctx.new_utf8_str(name); + let ext = HeapTypeExt { + qualname: PyRwLock::new(name.clone().into_wtf8()), + name: PyRwLock::new(name), + slots: None, + type_data: PyRwLock::new(None), + specialization_cache: TypeSpecializationCache::new(), + interpreter_id: HeapTypeExt::creating_interpreter_id(), + native_definition: Some(core::any::TypeId::of::()), + }; + let bases = PyTuple::new_ref_typed(vec![base.clone()], ctx); + Self::new_heap_inner( + base, + bases, + Default::default(), + slots, + ext, + metaclass, + ctx, + false, + ) + } + + /// Only for Rust payload validation. Python subclass checks use actual types. + pub fn is_native_subclass(&self) -> bool { + let key = core::any::TypeId::of::(); + self.mro.read().iter().any(|class| { + class + .heaptype_ext + .as_ref() + .is_some_and(|ext| ext.native_definition == Some(key)) + }) + } + /// Equivalent to CPython's PyType_Check macro /// Checks if obj is an instance of type (or its subclass) pub(crate) fn check(obj: &PyObject) -> Option<&Py> { @@ -861,7 +1038,7 @@ impl PyType { // Check each base in order and inherit the first collection flag found for base in bases { - let base_flags = base.slots.flags.load() & PyTypeFlags::COLLECTION; + let base_flags = base.effective_flags() & PyTypeFlags::COLLECTION; if !base_flags.is_empty() { slots.flags |= base_flags; return; @@ -870,34 +1047,55 @@ impl PyType { } pub fn has_patma_collection_flag(&self, flag: u8) -> bool { - debug_assert!(flag == PyTypeFlags::SEQUENCE || flag == PyTypeFlags::MAPPING); - self.slots.flags.has_feature(flag) + debug_assert!(matches!(flag, PyTypeFlags::SEQUENCE | PyTypeFlags::MAPPING)); + self.effective_flags().contains(&flag) } - pub fn set_is_abstract(&self, is_abstract: bool) { - if is_abstract { - self.slots.flags.insert(PyTypeFlags::IS_ABSTRACT); + pub fn effective_flags(&self) -> PyTypeFlags { + let flags = self.slots.flags.load(); + if self.heaptype_ext.is_some() || crate::gc_state::current_interpreter_id().is_none() { + return flags; + } + crate::vm::thread::try_with_current_vm(|vm| vm.state.native_types.runtime_flags(self)) + .flatten() + .map_or(flags, |local| { + (flags & !crate::vm::native_types::RUNTIME_FLAGS) | local + }) + } + + fn set_runtime_flags(&self, mask: PyTypeFlags, flags: PyTypeFlags) { + if self.heaptype_ext.is_some() { + self.slots.flags.replace_masked(mask, flags); } else { - self.slots.flags.remove(PyTypeFlags::IS_ABSTRACT); + crate::vm::thread::with_current_vm(|vm| { + vm.state.native_types.set_runtime_flags(self, mask, flags); + }); } self.modified(); } + pub fn set_is_abstract(&self, is_abstract: bool) { + self.set_runtime_flags( + PyTypeFlags::from_element(PyTypeFlags::IS_ABSTRACT), + if is_abstract { + PyTypeFlags::from_element(PyTypeFlags::IS_ABSTRACT) + } else { + PyTypeFlags::empty() + }, + ); + } + pub fn set_abc_collection_flags_recursive(&self, flags: PyTypeFlags) { let flags = flags & PyTypeFlags::COLLECTION; - if flags.is_empty() { + if flags.is_empty() + || self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) + || self.effective_flags() & PyTypeFlags::COLLECTION == flags + { return; } - self.slots - .flags - .replace_masked(PyTypeFlags::COLLECTION, flags); - self.modified(); - for weak_ref in self.subclasses.read().iter() { - if let Some(subclass) = weak_ref.upgrade() - && let Some(subclass) = subclass.downcast_ref::() - { - subclass.set_abc_collection_flags_recursive(flags); - } + self.set_runtime_flags(PyTypeFlags::COLLECTION, flags); + for subclass in self.subclass_snapshot() { + subclass.set_abc_collection_flags_recursive(flags); } } @@ -970,6 +1168,9 @@ impl PyType { ctx: &Context, defer_mro: bool, ) -> Result, String> { + if slots.native_layout_id.is_none() { + slots.native_layout_id = base.slots.native_layout_id; + } let mro = if defer_mro { // Leave tp_mro unset so a custom metaclass mro() sees __mro__ is None. Vec::new() @@ -1042,17 +1243,22 @@ impl PyType { ); if !defer_mro { new_type.mro.write().insert(0, new_type.clone()); - new_type.init_slots(ctx); + if new_type + .heaptype_ext + .as_ref() + .unwrap() + .native_definition + .is_none() + { + new_type.init_slots(ctx); + } else { + Self::set_new(&new_type.slots, new_type.base.deref()); + Self::set_alloc(&new_type.slots, new_type.base.deref()); + } } - let weakref_type = super::PyWeak::static_type(); for base in new_type.bases.read().as_slice() { - base.subclasses.write().push( - new_type - .as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); + base.add_subclass(&new_type); } Ok(new_type) @@ -1084,8 +1290,10 @@ impl PyType { slots.flags.insert(PyTypeFlags::MANAGED_WEAKREF); } - let bases = - PyTuple::new_ref_typed_with_type(vec![base.clone()], PyTuple::static_type().to_owned()); + let bases = PyTuple::new_ref_typed_with_type( + vec![base.clone()], + unsafe { PyTuple::static_type() }.to_owned(), + ); let mro = base.mro_map_collect(|x| x.to_owned()); let new_type = PyRef::new_ref( @@ -1122,14 +1330,8 @@ impl PyType { Self::set_new(&new_type.slots, new_type.base.deref()); Self::set_alloc(&new_type.slots, new_type.base.deref()); - let weakref_type = super::PyWeak::static_type(); for base in new_type.bases.read().as_slice() { - base.subclasses.write().push( - new_type - .as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); + base.add_subclass(&new_type); } Ok(new_type) @@ -1152,7 +1354,7 @@ impl PyType { let mut slot_name_set = std::collections::HashSet::new(); // mro[0] is self, so skip it; self.attributes is checked separately below - for cls in &self.mro.read()[1..] { + for cls in &mro { for name in cls.attributes.interned_names(ctx) { if name.as_bytes().starts_with(b"__") && name.as_bytes().ends_with(b"__") { slot_name_set.insert(name); @@ -1265,10 +1467,13 @@ impl PyType { #[cfg(all(feature = "threading", debug_assertions))] crate::vm::thread::debug_assert_current_thread_attached(); + if !vm.state.type_cache.available(vm) { + return (self.find_name_in_mro_uncached(name), 0); + } let version = self.tp_version_tag.load(Ordering::Acquire); if version != 0 { let idx = type_cache_hash(version, name); - let entry = &TYPE_CACHE[idx]; + let entry = &vm.state.type_cache.entries[idx]; let name_ptr = name as *const _ as *mut _; loop { let seq1 = entry.begin_read(); @@ -1306,13 +1511,18 @@ impl PyType { }; let result = self.find_name_in_mro_uncached(name); if assigned != 0 - && !TYPE_CACHE_CLEARING.load(Ordering::Acquire) + && vm.state.type_cache.available(vm) + && !vm.state.type_cache.clearing.load(Ordering::Acquire) && self.tp_version_tag.load(Ordering::Acquire) == assigned { let idx = type_cache_hash(assigned, name); - let entry = &TYPE_CACHE[idx]; + let entry = &vm.state.type_cache.entries[idx]; let name_ptr = name as *const _ as *mut _; entry.begin_write(); + if !vm.state.type_cache.available(vm) { + entry.end_write(); + return (result, 0); + } entry.version.store(0, Ordering::Release); let new_ptr = result.as_ref().map_or(core::ptr::null_mut(), |found| { // Defer memory reclamation of cached values via QSBR so @@ -1459,12 +1669,41 @@ impl PyType { /// _PyType_LookupRef: look up a name through the MRO without setting an exception. pub fn lookup_ref(&self, name: &Py, vm: &VirtualMachine) -> Option { - let interned_name = vm.ctx.interned_str(name)?; + let interned_name = self.interned_attr_name(name, vm)?; self.lookup_ref_and_version_interned(interned_name, vm).0 } + pub(crate) fn interned_attr_name( + &self, + name: &S, + vm: &VirtualMachine, + ) -> Option<&'static PyStrInterned> { + vm.ctx + .interned_str(name) + .or_else(|| self.interned_attr_name_cold(name.as_ref(), vm)) + } + + #[cold] + fn interned_attr_name_cold( + &self, + name: &Wtf8, + vm: &VirtualMachine, + ) -> Option<&'static PyStrInterned> { + // Lazy native namespaces may not have interned their attribute names + // yet. Populate them before treating an unknown name as absent, while + // keeping arbitrary missing names out of the immortal string pool. + let mro = self.mro.read().clone(); + for class in &mro { + drop(class.attributes.local()); + if let Some(name) = vm.ctx.interned_str(name) { + return Some(name); + } + } + None + } + pub fn get_super_attr(&self, attr_name: &'static PyStrInterned) -> Option { - let mro = self.mro.read(); + let mro = self.mro.read().clone(); mro.get(1..)? .iter() .find_map(|class| class.attributes.get(attr_name)) @@ -1478,36 +1717,39 @@ impl PyType { /// Check if attribute exists in MRO, using method cache for fast check. /// Unlike find_name_in_mro, avoids cloning the value on cache hit. fn has_name_in_mro(&self, name: &'static PyStrInterned) -> bool { - #[cfg(all(feature = "threading", debug_assertions))] - crate::vm::thread::debug_assert_current_thread_attached(); + crate::vm::thread::try_with_current_vm(|vm| { + #[cfg(all(feature = "threading", debug_assertions))] + crate::vm::thread::debug_assert_current_thread_attached(); - let version = self.tp_version_tag.load(Ordering::Acquire); - if version != 0 { - let idx = type_cache_hash(version, name); - let entry = &TYPE_CACHE[idx]; - let name_ptr = name as *const _ as *mut _; - loop { - let seq1 = entry.begin_read(); - let v1 = entry.version.load(Ordering::Acquire); - let type_version = self.tp_version_tag.load(Ordering::Acquire); - if v1 != type_version - || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) - { - break; - } - let ptr = entry.value.load(Ordering::Acquire); - if entry.end_read(seq1) { - if !ptr.is_null() { - return true; + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + let idx = type_cache_hash(version, name); + let entry = &vm.state.type_cache.entries[idx]; + let name_ptr = name as *const _ as *mut _; + loop { + let seq1 = entry.begin_read(); + let v1 = entry.version.load(Ordering::Acquire); + let type_version = self.tp_version_tag.load(Ordering::Acquire); + if v1 != type_version + || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) + { + break; } - break; + let ptr = entry.value.load(Ordering::Acquire); + if entry.end_read(seq1) { + if !ptr.is_null() { + return Some(true); + } + break; + } + continue; } - continue; } - } - // Cache miss — use find_name_in_mro which populates cache - self.find_name_in_mro(name).is_some() + None + }) + .flatten() + .unwrap_or_else(|| self.find_name_in_mro(name).is_some()) } pub fn get_attributes(&self, ctx: &Context) -> PyAttributes { @@ -1515,7 +1757,8 @@ impl PyType { let mut attributes = PyAttributes::default(); // mro[0] is self, so we iterate through the entire MRO in reverse - for bc in self.mro.read().iter().map(|cls| -> &Self { cls }).rev() { + let mro = self.mro.read().clone(); + for bc in mro.iter().rev() { attributes.extend(bc.attributes.attributes(ctx)); } @@ -1695,7 +1938,8 @@ impl Py { where F: Fn(&Self) -> R, { - self.mro.read().iter().map(|x| &**x).map(f).collect() + let mro = self.mro.read().clone(); + mro.iter().map(|x| &**x).map(f).collect() } pub fn mro_collect(&self) -> Vec> { @@ -1711,7 +1955,7 @@ impl Py { core::iter::successors(Some(self), |cls| cls.base.deref()) } - pub fn extend_methods(&'static self, method_defs: &'static [PyMethodDef], ctx: &Context) { + pub fn extend_methods(&self, method_defs: &'static [PyMethodDef], ctx: &Context) { for method_def in method_defs { let method = method_def.to_proper_method(self, ctx); self.set_attr(ctx.intern_str(method_def.name), method); @@ -2091,6 +2335,7 @@ impl Constructor for PyType { type_data: PyRwLock::new(None), specialization_cache: TypeSpecializationCache::new(), interpreter_id: HeapTypeExt::creating_interpreter_id(), + native_definition: None, }; (slots, heaptype_ext) }; @@ -2167,7 +2412,7 @@ impl Constructor for PyType { let attr_name = vm.ctx.intern_str(mangled_name.as_str()); let member_descriptor: PyRef = vm.ctx.new_pyref(PyMemberDescriptor { - common: PyDescriptorOwned { + common: PyDescriptor { typ: typ.clone(), name: attr_name, qualname: PyRwLock::new(None), @@ -2378,11 +2623,14 @@ impl GetAttr for PyType { )) } - let Some(name) = vm.ctx.interned_str(name_str) else { + let mcl = zelf.class(); + let Some(name) = zelf + .interned_attr_name(name_str, vm) + .or_else(|| mcl.interned_attr_name(name_str, vm)) + else { return Err(attribute_error(zelf, name_str.as_wtf8(), vm)); }; vm_trace!("type.__getattribute__({:?}, {:?})", zelf, name); - let mcl = zelf.class(); let mcl_attr = mcl.get_attr(name); if let Some(ref attr) = mcl_attr { @@ -2581,34 +2829,39 @@ impl Py { }; let add_as_subclass = |bases: &[PyTypeRef]| { - let weakref_type = super::PyWeak::static_type(); for base in bases { - base.subclasses.write().push( - zelf.as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); + base.add_subclass(zelf); } }; let remove_as_subclass = |bases: &[PyTypeRef], retired: &mut Vec| { for base in bases { - let mut subclasses = base.subclasses.write(); - let mut kept = Vec::with_capacity(subclasses.len()); - for weak in subclasses.drain(..) { - match weak.upgrade() { - Some(obj) if obj.is(zelf.as_object()) => { - retired.push(obj); + base.with_subclass_registry(|registry| { + let mut subclasses = registry.write(); + let mut kept = Vec::with_capacity(subclasses.len()); + for weak in subclasses.drain(..) { + if weak.is_dead() { retired.push(weak.into()); + continue; } - Some(obj) => { - retired.push(obj); + if !weak.is_visible() { kept.push(weak); + continue; + } + match weak.upgrade_visible() { + Some(obj) if obj.is(zelf.as_object()) => { + retired.push(obj); + retired.push(weak.into()); + } + Some(obj) => { + retired.push(obj); + kept.push(weak); + } + None => retired.push(weak.into()), } - None => retired.push(weak.into()), } - } - *subclasses = kept; + *subclasses = kept; + }); } }; @@ -2640,13 +2893,7 @@ impl Py { let old_mro = core::mem::replace(&mut *cls.mro.write(), new_mro); undo.push((cls.to_owned(), old_mro)); cls.modified_inner(); - let subclasses: Vec = cls - .subclasses - .read() - .iter() - .filter_map(|subclass| subclass.upgrade()) - .filter_map(|subclass| subclass.downcast::().ok()) - .collect(); + let subclasses = cls.subclass_snapshot(); for subclass in subclasses { update_mro_recursively(&subclass, undo, vm)?; } @@ -2712,6 +2959,11 @@ impl Py { .ok_or_else(|| vm.new_attribute_error("__abstractmethods__")) } + #[pygetset] + fn __flags__(&self) -> u64 { + self.effective_flags().bits() + } + #[pygetset(setter)] fn set___abstractmethods__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { let key = identifier!(vm, __abstractmethods__); @@ -3025,17 +3277,12 @@ impl Py { #[pymethod] fn __subclasses__(&self, vm: &VirtualMachine) -> PyList { - let mut subclasses = self.subclasses.write(); - subclasses.retain(|x| x.upgrade().is_some()); let interpreter_id = vm.state.interpreter_id; PyList::from( - subclasses - .iter() - .filter_map(|x| x.upgrade()) - .filter(|obj| { - obj.downcast_ref::() - .is_none_or(|typ| typ.is_visible_to_interpreter(interpreter_id)) - }) + self.subclass_snapshot() + .into_iter() + .filter(|typ| typ.is_visible_to_interpreter(interpreter_id)) + .map(Into::into) .collect::>(), ) } @@ -3468,6 +3715,7 @@ pub(crate) fn call_slot_new( .new .load() .is_some_and(|f| fn_addr(f) == fn_addr(new_wrapper as NewFunc)) + && staticbase.slots.native_new.is_none() { match staticbase.base.load_owned() { Some(base) => staticbase = base, @@ -3475,8 +3723,11 @@ pub(crate) fn call_slot_new( } } - let typ_new = typ.slots.new.load(); - let staticbase_new = staticbase.slots.new.load(); + let typ_new = typ.slots.native_new.or_else(|| typ.slots.new.load()); + let staticbase_new = staticbase + .slots + .native_new + .or_else(|| staticbase.slots.new.load()); if typ_new.map(fn_addr) != staticbase_new.map(fn_addr) { return Err(vm.new_type_error(format!( "{}.__new__({}) is not safe, use {}.__new__()", @@ -3486,11 +3737,7 @@ pub(crate) fn call_slot_new( ))); } - let slot_new = typ - .slots - .new - .load() - .expect("Should be able to find a new slot somewhere in the mro"); + let slot_new = typ_new.expect("Should be able to find a new slot somewhere in the mro"); slot_new(subtype, args, vm) } @@ -3798,7 +4045,8 @@ pub(crate) fn compatible_for_assignment( (None, None) => true, _ => false, }; - let compatible = newbase.is(oldbase) || (bases_equal && same_slots_added(newbase, oldbase)); + let compatible = old_to.slots.native_layout_id == new_to.slots.native_layout_id + && (newbase.is(oldbase) || (bases_equal && same_slots_added(newbase, oldbase))); if compatible { return Ok(()); } diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs index 83ea3bb54b3..76760cd66f1 100644 --- a/crates/vm/src/builtins/union.rs +++ b/crates/vm/src/builtins/union.rs @@ -35,8 +35,8 @@ impl fmt::Debug for PyUnion { impl PyPayload for PyUnion { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.union_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.union_type).to_owned() } } diff --git a/crates/vm/src/builtins/weakproxy.rs b/crates/vm/src/builtins/weakproxy.rs index efd8cb306c5..01661b266b5 100644 --- a/crates/vm/src/builtins/weakproxy.rs +++ b/crates/vm/src/builtins/weakproxy.rs @@ -24,14 +24,14 @@ impl PyPayload for PyWeakProxy { #[inline] unsafe fn validate_downcastable_from(obj: &PyObject) -> bool { ::BASICSIZE <= obj.class().slots().basicsize - && obj - .class() - .fast_issubclass(::static_type()) + && obj.class().fast_issubclass(unsafe { + ::static_type() + }) } #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.weakproxy_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.weakproxy_type).to_owned() } } diff --git a/crates/vm/src/builtins/weakref.rs b/crates/vm/src/builtins/weakref.rs index 135db6866cb..7a2d46e34e7 100644 --- a/crates/vm/src/builtins/weakref.rs +++ b/crates/vm/src/builtins/weakref.rs @@ -24,9 +24,13 @@ pub struct WeakNewArgs { } impl PyPayload for PyWeak { + fn supports_native_layout(layout: core::any::TypeId) -> bool { + layout == core::any::TypeId::of::() + || layout == core::any::TypeId::of::() + } #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.weakref_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.weakref_type).to_owned() } } diff --git a/crates/vm/src/builtins/zip.rs b/crates/vm/src/builtins/zip.rs index 3bdaca7f707..96e86552e74 100644 --- a/crates/vm/src/builtins/zip.rs +++ b/crates/vm/src/builtins/zip.rs @@ -19,8 +19,8 @@ pub struct PyZip { impl PyPayload for PyZip { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.zip_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.zip_type).to_owned() } } diff --git a/crates/vm/src/class.rs b/crates/vm/src/class.rs index 8a7a8306d16..d684c3e10f0 100644 --- a/crates/vm/src/class.rs +++ b/crates/vm/src/class.rs @@ -10,115 +10,144 @@ use crate::{ }; use rustpython_common::static_cell; -/// Add slot wrapper descriptors to a type's dict -/// -/// Iterates SLOT_DEFS and creates a PyWrapper for each slot that: -/// 1. Has a function set in the type's slots -/// 2. Doesn't already have an attribute in the type's dict -#[cfg(feature = "doc")] -pub fn add_operators(class: &'static Py, ctx: &Context, attr_docs: &[(&str, u32, u32)]) { - add_operators_inner(class, ctx, |name| match attr_doc(attr_docs, name) { - Some((offset, len)) if len != 0 => (offset, len), - _ => (0, 0), - }) +/// Immutable recipes captured before slot inheritance. Replaying a native +/// namespace must not install wrappers for slots inherited later. +pub struct NativeOperators { + wrappers: Vec, + new: bool, + unhashable: bool, } -#[cfg(not(feature = "doc"))] -pub fn add_operators(class: &'static Py, ctx: &Context, attr_docs: &[&str]) { - add_operators_inner(class, ctx, |name| { - if attr_name_present(attr_docs, name) { - // Present in the database: do not fall back to the slot text. - (0, 1) - } else { - (0, 0) - } - }) +struct NativeOperator { + name: &'static str, + wrapped: crate::builtins::descriptor::SlotFunc, + doc: &'static str, + plain_off: u32, + plain_len: u32, } -fn add_operators_inner( - class: &'static Py, - ctx: &Context, - mut plain_span: impl FnMut(&str) -> (u32, u32), -) { - for def in SLOT_DEFS { - let (plain_off, plain_len) = plain_span(def.name); - // Skip __new__ - it has special handling - if def.name == "__new__" { - continue; - } - - // Special handling for __hash__ = None - if def.name == "__hash__" - && class.slots().hash.load().is_some_and(|h| { +impl NativeOperators { + fn capture(class: &Py, ctx: &Context) -> Self { + let unhashable = + class.slots().hash.load().is_some_and(|h| { fn_addr(h) == fn_addr(hash_not_implemented as crate::types::HashFunc) + }); + let wrappers = SLOT_DEFS + .iter() + .filter_map(|def| { + // __getattr__ is not an alias for the __getattribute__ wrapper. + if matches!(def.name, "__new__" | "__getattr__") + || (def.name == "__hash__" && unhashable) + { + return None; + } + let wrapped = def.accessor.get_slot_func_with_op(class.slots(), def.op)?; + #[cfg(feature = "doc")] + let (plain_off, plain_len) = match attr_doc(T::ATTR_DOCS, def.name) { + Some((offset, len)) if len != 0 => (offset, len), + _ => (0, 0), + }; + #[cfg(not(feature = "doc"))] + let (plain_off, plain_len) = + (0, u32::from(attr_name_present(T::ATTR_DOCS, def.name))); + Some(NativeOperator { + name: def.name, + wrapped, + doc: def.doc, + plain_off, + plain_len, + }) }) - { - class.set_attr(ctx.names.__hash__, ctx.none.clone().into()); - continue; + .collect(); + let new = class.slots().native_new.is_some_and(|slot_new| { + let object_new = ctx.types.object_type.slots().new.load(); + core::ptr::eq(class, ctx.types.object_type) + || object_new.is_none_or(|f| fn_addr(slot_new) != fn_addr(f)) + }); + Self { + wrappers, + new, + unhashable, } + } - // __getattr__ should only have a wrapper if the type explicitly defines it. - // Unlike __getattribute__, __getattr__ is not present on object by default. - // Both map to TpGetattro, but only __getattribute__ gets a wrapper from the slot. - if def.name == "__getattr__" { - continue; + fn populate(&self, class: &Py, ctx: &Context) { + if self.new { + let bound_new = + ctx.slot_new_wrapper + .build_bound_method(ctx, class.to_owned().into(), class); + class.set_attr(ctx.names.__new__, bound_new.into()); } - - // Get the slot function wrapped in SlotFunc - let Some(slot_func) = def.accessor.get_slot_func_with_op(class.slots(), def.op) else { - continue; - }; - - // Check if attribute already exists in dict - let attr_name = ctx.intern_str(def.name); - if class.attributes().contains(attr_name) { - continue; + if self.unhashable { + class.set_attr(ctx.names.__hash__, ctx.none()); + } + for def in &self.wrappers { + let name = ctx.intern_str(def.name); + if class.attributes().contains(name) { + continue; + } + let wrapper = PyWrapper { + typ: class.to_owned(), + name, + wrapped: def.wrapped, + doc: Some(def.doc), + plain_off: def.plain_off, + plain_len: def.plain_len, + }; + class.set_attr(name, wrapper.into_ref(ctx).into()); } - - // Create and add the wrapper - let wrapper = PyWrapper { - typ: class, - name: attr_name, - wrapped: slot_func, - doc: Some(def.doc), - plain_off, - plain_len, - }; - class.set_attr(attr_name, wrapper.into_ref(ctx).into()); } } pub trait StaticType { // Ideally, saving PyType is better than PyTypeRef - fn static_cell() -> &'static static_cell::StaticCell; + /// # Safety + /// Raw types must be used under the native ownership and attachment + /// contract in [`crate::embedding`], including during bootstrap. + unsafe fn static_cell() -> &'static static_cell::StaticCell; #[inline] #[must_use] - fn static_metaclass() -> &'static Py { - PyType::static_type() + /// # Safety + /// Raw types must be used under the native ownership and attachment + /// contract in [`crate::embedding`], including during bootstrap. + unsafe fn static_metaclass() -> &'static Py { + unsafe { PyType::static_type() } } #[inline] #[must_use] - fn static_baseclass() -> &'static Py { - PyBaseObject::static_type() + /// # Safety + /// Raw types must be used under the native ownership and attachment + /// contract in [`crate::embedding`], including during bootstrap. + unsafe fn static_baseclass() -> &'static Py { + unsafe { PyBaseObject::static_type() } + } + + fn baseclass(_ctx: &Context) -> PyTypeRef { + unsafe { Self::static_baseclass() }.to_owned() } #[inline] #[must_use] - fn static_type() -> &'static Py { + /// # Safety + /// Raw types must be used under the native ownership and attachment + /// contract in [`crate::embedding`], including during bootstrap. + unsafe fn static_type() -> &'static Py { #[cold] fn fail() -> ! { panic!( "static type has not been initialized. e.g. the native types defined in different module may be used before importing library." ); } - Self::static_cell().get().unwrap_or_else(|| fail()) + unsafe { Self::static_cell() } + .get() + .unwrap_or_else(|| fail()) } #[must_use] fn init_manually(typ: PyTypeRef) -> &'static Py { - let cell = Self::static_cell(); + let cell = unsafe { Self::static_cell() }; cell.set(typ) .unwrap_or_else(|_| panic!("double initialization from init_manually")); let typ = cell.get().unwrap(); @@ -127,12 +156,15 @@ pub trait StaticType { } #[must_use] - fn init_builtin_type() -> &'static Py + /// # Safety + /// Raw types must be used under the native ownership and attachment + /// contract in [`crate::embedding`], including during bootstrap. + unsafe fn init_builtin_type() -> &'static Py where Self: PyClassImpl, { - let typ = Self::create_static_type(); - let cell = Self::static_cell(); + let typ = unsafe { Self::create_static_type() }; + let cell = unsafe { Self::static_cell() }; cell.set(typ) .unwrap_or_else(|_| panic!("double initialization of {}", Self::NAME)); let typ = cell.get().unwrap(); @@ -141,7 +173,10 @@ pub trait StaticType { } #[must_use] - fn create_static_type() -> PyTypeRef + /// # Safety + /// Raw types must be used under the native ownership and attachment + /// contract in [`crate::embedding`], including during bootstrap. + unsafe fn create_static_type() -> PyTypeRef where Self: PyClassImpl, { @@ -149,19 +184,19 @@ pub trait StaticType { // this type left the slot at 0. The base type object already exists. let mut slots = Self::make_slots(); if slots.itemsize == 0 { - slots.itemsize = Self::static_baseclass().slots.itemsize; + slots.itemsize = unsafe { Self::static_baseclass() }.slots.itemsize; } PyType::new_static( - Self::static_baseclass().to_owned(), + unsafe { Self::static_baseclass() }.to_owned(), Default::default(), slots, - Self::static_metaclass().to_owned(), + unsafe { Self::static_metaclass() }.to_owned(), ) .unwrap() } } -pub trait PyClassDef { +pub trait PyClassDef: 'static { const NAME: &'static str; const MODULE_NAME: Option<&'static str>; const TP_NAME: &'static str; @@ -175,6 +210,8 @@ pub trait PyClassDef { const ATTR_DOCS: &'static [&'static str] = &[]; const BASICSIZE: usize; const ITEMSIZE: usize = 0; + const INTERPRETER_LOCAL: bool = false; + const NATIVE_LAYOUT_ID: core::any::TypeId = core::any::TypeId::of::(); const UNHASHABLE: bool = false; const MEMBERS: &'static [crate::builtins::descriptor::PyMemberSpec] = &[]; @@ -291,11 +328,42 @@ pub trait PyClassImpl: PyClassDef { const METHOD_DEFS: &'static [PyMethodDef]; - fn impl_extend_class(ctx: &'static Context, class: &'static Py); + fn impl_extend_class(ctx: &Context, class: &Py); fn extend_slots(slots: &mut PyTypeSlots); - fn extend_class(ctx: &'static Context, class: &'static Py) + fn extend_class(ctx: &Context, class: &Py) + where + Self: Sized, + { + // Names must be available before a lazily populated namespace is read. + let _ = ctx.intern_str(Self::NAME); + let operators = NativeOperators::capture::(class, ctx); + if class.heaptype_ext.is_none() { + class + .attributes() + .define_native(crate::vm::native_types::NativeNamespaceDefinition { + class: class.to_owned(), + operators, + populate: Self::populate_namespace, + }); + } else { + Self::populate_namespace(ctx, class, &operators); + } + // Same walk as init_slots: a slot such as tp_init is copied only from + // a base that defines it, so a static grandchild must see that base + // in the MRO, not only its direct bases. + let mro = { + let guard = class.mro.read(); + guard[1..].to_vec() + }; + for base in &mro { + class.inherit_slots(base); + } + } + + #[doc(hidden)] + fn populate_namespace(ctx: &Context, class: &Py, operators: &NativeOperators) where Self: Sized, { @@ -307,8 +375,6 @@ pub trait PyClassImpl: PyClassDef { .contains(&PyTypeFlags::_CREATED_WITH_FLAGS) ); - let _ = ctx.intern_str(Self::NAME); // intern type name - if Self::TP_FLAGS.contains(&PyTypeFlags::HAS_DICT) && !Self::MEMBERS.iter().any(|member| member.name == "__dict__") { @@ -347,8 +413,14 @@ pub trait PyClassImpl: PyClassDef { // This matches CPython's behavior in type_dict_set_doc let doc_attr_name = identifier!(ctx, __doc__); if class.attributes().get(doc_attr_name).is_none() { - let doc = - plain_doc(Self::DOC).map_or_else(|| ctx.none(), |doc| ctx.new_str(doc).into()); + // Local heap types store the same rendered text that static types + // expose through type.__doc__, including native signature handling. + let text = if Self::INTERPRETER_LOCAL { + crate::builtins::type_::rendered_item_doc(Self::NAME, class.slots.doc) + } else { + plain_doc(Self::DOC) + }; + let doc = text.map_or_else(|| ctx.none(), |doc| ctx.new_str(doc).into()); class.set_attr(doc_attr_name, doc); } @@ -365,53 +437,58 @@ pub trait PyClassImpl: PyClassDef { } } - // Don't add __new__ attribute if slot_new is inherited from object - // (Python doesn't add __new__ to __dict__ for inherited slots) - // Exception: object itself should have __new__ in its dict - if let Some(slot_new) = class.slots().new.load() { - let object_new = ctx.types.object_type.slots().new.load(); - let is_object_itself = core::ptr::eq(class, ctx.types.object_type); - let is_inherited_from_object = !is_object_itself - && object_new.is_some_and(|obj_new| fn_addr(slot_new) == fn_addr(obj_new)); - - if !is_inherited_from_object { - let bound_new = - ctx.slot_new_wrapper - .build_bound_method(ctx, class.to_owned().into(), class); - class.set_attr(identifier!(ctx, __new__), bound_new.into()); - } - } + operators.populate(class, ctx); - // Add slot wrappers using SLOT_DEFS array - add_operators(class, ctx, Self::ATTR_DOCS); + class.extend_methods(class.slots().methods, ctx); + } - // Same walk as init_slots: a slot such as tp_init is copied only from - // a base that defines it, so a static grandchild must see that base - // in the MRO, not only its direct bases. - let mro = { - let guard = class.mro.read(); - guard[1..].to_vec() - }; - for base in &mro { - class.inherit_slots(base); + #[must_use] + fn make_class(ctx: &Context) -> PyTypeRef + where + Self: StaticType + Sized + 'static, + { + if !Self::INTERPRETER_LOCAL { + return unsafe { Self::make_static_type() }; } - - class.extend_methods(class.slots().methods, ctx); + crate::vm::thread::with_current_vm(|vm| { + vm.state.native_types.class::( + vm, + || { + let base = Self::baseclass(ctx); + let mut slots = Self::make_slots(); + if slots.itemsize == 0 { + slots.itemsize = base.slots.itemsize; + } + PyType::new_native_heap::( + Self::NAME, + base, + slots, + unsafe { Self::static_metaclass() }.to_owned(), + ctx, + ) + .expect("invalid native type definition") + }, + |class| Self::extend_class(ctx, class), + ) + }) } #[must_use] - fn make_static_type() -> PyTypeRef + /// # Safety + /// Raw types must be used under the native ownership and attachment + /// contract in [`crate::embedding`], including during bootstrap. + unsafe fn make_static_type() -> PyTypeRef where Self: StaticType + Sized, { - let typ = Self::static_cell().get_or_init(|| { - let typ = Self::create_static_type(); - Self::extend_class(Context::genesis(), unsafe { - // typ will be saved in static_cell - let r: &Py = &typ; - let r: &'static Py = core::mem::transmute(r); - r - }); + assert!( + !Self::INTERPRETER_LOCAL, + "interpreter-local type requires make_class" + ); + let typ = unsafe { Self::static_cell() }.get_or_init(|| { + let _owner = crate::gc_state::AllocationScope::shared(); + let typ = unsafe { Self::create_static_type() }; + Self::extend_class(Context::genesis(), &typ); typ }); // A static type is held by its `static_cell` for the life of the @@ -427,6 +504,7 @@ pub trait PyClassImpl: PyClassDef { flags: crate::types::AtomicPyTypeFlags::from_plain(Self::TP_FLAGS), name: Self::TP_NAME, basicsize: Self::BASICSIZE, + native_layout_id: Some(Self::NATIVE_LAYOUT_ID), itemsize: Self::ITEMSIZE, doc: { let internal = Self::INTERNAL_DOC; @@ -445,6 +523,11 @@ pub trait PyClassImpl: PyClassDef { } Self::extend_slots(&mut slots); + slots.native_new = slots.new.load(); + if !Self::INTERPRETER_LOCAL { + // Shared native layouts and slots cannot be changed by an interpreter. + slots.flags.insert(PyTypeFlags::IMMUTABLETYPE); + } slots } } diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index 587701d6528..9911ada1ffb 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -29,7 +29,9 @@ pub struct CodecsRegistry { inner: PyRwLock, } +#[derive(Default)] struct RegistryInner { + closed: bool, search_path: Vec, search_cache: HashMap, errors: HashMap, @@ -215,6 +217,7 @@ impl CodecsRegistry { .collect(); let inner = RegistryInner { + closed: false, search_path: Vec::new(), search_cache: HashMap::new(), errors, @@ -230,10 +233,26 @@ impl CodecsRegistry { return Err(vm.new_type_error("argument must be callable")); } - self.inner.write().search_path.push(search_function); + let mut inner = self.inner.write(); + if inner.closed { + drop(inner); + return Err(vm.new_runtime_error("codec registry is closed")); + } + inner.search_path.push(search_function); Ok(()) } + pub(crate) fn close(&self) { + let retired = core::mem::replace( + &mut *self.inner.write(), + RegistryInner { + closed: true, + ..Default::default() + }, + ); + drop(retired); + } + pub fn unregister(&self, search_function: &PyObject) { let mut inner = self.inner.write(); // Do nothing if search_path is not created yet or was cleared. @@ -241,29 +260,39 @@ impl CodecsRegistry { return; } - for (i, item) in inner.search_path.iter().enumerate() { - if item.get_id() == search_function.get_id() { - if !inner.search_cache.is_empty() { - inner.search_cache.clear(); - } - inner.search_path.remove(i); - return; - } + if let Some(i) = inner + .search_path + .iter() + .position(|item| item.get_id() == search_function.get_id()) + { + let cache = core::mem::take(&mut inner.search_cache); + let function = inner.search_path.remove(i); + drop(inner); + drop((cache, function)); } } pub(crate) fn register_manual(&self, name: &str, codec: PyCodec) { let name = normalize_encoding_name(name); - self.inner - .write() - .search_cache - .insert(name.into_owned(), codec); + let retired = { + let mut inner = self.inner.write(); + if inner.closed { + None + } else { + inner.search_cache.insert(name.into_owned(), codec) + } + }; + drop(retired); } pub fn lookup(&self, encoding: &str, vm: &VirtualMachine) -> PyResult { let encoding = normalize_encoding_name(encoding); let search_path = { let inner = self.inner.read(); + if inner.closed { + drop(inner); + return Err(vm.new_runtime_error("codec registry is closed")); + } if let Some(codec) = inner.search_cache.get(encoding.as_ref()) { // hit cache return Ok(codec.clone()); @@ -276,13 +305,21 @@ impl CodecsRegistry { let res = func.call((encoding.clone(),), vm)?; let res: Option = res.try_into_value(vm)?; if let Some(codec) = res { - let mut inner = self.inner.write(); - // someone might have raced us to this, so use theirs - let codec = inner - .search_cache - .entry(encoding.as_str().to_owned()) - .or_insert(codec); - return Ok(codec.clone()); + let winner = { + let mut inner = self.inner.write(); + // A search function can reenter the registry. Never release + // the losing tuple or refill a closed registry under lock. + if inner.closed { + codec + } else { + inner + .search_cache + .entry(encoding.as_str().to_owned()) + .or_insert_with(|| codec.clone()) + .clone() + } + }; + return Ok(winner); } } @@ -517,7 +554,11 @@ impl CodecsRegistry { } pub fn register_error(&self, name: String, handler: PyObjectRef) -> Option { - self.inner.write().errors.insert(name, handler) + let mut inner = self.inner.write(); + if inner.closed { + return Some(handler); + } + inner.errors.insert(name, handler) } pub fn unregister_error(&self, name: &str, vm: &VirtualMachine) -> PyResult { @@ -536,7 +577,8 @@ impl CodecsRegistry { "cannot un-register built-in error handler '{name}'" ))); } - Ok(self.inner.write().errors.remove(name).is_some()) + let retired = self.inner.write().errors.remove(name); + Ok(retired.is_some()) } pub fn lookup_error_opt(&self, name: &str) -> Option { diff --git a/crates/vm/src/convert/transmute_from.rs b/crates/vm/src/convert/transmute_from.rs index c1b4b79384e..859595c5209 100644 --- a/crates/vm/src/convert/transmute_from.rs +++ b/crates/vm/src/convert/transmute_from.rs @@ -16,14 +16,14 @@ pub unsafe trait TransmuteFromObject: Sized { unsafe impl TransmuteFromObject for PyRef { fn check(vm: &VirtualMachine, obj: &PyObject) -> PyResult<()> { let class = T::class(&vm.ctx); - if obj.fast_isinstance(class) { + if obj.fast_isinstance(&class) { if obj.downcastable::() { Ok(()) } else { - Err(vm.new_downcast_runtime_error(class, obj)) + Err(vm.new_downcast_runtime_error(&class, obj)) } } else { - Err(vm.new_downcast_type_error(class, obj)) + Err(vm.new_downcast_type_error(&class, obj)) } } } diff --git a/crates/vm/src/convert/try_from.rs b/crates/vm/src/convert/try_from.rs index 57d611f90cf..c87d0f75c8b 100644 --- a/crates/vm/src/convert/try_from.rs +++ b/crates/vm/src/convert/try_from.rs @@ -53,11 +53,11 @@ impl PyObject { F: Fn(&Py) -> PyResult, { let class = T::class(&vm.ctx); - let py_ref = if self.fast_isinstance(class) { + let py_ref = if self.fast_isinstance(&class) { self.downcast_ref() - .ok_or_else(|| vm.new_downcast_runtime_error(class, self))? + .ok_or_else(|| vm.new_downcast_runtime_error(&class, self))? } else { - return Err(vm.new_downcast_type_error(class, self)); + return Err(vm.new_downcast_type_error(&class, self)); }; f(py_ref) } @@ -79,11 +79,11 @@ where #[inline] fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let class = T::class(&vm.ctx); - if obj.fast_isinstance(class) { + if obj.fast_isinstance(&class) { T::try_downcast_from(&obj, vm)?; Ok(unsafe { obj.downcast_unchecked() }) } else { - Err(vm.new_downcast_type_error(class, &obj)) + Err(vm.new_downcast_type_error(&class, &obj)) } } } @@ -114,11 +114,11 @@ impl<'a, T: 'a + TryFromObject> TryFromBorrowedObject<'a> for Vec { impl<'a, T: PyPayload> TryFromBorrowedObject<'a> for &'a Py { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let class = T::class(&vm.ctx); - if obj.fast_isinstance(class) { + if obj.fast_isinstance(&class) { obj.downcast_ref() - .ok_or_else(|| vm.new_downcast_runtime_error(class, &obj)) + .ok_or_else(|| vm.new_downcast_runtime_error(&class, &obj)) } else { - Err(vm.new_downcast_type_error(class, &obj)) + Err(vm.new_downcast_type_error(&class, &obj)) } } } diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 4a901bfbf0e..a5f5c8c50d4 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -1840,7 +1840,7 @@ mod tests { #[test] fn clone_compacts_deleted_entries() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let dict = Dict::default(); for key in 0..128usize { dict.insert(vm, &key, key).unwrap(); @@ -1872,7 +1872,7 @@ mod tests { #[test] fn clone_shrinks_indices_after_pop_back() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let dict = Dict::default(); for key in 0..128usize { dict.insert(vm, &key, ()).unwrap(); @@ -1900,7 +1900,7 @@ mod tests { #[test] fn module_attr_cache_rejects_mixed_entries_and_unsafe_layouts() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let name = vm.ctx.intern_str("target"); let padding = vm.ctx.intern_str("padding"); let first: Dict = Dict::default(); @@ -1959,7 +1959,7 @@ mod tests { #[test] fn insert_basic() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let dict = Dict::default(); assert_eq!(0, dict.len()); @@ -1993,7 +1993,7 @@ mod tests { #[test] fn reserve_empty_avoids_growth() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let dict = Dict::default(); dict.reserve_for_empty(usize::MAX); assert_eq!(dict.read().indices.len(), 8); @@ -2047,7 +2047,7 @@ mod tests { } fn check_hash_equivalence(text: &str) { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let value1 = text; let value2 = vm.new_pyobj(value1.to_owned()); @@ -2059,7 +2059,7 @@ mod tests { #[test] fn replace_contents_invalidates_cached_indices() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let dict = Dict::default(); dict.insert(vm, &1usize, ()).unwrap(); let old_version = dict.assign_keys_version(); diff --git a/crates/vm/src/embedding.rs b/crates/vm/src/embedding.rs new file mode 100644 index 00000000000..303913a7050 --- /dev/null +++ b/crates/vm/src/embedding.rs @@ -0,0 +1,782 @@ +//! Scoped access to interpreter-owned objects. +//! +//! A `PyHandle` can be retained by the host or sent to another thread. Bind it +//! to its interpreter to access Python. Bound values never expose raw payloads. +//! +//! Native integration uses explicitly unsafe entry points. Their contract covers +//! the entire lifetime of raw `VirtualMachine`, `Context`, `PyObjectRef` and +//! `PyRef` values obtained through them: +//! +//! - Access and release private Python objects only while attached to their owner. +//! - Do not store foreign Python references in an interpreter's object graph. +//! - Do not access raw references, payloads or borrowed storage across detachment +//! or a nested entry into another interpreter. Native-only detached work must +//! not execute Python destructors when dropping its captures. +//! - Retain host values using `PyHandle`; raw references must not escape the entry. +//! - Native callbacks, including initialization hooks and buffer release, obey +//! the same rules. Process-wide definitions contain no mutable Python state. +//! +//! During bootstrap only the interpreter implementation may access the shared +//! native definitions before an owner is attached. +//! +//! ```compile_fail +//! let interpreter = rustpython_vm::Interpreter::without_stdlib(Default::default()); +//! let escaped = interpreter.enter(|vm| vm.new_int(1).unwrap()); +//! ``` +//! +//! ```compile_fail +//! let interpreter = rustpython_vm::Interpreter::without_stdlib(Default::default()); +//! interpreter.enter(|vm| { +//! let value = vm.new_int(1).unwrap(); +//! vm.detach(|| value.to_i64()).unwrap(); +//! }); +//! ``` +//! +//! Raw getters and native callbacks cannot bypass this boundary in safe Rust: +//! +//! ```compile_fail +//! let context = rustpython_vm::Context::genesis_unchecked(); +//! let object = context.new_list(Vec::new()); +//! ``` +//! +//! ```compile_fail +//! use rustpython_vm::class::StaticType; +//! let class = rustpython_vm::builtins::PyList::static_type(); +//! ``` +//! +//! ```compile_fail +//! rustpython_vm::vm::thread::with_current_vm_unchecked(|vm| vm.ctx.none()); +//! ``` +//! +//! ```compile_fail +//! rustpython_vm::Interpreter::builder(Default::default()).init_hook(|vm| { +//! let _object = vm.ctx.new_list(Vec::new()); +//! }); +//! ``` +//! +//! ```compile_fail +//! rustpython_vm::vm::thread::update_thread_exception(None); +//! ``` +//! +//! ```compile_fail +//! rustpython_vm::vm::runtime::destroy_owned_interpreter(0); +//! ``` + +use crate::{ + PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, + builtins::PyBaseException, + common::rc::PyRc, + vm::{roots::RootLease, thread}, +}; +use core::{fmt, marker::PhantomData}; + +/// An opaque persistent root in one interpreter. Dropping the last clone queues +/// release on the owner; it never runs Python on the dropping native thread. +pub struct PyHandle { + lease: PyRc, + kind: PhantomData T>, +} + +impl Clone for PyHandle { + fn clone(&self) -> Self { + Self { + lease: self.lease.clone(), + kind: PhantomData, + } + } +} + +impl fmt::Debug for PyHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyHandle") + .field("interpreter", &self.lease.owner()) + .finish_non_exhaustive() + } +} + +impl PyHandle { + #[must_use] + pub fn interpreter_id(&self) -> i64 { + self.lease.owner() + } + + /// Obtain an owner-local reference for a native callback. + /// + /// # Safety + /// Release the returned reference before leaving this owner's attachment. + /// Follow [`crate::Interpreter::enter_unchecked`]'s contract. + pub unsafe fn to_object_unchecked(&self, vm: &VirtualMachine) -> Result { + let vm = Vm::new(vm); + vm.check()?; + if self.interpreter_id() != vm.raw.state.interpreter_id { + return Err(Error::WrongInterpreter); + } + self.lease.object().ok_or(Error::InterpreterClosed) + } +} + +impl PyHandle { + /// Transfer a native callback's owned reference into an opaque host root. + /// + /// # Safety + /// The object must be owned by the currently attached interpreter, and its + /// reachable graph must obey the native ownership contract of this module. + pub unsafe fn from_object_unchecked(object: PyObjectRef, vm: &VirtualMachine) -> Result { + Vm::new(vm).root(object) + } +} + +impl PyHandle { + pub(crate) fn exception( + &self, + vm: &VirtualMachine, + ) -> Option { + if self.interpreter_id() != vm.state.interpreter_id { + return Some(vm.new_runtime_error("exception belongs to another interpreter")); + } + self.lease + .object() + .and_then(|object| object.downcast().ok()) + } +} + +/// Access failures contain no unscoped Python references. +#[derive(Debug)] +pub enum Error { + WrongInterpreter, + InterpreterClosed, + ForkedOwner, + NotAttached, + Python(PyHandle), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::WrongInterpreter => "object belongs to another interpreter", + Self::InterpreterClosed => "interpreter has been closed", + Self::ForkedOwner => "VM owner was inactive or on another thread at fork", + Self::NotAttached => "this interpreter scope is not currently attached", + Self::Python(_) => "Python raised an exception", + }) + } +} + +impl core::error::Error for Error {} + +pub type Result = core::result::Result; + +/// A fresh, invariant scope for one entry into Python. Neither this token nor a +/// bound object can cross native threads, outlive the entry, or be used while +/// the thread is detached or running a nested entry into another VM. +#[derive(Clone, Copy)] +pub struct Vm<'vm> { + raw: &'vm VirtualMachine, + scope: PhantomData<&'vm mut &'vm ()>, + thread: PhantomData>, +} + +pub struct Bound<'vm, T = PyObject> { + handle: PyHandle, + vm: Vm<'vm>, +} + +impl Clone for Bound<'_, T> { + fn clone(&self) -> Self { + Self { + handle: self.handle.clone(), + vm: self.vm, + } + } +} + +impl fmt::Debug for Bound<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.handle.fmt(f) + } +} + +impl<'vm> Vm<'vm> { + pub(crate) fn new(raw: &'vm VirtualMachine) -> Self { + Self { + raw, + scope: PhantomData, + thread: PhantomData, + } + } + + fn check(self) -> Result<()> { + if !self.raw.owner_is_valid() { + return Err(Error::ForkedOwner); + } + if self + .raw + .state + .closed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(Error::InterpreterClosed); + } + if thread::is_current_attached(self.raw) { + if self.raw.state.roots.is_closed() { + return Err(Error::InterpreterClosed); + } + self.raw.state.roots.drain_pending(); + Ok(()) + } else { + Err(Error::NotAttached) + } + } + + fn root(self, object: PyObjectRef) -> Result> { + self.check()?; + if object + .gc_handle() + .heap() + .and_then(|heap| heap.owner) + .is_some_and(|owner| owner != self.raw.state.interpreter_id) + { + return Err(Error::WrongInterpreter); + } + self.raw + .state + .roots + .insert_object(object) + .map(|lease| PyHandle { + lease, + kind: PhantomData, + }) + .ok_or(Error::InterpreterClosed) + } + + fn exception(self, exception: crate::builtins::PyBaseExceptionRef) -> Error { + match self.root(exception.into()) { + Ok(handle) => Error::Python(handle), + Err(error) => error, + } + } + + fn value(self, result: PyResult) -> Result> { + let object = result.map_err(|exception| self.exception(exception))?; + Ok(Bound { + handle: self.root(object)?, + vm: self, + }) + } + + pub fn bind(self, handle: &PyHandle) -> Result> { + self.check()?; + if handle.lease.owner() != self.raw.state.interpreter_id { + return Err(Error::WrongInterpreter); + } + let object = handle.lease.object().ok_or(Error::InterpreterClosed)?; + drop(object); + Ok(Bound { + handle: handle.clone(), + vm: self, + }) + } + + pub fn none(self) -> Result> { + self.check()?; + self.value(Ok(self.raw.ctx.none())) + } + + pub fn new_int(self, value: i64) -> Result> { + self.check()?; + self.value(Ok(self.raw.ctx.new_int(value).into())) + } + + pub fn new_str(self, value: &str) -> Result> { + self.check()?; + self.value(Ok(self.raw.ctx.new_str(value).into())) + } + + pub fn new_bytes(self, value: Vec) -> Result> { + self.check()?; + self.value(Ok(self.raw.ctx.new_bytes(value).into())) + } + + pub fn new_list(self, values: &[Bound<'vm>]) -> Result> { + self.check()?; + let objects = values + .iter() + .map(|value| value.object_for(self)) + .collect::>()?; + self.value(Ok(self.raw.ctx.new_list(objects).into())) + } + + pub fn import(self, name: &str) -> Result> { + self.check()?; + self.value(self.raw.import(&self.raw.ctx.new_str(name), 0)) + } + + /// Execute in the interpreter's persistent `__main__` namespace. + #[cfg(feature = "rustpython-compiler")] + pub fn exec(self, source: &str) -> Result<()> { + self.run(source, crate::compiler::Mode::Exec).map(drop) + } + + #[cfg(feature = "rustpython-compiler")] + pub fn eval(self, source: &str) -> Result> { + self.run(source, crate::compiler::Mode::Eval) + } + + #[cfg(feature = "rustpython-compiler")] + fn run(self, source: &str, mode: crate::compiler::Mode) -> Result> { + self.check()?; + let result = (|| { + let globals = self.raw.main_namespace()?; + let scope = crate::scope::Scope::with_builtins(None, globals, self.raw); + let code = self + .raw + .compile(source, mode, "") + .map_err(|error| error.into_pyexception(self.raw, Some(source)))?; + self.raw.run_code_obj(code, scope) + })(); + self.value(result) + } + + /// Run a host wait without blocking collection. Scoped VM and object tokens + /// are not Send/Sync, so the callback cannot capture and use them detached. + pub fn detach(self, f: impl FnOnce() -> R + Send) -> Result { + self.check()?; + Ok(self.raw.allow_threads(f)) + } + + /// Release handles dropped by host threads since the preceding drain. + pub fn drain_releases(self) -> Result<()> { + self.check()?; + self.raw.state.roots.drain_pending(); + Ok(()) + } +} + +impl<'vm, T> Bound<'vm, T> { + fn object_for(&self, vm: Vm<'vm>) -> Result { + vm.check()?; + if self.handle.lease.owner() != vm.raw.state.interpreter_id { + return Err(Error::WrongInterpreter); + } + self.handle.lease.object().ok_or(Error::InterpreterClosed) + } + + /// Keep this object beyond the current entry without exposing its payload. + #[must_use] + pub fn unbind(self) -> PyHandle { + self.handle + } + + pub fn get_attr(&self, name: &str) -> Result> { + let object = self.object_for(self.vm)?; + self.vm + .value(object.get_attr(&self.vm.raw.ctx.new_str(name), self.vm.raw)) + } + + pub fn set_attr(&self, name: &str, value: &Bound<'vm, U>) -> Result<()> { + let object = self.object_for(self.vm)?; + let value = value.object_for(self.vm)?; + object + .set_attr(&self.vm.raw.ctx.new_str(name), value, self.vm.raw) + .map_err(|error| self.vm.exception(error)) + } + + pub fn call(&self, args: &[Bound<'vm>]) -> Result> { + let object = self.object_for(self.vm)?; + let args = args + .iter() + .map(|arg| arg.object_for(self.vm)) + .collect::>>()?; + self.vm + .value(object.call(crate::function::FuncArgs::from(args), self.vm.raw)) + } + + pub fn repr(&self) -> Result { + let object = self.object_for(self.vm)?; + object + .repr(self.vm.raw) + .map(|value| value.as_wtf8().to_string_lossy().into_owned()) + .map_err(|error| self.vm.exception(error)) + } + + pub fn to_string(&self) -> Result { + let object = self.object_for(self.vm)?; + object + .str(self.vm.raw) + .map(|value| value.as_wtf8().to_string_lossy().into_owned()) + .map_err(|error| self.vm.exception(error)) + } + + pub fn next(&self) -> Result>> { + let object = self.object_for(self.vm)?; + let result = crate::protocol::PyIter::new(object) + .next(self.vm.raw) + .map_err(|error| self.vm.exception(error))?; + match result { + crate::protocol::PyIterReturn::Return(value) => self.vm.value(Ok(value)).map(Some), + crate::protocol::PyIterReturn::StopIteration(_) => Ok(None), + } + } + + pub fn to_i64(&self) -> Result { + let object = self.object_for(self.vm)?; + i64::try_from_borrowed_object(self.vm.raw, &object) + .map_err(|error| self.vm.exception(error)) + } + + pub fn to_bool(&self) -> Result { + let object = self.object_for(self.vm)?; + object + .try_to_bool(self.vm.raw) + .map_err(|error| self.vm.exception(error)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Interpreter; + + #[test] + fn shutdown_retires_all_root_groups_after_native_panics() { + use alloc::sync::Arc; + use core::sync::atomic::{AtomicUsize, Ordering}; + + struct PanicDrop(Arc); + impl Drop for PanicDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + panic!("root destructor"); + } + } + struct CacheKey; + + let panics = Arc::new(AtomicUsize::new(0)); + let interpreter = Interpreter::without_stdlib(Default::default()); + let (state, lease) = interpreter.enter_raw(|vm| { + let lease = vm.state.roots.insert_resource(PanicDrop(panics.clone())); + vm.__replace_native::(PanicDrop(panics.clone())); + (vm.state.clone(), lease) + }); + let result = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| drop(interpreter))); + assert!(result.is_err()); + assert_eq!(panics.load(Ordering::Relaxed), 2); + assert!(state.roots.is_closed()); + assert!(state.gc.garbage_list().is_none()); + assert!(state.gc.callbacks_list().is_none()); + drop(lease); + #[cfg(feature = "threading")] + std::thread::spawn(move || drop(state)).join().unwrap(); + } + + #[test] + fn shutdown_releases_native_resources_before_disabling_callbacks() { + use alloc::sync::Arc; + use core::sync::atomic::{AtomicUsize, Ordering}; + + unsafe extern "C" fn release(object: *mut crate::PyObject) { + let capsule = unsafe { &*object } + .downcast_ref::() + .unwrap(); + let released = unsafe { Box::from_raw(capsule.pointer().cast::>()) }; + assert!(thread::try_with_attached_vm(|_| ()).is_some()); + released.fetch_add(1, Ordering::Relaxed); + } + + struct CacheKey; + let released = Arc::new(AtomicUsize::new(0)); + let interpreter = Interpreter::without_stdlib(Default::default()); + let retained_state = interpreter.enter_raw(|vm| { + let capsule = || -> PyObjectRef { + vm.ctx + .new_capsule( + Box::into_raw(Box::new(released.clone())).cast(), + None, + Some(release), + ) + .into() + }; + vm.__replace_native::(capsule()); + vm.ctx + .types + .list_type + .set_str_attr("_shutdown_resource", capsule(), &vm.ctx); + let len = vm.callable_cache.len.get().unwrap(); + len.set_attr("__module__", capsule(), vm).unwrap(); + let signum = unsafe { crate::signal::SignalNum::new_unchecked(2) }; + vm.signal_handlers.get().unwrap().borrow_mut()[signum] = Some(capsule()); + vm.state + .warnings + .once_registry() + .set_item("resource", capsule(), vm) + .unwrap(); + #[cfg(feature = "threading")] + drop( + thread::current_thread_slot() + .unwrap() + .replace_profile(capsule()), + ); + vm.state + .gc + .garbage_list() + .unwrap() + .borrow_vec_mut() + .push(capsule()); + vm.state.clone() + }); + drop(interpreter); + assert_eq!( + released.load(Ordering::Relaxed), + if cfg!(feature = "threading") { 7 } else { 6 } + ); + drop(retained_state); + } + + #[cfg(feature = "threading")] + #[test] + fn busy_finalize_preserves_the_owner_and_can_be_retried() { + let interpreter = Interpreter::without_stdlib(Default::default()); + let worker = interpreter.new_thread(); + let value = interpreter.enter(|vm| vm.new_int(42).unwrap().unbind()); + let busy = interpreter.finalize().unwrap_err(); + busy.interpreter().enter(|vm| { + assert_eq!(vm.bind(&value).unwrap().to_i64().unwrap(), 42); + }); + drop(worker); + assert_eq!(busy.retry().unwrap(), 0); + drop(value); + } + + #[cfg(feature = "threading")] + #[test] + fn scoped_shutdown_joins_python_workers_after_native_owners_leave() { + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + + let interpreter = Interpreter::without_stdlib(Default::default()); + let native_worker = interpreter.new_thread(); + let pending = Arc::new(Mutex::new(Some( + interpreter.enter_raw(VirtualMachine::new_python_thread), + ))); + let joined = Arc::new(AtomicBool::new(false)); + interpreter.enter_raw(|vm| { + let pending = pending.clone(); + let joined = joined.clone(); + let shutdown = vm.new_function("_shutdown", move |vm: &VirtualMachine| { + let worker = pending.lock().unwrap().take().unwrap(); + let parent = std::thread::spawn(move || { + worker.run_raw(|vm| { + let child = vm.new_python_thread(); + let child = std::thread::spawn(move || { + child.run(|vm| { + assert_eq!(vm.new_int(42).unwrap().to_i64().unwrap(), 42); + }); + }); + vm.allow_threads(|| child.join().unwrap()); + }); + }); + vm.allow_threads(|| parent.join().unwrap()); + joined.store(true, Ordering::Release); + }); + let attrs = vm.ctx.new_dict(); + attrs.set_item("_shutdown", shutdown.into(), vm).unwrap(); + let module = vm.new_module("threading", attrs, None); + vm.sys_module + .get_attr("modules", vm) + .unwrap() + .set_item("threading", module.into(), vm) + .unwrap(); + }); + let busy = interpreter.run(|_| Ok(())).unwrap_err(); + assert!(!joined.load(Ordering::Acquire)); + busy.interpreter().enter(|vm| { + assert_eq!(vm.new_int(7).unwrap().to_i64().unwrap(), 7); + }); + drop(native_worker); + let result = busy.retry(); + // On a regression, release the pending worker before asserting so + // its capture cannot retain the interpreter through the module. + drop(pending.lock().unwrap().take()); + assert_eq!(result.unwrap(), 0); + assert!(joined.load(Ordering::Acquire)); + } + + #[cfg(feature = "threading")] + #[test] + fn last_worker_closes_roots_before_foreign_state_snapshot_is_released() { + let interpreter = Interpreter::without_stdlib(Default::default()); + let state = interpreter.enter_raw(|vm| vm.state.clone()); + let worker = interpreter.new_thread(); + drop(interpreter); + assert!(!state.roots.is_closed()); + let other = Interpreter::without_stdlib(Default::default()); + other.enter(|outer| { + drop(worker); + assert!(state.roots.is_closed()); + assert_eq!(outer.new_int(7).unwrap().to_i64().unwrap(), 7); + }); + std::thread::spawn(move || drop(state)).join().unwrap(); + } + + #[cfg(feature = "threading")] + #[test] + fn cancelled_worker_after_native_finalize_never_enters_shutdown_wait() { + let interpreter = Interpreter::without_stdlib(Default::default()); + let worker = interpreter.new_thread(); + let _ = interpreter.finalize_raw(None); + worker.run(|vm| { + assert!(matches!(vm.new_int(1), Err(Error::InterpreterClosed))); + }); + std::thread::spawn(move || drop(worker)).join().unwrap(); + } + + #[test] + fn bootstrap_panic_retires_roots_in_their_owner() { + use alloc::sync::Arc; + use core::sync::atomic::{AtomicI64, Ordering}; + + struct Probe(Arc); + impl Drop for Probe { + fn drop(&mut self) { + self.0.store( + thread::try_with_attached_vm(|vm| vm.state.interpreter_id).unwrap_or(-1), + Ordering::Release, + ); + } + } + + let released_by = Arc::new(AtomicI64::new(-1)); + let expected_owner = Arc::new(AtomicI64::new(-2)); + let retained = PyRc::new(std::sync::Mutex::new(None)); + let release = released_by.clone(); + let expected = expected_owner.clone(); + let root = retained.clone(); + let failed = std::panic::catch_unwind(move || { + // SAFETY: the hook retains only native integers and owner roots. + unsafe { + Interpreter::builder(Default::default()).init_hook(move |vm| { + expected.store(vm.state.interpreter_id, Ordering::Release); + *root.lock().unwrap() = vm.state.roots.insert_resource(Probe(release)); + panic!("bootstrap rollback"); + }) + } + .build() + }); + assert!(failed.is_err()); + assert_eq!( + released_by.load(Ordering::Acquire), + expected_owner.load(Ordering::Acquire) + ); + drop(retained); + } + + #[test] + fn handles_bind_only_to_the_owner_and_nested_entries_invalidate_outer_tokens() { + let first = Interpreter::without_stdlib(Default::default()); + let second = first.create_subinterpreter(); + let handle = first.enter(|vm| vm.new_int(42).unwrap().unbind()); + second.enter(|vm| { + assert!(matches!(vm.bind(&handle), Err(Error::WrongInterpreter))); + }); + first.enter(|vm| { + let value = vm.bind(&handle).unwrap(); + second.enter(|_| { + assert!(matches!(value.to_i64(), Err(Error::NotAttached))); + assert!(matches!(vm.new_int(1), Err(Error::NotAttached))); + }); + assert_eq!(value.to_i64().unwrap(), 42); + }); + #[cfg(feature = "threading")] + { + let thread = first.new_thread(); + drop(first); + thread.run_raw(|raw| { + assert_eq!(Vm::new(raw).bind(&handle).unwrap().to_i64().unwrap(), 42); + }); + } + drop(handle); + } + + #[cfg(all(feature = "threading", feature = "rustpython-compiler"))] + #[test] + fn foreign_handle_drop_defers_finalizers_until_owner_entry() { + let interpreter = Interpreter::without_stdlib(Default::default()); + let handle = interpreter.enter(|vm| { + vm.exec("events = []\nclass C:\n def __del__(self): events.append(1)\n") + .unwrap(); + vm.eval("C()").unwrap().unbind() + }); + let other = interpreter.create_subinterpreter(); + other.enter(|_| std::thread::spawn(move || drop(handle)).join().unwrap()); + interpreter.enter(|vm| { + assert_eq!(vm.eval("len(events)").unwrap().to_i64().unwrap(), 1); + }); + } + + #[cfg(all(feature = "threading", feature = "rustpython-compiler"))] + #[test] + fn host_thread_exit_releases_locals_in_the_owner_and_unlocks_sentinels() { + let interpreter = Interpreter::without_stdlib(Default::default()); + interpreter.enter(|vm| { + vm.exec("import _thread\nevents = []\nlocal = _thread._local()\nclass C:\n def __del__(self): events.append(1)\n").unwrap(); + }); + let thread = interpreter.new_thread(); + std::thread::spawn(move || thread.run_raw(|raw| { + Vm::new(raw).exec("local.value = C()\nsentinel = _thread._set_sentinel()\nsentinel.acquire()\n").unwrap(); + })).join().unwrap(); + interpreter.enter(|vm| { + assert_eq!(vm.eval("len(events)").unwrap().to_i64().unwrap(), 1); + assert!(!vm.eval("sentinel.locked()").unwrap().to_bool().unwrap()); + }); + } + + #[cfg(all(feature = "threading", feature = "rustpython-compiler"))] + #[test] + fn local_initialization_failure_is_retired_during_a_root_release() { + let interpreter = Interpreter::without_stdlib(Default::default()); + interpreter.enter(|vm| { + vm.exec("import _thread\ncalls = []\nfail = False\nclass Local(_thread._local):\n def __init__(self):\n calls.append(1)\n if fail: raise ValueError\nclass C:\n def __del__(self):\n for _ in range(2):\n try: local.value\n except ValueError: pass\n").unwrap(); + }); + let thread = interpreter.new_thread(); + std::thread::spawn(move || { + thread.run_raw(|raw| { + Vm::new(raw).exec("local = Local()\n").unwrap(); + }) + }) + .join() + .unwrap(); + let handle = interpreter.enter(|vm| { + vm.exec("fail = True\n").unwrap(); + vm.eval("C()").unwrap().unbind() + }); + drop(handle); + interpreter.enter(|vm| { + assert_eq!(vm.eval("len(calls)").unwrap().to_i64().unwrap(), 3); + }); + } + + #[cfg(feature = "rustpython-compiler")] + #[test] + fn exceptions_remain_bound_and_functions_accept_bound_arguments() { + let interpreter = Interpreter::without_stdlib(Default::default()); + let error = interpreter.enter(|vm| { + let function = vm.eval("lambda x: x + 1").unwrap(); + let value = vm.new_int(41).unwrap(); + assert_eq!(function.call(&[value]).unwrap().to_i64().unwrap(), 42); + vm.eval("1 / 0").unwrap_err() + }); + let Error::Python(exception) = error else { + panic!("expected Python exception") + }; + interpreter.enter(|vm| { + assert!( + vm.bind(&exception) + .unwrap() + .repr() + .unwrap() + .contains("ZeroDivisionError") + ); + }); + } +} diff --git a/crates/vm/src/eval.rs b/crates/vm/src/eval.rs index 5a3a804688f..34d03fad2f7 100644 --- a/crates/vm/src/eval.rs +++ b/crates/vm/src/eval.rs @@ -17,7 +17,7 @@ mod tests { #[test] fn print_42() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let source = String::from("print('Hello world')"); let vars = vm.new_scope_with_builtins(); let result = eval(vm, &source, vars, "").expect("this should pass"); diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index 2c61d8ba7c1..6195dd54083 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -41,11 +41,9 @@ fn create_exception_group(ctx: &Context) -> PyRef { } #[must_use] -pub fn exception_group() -> &'static Py { - ::rustpython_vm::common::static_cell! { - static CELL: ::rustpython_vm::builtins::PyTypeRef; - } - CELL.get_or_init(|| create_exception_group(Context::genesis())) +pub fn exception_group(vm: &VirtualMachine) -> PyTypeRef { + struct ExceptionGroup; + vm.__cached_native::(|| create_exception_group(&vm.ctx)) } pub(super) mod types { @@ -336,9 +334,9 @@ pub(super) mod types { } } - let exception_group_type = crate::exception_group::exception_group(); + let exception_group_type = crate::exception_group::exception_group(vm); - let actual_cls = if cls.is(exception_group_type) { + let actual_cls = if cls.is(&exception_group_type) { if has_non_exception { return Err( vm.new_type_error("Cannot nest BaseExceptions in an ExceptionGroup") @@ -347,7 +345,7 @@ pub(super) mod types { cls } else if cls.is(vm.ctx.exceptions.base_exception_group) { if !has_non_exception { - exception_group_type.to_owned() + exception_group_type } else { cls } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 9562ad7a8b5..bfb57f8bbe4 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -51,24 +51,27 @@ static MEMORY_ERROR_FREELIST: Mutex<[Option; MEMORY_ERROR_FREEL impl PyPayload for PyBaseException { #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.exceptions.base_exception_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.exceptions.base_exception_type).to_owned() } } impl PyPayload for PyMemoryError { const PAYLOAD_TYPE_ID: core::any::TypeId = ::PAYLOAD_TYPE_ID; + fn supports_native_layout(layout: core::any::TypeId) -> bool { + ::supports_native_layout(layout) + } const HAS_FREELIST: bool = true; const MAX_FREELIST: usize = MEMORY_ERROR_FREELIST_SIZE; #[inline] unsafe fn validate_downcastable_from(obj: &PyObject) -> bool { obj.class() - .fast_issubclass(::static_type()) + .fast_issubclass(unsafe { ::static_type() }) } - fn class(_ctx: &Context) -> &'static Py { - ::static_type() + fn class(_ctx: &Context) -> crate::builtins::PyTypeRef { + (unsafe { ::static_type() }).to_owned() } unsafe fn freelist_push(obj: *mut PyObject) -> bool { @@ -869,7 +872,7 @@ impl Constructor for PyBaseException { type Args = FuncArgs; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - if cls.is(Self::class(&vm.ctx)) && !args.kwargs.is_empty() { + if cls.is(&Self::class(&vm.ctx)) && !args.kwargs.is_empty() { return Err(vm.new_type_error("BaseException() takes no keyword arguments")); } Self::new(args.args, vm) @@ -904,94 +907,95 @@ impl ExceptionZoo { pub(crate) fn init() -> Self { use self::types::*; - let base_exception_type = PyBaseException::init_builtin_type(); + let base_exception_type = unsafe { PyBaseException::init_builtin_type() }; // Sorted By Hierarchy then alphabetized. - let base_exception_group = PyBaseExceptionGroup::init_builtin_type(); - let system_exit = PySystemExit::init_builtin_type(); - let keyboard_interrupt = PyKeyboardInterrupt::init_builtin_type(); - let generator_exit = PyGeneratorExit::init_builtin_type(); + let base_exception_group = unsafe { PyBaseExceptionGroup::init_builtin_type() }; + let system_exit = unsafe { PySystemExit::init_builtin_type() }; + let keyboard_interrupt = unsafe { PyKeyboardInterrupt::init_builtin_type() }; + let generator_exit = unsafe { PyGeneratorExit::init_builtin_type() }; - let exception_type = PyException::init_builtin_type(); - let stop_iteration = PyStopIteration::init_builtin_type(); - let stop_async_iteration = PyStopAsyncIteration::init_builtin_type(); - let arithmetic_error = PyArithmeticError::init_builtin_type(); - let floating_point_error = PyFloatingPointError::init_builtin_type(); - let overflow_error = PyOverflowError::init_builtin_type(); - let zero_division_error = PyZeroDivisionError::init_builtin_type(); + let exception_type = unsafe { PyException::init_builtin_type() }; + let stop_iteration = unsafe { PyStopIteration::init_builtin_type() }; + let stop_async_iteration = unsafe { PyStopAsyncIteration::init_builtin_type() }; + let arithmetic_error = unsafe { PyArithmeticError::init_builtin_type() }; + let floating_point_error = unsafe { PyFloatingPointError::init_builtin_type() }; + let overflow_error = unsafe { PyOverflowError::init_builtin_type() }; + let zero_division_error = unsafe { PyZeroDivisionError::init_builtin_type() }; - let assertion_error = PyAssertionError::init_builtin_type(); - let attribute_error = PyAttributeError::init_builtin_type(); - let buffer_error = PyBufferError::init_builtin_type(); - let eof_error = PyEOFError::init_builtin_type(); + let assertion_error = unsafe { PyAssertionError::init_builtin_type() }; + let attribute_error = unsafe { PyAttributeError::init_builtin_type() }; + let buffer_error = unsafe { PyBufferError::init_builtin_type() }; + let eof_error = unsafe { PyEOFError::init_builtin_type() }; - let import_error = PyImportError::init_builtin_type(); - let module_not_found_error = PyModuleNotFoundError::init_builtin_type(); + let import_error = unsafe { PyImportError::init_builtin_type() }; + let module_not_found_error = unsafe { PyModuleNotFoundError::init_builtin_type() }; - let lookup_error = PyLookupError::init_builtin_type(); - let index_error = PyIndexError::init_builtin_type(); - let key_error = PyKeyError::init_builtin_type(); + let lookup_error = unsafe { PyLookupError::init_builtin_type() }; + let index_error = unsafe { PyIndexError::init_builtin_type() }; + let key_error = unsafe { PyKeyError::init_builtin_type() }; - let memory_error = PyMemoryError::init_builtin_type(); + let memory_error = unsafe { PyMemoryError::init_builtin_type() }; - let name_error = PyNameError::init_builtin_type(); - let unbound_local_error = PyUnboundLocalError::init_builtin_type(); + let name_error = unsafe { PyNameError::init_builtin_type() }; + let unbound_local_error = unsafe { PyUnboundLocalError::init_builtin_type() }; // os errors - let os_error = PyOSError::init_builtin_type(); - let blocking_io_error = PyBlockingIOError::init_builtin_type(); - let child_process_error = PyChildProcessError::init_builtin_type(); - - let connection_error = PyConnectionError::init_builtin_type(); - let broken_pipe_error = PyBrokenPipeError::init_builtin_type(); - let connection_aborted_error = PyConnectionAbortedError::init_builtin_type(); - let connection_refused_error = PyConnectionRefusedError::init_builtin_type(); - let connection_reset_error = PyConnectionResetError::init_builtin_type(); - - let file_exists_error = PyFileExistsError::init_builtin_type(); - let file_not_found_error = PyFileNotFoundError::init_builtin_type(); - let interrupted_error = PyInterruptedError::init_builtin_type(); - let is_a_directory_error = PyIsADirectoryError::init_builtin_type(); - let not_a_directory_error = PyNotADirectoryError::init_builtin_type(); - let permission_error = PyPermissionError::init_builtin_type(); - let process_lookup_error = PyProcessLookupError::init_builtin_type(); - let timeout_error = PyTimeoutError::init_builtin_type(); - - let reference_error = PyReferenceError::init_builtin_type(); - - let runtime_error = PyRuntimeError::init_builtin_type(); - let not_implemented_error = PyNotImplementedError::init_builtin_type(); - let recursion_error = PyRecursionError::init_builtin_type(); - let python_finalization_error = PyPythonFinalizationError::init_builtin_type(); - - let syntax_error = PySyntaxError::init_builtin_type(); - let incomplete_input_error = PyIncompleteInputError::init_builtin_type(); - let indentation_error = PyIndentationError::init_builtin_type(); - let tab_error = PyTabError::init_builtin_type(); - - let system_error = PySystemError::init_builtin_type(); - let type_error = PyTypeError::init_builtin_type(); - let value_error = PyValueError::init_builtin_type(); - let unicode_error = PyUnicodeError::init_builtin_type(); - let unicode_decode_error = PyUnicodeDecodeError::init_builtin_type(); - let unicode_encode_error = PyUnicodeEncodeError::init_builtin_type(); - let unicode_translate_error = PyUnicodeTranslateError::init_builtin_type(); + let os_error = unsafe { PyOSError::init_builtin_type() }; + let blocking_io_error = unsafe { PyBlockingIOError::init_builtin_type() }; + let child_process_error = unsafe { PyChildProcessError::init_builtin_type() }; + + let connection_error = unsafe { PyConnectionError::init_builtin_type() }; + let broken_pipe_error = unsafe { PyBrokenPipeError::init_builtin_type() }; + let connection_aborted_error = unsafe { PyConnectionAbortedError::init_builtin_type() }; + let connection_refused_error = unsafe { PyConnectionRefusedError::init_builtin_type() }; + let connection_reset_error = unsafe { PyConnectionResetError::init_builtin_type() }; + + let file_exists_error = unsafe { PyFileExistsError::init_builtin_type() }; + let file_not_found_error = unsafe { PyFileNotFoundError::init_builtin_type() }; + let interrupted_error = unsafe { PyInterruptedError::init_builtin_type() }; + let is_a_directory_error = unsafe { PyIsADirectoryError::init_builtin_type() }; + let not_a_directory_error = unsafe { PyNotADirectoryError::init_builtin_type() }; + let permission_error = unsafe { PyPermissionError::init_builtin_type() }; + let process_lookup_error = unsafe { PyProcessLookupError::init_builtin_type() }; + let timeout_error = unsafe { PyTimeoutError::init_builtin_type() }; + + let reference_error = unsafe { PyReferenceError::init_builtin_type() }; + + let runtime_error = unsafe { PyRuntimeError::init_builtin_type() }; + let not_implemented_error = unsafe { PyNotImplementedError::init_builtin_type() }; + let recursion_error = unsafe { PyRecursionError::init_builtin_type() }; + let python_finalization_error = unsafe { PyPythonFinalizationError::init_builtin_type() }; + + let syntax_error = unsafe { PySyntaxError::init_builtin_type() }; + let incomplete_input_error = unsafe { PyIncompleteInputError::init_builtin_type() }; + let indentation_error = unsafe { PyIndentationError::init_builtin_type() }; + let tab_error = unsafe { PyTabError::init_builtin_type() }; + + let system_error = unsafe { PySystemError::init_builtin_type() }; + let type_error = unsafe { PyTypeError::init_builtin_type() }; + let value_error = unsafe { PyValueError::init_builtin_type() }; + let unicode_error = unsafe { PyUnicodeError::init_builtin_type() }; + let unicode_decode_error = unsafe { PyUnicodeDecodeError::init_builtin_type() }; + let unicode_encode_error = unsafe { PyUnicodeEncodeError::init_builtin_type() }; + let unicode_translate_error = unsafe { PyUnicodeTranslateError::init_builtin_type() }; #[cfg(feature = "jit")] - let jit_error = PyJitError::init_builtin_type(); - - let warning = PyWarning::init_builtin_type(); - let deprecation_warning = PyDeprecationWarning::init_builtin_type(); - let pending_deprecation_warning = PyPendingDeprecationWarning::init_builtin_type(); - let runtime_warning = PyRuntimeWarning::init_builtin_type(); - let syntax_warning = PySyntaxWarning::init_builtin_type(); - let user_warning = PyUserWarning::init_builtin_type(); - let future_warning = PyFutureWarning::init_builtin_type(); - let import_warning = PyImportWarning::init_builtin_type(); - let unicode_warning = PyUnicodeWarning::init_builtin_type(); - let bytes_warning = PyBytesWarning::init_builtin_type(); - let resource_warning = PyResourceWarning::init_builtin_type(); - let encoding_warning = PyEncodingWarning::init_builtin_type(); + let jit_error = unsafe { PyJitError::init_builtin_type() }; + + let warning = unsafe { PyWarning::init_builtin_type() }; + let deprecation_warning = unsafe { PyDeprecationWarning::init_builtin_type() }; + let pending_deprecation_warning = + unsafe { PyPendingDeprecationWarning::init_builtin_type() }; + let runtime_warning = unsafe { PyRuntimeWarning::init_builtin_type() }; + let syntax_warning = unsafe { PySyntaxWarning::init_builtin_type() }; + let user_warning = unsafe { PyUserWarning::init_builtin_type() }; + let future_warning = unsafe { PyFutureWarning::init_builtin_type() }; + let import_warning = unsafe { PyImportWarning::init_builtin_type() }; + let unicode_warning = unsafe { PyUnicodeWarning::init_builtin_type() }; + let bytes_warning = unsafe { PyBytesWarning::init_builtin_type() }; + let resource_warning = unsafe { PyResourceWarning::init_builtin_type() }; + let encoding_warning = unsafe { PyEncodingWarning::init_builtin_type() }; Self { base_exception_type, @@ -3638,7 +3642,7 @@ pub fn exception_group_match( } else { // Naked exception - wrap it in ExceptionGroup let excs = vm.ctx.new_tuple(vec![exc_value.to_owned()]); - let eg_type: PyObjectRef = crate::exception_group::exception_group().to_owned().into(); + let eg_type: PyObjectRef = crate::exception_group::exception_group(vm).into(); let wrapped = eg_type.call((vm.ctx.new_str(""), excs), vm)?; // Copy traceback from original exception if let Ok(exc) = exc_value.to_owned().downcast::() @@ -3744,7 +3748,7 @@ pub fn prep_reraise_star(orig: &PyObject, excs: &PyObject, vm: &VirtualMachine) // Create new ExceptionGroup for multiple exceptions let excs_tuple = vm.ctx.new_tuple(raised); - let eg_type: PyObjectRef = crate::exception_group::exception_group().to_owned().into(); + let eg_type: PyObjectRef = crate::exception_group::exception_group(vm).into(); eg_type.call((vm.ctx.new_str(""), excs_tuple), vm) } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index cc54ad6ee18..d6784349e90 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -79,7 +79,10 @@ unsafe fn owned_chain_frame(iframe: *const InterpreterFrame) -> Option Option { +/// # Safety +/// Keep the returned frame attached to its owning interpreter for its entire +/// lifetime, following [`crate::embedding`]'s native ownership contract. +pub unsafe fn current_thread_frame() -> Option { let ptr = crate::vm::thread::get_current_frame(); unsafe { owned_chain_frame(ptr) } } @@ -107,7 +110,7 @@ pub fn current_thread_frame_materialize(vm: &VirtualMachine) -> Option Option { +pub(crate) fn current_globals() -> Option { let ptr = crate::vm::thread::get_current_frame(); if ptr.is_null() { return None; @@ -118,7 +121,7 @@ pub fn current_globals() -> Option { /// Read the code object from the topmost frame on this thread's chain. /// Returns `None` if the chain is empty. #[must_use] -pub fn current_code() -> Option> { +pub(crate) fn current_code() -> Option> { let ptr = crate::vm::thread::get_current_frame(); if ptr.is_null() { return None; @@ -128,7 +131,7 @@ pub fn current_code() -> Option> { /// Read the builtins object from the topmost frame on this thread's chain. #[must_use] -pub fn current_builtins() -> Option { +pub(crate) fn current_builtins() -> Option { let ptr = crate::vm::thread::get_current_frame(); if ptr.is_null() { return None; @@ -157,7 +160,12 @@ pub fn frame_at_offset(offset: usize, vm: &VirtualMachine) -> Option Option { +/// # Safety +/// Keep the returned frame attached to its owning interpreter for its entire +/// lifetime, following [`crate::embedding`]'s native ownership contract. +pub unsafe fn find_owned_chain_frame_by_iframe( + target: *const InterpreterFrame, +) -> Option { let mut cur = crate::vm::thread::get_current_frame(); while !cur.is_null() { if core::ptr::eq(cur, target) { @@ -171,7 +179,10 @@ pub fn find_owned_chain_frame_by_iframe(target: *const InterpreterFrame) -> Opti /// If `target` FrameObject is on the current thread's chain, return an /// owned reference to it; otherwise `None`. Presence on the chain proves liveness. #[must_use] -pub fn find_owned_chain_frame(target: *const FrameObject) -> Option { +/// # Safety +/// Keep the returned frame attached to its owning interpreter for its entire +/// lifetime, following [`crate::embedding`]'s native ownership contract. +pub unsafe fn find_owned_chain_frame(target: *const FrameObject) -> Option { let mut cur = crate::vm::thread::get_current_frame(); while !cur.is_null() { let iframe_ref = unsafe { &*cur }; @@ -188,7 +199,7 @@ pub fn find_owned_chain_frame(target: *const FrameObject) -> Option)) { +pub(crate) fn for_each_current_frame(mut f: impl FnMut(&Py)) { let mut cur = crate::vm::thread::get_current_frame(); while !cur.is_null() { let iframe_ref = unsafe { &*cur }; @@ -1621,8 +1632,8 @@ impl PyPayload for FrameObject { const NEW_REF_UNTRACKED: bool = true; #[inline] - fn class(ctx: &Context) -> &'static Py { - ctx.types.frame_type + fn class(ctx: &Context) -> crate::builtins::PyTypeRef { + (ctx.types.frame_type).to_owned() } #[inline] @@ -5033,12 +5044,14 @@ impl ExecutingFrame<'_> { CommonConstant::BuiltinAll => vm .callable_cache .builtin_all - .clone() + .get() + .cloned() .expect("builtin_all not initialized"), CommonConstant::BuiltinAny => vm .callable_cache .builtin_any - .clone() + .get() + .cloned() .expect("builtin_any not initialized"), CommonConstant::BuiltinList => vm.ctx.types.list_type.to_owned().into(), CommonConstant::BuiltinSet => vm.ctx.types.set_type.to_owned().into(), @@ -6753,7 +6766,7 @@ impl ExecutingFrame<'_> { && vm .callable_cache .len - .as_ref() + .get() .is_some_and(|len_callable| callable.is(len_callable)) { let len = obj.length(vm)?; @@ -6780,7 +6793,7 @@ impl ExecutingFrame<'_> { if vm .callable_cache .isinstance - .as_ref() + .get() .is_some_and(|isinstance_callable| callable.is(isinstance_callable)) { // Stack: [callable, self_or_null, args...]; effective_nargs == 2, @@ -7005,7 +7018,7 @@ impl ExecutingFrame<'_> { if vm .callable_cache .list_append - .as_ref() + .get() .is_some_and(|list_append| callable.is(list_append)) && self_or_null_is_some && self_is_list @@ -7055,7 +7068,7 @@ impl ExecutingFrame<'_> { .localsplus .stack_index(self_index) .as_ref() - .is_some_and(|self_obj| self_obj.class().is(descr.common.typ)) + .is_some_and(|self_obj| self_obj.class().is(&descr.common.typ)) { let func = descr.method.func; let callee = Callee::named(descr.method.name).with_instance_arg(true); @@ -7096,7 +7109,7 @@ impl ExecutingFrame<'_> { .localsplus .stack_index(self_index) .as_ref() - .is_some_and(|self_obj| self_obj.class().is(descr.common.typ)) + .is_some_and(|self_obj| self_obj.class().is(&descr.common.typ)) { let func = descr.method.func; let callee = Callee::named(descr.method.name).with_instance_arg(true); @@ -7137,7 +7150,7 @@ impl ExecutingFrame<'_> { .localsplus .stack_index(self_index) .as_ref() - .is_some_and(|self_obj| self_obj.class().is(descr.common.typ)) + .is_some_and(|self_obj| self_obj.class().is(&descr.common.typ)) { let func = descr.method.func; let callee = Callee::named(descr.method.name).with_instance_arg(true); @@ -7243,7 +7256,7 @@ impl ExecutingFrame<'_> { .localsplus .stack_index(self_index) .as_ref() - .is_some_and(|self_obj| self_obj.class().is(descr.common.typ)) + .is_some_and(|self_obj| self_obj.class().is(&descr.common.typ)) { let func = descr.method.func; let callee = Callee::named(descr.method.name).with_instance_arg(true); @@ -11014,7 +11027,7 @@ impl ExecutingFrame<'_> { && vm .callable_cache .list_append - .as_ref() + .get() .is_some_and(|list_append| callable.is(list_append)) { Instruction::CallListAppend @@ -11058,7 +11071,7 @@ impl ExecutingFrame<'_> { && vm .callable_cache .len - .as_ref() + .get() .is_some_and(|len_callable| callable.is(len_callable)) { Instruction::CallLen @@ -11071,7 +11084,7 @@ impl ExecutingFrame<'_> { && vm .callable_cache .isinstance - .as_ref() + .get() .is_some_and(|isinstance_callable| callable.is(isinstance_callable)) { Instruction::CallIsinstance @@ -11525,13 +11538,13 @@ impl ExecutingFrame<'_> { let new_op = if cls.is(vm.ctx.types.bool_type) { Some(Instruction::ToBoolBool) - } else if cls.is(PyInt::class(&vm.ctx)) { + } else if cls.is(&PyInt::class(&vm.ctx)) { Some(Instruction::ToBoolInt) } else if cls.is(vm.ctx.types.none_type) { Some(Instruction::ToBoolNone) - } else if cls.is(PyList::class(&vm.ctx)) { + } else if cls.is(&PyList::class(&vm.ctx)) { Some(Instruction::ToBoolList) - } else if cls.is(PyStr::class(&vm.ctx)) { + } else if cls.is(&PyStr::class(&vm.ctx)) { Some(Instruction::ToBoolStr) } else if cls.slots().flags.has_feature(PyTypeFlags::HEAPTYPE) { // Capture the version before inspecting the bool/len slots so a diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index 2106a668f77..49f091c02c7 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -376,7 +376,7 @@ impl Callee { /// The same, for the type a slot was written for. #[must_use] pub fn of(vm: &VirtualMachine) -> Self { - Self::for_type(T::class(&vm.ctx)) + Self::for_type(&T::class(&vm.ctx)) } /// Marks a call whose leading argument fills the method's instance parameter. diff --git a/crates/vm/src/function/buffer.rs b/crates/vm/src/function/buffer.rs index 51f9dc28ac5..6413382f13b 100644 --- a/crates/vm/src/function/buffer.rs +++ b/crates/vm/src/function/buffer.rs @@ -60,15 +60,7 @@ impl ArgBytesLike { /// object's -- are borrowed where they lie, which is all CPython holds in /// either case. pub fn borrow_buf_unlocked(&self, vm: &VirtualMachine) -> PyResult> { - let borrowed = self.borrow_buf(); - if !borrowed.is_locked() { - return Ok(UnlockedBuf::Borrowed(borrowed)); - } - let mut copy = Vec::new(); - copy.try_reserve_exact(borrowed.len()) - .map_err(|_| vm.no_memory_error())?; - copy.extend_from_slice(&borrowed); - Ok(UnlockedBuf::Copied(copy)) + UnlockedBuf::new(self.borrow_buf(), vm) } #[must_use] @@ -160,6 +152,19 @@ pub enum UnlockedBuf<'a> { Copied(Vec), } +impl<'a> UnlockedBuf<'a> { + pub(crate) fn new(borrowed: BorrowedValue<'a, [u8]>, vm: &VirtualMachine) -> PyResult { + if !borrowed.is_locked() { + return Ok(Self::Borrowed(borrowed)); + } + let mut copy = Vec::new(); + copy.try_reserve_exact(borrowed.len()) + .map_err(|_| vm.no_memory_error())?; + copy.extend_from_slice(&borrowed); + Ok(Self::Copied(copy)) + } +} + impl core::ops::Deref for UnlockedBuf<'_> { type Target = [u8]; diff --git a/crates/vm/src/function/method.rs b/crates/vm/src/function/method.rs index 0a80efc939e..dd40b703452 100644 --- a/crates/vm/src/function/method.rs +++ b/crates/vm/src/function/method.rs @@ -156,11 +156,7 @@ impl PyMethodDef { } } - pub fn to_proper_method( - &'static self, - class: &'static Py, - ctx: &Context, - ) -> PyObjectRef { + pub fn to_proper_method(&'static self, class: &Py, ctx: &Context) -> PyObjectRef { if self.flags.contains(PyMethodFlags::METHOD) { self.build_method(ctx, class).into() } else if self.flags.contains(PyMethodFlags::CLASS) { @@ -183,19 +179,11 @@ impl PyMethodDef { } } - pub fn to_method( - &'static self, - class: &'static Py, - ctx: &Context, - ) -> PyMethodDescriptor { + pub fn to_method(&'static self, class: &Py, ctx: &Context) -> PyMethodDescriptor { PyMethodDescriptor::new(self, class, ctx) } - pub const fn to_bound_method( - &'static self, - obj: PyObjectRef, - class: &'static Py, - ) -> PyNativeMethod { + pub fn to_bound_method(&'static self, obj: PyObjectRef, class: &Py) -> PyNativeMethod { PyNativeMethod { func: PyNativeFunction { zelf: Some(obj), @@ -204,7 +192,7 @@ impl PyMethodDef { module: crate::object::PyAtomicRef::new_empty(), _method_def_owner: None, }, - class, + class: class.to_owned(), } } @@ -234,7 +222,7 @@ impl PyMethodDef { pub fn build_method( &'static self, ctx: &Context, - class: &'static Py, + class: &Py, ) -> PyRef { debug_assert!(self.flags.contains(PyMethodFlags::METHOD)); let method = self.to_method(class, ctx); @@ -245,7 +233,7 @@ impl PyMethodDef { &'static self, ctx: &Context, obj: PyObjectRef, - class: &'static Py, + class: &Py, ) -> PyRef { PyRef::new_ref( self.to_bound_method(obj, class), @@ -257,7 +245,7 @@ impl PyMethodDef { pub fn build_classmethod( &'static self, ctx: &Context, - class: &'static Py, + class: &Py, ) -> PyRef { debug_assert!(self.flags.contains(PyMethodFlags::CLASS)); PyClassMethodDescriptor::new(self, class, ctx).into_ref(ctx) @@ -266,7 +254,7 @@ impl PyMethodDef { pub fn build_staticmethod( &'static self, ctx: &Context, - class: &'static Py, + class: &Py, ) -> PyRef { debug_assert!(self.flags.contains(PyMethodFlags::STATIC)); // Set zelf to the class (m_self = type for static methods). @@ -278,7 +266,11 @@ impl PyMethodDef { module: crate::object::PyAtomicRef::new_empty(), _method_def_owner: None, }; - PyNativeMethod { func, class }.into_ref(ctx) + PyNativeMethod { + func, + class: class.to_owned(), + } + .into_ref(ctx) } /// Concatenate method groups. A pending body is copied from `docs`, then cleared. @@ -375,15 +367,51 @@ impl core::fmt::Debug for PyMethodDef { // This is not a part of CPython API. // But useful to support dynamically generated methods #[pyclass(name, module = false, ctx = "method_def")] -#[derive(Debug)] pub struct HeapMethodDef { method: PyMethodDef, + _function_owner: Option>, +} + +impl core::fmt::Debug for HeapMethodDef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.method.fmt(f) + } } impl HeapMethodDef { #[must_use] pub const fn new(method: PyMethodDef) -> Self { - Self { method } + Self { + method, + _function_owner: None, + } + } + + pub(crate) fn with_owned_function( + name: &'static str, + function: Box, + flags: PyMethodFlags, + doc: super::ItemDoc, + ) -> Self { + // SAFETY: the box has a stable address and is retained by this payload. + // The definition is private; all functions/descriptors referring to it + // retain this HeapMethodDef and expose no static reference to callers. + let func = unsafe { &*core::ptr::from_ref(&*function) }; + Self { + method: PyMethodDef { + name, + func, + flags, + #[cfg(feature = "doc")] + doc_off: doc.offset, + #[cfg(feature = "doc")] + doc_len: doc.len, + #[cfg(feature = "doc")] + doc_body_pending: false, + doc: doc.text, + }, + _function_owner: Some(function), + } } } @@ -409,7 +437,7 @@ impl Py { pub fn build_method( &self, - class: &'static Py, + class: &Py, vm: &VirtualMachine, ) -> PyRef { let mut function = unsafe { self.method() }.to_method(class, &vm.ctx); @@ -431,3 +459,41 @@ pub(crate) fn init(ctx: &'static Context) { let _ = ctx.intern_str(HeapMethodDef::NAME); } + +#[cfg(test)] +mod tests { + use super::*; + use crate::AsObject; + + #[test] + fn generated_functions_own_and_release_their_native_captures() { + use alloc::sync::Arc; + let captured = Arc::new(17); + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let capture = captured.clone(); + let definition = vm.ctx.new_method_def( + "captured", + move || *capture, + PyMethodFlags::empty(), + super::super::ItemDoc::NONE, + ); + let function = definition.build_function(vm, None); + let descriptor = definition.build_method(vm.ctx.types.object_type, vm); + drop(definition); + assert_eq!(Arc::strong_count(&captured), 2); + assert_eq!( + function + .as_object() + .call((), vm) + .unwrap() + .try_into_value::(vm) + .unwrap(), + 17 + ); + drop(function); + assert_eq!(Arc::strong_count(&captured), 2); + drop(descriptor); + assert_eq!(Arc::strong_count(&captured), 1); + }); + } +} diff --git a/crates/vm/src/function/mod.rs b/crates/vm/src/function/mod.rs index 0fcbffc96bb..2cdba55a832 100644 --- a/crates/vm/src/function/mod.rs +++ b/crates/vm/src/function/mod.rs @@ -23,6 +23,7 @@ pub(crate) use argument::{arity_message, unexpected_keyword_message}; pub use arithmetic::{PyArithmeticValue, PyComparisonValue}; pub use buffer::{ ArgAsciiBuffer, ArgBytesLike, ArgContiguousBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike, + UnlockedBuf, }; pub use builtin::{IntoPyNativeFn, PyNativeFn, static_func, static_raw_func}; pub use doctext::{ItemDoc, db_doc, plain_doc}; diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index 774f2ff39ba..92a3697cb2b 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -1,13 +1,12 @@ //! Garbage Collection State and Algorithm //! -//! Generational garbage collection using an intrusive doubly-linked list. +//! Interpreter-local heaps and a compact, revalidated cycle-collection snapshot. -use crate::common::linked_list::LinkedList; -use crate::common::lock::{PyMutex, PyRwLock}; -use crate::object::{GC_NO_OWNER, GC_PERMANENT, GC_REACHABLE, GC_UNTRACKED, GcLink, GcOwner}; +use crate::common::{lock::PyMutex, rc::PyRc}; +use crate::object::{GC_PERMANENT, GC_UNTRACKED}; use crate::{AsObject, PyObject, PyObjectRef}; use core::ptr::NonNull; -use core::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; fn elapsed_secs( #[cfg(target_arch = "wasm32")] _start: (), @@ -57,9 +56,7 @@ pub struct GcStats { /// One generation's collection policy and statistics, per interpreter. /// -/// The objects themselves live in the process-wide lists on [`GcState`], so the -/// occupancy count sits there; what an interpreter owns is when to collect and -/// what its own collections have done. +/// Object membership and allocation pressure belong to this interpreter's heap. pub struct GcGeneration { /// Threshold for triggering collection threshold: AtomicU32, @@ -129,203 +126,125 @@ impl GcGeneration { } } -/// Drop one from a generation's occupancy. -/// -/// A collection resets the counts of the generations it emptied, but it only -/// empties its own interpreter's objects; another interpreter's stay behind with -/// the count already zeroed, and untracking one of those must not wrap. -fn release_count(count: &AtomicUsize) { - let _ = count.try_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1)); -} +mod collector; +mod graph; +mod heap; -/// Whether `owner`'s collections act on `obj`. -/// -/// Objects with no owner — everything the shared context allocates, and anything -/// allocated with no interpreter current — belong to all of them. -fn is_owned_by(obj: &PyObject, owner: GcOwner) -> bool { - let obj_owner = obj.gc_owner(); - obj_owner == owner || obj_owner == GC_NO_OWNER -} +use graph::Graph; +pub(crate) use heap::GcHandle; +pub use heap::GcHeap; -/// Wrapper for NonNull to impl Hash/Eq for use in temporary collection sets. -/// Only used within collect_inner, never shared across threads. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -struct GcPtr(NonNull); +type HeapWeak = cfg_select! { + feature = "threading" => alloc::sync::Weak::, + _ => alloc::rc::Weak::, +}; -/// Hashing for the tables a collection keys by an object's address. -/// -/// The default hasher is SipHash, which buys resistance against a caller -/// choosing keys that collide. Nothing chooses these keys: they are addresses -/// this process handed out, and the tables live and die inside one collection. -/// What a collection needs from them is speed -- it hashes every tracked -/// object -- so this runs the address through a handful of multiplies and -/// shifts instead. The shifts are what earns the speed: a table picks its -/// bucket from the low bits, and an address arrives with its low bits zeroed -/// by alignment, so entropy has to be carried downward or every object lands -/// in the same few buckets. +/// Collection frequency is independent of heap occupancy. #[derive(Default)] -struct GcPtrHasher(u64); +struct GcSchedule { + young_collections: usize, + middle_collections: usize, + long_lived_pending: usize, + long_lived_total: usize, +} -impl core::hash::Hasher for GcPtrHasher { - fn finish(&self) -> u64 { - self.0 +impl GcSchedule { + fn generation(&self, thresholds: (u32, u32, u32)) -> usize { + // As in CPython, require growth before scanning the entire old heap. + // A full collection every fixed number of allocations would make + // building a large, retained heap quadratic. + if self.middle_collections > thresholds.2 as usize + && self.long_lived_pending >= self.long_lived_total / 4 + { + 2 + } else { + usize::from(self.young_collections > thresholds.1 as usize) + } } - fn write_usize(&mut self, value: usize) { - let mut z = (value as u64).wrapping_add(0x9E37_79B9_7F4A_7C15); - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - self.0 = z ^ (z >> 31); + fn start_collection(&mut self, generation: usize) { + if generation == 0 { + self.young_collections = self.young_collections.saturating_add(1); + } else { + self.young_collections = 0; + self.middle_collections = if generation == 1 { + self.middle_collections.saturating_add(1) + } else { + 0 + }; + } } - fn write(&mut self, bytes: &[u8]) { - // Addresses reach this hasher through `write_usize`; a key hashed any - // other way still has to land somewhere sensible. - for &byte in bytes { - self.0 = (self.0 ^ u64::from(byte)).wrapping_mul(0x0100_0000_01B3); + fn record_survivors(&mut self, generation: usize, survivors: usize) { + if generation == 1 { + self.long_lived_pending = self.long_lived_pending.saturating_add(survivors); + } else if generation == 2 { + self.long_lived_pending = 0; + self.long_lived_total = survivors; } } } -type GcBuildHasher = core::hash::BuildHasherDefault; -type GcSet = std::collections::HashSet; -type GcMap = std::collections::HashMap; - -/// RAII barrier that parks every other thread for the pointer-reading phases -/// of a collection and lets them run again before finalizers execute. -/// -/// Reference subtraction, the reachability walk and the strong-reference -/// snapshot dereference the interpreter state of every tracked object, -/// including the `localsplus` of frames that other threads are actively -/// executing. Those writes carry no synchronization, so the reads are only -/// well-defined while all other threads are parked at a safepoint. Restarting -/// happens explicitly once the snapshot has pinned every object; `Drop` is a -/// backstop that also restarts on the early-return paths. -/// -/// A collection acts on one interpreter's objects, but its candidates include -/// the ones no interpreter owns, which every interpreter can reference and so -/// incref. Reading a refcount that another interpreter is changing is what -/// makes an object look unreachable when it is not, so every live interpreter -/// is stopped, not just the collecting one. Stopping in `runtime` id order -/// keeps exclusion acquisition ordered; the `collecting` mutex additionally -/// serializes collections process-wide, so no second collector can take these -/// exclusions in another order. +/// Parks only the candidate heap's owner while reading Python pointers. +/// Graph analysis and finalizers run after its threads resume. #[cfg(feature = "threading")] struct CollectStopTheWorld { - /// Stopped interpreter states, in stop order. Held as strong references so - /// an interpreter cannot be dropped between stop and restart, and kept past - /// the restart so that releasing the last one — which frees that - /// interpreter's objects, and so removes them from these lists — happens - /// after the collection has let go of the generation locks. - stopped: Vec>, - /// Keeps interpreters from registering between the snapshot below and the - /// restart. One registered in that window would be missing from `stopped`, - /// so its bootstrap would keep running — and mutating the shared generation - /// lists — while this collection reads them. - admission: Option>, + state: Option>, restarted: bool, } #[cfg(feature = "threading")] impl CollectStopTheWorld { - /// Request stop-the-world on every live interpreter when the current thread - /// has an attached VM. Falls back to no barrier when no VM is attached (the - /// tracked-object reads then run without other threads only if the caller - /// guarantees it). - fn new() -> Self { - // No attached VM means no interpreter is running Python on this thread; - // keep the historical no-barrier fallback. - if !crate::vm::thread::current_vm_is_set() { - return Self { - stopped: Vec::new(), - admission: None, - restarted: true, - }; - } - - // Accumulate into a live `Self` rather than a bare Vec: if a later - // `stop_the_world` unwinds, dropping this guard restarts the - // interpreters already stopped, instead of leaving their threads parked - // and their exclusion held forever. - let mut guard = Self { - stopped: Vec::new(), - admission: Some(crate::vm::runtime::lock_admission_for_stop()), + fn new(gc: &GcInterpreterState) -> Self { + let state = gc + .heap + .owner + .and_then(crate::vm::runtime::lookup_interpreter); + if let Some(state) = &state { + debug_assert!(core::ptr::eq(gc, &state.gc)); + state.stop_the_world.stop_the_world(state); + } + // A standalone, unpublished native heap has no executing interpreter. + Self { + state, restarted: false, - }; - for state in crate::vm::runtime::live_interpreter_states() { - state.stop_the_world.stop_the_world(&state); - guard.stopped.push(state); } - guard } - /// Restart the world. Idempotent. fn restart(&mut self) { - if self.restarted { - return; - } - self.restarted = true; - // Reverse of the stop order. The references stay until this guard is - // dropped; see the field comment. - for state in self.stopped.iter().rev() { - state.stop_the_world.start_the_world(state); + if !self.restarted { + self.restarted = true; + if let Some(state) = &self.state { + state.stop_the_world.start_the_world(state); + } } - // Nothing is parked any more, so registration may resume. - self.admission = None; - } - - /// Whether this collection actually stopped the world. - #[cfg(all(unix, debug_assertions))] - fn is_stopped(&self) -> bool { - !self.stopped.is_empty() } } -#[cfg(feature = "threading")] impl Drop for CollectStopTheWorld { fn drop(&mut self) { self.restart(); } } -/// The process-wide object lists every interpreter's collections walk. -/// -/// Interpreter-owned policy and results live in [`GcInterpreterState`]; what is -/// here is shared because the lists are: an object is untracked from -/// `default_dealloc`, where no interpreter is in scope, so it has to be findable -/// without one. +#[cfg(not(feature = "threading"))] +struct CollectStopTheWorld; +#[cfg(not(feature = "threading"))] +impl CollectStopTheWorld { + fn new(_gc: &GcInterpreterState) -> Self { + Self + } + fn restart(&mut self) {} +} + +/// Process coordination and weak heap directory. Object operations go directly +/// through the owning heap, including deallocation outside an entered VM. pub struct GcState { - /// Per-generation intrusive linked lists for object tracking. - /// Objects start in gen0, survivors are promoted to gen1, then gen2. - generation_lists: [PyRwLock>; 3], - /// Frozen/permanent objects (excluded from normal GC) - permanent_list: PyRwLock>, - /// Number of tracked objects per generation, across all interpreters. - /// - /// Advisory: they drive the collection threshold and `gc.get_count()`, and - /// the generation locks — not these counters — order the list changes they - /// describe. Every access is therefore relaxed, which keeps the tracking and - /// untracking of every object off the barrier path. - counts: [AtomicUsize; 3], - /// Number of frozen objects. Advisory, like `counts`. - permanent_count: AtomicUsize, - /// Mutex for collection (prevents concurrent collections) + shared: PyRc, + heaps: PyMutex>, collecting: PyMutex<()>, - /// Next `gc_owner` tag to hand to an interpreter. - next_owner: AtomicU16, - /// Tags of interpreters that are gone. Their objects outlived them, so a - /// collection adopts them — tags them `GC_NO_OWNER` again — as it walks, - /// rather than leaving them for a collector that will never come. - retired: PyMutex>, } -// SAFETY: All fields are either inherently Send/Sync (atomics, RwLock, Mutex) or protected by PyMutex. -// LinkedList is Send+Sync because GcLink's Target (PyObject) is Send+Sync. -#[cfg(feature = "threading")] -unsafe impl Send for GcState {} -#[cfg(feature = "threading")] -unsafe impl Sync for GcState {} - impl Default for GcState { fn default() -> Self { Self::new() @@ -334,1023 +253,200 @@ impl Default for GcState { impl GcState { #[must_use] - pub const fn new() -> Self { + pub fn new() -> Self { Self { - generation_lists: [ - PyRwLock::new(LinkedList::new()), - PyRwLock::new(LinkedList::new()), - PyRwLock::new(LinkedList::new()), - ], - permanent_list: PyRwLock::new(LinkedList::new()), - counts: [ - AtomicUsize::new(0), - AtomicUsize::new(0), - AtomicUsize::new(0), - ], - permanent_count: AtomicUsize::new(0), + shared: PyRc::new(GcHeap::new(None)), + heaps: PyMutex::new(Vec::new()), collecting: PyMutex::new(()), - next_owner: AtomicU16::new(GC_NO_OWNER + 1), - retired: PyMutex::new(Vec::new()), } } - /// Reserve a tag for a new interpreter. Tags are never reused; exhausting - /// the tag space falls back to `GC_NO_OWNER`, which costs isolation but - /// stays correct, rather than aliasing a live interpreter. - fn alloc_owner(&self) -> GcOwner { - self.next_owner - .try_update(Ordering::Relaxed, Ordering::Relaxed, |next| { - next.checked_add(1) - }) - .unwrap_or(GC_NO_OWNER) - } - - /// Record that `owner`'s interpreter is gone, so the next collection adopts - /// whatever it left behind. Retagging the objects here would mean walking - /// every list under an interpreter drop, which happens while a collection - /// holds the collecting lock. - fn retire_owner(&self, owner: GcOwner) { - if owner == GC_NO_OWNER { - return; - } - self.retired.lock().push(owner); - } - - /// Get counts for all generations. Tracked objects are shared, so these are - /// process-wide even though the thresholds they are compared against are - /// per interpreter. - pub fn get_count(&self) -> (usize, usize, usize) { - ( - self.counts[0].load(Ordering::Relaxed), - self.counts[1].load(Ordering::Relaxed), - self.counts[2].load(Ordering::Relaxed), - ) - } - - /// Track a new object (add to gen0) as owned by `owner`. - /// O(1) — intrusive linked list push_front, no hashing. - /// - /// # Safety - /// obj must be a valid pointer to a PyObject - pub unsafe fn track_object(&self, obj: NonNull, owner: GcOwner) { - let obj_ref = unsafe { obj.as_ref() }; - obj_ref.set_gc_tracked(); - obj_ref.set_gc_generation(0); - obj_ref.set_gc_owner(owner); - - self.generation_lists[0].write().push_front(obj); - self.counts[0].fetch_add(1, Ordering::Relaxed); + fn new_heap(&self, owner: i64) -> PyRc { + let heap = PyRc::new(GcHeap::new(Some(owner))); + let mut heaps = self.heaps.lock(); + heaps.retain(|h| h.strong_count() != 0); + heaps.push(PyRc::downgrade(&heap)); + heap } - /// Track a freshly allocated object (add to gen0) as owned by `owner`. - /// - /// Like [`Self::track_object`], but for the hot allocation path only: - /// `obj`'s `gc_bits` must still hold its freshly-initialized value of `0` - /// (true right after `Py::new` or a freelist pop, both of which zero - /// it), so the tracked bit can go in with a plain store instead of the - /// `fetch_or` `set_gc_tracked()` needs to be safe for the general case - /// (e.g. re-tracking a resurrected object, whose bits are not zero — it - /// may carry `FINALIZED`). A plain relaxed store compiles to a single - /// store instruction; `fetch_or` is a read-modify-write that, even - /// without contention, is measurably pricier on a hot per-allocation path. - /// - /// # Safety - /// obj must be a valid pointer to a PyObject whose `gc_bits` is still `0`. - unsafe fn track_object_fresh(&self, obj: NonNull, owner: GcOwner) { - let obj_ref = unsafe { obj.as_ref() }; - obj_ref.init_gc_tracked_bit(); - obj_ref.set_gc_generation(0); - obj_ref.set_gc_owner(owner); - - self.generation_lists[0].write().push_front(obj); - self.counts[0].fetch_add(1, Ordering::Relaxed); + fn retired_heaps(&self) -> Vec> { + let mut result = Vec::new(); + self.heaps.lock().retain(|weak| { + if let Some(heap) = weak.upgrade() { + if heap.retired.load(Ordering::Acquire) { + result.push(heap); + } + true + } else { + false + } + }); + result } - /// Track two freshly allocated objects as one push under one list lock. - /// - /// Like [`Self::track_object_fresh`] twice over, for a pair that is always - /// born together: a generator and the frame it owns. Doing it in one go - /// saves the second lock round trip on a path that runs per generator. + /// Track a valid untracked object. A previously bound object retains its + /// original heap even if it is re-tracked from a different interpreter. /// /// # Safety - /// Both must be valid pointers to PyObjects whose `gc_bits` is still `0`. - unsafe fn track_pair_fresh(&self, a: NonNull, b: NonNull, owner: GcOwner) { - debug_assert_ne!(a, b); - for obj in [a, b] { - let obj_ref = unsafe { obj.as_ref() }; - obj_ref.init_gc_tracked_bit(); - obj_ref.set_gc_generation(0); - obj_ref.set_gc_owner(owner); - } - - { - let mut list = self.generation_lists[0].write(); - list.push_front(a); - list.push_front(b); - } - self.counts[0].fetch_add(2, Ordering::Relaxed); + /// The caller must exclude concurrent tracking of the object. + pub unsafe fn track_object(&self, obj: NonNull, heap: PyRc) { + unsafe { GcHeap::track(&heap, obj, false) }; } - /// Untrack an object (remove from GC lists). - /// O(1) — intrusive linked list remove by node pointer. - /// /// # Safety - /// obj must be a valid pointer to a PyObject that is currently tracked. - /// The object's memory must still be valid (pointers are read). + /// The object's memory must remain valid throughout removal. pub unsafe fn untrack_object(&self, obj: NonNull) { - let obj_ref = unsafe { obj.as_ref() }; - - loop { - let obj_gen = obj_ref.gc_generation(); - - let (list_lock, count) = if obj_gen <= 2 { - ( - &self.generation_lists[obj_gen as usize] as &PyRwLock>, - &self.counts[obj_gen as usize], - ) - } else if obj_gen == GC_PERMANENT { - (&self.permanent_list, &self.permanent_count) - } else { - return; // GC_UNTRACKED or unknown — already untracked - }; - - let mut list = list_lock.write(); - // Re-check generation under lock (may have changed due to promotion) - if obj_ref.gc_generation() != obj_gen { - drop(list); - continue; // Retry with the updated generation - } - if unsafe { list.remove(obj) }.is_some() { - release_count(count); - obj_ref.clear_gc_tracked(); - obj_ref.set_gc_generation(GC_UNTRACKED); - } else { - // Object claims to be in this generation but wasn't found in the list. - // This indicates a bug: the object was already removed from the list - // without updating gc_generation, or was never inserted. - eprintln!( - "GC WARNING: untrack_object failed to remove obj={obj:p} from gen={obj_gen}, \ - tracked={}, gc_gen={}", - obj_ref.is_gc_tracked(), - obj_ref.gc_generation() - ); - } - return; + let obj = unsafe { obj.as_ref() }; + if let Some(heap) = obj.gc_handle().heap() { + heap.untrack(obj); } } - /// Get the objects `owner` tracks (for gc.get_objects), plus the ones no - /// interpreter owns. - /// If generation is None, returns all such objects. - /// If generation is Some(n), returns those in generation n only. - pub fn get_objects(&self, generation: Option, owner: GcOwner) -> Vec { - fn collect_from_list( - list: &LinkedList, - owner: GcOwner, - ) -> impl Iterator + '_ { - list.iter() - .filter(move |obj| is_owned_by(obj, owner)) - .filter_map(|obj| obj.try_to_owned()) - } - - match generation { - None => { - // Return all tracked objects from all generations + permanent - let mut result = Vec::new(); - for gen_list in &self.generation_lists { - result.extend(collect_from_list(&gen_list.read(), owner)); - } - result.extend(collect_from_list(&self.permanent_list.read(), owner)); - result - } - Some(g) if (0..=2).contains(&g) => { - let guard = self.generation_lists[g as usize].read(); - collect_from_list(&guard, owner).collect() - } - _ => Vec::new(), - } - } - - /// Check if automatic GC should run and run it if needed. - /// Called after object allocation. - /// Returns true if GC was run, false otherwise. fn maybe_collect(&self, gc: &GcInterpreterState) -> bool { - if !gc.is_enabled() { - return false; - } - - // Check gen0 threshold - let count0 = self.counts[0].load(Ordering::Relaxed) as u32; - let threshold0 = gc.generations[0].threshold(); - if threshold0 > 0 && count0 >= threshold0 { + let threshold = gc.generations[0].threshold(); + if gc.is_enabled() + && threshold != 0 + && gc.heap.allocations.load(Ordering::Relaxed) as i128 >= threshold as i128 + { #[cfg(feature = "threading")] - { - // Defer to the next bytecode safepoint. Collecting here would - // stop the world while this thread may hold an internal lock - // (e.g. a lazily-initialized frame locals cell) that another - // thread is blocked on with no way to reach a safepoint — - // a deadlock. At a safepoint no such lock is held. - gc.scheduled.store(true, Ordering::Relaxed); - return false; - } - // Without threading there is no safepoint to defer to and no other - // thread whose frames could be read mid-mutation, so collect inline. + gc.scheduled.store(true, Ordering::Relaxed); #[cfg(not(feature = "threading"))] { - self.collect_inner(gc, 0, false); + self.collect_inner(gc, None, false); return true; } } - false } - fn collect_inner( - &self, - gc: &GcInterpreterState, - generation: usize, - force: bool, - ) -> CollectResult { - if !force && !gc.is_enabled() { - #[cfg(feature = "threading")] - gc.scheduled.store(false, Ordering::Relaxed); - return CollectResult::default(); - } - - // Try to acquire the collecting lock - let Some(_guard) = self.collecting.try_lock() else { - return CollectResult::default(); - }; - - // A busy collector must not consume another interpreter's request. - // Clear only after acquiring the lock, before callbacks can request - // a later collection. - #[cfg(feature = "threading")] - gc.scheduled.store(false, Ordering::Relaxed); - - let start_time = cfg_select! { - target_arch = "wasm32" => (), - _ => std::time::Instant::now(), - }; - - // Memory barrier to ensure visibility of all reference count updates - // from other threads before we start analyzing the object graph. - core::sync::atomic::fence(Ordering::SeqCst); - - let generation = generation.min(2); - let debug = gc.get_debug(); - - // Clear the method cache to release strong references that - // might prevent cycle collection (_PyType_ClearCache). - crate::builtins::type_::type_cache_clear(); - - // Backstop for QSBR reclamation (threads may have missed requests). - #[cfg(feature = "threading")] - crate::object::qsbr::QSBR.process(); - - // Stop the world before reading any tracked object's interpreter - // state. Requested *before* the generation read locks are taken: a - // thread parking at a safepoint may still hold a generation write lock - // (track/untrack/promote) and must be able to release it to reach the - // safepoint. It could not do so if this thread already held a read - // lock it was waiting behind — hence the ordering. - // - // Auto-collection is deferred to a bytecode safepoint (see - // `maybe_collect`), where no internal lock is held, so it never stops - // the world under a lock. Explicit `gc.collect()` runs synchronously - // here; a re-entrant call from a finalizer during an in-progress - // collection is turned into a no-op by the `collecting` try_lock above. - // The one residual is an explicit `gc.collect()` reached from a - // finalizer/`__del__` that runs inline while a non-generation internal - // lock is still held (e.g. a container write lock during element - // replacement) with another thread blocked on that same lock: stopping - // the world then waits for a thread that cannot reach a safepoint. - // Closing it fully would require making those locks stop-the-world - // aware; the exclusion above only serializes the fork/GC requesters. - #[cfg(feature = "threading")] - let mut stw = CollectStopTheWorld::new(); - - // Step 1: Gather objects from generations 0..=generation - // Hold read locks for the entire scan to prevent concurrent modifications. - let gen_locks: Vec<_> = (0..=generation) - .map(|i| self.generation_lists[i].read()) - .collect(); - - // Only this interpreter's objects, plus the ones no interpreter owns. - // Another interpreter's objects stay out of the candidate set, so they - // act as external roots: anything they reference survives this pass. - let owner = gc.owner; - // Sorted so that the test below, which every scanned object pays for, - // stays logarithmic in the number of interpreters that have been - // dropped instead of linear. - let retired = { - let mut retired = self.retired.lock().clone(); - retired.sort_unstable(); - retired - }; - // Each candidate carries its own count, with `GcBits::COLLECTING` - // saying the count is there. Every edge in the heap is answered from - // that bit and that field; a table keyed by address turned each of - // those answers into a hash of the address instead. `candidate_ptrs` - // keeps the candidates in a walkable order, and the bit is what keeps - // an object that appears in two generation lists out of it twice. - let mut candidate_ptrs: Vec = Vec::new(); - for gen_list in &gen_locks { - for obj in gen_list.iter() { - if retired.binary_search(&obj.gc_owner()).is_ok() { - obj.set_gc_owner(GC_NO_OWNER); - } - let strong_count = obj.strong_count(); - if strong_count > 0 && is_owned_by(obj, owner) && !obj.is_gc_collecting() { - obj.start_gc_refs(strong_count); - candidate_ptrs.push(GcPtr(NonNull::from(obj))); - } - } - } - - // A full collection is the only one that sees every generation, so it - // is where adoption finishes and the tags stop being tracked. - if generation == 2 && !retired.is_empty() { - for obj in self.permanent_list.read().iter() { - if retired.binary_search(&obj.gc_owner()).is_ok() { - obj.set_gc_owner(GC_NO_OWNER); - } - } - // Only the tags this scan saw: one retired while it ran still has - // objects nobody has adopted. - self.retired - .lock() - .retain(|tag| retired.binary_search(tag).is_err()); - } - - if candidate_ptrs.is_empty() { - // Reset counts for generations whose objects were promoted away. - // For gen2 (oldest), survivors stay in-place so don't reset gen2 count. - let reset_end = if generation >= 2 { 2 } else { generation + 1 }; - for count in self.counts.iter().take(reset_end) { - count.store(0, Ordering::Relaxed); - } - - let duration = elapsed_secs(start_time); - - gc.generations[generation].update_stats(0, 0, 0, duration); - return CollectResult { - collected: 0, - uncollectable: 0, - candidates: 0, - duration, - }; - } - - let candidates = candidate_ptrs.len(); - - if debug.contains(GcDebugFlags::STATS) { - eprintln!("gc: collecting {candidates} objects from generations 0..={generation}"); - } - - // Step 3: Subtract internal references - // Pre-compute referent pointers once per object so that both step 3 - // (subtract refs) and step 4 (BFS reachability) see the same snapshot - // of each object's children. Without this, a dict whose write lock is - // held during one traversal but not the other can yield inconsistent - // results, causing live objects to be incorrectly collected. - // - // Every object's referents go in one buffer, with each object holding - // the range that is its own: a vector each would be an allocation per - // tracked object, and the collection wants them all at once anyway. - let mut referent_ptrs: Vec> = Vec::new(); - let mut referent_ranges: GcMap = GcMap::default(); - - for &ptr in &candidate_ptrs { - let obj = unsafe { ptr.0.as_ref() }; - if obj.strong_count() == 0 { - continue; - } - let start = referent_ptrs.len(); - unsafe { obj.gc_extend_referent_ptrs(&mut referent_ptrs) }; - let end = referent_ptrs.len(); - for &child_ptr in &referent_ptrs[start..end] { - // SAFETY: the referents came from `traverse`, which handed out - // live references to them, and the world is stopped. - let child = unsafe { child_ptr.as_ref() }; - if child.is_gc_collecting() { - child.subtract_gc_ref(); - } - } - referent_ranges.insert(ptr, (start, end)); - } - - // Step 4: Find reachable objects (gc_refs > 0) and traverse from them - let mut worklist: Vec = Vec::new(); - - for &ptr in &candidate_ptrs { - let obj = unsafe { ptr.0.as_ref() }; - if obj.gc_refs() > 0 { - obj.mark_gc_reachable(); - worklist.push(ptr); - } - } - - while let Some(ptr) = worklist.pop() { - let obj = unsafe { ptr.0.as_ref() }; - if obj.is_gc_tracked() { - // Reuse the pre-computed referent pointers from step 3, in - // place: copying them out again costs a second pass over every - // edge in the heap. Objects skipped in step 3 (strong_count was - // 0) have none stored and are traversed here instead. - let computed; - let children: &[NonNull] = match referent_ranges.get(&ptr) { - Some(&(start, end)) => &referent_ptrs[start..end], - None => { - computed = unsafe { obj.gc_get_referent_ptrs() }; - &computed - } - }; - for &child_ptr in children { - // SAFETY: as in step 3, the referents are live. - let child = unsafe { child_ptr.as_ref() }; - if child.is_gc_collecting() && child.mark_gc_reachable() { - worklist.push(GcPtr(child_ptr)); - } - } - } - } - - // Step 5: Split the candidates on what step 4 concluded, and hand the - // headers back: nothing past here reads `gc_refs`, and a candidate that - // kept the bit would be passed over by every later collection. - let mut reachable: Vec = Vec::new(); - let mut unreachable: Vec = Vec::new(); - for &ptr in &candidate_ptrs { - let obj = unsafe { ptr.0.as_ref() }; - if obj.gc_refs() == GC_REACHABLE { - reachable.push(ptr); - } else { - unreachable.push(ptr); - } - obj.end_gc_refs(); - } - - // With the world stopped, every frame on any thread's call stack is a - // live root that is externally referenced and must have been - // classified reachable. A running frame appearing in `unreachable` - // would mean the reachability analysis observed its interpreter state - // as garbage — the exact hazard the barrier exists to prevent. - // Verify no running frame is classified unreachable. - // Walk the TLS frame chain (CURRENT_FRAME) instead of top_frame, - // because stack-allocated frames update only CURRENT_FRAME (via - // set_current_frame_nosave), not top_frame. - #[cfg(all(unix, feature = "threading", debug_assertions))] - if stw.is_stopped() { - let unreachable_set: GcSet = unreachable.iter().copied().collect(); - let mut cur = crate::vm::thread::get_current_frame(); - while !cur.is_null() { - let iframe = unsafe { &*cur }; - if let Some(fo) = iframe.frame_obj() { - let obj = fo.as_object(); - let ptr = GcPtr(NonNull::from(obj)); - debug_assert!( - !unreachable_set.contains(&ptr), - "running frame {obj:p} classified unreachable during GC" - ); - } - cur = iframe.previous(); - } - } - - if debug.contains(GcDebugFlags::STATS) { - eprintln!( - "gc: {} reachable, {} unreachable", - reachable.len(), - unreachable.len() - ); - } - - // Create strong references while read locks are still held. - // After dropping gen_locks, other threads can untrack+free objects, - // making the raw pointers in `reachable`/`unreachable` dangling. - // Strong refs keep objects alive for later phases. - // - // Use try_to_owned() (CAS-based) instead of strong_count()+to_owned() - // to prevent a TOCTOU race: another thread can dec() the count to 0 - // between the check and the increment, causing a use-after-free when - // the destroying thread eventually frees the memory. - let survivor_refs: Vec = reachable - .iter() - .filter_map(|ptr| { - let obj = unsafe { ptr.0.as_ref() }; - obj.try_to_owned() - }) - .collect(); - - let unreachable_refs: Vec = unreachable - .iter() - .filter_map(|ptr| { - let obj = unsafe { ptr.0.as_ref() }; - obj.try_to_owned() - }) - .collect(); - - // The pointer-reading phases are done: strong references now pin every - // survivor and unreachable object, so the remaining phases can run with - // the world restarted. Finalizers and tp_clear must not run under - // stop-the-world — they execute arbitrary Python — and they only touch - // dead/husk objects, never a running frame. - #[cfg(feature = "threading")] - stw.restart(); - - if unreachable.is_empty() { - drop(gen_locks); - self.promote_survivors(generation, &survivor_refs); - let reset_end = if generation >= 2 { 2 } else { generation + 1 }; - for count in self.counts.iter().take(reset_end) { - count.store(0, Ordering::Relaxed); - } - - let duration = elapsed_secs(start_time); - - gc.generations[generation].update_stats(0, 0, candidates, duration); - return CollectResult { - collected: 0, - uncollectable: 0, - candidates, - duration, - }; - } - - // Release read locks before finalization phase. - drop(gen_locks); - - // Step 6: Finalize unreachable objects and handle resurrection - - if unreachable_refs.is_empty() { - self.promote_survivors(generation, &survivor_refs); - let reset_end = if generation >= 2 { 2 } else { generation + 1 }; - for count in self.counts.iter().take(reset_end) { - count.store(0, Ordering::Relaxed); - } - - let duration = elapsed_secs(start_time); - - gc.generations[generation].update_stats(0, 0, candidates, duration); - return CollectResult { - collected: 0, - uncollectable: 0, - candidates, - duration, - }; - } - - // 6b: Record initial strong counts (for resurrection detection) - let initial_counts: GcMap = unreachable_refs - .iter() - .map(|obj| { - let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); - (ptr, obj.strong_count()) - }) - .collect(); - - // 6c: Clear existing weakrefs BEFORE calling __del__ - let mut all_callbacks: Vec<(crate::PyRef, crate::PyObjectRef)> = - Vec::new(); - for obj_ref in &unreachable_refs { - let callbacks = obj_ref.gc_clear_weakrefs_collect_callbacks(); - all_callbacks.extend(callbacks); - } - for (wr, cb) in all_callbacks { - if let Some(Err(e)) = crate::vm::thread::with_vm(&cb, |vm| cb.call((wr.clone(),), vm)) { - crate::vm::thread::with_vm(&cb, |vm| { - vm.run_unraisable(e.clone(), Some("weakref callback".to_owned()), cb.clone()); - }); - } - } - - // 6d: Call __del__ on unreachable objects (skip already-finalized). - // try_call_finalizer() internally checks gc_finalized() and sets it, - // so we must NOT set it beforehand. - for obj_ref in &unreachable_refs { - obj_ref.try_call_finalizer(); - } - - // Detect resurrection - let mut resurrected_set: GcSet = GcSet::default(); - let unreachable_set: GcSet = unreachable.iter().copied().collect(); - - for obj in &unreachable_refs { - let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); - let initial = initial_counts.get(&ptr).copied().unwrap_or(1); - if obj.strong_count() > initial { - resurrected_set.insert(ptr); - } - } - - // Transitive resurrection - let mut worklist: Vec = resurrected_set.iter().copied().collect(); - while let Some(ptr) = worklist.pop() { - let obj = unsafe { ptr.0.as_ref() }; - let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; - for child_ptr in referent_ptrs { - let child_gc_ptr = GcPtr(child_ptr); - if unreachable_set.contains(&child_gc_ptr) && resurrected_set.insert(child_gc_ptr) { - worklist.push(child_gc_ptr); - } - } - } - - // Partition into resurrected and truly dead - let (resurrected, truly_dead): (Vec<_>, Vec<_>) = - unreachable_refs.into_iter().partition(|obj| { - let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); - resurrected_set.contains(&ptr) - }); - - if debug.contains(GcDebugFlags::STATS) { - eprintln!( - "gc: {} resurrected, {} truly dead", - resurrected.len(), - truly_dead.len() - ); - } - - // Compute collected count (exclude instance dicts in truly_dead) - let collected = { - let dead_ptrs: GcSet = truly_dead - .iter() - .map(|obj| obj.as_ref() as *const PyObject as usize) - .collect(); - let instance_dict_count = truly_dead - .iter() - .filter(|obj| { - if let Some(dict_ref) = obj.dict() { - dead_ptrs.contains(&(dict_ref.as_object() as *const PyObject as usize)) - } else { - false - } - }) - .count(); - truly_dead.len() - instance_dict_count - }; - - // Promote survivors to next generation BEFORE tp_clear. - // move_legacy_finalizer_reachable → delete_garbage order ensures - // survivor_refs are dropped before tp_clear, so reachable objects - // aren't kept alive beyond the deferred-drop phase. - self.promote_survivors(generation, &survivor_refs); - drop(survivor_refs); - - // Resurrected objects stay tracked and survive this collection too. - self.promote_survivors(generation, &resurrected); - drop(resurrected); - - if debug.contains(GcDebugFlags::COLLECTABLE) { - for obj in &truly_dead { - eprintln!( - "gc: collectable <{} {:p}>", - obj.class().name(), - obj.as_ref() - ); - } - } - - if debug.contains(GcDebugFlags::SAVEALL) { - self.promote_survivors(generation, &truly_dead); - let mut garbage_guard = gc.garbage.lock(); - for obj_ref in &truly_dead { - garbage_guard.push(obj_ref.clone()); - } - } - - if !truly_dead.is_empty() { - // Break cycles by clearing references (tp_clear) - // Use deferred drop context to prevent stack overflow. - // With DEBUG_SAVEALL the objects stay reachable through - // gc.garbage, so they must not be cleared (delete_garbage - // skips tp_clear for saved objects). - let save_all = debug.contains(GcDebugFlags::SAVEALL); - - // Untrack dead objects BEFORE clearing them, mirroring the - // untrack-then-clear ordering of the refcount dealloc path. - // A cleared object (e.g. a frame husk with iframe == None) must - // never be observable through the generation lists, or another - // thread could obtain a strong reference via gc.get_objects() - // and access the cleared payload. - let mut late_resurrected: GcSet = GcSet::default(); - if !save_all { - let mut expected_counts: GcMap = GcMap::default(); - for obj_ref in &truly_dead { - let obj = obj_ref.as_ref(); - if obj.is_gc_tracked() { - unsafe { self.untrack_object(NonNull::from(obj)) }; - } - // One strong reference held by the `truly_dead` vec itself. - expected_counts.insert(GcPtr(NonNull::from(obj)), 1); - } - // With the objects out of the generation lists, no new external - // reference can appear. Count the references coming from within - // the dead set; any surplus in strong_count means another thread - // grabbed a reference before untracking (late resurrection) and - // the object must not be cleared. - let mut referents: GcMap>> = GcMap::default(); - for obj_ref in &truly_dead { - let referent_ptrs = unsafe { obj_ref.gc_get_referent_ptrs() }; - for child_ptr in &referent_ptrs { - if let Some(n) = expected_counts.get_mut(&GcPtr(*child_ptr)) { - *n += 1; - } - } - referents.insert(GcPtr(NonNull::from(obj_ref.as_ref())), referent_ptrs); - } - let mut worklist: Vec = Vec::new(); - for obj_ref in &truly_dead { - let ptr = GcPtr(NonNull::from(obj_ref.as_ref())); - if obj_ref.strong_count() > expected_counts[&ptr] - && late_resurrected.insert(ptr) - { - worklist.push(ptr); - } - } - // A holder of a late-resurrected object can reach its referents, - // so everything reachable from it must stay intact as well. - while let Some(ptr) = worklist.pop() { - let Some(referent_ptrs) = referents.get(&ptr) else { - continue; - }; - for child_ptr in referent_ptrs { - let child = GcPtr(*child_ptr); - if expected_counts.contains_key(&child) && late_resurrected.insert(child) { - worklist.push(child); - } - } - } - // Re-track late-resurrected objects so a future collection can - // retry once the external references are released. - #[expect( - clippy::iter_over_hash_type, - reason = "Iteration order doesn't matter here" - )] - for &ptr in &late_resurrected { - // Re-tracking a resurrected object: it keeps the owner it - // was allocated under. - let owner = unsafe { ptr.0.as_ref() }.gc_owner(); - unsafe { self.track_object(ptr.0, owner) }; - } - } - rustpython_common::refcount::with_deferred_drops(|| { - if !save_all { - for obj_ref in &truly_dead { - let obj = obj_ref.as_ref(); - if late_resurrected.contains(&GcPtr(NonNull::from(obj))) { - continue; - } - if obj.gc_has_clear() { - let edges = unsafe { obj.gc_clear() }; - drop(edges); - } - } - } - drop(truly_dead); - }); - } - - // Reset counts for generations whose objects were promoted away. - // For gen2 (oldest), survivors stay in-place so don't reset gen2 count. - let reset_end = if generation >= 2 { 2 } else { generation + 1 }; - for count in self.counts.iter().take(reset_end) { - count.store(0, Ordering::Relaxed); - } - - let duration = elapsed_secs(start_time); - - gc.generations[generation].update_stats(collected, 0, candidates, duration); - - CollectResult { - collected, - uncollectable: 0, - candidates, - duration, - } - } - - /// Promote surviving objects to the next generation, or gen2 for a full collection. - /// - /// `survivors` must be strong references (`PyObjectRef`) to keep objects alive, - /// since the generation read locks are released before this is called. - /// - /// Holds both source and destination list locks simultaneously to prevent - /// a race where concurrent `untrack_object` reads a stale `gc_generation` - /// and operates on the wrong list. - fn promote_survivors(&self, from_gen: usize, survivors: &[PyObjectRef]) { - let next_gen = (from_gen + 1).min(2); - - // The world has restarted by this point. Batch lock acquisition and - // counter updates, but bound each batch so other interpreters can keep - // tracking and untracking objects between batches. - for batch in survivors.chunks(256) { - for src_gen in 0..next_gen { - // Lock both source and destination lists simultaneously. - // Always ascending order (src_gen < next_gen) → no deadlock. - let mut src = self.generation_lists[src_gen].write(); - let mut dst = self.generation_lists[next_gen].write(); - let mut promoted = 0; - - for obj_ref in batch { - let obj = obj_ref.as_ref(); - // Re-check under locks: object might have been untracked concurrently - if obj.gc_generation() as usize != src_gen || !obj.is_gc_tracked() { - continue; - } - - let ptr = NonNull::from(obj); - if unsafe { src.remove(ptr) }.is_some() { - dst.push_front(ptr); - obj.set_gc_generation(next_gen as u8); - promoted += 1; - } - } - - if promoted != 0 { - let _ = self.counts[src_gen].try_update( - Ordering::Relaxed, - Ordering::Relaxed, - |count| Some(count.saturating_sub(promoted)), - ); - self.counts[next_gen].fetch_add(promoted, Ordering::Relaxed); - } - } - } - } - - /// Get count of frozen objects - pub fn get_freeze_count(&self) -> usize { - self.permanent_count.load(Ordering::Relaxed) - } - - /// Freeze the objects `owner` could collect (move them to the permanent - /// generation). - /// Lock order: generation_lists[i] → permanent_list (consistent with unfreeze). - fn freeze(&self, owner: GcOwner) { - let mut count = 0usize; - - for (gen_idx, gen_list) in self.generation_lists.iter().enumerate() { - let mut list = gen_list.write(); - let mut perm = self.permanent_list.write(); - let moving: Vec<_> = list - .iter() - .filter(|obj| is_owned_by(obj, owner)) - .map(NonNull::from) - .collect(); - for ptr in moving { - if unsafe { list.remove(ptr) }.is_none() { - continue; - } - perm.push_front(ptr); - unsafe { ptr.as_ref().set_gc_generation(GC_PERMANENT) }; - count += 1; - release_count(&self.counts[gen_idx]); - } - } - - self.permanent_count.fetch_add(count, Ordering::Relaxed); + #[cfg(all(unix, feature = "threading"))] + pub(crate) fn lock_retired_for_fork(&self) -> crate::common::lock::PyMutexGuard<'_, ()> { + self.collecting.lock() } - /// Unfreeze the objects `owner` froze (move them from permanent to gen2). - /// Lock order: generation_lists[2] → permanent_list (consistent with freeze). - fn unfreeze(&self, owner: GcOwner) { - let mut count = 0usize; - - { - let mut gen2 = self.generation_lists[2].write(); - let mut perm_list = self.permanent_list.write(); - let moving: Vec<_> = perm_list - .iter() - .filter(|obj| is_owned_by(obj, owner)) - .map(NonNull::from) - .collect(); - for ptr in moving { - if unsafe { perm_list.remove(ptr) }.is_none() { - continue; - } - gen2.push_front(ptr); - unsafe { ptr.as_ref().set_gc_generation(2) }; - count += 1; - } - let _ = self.permanent_count.try_update( - Ordering::Relaxed, - Ordering::Relaxed, - |permanent| Some(permanent.saturating_sub(count)), - ); - } - - self.counts[2].fetch_add(count, Ordering::Relaxed); + #[cfg(all(unix, feature = "threading"))] + pub(crate) fn heaps_for_fork(&self) -> Vec> { + core::iter::once(self.shared.clone()) + .chain(self.heaps.lock().iter().filter_map(HeapWeak::upgrade)) + .collect() } - /// Reset all locks to unlocked state after fork(). - /// - /// After fork(), only the forking thread survives. Any lock held by another - /// thread is permanently stuck. This resets them by zeroing the raw bytes. - /// - /// # Safety - /// Must only be called after fork() in the child process when no other - /// threads exist. The calling thread must NOT hold any of these locks. #[cfg(all(unix, feature = "threading"))] + /// # Safety + /// Call only in the fork child after the fork coordinator stabilized all + /// heaps and released its inherited guards. pub unsafe fn reinit_after_fork(&self) { - use crate::common::lock::{reinit_mutex_after_fork, reinit_rwlock_after_fork}; - unsafe { - reinit_mutex_after_fork(&self.collecting); - reinit_mutex_after_fork(&self.retired); - - for rw in &self.generation_lists { - reinit_rwlock_after_fork(rw); + crate::common::lock::reinit_mutex_after_fork(&self.collecting); + crate::common::lock::reinit_mutex_after_fork(&self.heaps); + self.shared.reinit_after_fork(); + for heap in self.heaps.lock().iter().filter_map(HeapWeak::upgrade) { + heap.reinit_after_fork(); } - reinit_rwlock_after_fork(&self.permanent_list); } } } -/// Per-interpreter garbage collector state (≈ `PyInterpreterState.gc`). -/// -/// The generation lists are process-wide (see [`GcState`]); what an interpreter -/// owns is the policy applied to them and the results — which objects its -/// collections consider, whether they run automatically, and where uncollectable -/// objects end up. +/// Policy, allocation pressure and collection results for one interpreter. pub struct GcInterpreterState { - /// Tag written into every object this interpreter tracks. - owner: GcOwner, - /// Per-generation thresholds and statistics. + heap: PyRc, + collecting: PyMutex<()>, + #[cfg(all(unix, feature = "threading"))] + collecting_thread: core::sync::atomic::AtomicU64, pub generations: [GcGeneration; 3], - /// GC enabled flag + schedule: PyMutex, enabled: AtomicBool, - /// Automatic collection requested at this interpreter's next safepoint. #[cfg(feature = "threading")] scheduled: AtomicBool, - /// Debug flags debug: AtomicU32, - /// Uncollectable objects saved by this interpreter's collections, drained - /// into `py_garbage` by `gc.collect()`. pub garbage: PyMutex>, - /// `gc.garbage` - pub py_garbage: crate::builtins::PyListRef, - /// `gc.callbacks` - pub py_callbacks: crate::builtins::PyListRef, + py_garbage: PyMutex>, + py_callbacks: PyMutex>, } impl GcInterpreterState { + #[cfg(feature = "threading")] + pub(crate) fn qsbr(&self) -> &alloc::sync::Arc { + &self.heap.qsbr + } + pub fn new(ctx: &crate::vm::Context) -> Self { - Self { - owner: gc_state().alloc_owner(), + Self::new_for_interpreter(ctx, crate::vm::runtime::alloc_interpreter_id()) + } + + pub(crate) fn new_for_interpreter(ctx: &crate::vm::Context, owner: i64) -> Self { + let heap = gc_state().new_heap(owner); + let _owner = AllocationScope::new(heap.clone()); + let state = Self { + heap, + collecting: PyMutex::new(()), + #[cfg(all(unix, feature = "threading"))] + collecting_thread: core::sync::atomic::AtomicU64::new(0), generations: [ - GcGeneration::new(2000), // young - GcGeneration::new(10), // old[0] - GcGeneration::new(0), // old[1] + GcGeneration::new(2000), + GcGeneration::new(10), + GcGeneration::new(10), ], + schedule: PyMutex::new(GcSchedule::default()), enabled: AtomicBool::new(true), #[cfg(feature = "threading")] scheduled: AtomicBool::new(false), debug: AtomicU32::new(0), garbage: PyMutex::new(Vec::new()), - py_garbage: ctx.new_list(Vec::new()), - py_callbacks: ctx.new_list(Vec::new()), - } + py_garbage: PyMutex::new(Some(ctx.new_list(Vec::new()))), + py_callbacks: PyMutex::new(Some(ctx.new_list(Vec::new()))), + }; + state.heap.allocations.store(0, Ordering::Relaxed); + state } - /// Check if GC is enabled. - /// - /// Relaxed, like [`GcGeneration::threshold`]: it is read once per - /// allocation, and an allocation racing `gc.disable()` may use either value. - pub fn is_enabled(&self) -> bool { - self.enabled.load(Ordering::Relaxed) + pub(crate) fn allocation_scope(&self) -> AllocationScope { + AllocationScope::new(self.heap.clone()) } - /// Leave requests pending while a collector is busy, without repeatedly - /// entering the bytecode loop's slow path during its Python callbacks. - #[cfg(feature = "threading")] - #[inline] - pub(crate) fn collection_ready(&self) -> bool { - self.scheduled.load(Ordering::Relaxed) && !gc_state().collecting.is_locked() + pub(crate) fn garbage_list(&self) -> Option { + self.py_garbage.lock().clone() + } + + pub(crate) fn callbacks_list(&self) -> Option { + self.py_callbacks.lock().clone() + } + + pub(crate) fn close_python_roots(&self) { + self.disable(); + self.set_debug(GcDebugFlags::empty()); + let garbage = self.py_garbage.lock().take(); + let callbacks = self.py_callbacks.lock().take(); + let retained = core::mem::take(&mut *self.garbage.lock()); + drop((garbage, callbacks, retained)); } - /// Enable GC + pub(crate) fn retire_python_roots(&self, ctx: &crate::Context) { + self.disable(); + self.set_debug(GcDebugFlags::empty()); + let empty_garbage = ctx.new_list(Vec::new()); + let empty_callbacks = ctx.new_list(Vec::new()); + let garbage = self.py_garbage.lock().replace(empty_garbage); + let callbacks = self.py_callbacks.lock().replace(empty_callbacks); + let retained = core::mem::take(&mut *self.garbage.lock()); + drop((garbage, callbacks, retained)); + } + + pub fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::Relaxed) + } pub fn enable(&self) { self.enabled.store(true, Ordering::Relaxed); } - - /// Disable GC pub fn disable(&self) { self.enabled.store(false, Ordering::Relaxed); } - - /// Get debug flags pub fn get_debug(&self) -> GcDebugFlags { - GcDebugFlags::from_bits_truncate(self.debug.load(Ordering::SeqCst)) + GcDebugFlags::from_bits_truncate(self.debug.load(Ordering::Relaxed)) } - - /// Set debug flags pub fn set_debug(&self, flags: GcDebugFlags) { - self.debug.store(flags.bits(), Ordering::SeqCst); + self.debug.store(flags.bits(), Ordering::Relaxed); } - - /// Get thresholds for all generations pub fn get_threshold(&self) -> (u32, u32, u32) { ( self.generations[0].threshold(), @@ -1358,8 +454,6 @@ impl GcInterpreterState { self.generations[2].threshold(), ) } - - /// Set thresholds pub fn set_threshold(&self, t0: u32, t1: Option, t2: Option) { self.generations[0].set_threshold(t0); if let Some(t1) = t1 { @@ -1369,51 +463,62 @@ impl GcInterpreterState { self.generations[2].set_threshold(t2); } } - - /// Get statistics for all generations + pub fn get_count(&self) -> (isize, usize, usize) { + let schedule = self.schedule.lock(); + ( + self.heap.allocations.load(Ordering::Relaxed), + schedule.young_collections, + schedule.middle_collections, + ) + } pub fn get_stats(&self) -> [GcStats; 3] { - [ - self.generations[0].stats(), - self.generations[1].stats(), - self.generations[2].stats(), - ] + core::array::from_fn(|i| self.generations[i].stats()) } - - /// Perform garbage collection on the given generation pub fn collect(&self, generation: usize) -> CollectResult { - gc_state().collect_inner(self, generation, false) + gc_state().collect_inner(self, Some(generation), false) } - - /// Force collection even if GC is disabled (for manual gc.collect() calls) pub fn collect_force(&self, generation: usize) -> CollectResult { - gc_state().collect_inner(self, generation, true) + gc_state().collect_inner(self, Some(generation), true) + } + #[cfg(feature = "threading")] + pub(crate) fn collect_automatic(&self) -> CollectResult { + gc_state().collect_inner(self, None, false) + } + #[cfg(feature = "threading")] + #[inline] + pub(crate) fn collection_ready(&self) -> bool { + self.scheduled.load(Ordering::Relaxed) && !self.collecting.is_locked() } - - /// The tracked objects this interpreter can reach (for gc.get_objects). pub fn get_objects(&self, generation: Option) -> Vec { - gc_state().get_objects(generation, self.owner) + self.heap.get_objects(generation) } - - /// Move the objects this interpreter could collect into the permanent - /// generation. pub fn freeze(&self) { - gc_state().freeze(self.owner); + self.heap.freeze(); + self.schedule.lock().start_collection(2); } - - /// Move them back out of it. pub fn unfreeze(&self) { - gc_state().unfreeze(self.owner); + self.heap.unfreeze(); + } + pub fn get_freeze_count(&self) -> usize { + self.heap.freeze_count() } - - /// Reset this interpreter's GC locks to unlocked state after fork(). - /// - /// # Safety - /// Must only be called after fork() in the child process when no other - /// threads exist. The calling thread must NOT hold any of these locks. #[cfg(all(unix, feature = "threading"))] + /// # Safety + /// Only the surviving fork thread may execute. Heap and policy mutations + /// must have been quiescent at the syscall. pub unsafe fn reinit_after_fork(&self) { unsafe { + // fork() can run inside a callback of this thread's collection. + // Its native guard survives and must unlock the original mutex. + if self.collecting_thread.load(Ordering::Relaxed) != crate::stdlib::_thread::get_ident() + { + crate::common::lock::reinit_mutex_after_fork(&self.collecting); + self.collecting_thread.store(0, Ordering::Relaxed); + } crate::common::lock::reinit_mutex_after_fork(&self.garbage); + crate::common::lock::reinit_mutex_after_fork(&self.py_garbage); + crate::common::lock::reinit_mutex_after_fork(&self.py_callbacks); + crate::common::lock::reinit_mutex_after_fork(&self.schedule); for generation in &self.generations { generation.reinit_stats_after_fork(); } @@ -1423,277 +528,127 @@ impl GcInterpreterState { impl Drop for GcInterpreterState { fn drop(&mut self) { - // Objects this interpreter tracked can outlive it (another interpreter - // may still hold one). Clearing the tag hands them to every collection - // instead of stranding them. The tag itself is not handed back: it stays - // retired so that a later interpreter cannot inherit these objects. - gc_state().retire_owner(self.owner); + crate::vm::thread::native_sweep(|| { + #[cfg(all(unix, feature = "threading"))] + let _retirement = crate::vm::fork::native_phase(); + let _owner = self.allocation_scope(); + let mut cleanup = crate::vm::owned::Cleanup::default(); + cleanup.run(|| self.close_python_roots()); + self.heap.retired.store(true, Ordering::Release); + cleanup.run(|| gc_state().collect_retired()); + cleanup.finish(); + }); } } -/// The tag `track_object` should write for the interpreter running now. +/// Heap for objects allocated on the currently entered interpreter. #[must_use] -pub fn current_owner() -> GcOwner { - // SAFETY: the pointee is owned by the `PyGlobalState` of the VM on top of - // this thread's VM stack, which outlives the section this call runs in. - crate::vm::thread::current_gc_state().map_or(GC_NO_OWNER, |gc| unsafe { gc.as_ref() }.owner) +pub(crate) fn current_owner() -> PyRc { + if let Some(owner) = ALLOCATION_OWNER.with(|owner| owner.borrow().clone()) { + return owner; + } + crate::vm::thread::current_gc_state().map_or_else( + || gc_state().shared.clone(), + |gc| unsafe { gc.as_ref() }.heap.clone(), + ) } -/// Track a freshly allocated object under the interpreter running now, and let -/// it collect if the allocation pushed gen0 past its threshold. -/// -/// # Safety -/// obj must be a valid pointer to a PyObject that is not already tracked. -pub(crate) unsafe fn track_new_object(obj: NonNull) { - let state = gc_state(); - let Some(gc) = crate::vm::thread::current_gc_state() else { - // No interpreter is running: the shared context builds its own objects - // this way. They are left unowned, so every interpreter collects them. - unsafe { state.track_object_fresh(obj, GC_NO_OWNER) }; - return; - }; - // SAFETY: as in `current_owner`. - let gc = unsafe { gc.as_ref() }; - unsafe { state.track_object_fresh(obj, gc.owner) }; - state.maybe_collect(gc); +/// The allocation domain, including bootstrap and shared-definition scopes. +pub(crate) fn current_interpreter_id() -> Option { + ALLOCATION_OWNER.with(|owner| { + if let Some(owner) = owner.borrow().as_ref() { + owner.owner + } else { + crate::vm::thread::try_with_current_vm(|vm| vm.state.interpreter_id) + } + }) } -/// Track a generator (or coroutine, or async generator) together with the -/// frame it owns, and let the pair collect if it pushed gen0 past its -/// threshold. -/// -/// # Safety -/// Both must be valid pointers to distinct PyObjects that are not already -/// tracked and whose `gc_bits` is still `0`. -pub(crate) unsafe fn track_new_pair(obj: NonNull, frame: NonNull) { - let state = gc_state(); - let Some(gc) = crate::vm::thread::current_gc_state() else { - unsafe { state.track_pair_fresh(obj, frame, GC_NO_OWNER) }; - return; +thread_local! { + static ALLOCATION_OWNER: core::cell::RefCell>> = const { + core::cell::RefCell::new(None) }; - // SAFETY: as in `current_owner`. - let gc = unsafe { gc.as_ref() }; - unsafe { state.track_pair_fresh(obj, frame, gc.owner) }; - state.maybe_collect(gc); } -/// Get a reference to the GC state. -/// -/// In threading mode this is a true global (OnceLock). -/// In non-threading mode this is thread-local, because PyRwLock/PyMutex -/// use Cell-based locks that are not Sync. -/// -/// Every interpreter's tracked objects live in these lists, because untracking -/// happens in `default_dealloc`, where no interpreter is in scope to route to. -/// What a collection *acts on* is still one interpreter's own objects, selected -/// by the `gc_owner` tag; [`GcInterpreterState`] holds the rest of the state -/// that goes with that. The counts here, and so `gc.get_count()` and -/// `gc.get_freeze_count()`, stay process-wide: they measure how full these -/// lists are. -pub fn gc_state() -> &'static GcState { - rustpython_common::static_cell! { - static GC_STATE: GcState; - } - GC_STATE.get_or_init(GcState::new) +/// Bootstrap and native-definition construction can allocate before a VM is +/// attached, or while a different interpreter is entered. Scope the owner +/// explicitly without pretending that those allocations have an active VM. +pub(crate) struct AllocationScope { + previous: Option>, + _thread: core::marker::PhantomData>, } -#[cfg(test)] -mod tests { - use super::*; - - fn interpreter_state() -> GcInterpreterState { - GcInterpreterState::new(crate::vm::Context::genesis()) - } - - #[test] - fn gc_state_default() { - let state = interpreter_state(); - assert!(state.is_enabled()); - assert_eq!(state.get_debug(), GcDebugFlags::empty()); - assert_eq!(state.get_threshold(), (2000, 10, 0)); - } - - #[test] - fn gc_enable_disable() { - let state = interpreter_state(); - assert!(state.is_enabled()); - state.disable(); - assert!(!state.is_enabled()); - state.enable(); - assert!(state.is_enabled()); - } - - #[test] - fn gc_threshold() { - let state = interpreter_state(); - state.set_threshold(100, Some(20), Some(30)); - assert_eq!(state.get_threshold(), (100, 20, 30)); - } - - #[test] - fn gc_debug_flags() { - let state = interpreter_state(); - state.set_debug(GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE); - assert_eq!( - state.get_debug(), - GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE - ); - } - - /// Live interpreters never share an owner tag, or their collections would - /// reach each other's objects. - #[test] - fn gc_owner_tags_are_distinct_while_live() { - let first = interpreter_state(); - let second = interpreter_state(); - assert_ne!(first.owner, second.owner); - assert_ne!(first.owner, GC_NO_OWNER); - assert_ne!(second.owner, GC_NO_OWNER); - } - - #[cfg(feature = "threading")] - #[test] - fn automatic_gc_request_stays_with_allocating_interpreter() { - let first = crate::Interpreter::without_stdlib(Default::default()); - let second = crate::Interpreter::without_stdlib(Default::default()); - first.enter(|vm| vm.state.gc.scheduled.store(false, Ordering::Relaxed)); - second.enter(|vm| vm.state.gc.scheduled.store(false, Ordering::Relaxed)); - // An isolated allocation counter makes crossing the threshold - // deterministic without depending on the rest of the test process. - let allocations = GcState::new(); - allocations.counts[0].store(1, Ordering::Relaxed); - first.enter(|vm| { - vm.state.gc.set_threshold(1, None, None); - assert!(!allocations.maybe_collect(&vm.state.gc)); - }); - second.enter(|vm| { - assert!(!vm.state.gc.scheduled.load(Ordering::Relaxed)); - vm.run_scheduled_gc(); - }); - first.enter(|vm| { - assert!(vm.state.gc.scheduled.swap(false, Ordering::Relaxed)); - }); +impl AllocationScope { + fn new(heap: PyRc) -> Self { + Self { + previous: ALLOCATION_OWNER.with(|owner| owner.replace(Some(heap))), + _thread: core::marker::PhantomData, + } } - #[cfg(feature = "threading")] - #[test] - fn automatic_gc_request_survives_busy_collector() { - let state = interpreter_state(); - let _guard = gc_state().collecting.lock(); - state.scheduled.store(true, Ordering::Relaxed); - state.collect(0); - assert!(state.scheduled.load(Ordering::Relaxed)); - assert!(!state.collection_ready()); - - state.disable(); - state.collect(0); - assert!(!state.scheduled.load(Ordering::Relaxed)); + pub(crate) fn shared() -> Self { + Self::new(gc_state().shared.clone()) } +} - #[test] - fn release_count_does_not_wrap_during_reset() { - use std::sync::Barrier; - - let count = AtomicUsize::new(1); - let start = Barrier::new(3); - let finish = Barrier::new(3); - let mut underflows = 0; - std::thread::scope(|scope| { - for reset in [false, true] { - let (count, start, finish) = (&count, &start, &finish); - scope.spawn(move || { - for _ in 0..10_000 { - start.wait(); - if reset { - count.store(0, Ordering::Relaxed); - } else { - release_count(count); - } - finish.wait(); - } - }); - } - for _ in 0..10_000 { - count.store(1, Ordering::Relaxed); - start.wait(); - finish.wait(); - underflows += usize::from(count.load(Ordering::Relaxed) > 1); - } - }); - assert_eq!(underflows, 0); +impl Drop for AllocationScope { + fn drop(&mut self) { + ALLOCATION_OWNER.with(|owner| owner.replace(self.previous.take())); } +} - #[test] - fn survivor_promotion_handles_mixed_and_stale_generations() { - // Isolate list membership and counters from other tests' collections. - // Detach every object before its normal deallocator consults gc_state(). - struct Heap { - state: GcState, - objects: Vec, +/// # Safety +/// obj must be valid, freshly initialized and not yet tracked. +pub(crate) unsafe fn track_new_object(obj: NonNull) { + let state = gc_state(); + let heap = current_owner(); + unsafe { GcHeap::track(&heap, obj, true) }; + if let Some(gc) = crate::vm::thread::current_gc_state() { + let gc = unsafe { gc.as_ref() }; + if PyRc::ptr_eq(&heap, &gc.heap) { + state.maybe_collect(gc); } + } +} - impl Drop for Heap { - fn drop(&mut self) { - for obj in &self.objects { - unsafe { self.state.untrack_object(NonNull::from(obj.as_ref())) }; - } - } +/// # Safety +/// Both objects must be valid, distinct, freshly initialized and untracked. +pub(crate) unsafe fn track_new_pair(obj: NonNull, frame: NonNull) { + let state = gc_state(); + let gc = crate::vm::thread::current_gc_state(); + let heap = current_owner(); + unsafe { GcHeap::track_pair(&heap, obj, frame) }; + if let Some(gc) = gc { + let gc = unsafe { gc.as_ref() }; + if PyRc::ptr_eq(&heap, &gc.heap) { + state.maybe_collect(gc); } + } +} - let ctx = crate::vm::Context::genesis(); - // A global collector must not retain a promotion snapshot containing - // objects that this test has moved into its private lists. - let _collector = gc_state().collecting.lock(); - let mut heap = Heap { - state: GcState::new(), - objects: Vec::new(), - }; - for _ in 0..600 { - let obj: PyObjectRef = ctx.new_list(Vec::new()).into(); - let ptr = NonNull::from(obj.as_ref()); - unsafe { - gc_state().untrack_object(ptr); - heap.state.track_object(ptr, 1); - } - heap.objects.push(obj); - } +pub(crate) fn gc_state() -> &'static GcState { + rustpython_common::static_cell! { static GC_STATE: GcState; } + GC_STATE.get_or_init(GcState::new) +} - let Heap { state, objects } = &heap; - // More than one batch, with survivors initially in all three generations. - state.promote_survivors(0, &objects[..200]); - state.promote_survivors(1, &objects[..100]); - assert_eq!(state.get_count(), (400, 100, 100)); +#[cfg(test)] +mod tests; - // Finalizers or another thread may freeze/untrack a survivor after the - // collection took its snapshot, but before promotion acquires the locks. - objects[250].set_gc_owner(2); - state.freeze(2); - unsafe { state.untrack_object(NonNull::from(objects[500].as_ref())) }; - // Counts are advisory and may have been reset by an earlier collection. - state.counts[0].store(0, Ordering::Relaxed); +/// Access native collection bookkeeping. +/// +/// # Safety +/// Tracking and untracking must obey the attached owner and heap lifetime rules +/// documented in [`crate::embedding`]. +#[must_use] +pub unsafe fn gc_state_unchecked() -> &'static GcState { + gc_state() +} - state.promote_survivors(2, objects); - assert_eq!(state.get_count(), (0, 0, 598)); - assert_eq!(state.get_freeze_count(), 1); - for (index, obj) in objects.iter().enumerate() { - let expected = match index { - 250 => GC_PERMANENT, - 500 => GC_UNTRACKED, - _ => 2, - }; - assert_eq!(obj.gc_generation(), expected); - } - assert_eq!(state.generation_lists[0].read().iter().count(), 0); - assert_eq!(state.generation_lists[1].read().iter().count(), 0); - assert_eq!(state.generation_lists[2].read().iter().count(), 598); - // Each node must occur exactly once, with its intrusive links intact. - let promoted: GcSet<_> = state.generation_lists[2] - .read() - .iter() - .map(NonNull::from) - .collect(); - for obj in objects.iter().filter(|obj| obj.gc_generation() == 2) { - assert!(promoted.contains(&NonNull::from(obj.as_ref()))); - } - } +/// Access the current native allocation owner. +/// +/// # Safety +/// Do not access this heap's objects outside their owner's attachment. +#[must_use] +pub unsafe fn current_owner_unchecked() -> PyRc { + current_owner() } diff --git a/crates/vm/src/gc_state/collector.rs b/crates/vm/src/gc_state/collector.rs new file mode 100644 index 00000000000..1f4d1f237cc --- /dev/null +++ b/crates/vm/src/gc_state/collector.rs @@ -0,0 +1,324 @@ +//! Snapshot, analysis, finalization and commit of a collection. + +use super::*; + +type WeakCallbacks = Vec<(crate::PyRef, PyObjectRef)>; + +fn promote(objects: &[PyObjectRef], generation: usize) { + // Snapshot order groups objects by heap. Partitioning preserves that order, + // so promotion needs neither a heap lookup table nor one lock per object. + let mut start = 0; + while start < objects.len() { + let heap = objects[start].gc_handle().heap().unwrap(); + let mut end = start + 1; + while end < objects.len() + && objects[end] + .gc_handle() + .heap() + .is_some_and(|h| core::ptr::eq(h, heap)) + { + end += 1; + } + heap.promote(&objects[start..end], generation); + start = end; + } +} + +fn clear_weakrefs(objects: &[PyObjectRef], callbacks: &mut WeakCallbacks) { + // Weakrefs within the same unreachable graph must not run callbacks that + // could resurrect that graph. A reachable weakref still gets its callback. + for obj in objects { + obj.start_gc_index(0); + } + scopeguard::defer! { + for obj in objects { + obj.end_gc_refs(); + } + } + for obj in objects { + callbacks.extend(obj.gc_clear_weakrefs_collect_callbacks()); + } +} + +fn run_weak_callbacks(callbacks: WeakCallbacks) { + for (wr, callback) in callbacks { + if let Some(Err(error)) = + crate::vm::thread::with_vm(&callback, |vm| callback.call((wr.clone(),), vm)) + { + crate::vm::thread::with_vm(&callback, |vm| { + vm.run_unraisable( + error.clone(), + Some("weakref callback".to_owned()), + callback.clone(), + ); + }); + } + } +} + +fn callbacks(gc: &GcInterpreterState, phase: &str, generation: usize, result: &CollectResult) { + crate::vm::thread::try_with_current_vm(|vm| { + if core::ptr::eq(gc, &vm.state.gc) { + crate::stdlib::gc::invoke_callbacks(vm, phase, generation, result); + } + }); +} + +/// The caller holds stop-the-world. Only references in `objects` count as +/// collector pins; references from outside this candidate set are live roots. +pub(super) unsafe fn revalidate(objects: Vec) -> (Vec, Vec) { + // Another thread may freeze the heap while snapshot analysis runs. Those + // objects are roots, and their edges must not be subtracted as candidates. + let (mut retained, objects): (Vec<_>, Vec<_>) = objects + .into_iter() + .partition(|obj| obj.gc_generation() >= GC_PERMANENT); + let mut graph = unsafe { Graph::capture(objects) }; + graph.analyze(); + let (live, dead) = graph.partition(); + retained.extend(live); + (retained, dead) +} + +impl GcState { + pub(super) fn collect_inner( + &self, + gc: &GcInterpreterState, + generation: Option, + force: bool, + ) -> CollectResult { + if gc + .heap + .owner + .is_some_and(crate::vm::native_types::initializing) + { + return CollectResult::default(); + } + if !force && !gc.is_enabled() { + #[cfg(feature = "threading")] + gc.scheduled.store(false, Ordering::Relaxed); + return CollectResult::default(); + } + let Some(_collecting) = gc.collecting.try_lock() else { + return CollectResult::default(); + }; + #[cfg(all(unix, feature = "threading"))] + let _collecting_thread = { + gc.collecting_thread + .store(crate::stdlib::_thread::get_ident(), Ordering::Relaxed); + scopeguard::guard(&gc.collecting_thread, |thread| { + thread.store(0, Ordering::Relaxed) + }) + }; + #[cfg(feature = "threading")] + gc.scheduled.store(false, Ordering::Relaxed); + let generation = { + let mut schedule = gc.schedule.lock(); + let generation = match generation { + Some(g) => g.min(2), + None => { + let thresholds = gc.get_threshold(); + if thresholds.0 == 0 { + return CollectResult::default(); + } + schedule.generation(thresholds) + } + }; + schedule.start_collection(generation); + generation + }; + let start_time = cfg_select! { + target_arch = "wasm32" => (), + _ => std::time::Instant::now(), + }; + // Reset only this interpreter, before callbacks can allocate. Never + // erase another VM's pressure or requests made during this collection. + gc.heap.allocations.store(0, Ordering::Relaxed); + callbacks(gc, "start", generation, &CollectResult::default()); + if let Some(state) = gc + .heap + .owner + .and_then(crate::vm::runtime::lookup_interpreter) + { + state.type_cache.clear(); + } + #[cfg(feature = "threading")] + { + gc.qsbr().process(); + crate::object::qsbr::shared().process(); + } + + let (mut graph, candidates) = { + let _world = CollectStopTheWorld::new(gc); + let objects = gc.heap.snapshot(Some(generation)); + let candidates = objects.len(); + (unsafe { Graph::capture(objects) }, candidates) + }; + // No Python object reads: the expensive graph walk does not stop peers. + graph.analyze(); + let (live, mut suspects) = graph.partition(); + gc.schedule.lock().record_survivors(generation, live.len()); + promote(&live, generation); + drop(live); + + if !suspects.is_empty() { + let mut world = CollectStopTheWorld::new(gc); + let (live, dead) = unsafe { revalidate(suspects) }; + let mut weak_callbacks = Vec::new(); + clear_weakrefs(&dead, &mut weak_callbacks); + world.restart(); + drop(world); + promote(&live, generation); + drop(live); + run_weak_callbacks(weak_callbacks); + for obj in &dead { + obj.try_call_finalizer(); + } + suspects = dead; + } + + let debug = gc.get_debug(); + let mut collected = 0; + if !suspects.is_empty() { + let mut world = CollectStopTheWorld::new(gc); + let (mut resurrected, mut dead) = unsafe { revalidate(suspects) }; + let mut weak_callbacks = Vec::new(); + // Finalizers may have published fresh weakrefs. Invalidate those + // before exposing untracked objects to concurrent weakref upgrades. + clear_weakrefs(&dead, &mut weak_callbacks); + if !weak_callbacks.is_empty() { + // Callback references themselves are now held externally by + // this collector. Account for those pins before committing. + let (live, remaining) = unsafe { revalidate(dead) }; + resurrected.extend(live); + dead = remaining; + } + + // Match Python's object count: the separately allocated instance + // dictionary is storage of its instance, not another Python object. + for obj in &dead { + obj.start_gc_index(0); + } + let instance_dicts = dead + .iter() + .filter(|obj| { + obj.dict() + .is_some_and(|dict| dict.as_object().is_gc_collecting()) + }) + .count(); + collected = dead.len().saturating_sub(instance_dicts); + for obj in &dead { + obj.end_gc_refs(); + } + + if !debug.contains(GcDebugFlags::SAVEALL) { + // No external strong roots remain. Once weakrefs and heap + // membership are removed, no new reference can be acquired via + // Python introspection after the world resumes. + for obj in &dead { + // A dead weakref can still be found through its live + // referent's list. Remove that ingress before restarting. + obj.detach_weakref(); + if let Some(heap) = obj.gc_handle().heap() { + heap.untrack(obj); + } + } + } + world.restart(); + drop(world); + promote(&resurrected, generation); + drop(resurrected); + run_weak_callbacks(weak_callbacks); + if debug.contains(GcDebugFlags::SAVEALL) { + promote(&dead, generation); + gc.garbage.lock().extend(dead); + } else { + // tp_clear may release children and execute finalizers; it must + // run after restarting other interpreters and releasing locks. + rustpython_common::refcount::with_deferred_drops(|| { + for obj in &dead { + if debug.contains(GcDebugFlags::COLLECTABLE) { + eprintln!( + "gc: collectable <{} {:p}>", + obj.class().name(), + obj.as_ref() + ); + } + if obj.gc_has_clear() { + drop(unsafe { obj.gc_clear() }); + } + } + drop(dead); + }); + } + } + if generation == 2 { + gc.heap.trim(); + } + let result = CollectResult { + collected, + uncollectable: 0, + candidates, + duration: elapsed_secs(start_time), + }; + gc.generations[generation].update_stats(collected, 0, candidates, result.duration); + if debug.contains(GcDebugFlags::STATS) { + eprintln!( + "gc: generation {generation}, {candidates} candidates, {collected} collected" + ); + } + let saved = core::mem::take(&mut *gc.garbage.lock()); + if !saved.is_empty() + && let Some(garbage) = gc.garbage_list() + { + garbage.borrow_vec_mut().extend(saved); + } + callbacks(gc, "stop", generation, &result); + // Retired heaps need a full graph scan. Keep that work out of young + // collections; teardown also sweeps them when an interpreter retires. + if generation == 2 { + self.collect_retired(); + } + result + } +} + +impl GcState { + /// No interpreter can execute in these heaps. Mask callbacks for the whole + /// sweep, including snapshot pins and unwinding: releasing a pin can itself + /// become the last reference and invoke native destruction. + pub(super) fn collect_retired(&self) { + crate::vm::thread::native_sweep(|| { + #[cfg(all(unix, feature = "threading"))] + let _retirement = crate::vm::fork::native_phase(); + let Some(_collecting) = self.collecting.try_lock() else { + return; + }; + for heap in self.retired_heaps() { + let _owner = AllocationScope::new(heap.clone()); + heap.unfreeze(); + let mut graph = unsafe { Graph::capture(heap.snapshot(None)) }; + graph.analyze(); + let (live, dead) = graph.partition(); + drop(live); + let mut callbacks = Vec::new(); + clear_weakrefs(&dead, &mut callbacks); + for obj in &dead { + obj.detach_weakref(); + heap.untrack(obj); + } + drop(callbacks); + rustpython_common::refcount::with_deferred_drops(|| { + for obj in &dead { + if obj.gc_has_clear() { + drop(unsafe { obj.gc_clear() }); + } + } + drop(dead); + }); + heap.trim(); + #[cfg(feature = "threading")] + heap.qsbr.process(); + } + }); + } +} diff --git a/crates/vm/src/gc_state/graph.rs b/crates/vm/src/gc_state/graph.rs new file mode 100644 index 00000000000..d49f7f8a04e --- /dev/null +++ b/crates/vm/src/gc_state/graph.rs @@ -0,0 +1,135 @@ +//! A collection snapshot: compact indices, with no object access during analysis. + +use super::*; + +struct Node { + object: PyObjectRef, + refs: usize, + end: usize, +} + +pub(super) struct Graph { + nodes: Vec, + edges: Vec, +} + +struct ResetIndices<'a>(&'a [Node]); + +impl Drop for ResetIndices<'_> { + fn drop(&mut self) { + for node in self.0 { + node.object.end_gc_refs(); + } + } +} + +impl Graph { + /// All candidates must have exactly one collector-owned reference in + /// `objects`, and belong to one heap. Its payloads must be stable under + /// the owner's stop-the-world or permanent retirement. Each child's + /// temporary header index is valid only for this capture. Duplicate owned edges are + /// retained, since each contributes one strong reference. + pub(super) unsafe fn capture(objects: Vec) -> Self { + let nodes: Vec<_> = objects + .into_iter() + .map(|object| Node { + refs: object.strong_count().saturating_sub(1), + object, + end: 0, + }) + .collect(); + let mut graph = Self { + nodes, + edges: Vec::new(), + }; + let heap = graph + .nodes + .first() + .map(|node| core::ptr::from_ref(node.object.gc_handle().heap().unwrap())); + debug_assert!( + graph + .nodes + .iter() + .all(|node| { node.object.gc_handle().heap().map(core::ptr::from_ref) == heap }) + ); + if graph.nodes.len() > u32::MAX as usize { + // More nodes than the compact index can represent: conservatively + // retain them, rather than wrap and subtract from another object. + for node in &mut graph.nodes { + node.refs = usize::MAX; + } + return graph; + } + { + let reset = ResetIndices(&graph.nodes); + for (index, node) in reset.0.iter().enumerate() { + node.object.start_gc_index(index as u32); + } + // Offsets live separately while ResetIndices borrows all nodes. + let mut ends = Vec::with_capacity(graph.nodes.len()); + for node in reset.0 { + unsafe { + node.object.gc_visit_referents(&mut |child| { + if child.gc_handle().heap().map(core::ptr::from_ref) == heap + && child.is_gc_collecting() + { + graph.edges.push(child.gc_refs()); + } + }); + } + ends.push(graph.edges.len()); + } + drop(reset); + for (node, end) in graph.nodes.iter_mut().zip(ends) { + node.end = end; + } + } + graph + } + + /// Only integer arrays are read here. Python threads may run concurrently: + /// object lifetimes are pinned, and later revalidation handles mutations. + pub(super) fn analyze(&mut self) { + for &child in &self.edges { + let refs = &mut self.nodes[child as usize].refs; + if *refs != usize::MAX { + *refs = refs.saturating_sub(1); + } + } + let mut work = Vec::new(); + for (index, node) in self.nodes.iter_mut().enumerate() { + if node.refs != 0 { + node.refs = usize::MAX; + work.push(index); + } + } + while let Some(index) = work.pop() { + let start = if index == 0 { + 0 + } else { + self.nodes[index - 1].end + }; + let end = self.nodes[index].end; + for &child in &self.edges[start..end] { + let child = child as usize; + if self.nodes[child].refs != usize::MAX { + self.nodes[child].refs = usize::MAX; + work.push(child); + } + } + } + } + + pub(super) fn partition(self) -> (Vec, Vec) { + let mut live = Vec::new(); + let mut dead = Vec::new(); + for node in self.nodes { + if node.refs == usize::MAX { + live.push(node.object); + } else { + dead.push(node.object); + } + } + (live, dead) + } +} diff --git a/crates/vm/src/gc_state/heap.rs b/crates/vm/src/gc_state/heap.rs new file mode 100644 index 00000000000..a1c7a1af616 --- /dev/null +++ b/crates/vm/src/gc_state/heap.rs @@ -0,0 +1,288 @@ +//! Object membership and lifetime, independent of an interpreter's lifetime. + +use super::*; +use crate::common::rc::PyRc; +use core::sync::atomic::{AtomicIsize, AtomicPtr}; + +const NO_SLOT: usize = usize::MAX; + +/// A strong heap handle and a slot, replacing the two intrusive list pointers. +/// The heap owns only raw object pointers: this strong handle creates no cycle. +/// Keeping it until object destruction makes lookup safe even after untracking +/// or interpreter shutdown, without a registry lookup on the deallocation path. +pub(crate) struct GcHandle { + heap: AtomicPtr, + slot: AtomicUsize, +} + +impl GcHandle { + pub(crate) const fn new() -> Self { + Self { + heap: AtomicPtr::new(core::ptr::null_mut()), + slot: AtomicUsize::new(NO_SLOT), + } + } + + pub(crate) fn heap(&self) -> Option<&GcHeap> { + // The pointer is assigned once before publishing the object and owns + // a strong reference until destruction. A borrow of the object pins it. + unsafe { self.heap.load(Ordering::Acquire).as_ref() } + } + + /// Bind a fresh allocation before it is published, including objects that + /// never enter the cycle collector but still need owner-bound destruction. + pub(crate) fn bind_new(&self, heap: PyRc) { + debug_assert!(self.heap.load(Ordering::Relaxed).is_null()); + self.heap + .store(PyRc::into_raw(heap).cast_mut(), Ordering::Release); + } + + fn bind(&self, heap: &PyRc) -> &GcHeap { + if self.heap().is_none() { + let raw = PyRc::into_raw(heap.clone()).cast_mut(); + if self + .heap + .compare_exchange( + core::ptr::null_mut(), + raw, + Ordering::Release, + Ordering::Relaxed, + ) + .is_err() + { + unsafe { drop(PyRc::from_raw(raw)) }; + } + } + self.heap().unwrap() + } + + /// Release before publishing a husk in a freelist; Drop is the backstop + /// for objects that bypass freelists, including bootstrap allocations. + pub(crate) fn release(&self) { + debug_assert_eq!(self.slot.load(Ordering::Relaxed), NO_SLOT); + let ptr = self.heap.swap(core::ptr::null_mut(), Ordering::Relaxed); + if !ptr.is_null() { + unsafe { drop(PyRc::from_raw(ptr)) }; + } + } +} + +impl Drop for GcHandle { + fn drop(&mut self) { + self.release(); + } +} + +#[derive(Clone, Copy)] +pub(super) struct ObjectPtr(pub NonNull); + +// Access is protected by the heap lock and collection's stop-the-world barrier. +#[cfg(feature = "threading")] +unsafe impl Send for ObjectPtr {} +#[cfg(feature = "threading")] +unsafe impl Sync for ObjectPtr {} + +pub(super) struct HeapObjects { + generations: [Vec; 4], +} + +impl HeapObjects { + fn insert(&mut self, obj: &PyObject, generation: usize) { + let objects = &mut self.generations[generation]; + obj.gc_handle().slot.store(objects.len(), Ordering::Relaxed); + objects.push(ObjectPtr(NonNull::from(obj))); + obj.set_gc_generation(generation as u8); + } + + fn remove(&mut self, obj: &PyObject) -> bool { + let generation = obj.gc_generation() as usize; + if generation >= self.generations.len() { + return false; + } + let objects = &mut self.generations[generation]; + let slot = obj.gc_handle().slot.load(Ordering::Relaxed); + assert_eq!(objects[slot].0, NonNull::from(obj)); + objects.swap_remove(slot); + if let Some(moved) = objects.get(slot) { + unsafe { moved.0.as_ref() } + .gc_handle() + .slot + .store(slot, Ordering::Relaxed); + } + obj.gc_handle().slot.store(NO_SLOT, Ordering::Relaxed); + obj.set_gc_generation(GC_UNTRACKED); + true + } + + fn move_all(&mut self, from: usize, to: usize) { + let mut objects = core::mem::take(&mut self.generations[from]); + self.generations[to].reserve(objects.len()); + for ptr in objects.drain(..) { + self.insert(unsafe { ptr.0.as_ref() }, to); + } + // Empty generations do not retain a previous full heap's allocation. + } +} + +/// A heap outlives its VM while any object allocated into it still exists. +pub struct GcHeap { + pub(crate) owner: Option, + #[cfg(feature = "threading")] + pub(crate) qsbr: alloc::sync::Arc, + objects: PyMutex, + // Shared immortal referents never link weakrefs from different owners. + // Keeping their native list heads here leaves ordinary object prefixes + // unchanged and releases the directory with its owner heap. + weakrefs: PyMutex>>, + pub(super) allocations: AtomicIsize, + pub(super) retired: AtomicBool, +} + +impl GcHeap { + pub(super) fn new(owner: Option) -> Self { + Self { + owner, + #[cfg(feature = "threading")] + qsbr: if owner.is_some() { + alloc::sync::Arc::new(crate::object::qsbr::Qsbr::new()) + } else { + crate::object::qsbr::shared().clone() + }, + objects: PyMutex::new(HeapObjects { + generations: core::array::from_fn(|_| Vec::new()), + }), + weakrefs: PyMutex::new(std::collections::HashMap::new()), + allocations: AtomicIsize::new(0), + retired: AtomicBool::new(false), + } + } + + pub(crate) fn weakrefs(&self, referent: &PyObject) -> PyRc { + debug_assert!(referent.is_immortal()); + self.weakrefs + .lock() + .entry(core::ptr::from_ref(referent).addr()) + .or_default() + .clone() + } + + /// The caller owns a valid, untracked object and excludes concurrent tracking. + pub(super) unsafe fn track(heap: &PyRc, ptr: NonNull, fresh: bool) { + let obj = unsafe { ptr.as_ref() }; + let heap = obj.gc_handle().bind(heap); + let mut objects = heap.objects.lock(); + assert!(!obj.is_gc_tracked()); + if fresh { + obj.init_gc_tracked_bit(); + } else { + obj.set_gc_tracked(); + } + objects.insert(obj, 0); + heap.allocations.fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn untrack(&self, obj: &PyObject) { + let mut objects = self.objects.lock(); + if objects.remove(obj) { + obj.clear_gc_tracked(); + self.allocations.fetch_sub(1, Ordering::Relaxed); + } + } + + /// Publish a freshly allocated generator and frame under one heap lock. + pub(super) unsafe fn track_pair( + heap: &PyRc, + first: NonNull, + second: NonNull, + ) { + let first = unsafe { first.as_ref() }; + let second = unsafe { second.as_ref() }; + first.gc_handle().bind(heap); + second.gc_handle().bind(heap); + let mut objects = heap.objects.lock(); + for obj in [first, second] { + debug_assert!(core::ptr::eq(obj.gc_handle().heap().unwrap(), &**heap)); + obj.init_gc_tracked_bit(); + objects.insert(obj, 0); + } + heap.allocations.fetch_add(2, Ordering::Relaxed); + } + + pub(super) fn snapshot(&self, generation: Option) -> Vec { + let objects = self.objects.lock(); + let end = generation.map_or(GC_PERMANENT as usize, |g| g + 1); + objects.generations[..end] + .iter() + .flatten() + .filter_map(|p| unsafe { p.0.as_ref() }.try_to_owned()) + .collect() + } + + pub(super) fn get_objects(&self, generation: Option) -> Vec { + let objects = self.objects.lock(); + let range = match generation { + None => 0..GC_PERMANENT as usize, + Some(g) if (0..=2).contains(&g) => g as usize..g as usize + 1, + _ => return Vec::new(), + }; + objects.generations[range] + .iter() + .flatten() + .filter_map(|p| unsafe { p.0.as_ref() }.try_to_owned()) + .collect() + } + + pub(super) fn promote(&self, objects: &[PyObjectRef], generation: usize) { + let destination = (generation + 1).min(2); + for batch in objects.chunks(256) { + let mut storage = self.objects.lock(); + for obj in batch { + if obj.gc_generation() as usize >= destination { + continue; + } + if storage.remove(obj) { + storage.insert(obj, destination); + } + } + } + } + + pub(super) fn freeze(&self) { + let mut objects = self.objects.lock(); + for generation in 0..GC_PERMANENT as usize { + objects.move_all(generation, GC_PERMANENT as usize); + } + self.allocations.store(0, Ordering::Relaxed); + } + + pub(super) fn unfreeze(&self) { + self.objects.lock().move_all(GC_PERMANENT as usize, 2); + } + + pub(super) fn freeze_count(&self) -> usize { + self.objects.lock().generations[GC_PERMANENT as usize].len() + } + + pub(super) fn trim(&self) { + let mut objects = self.objects.lock(); + for generation in &mut objects.generations { + if generation.capacity() > generation.len().saturating_mul(4).max(256) { + generation.shrink_to_fit(); + } + } + } + + #[cfg(all(unix, feature = "threading"))] + pub(super) unsafe fn reinit_after_fork(&self) { + unsafe { + crate::common::lock::reinit_mutex_after_fork(&self.objects); + crate::common::lock::reinit_mutex_after_fork(&self.weakrefs); + } + } +} + +impl Drop for GcHeap { + fn drop(&mut self) { + debug_assert!(self.objects.get_mut().generations.iter().all(Vec::is_empty)); + } +} diff --git a/crates/vm/src/gc_state/tests.rs b/crates/vm/src/gc_state/tests.rs new file mode 100644 index 00000000000..e3b68bdfd11 --- /dev/null +++ b/crates/vm/src/gc_state/tests.rs @@ -0,0 +1,314 @@ +use super::*; + +#[cfg(feature = "threading")] +#[test] +fn collection_does_not_wait_for_an_attached_foreign_interpreter() { + use core::time::Duration; + use std::sync::mpsc; + let first = crate::Interpreter::without_stdlib(Default::default()); + let second = first.create_subinterpreter(); + let state = first.enter_raw(|vm| vm.state.clone()); + let before = state.stop_the_world.stats_snapshot().stop_calls; + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let first_worker = first.new_thread(); + let waiting = std::thread::spawn(move || { + first_worker.run_raw(|_| { + entered_tx.send(()).unwrap(); + // Deliberately remain attached. Rescue the test if a regression + // incorrectly waits for this unrelated interpreter's safepoint. + let _ = release_rx.recv_timeout(Duration::from_secs(10)); + }); + }); + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + let second_worker = second.new_thread(); + let collecting = std::thread::spawn(move || { + second_worker.run_raw(|vm| { + let node = vm.ctx.new_list(Vec::new()); + node.borrow_vec_mut().push(node.clone().into()); + drop(node); + let collected = vm.state.gc.collect_force(2).collected; + done_tx.send(collected).unwrap(); + }); + }); + let result = done_rx.recv_timeout(Duration::from_secs(5)); + let _ = release_tx.send(()); + waiting.join().unwrap(); + collecting.join().unwrap(); + assert!(result.expect("foreign VM delayed collection") >= 1); + assert_eq!(state.stop_the_world.stats_snapshot().stop_calls, before); +} + +#[test] +fn retired_cycles_are_reclaimed_without_a_live_owner() { + let ctx = crate::Context::genesis(); + let state = GcInterpreterState::new(ctx); + let heap = PyRc::downgrade(&state.heap); + { + let _owner = state.allocation_scope(); + let node = ctx.new_list(Vec::new()); + node.borrow_vec_mut().push(node.clone().into()); + } + // No VM ever entered this heap, so no Python execution or callbacks are + // available to collect its cycle after the native state goes away. + drop(state); + let deadline = std::time::Instant::now() + core::time::Duration::from_secs(5); + while heap.strong_count() != 0 { + gc_state().collect_retired(); + assert!( + std::time::Instant::now() < deadline, + "retired heap was retained" + ); + std::thread::yield_now(); + } + assert!(heap.upgrade().is_none()); +} + +#[test] +fn retired_cycles_wait_for_full_collection_and_recheck_external_roots() { + let ctx = crate::Context::genesis(); + let collector = GcState::new(); + let heap = collector.new_heap(crate::vm::runtime::alloc_interpreter_id()); + let state = GcInterpreterState::new(ctx); + let node = { + let _owner = AllocationScope::new(heap.clone()); + let node = ctx.new_list(Vec::new()); + node.borrow_vec_mut().push(node.clone().into()); + node + }; + heap.retired.store(true, Ordering::Release); + collector.collect_retired(); + assert_eq!(heap.snapshot(None).len(), 1); + // Dropping an external root leaves both membership and the cycle intact. + // A membership-only dirty flag would miss the next chance to collect it. + drop(node); + collector.collect_inner(&state, Some(0), true); + collector.collect_inner(&state, Some(1), true); + let retained_after_young_collections = heap.snapshot(None).len(); + collector.collect_inner(&state, Some(2), true); + assert!(heap.snapshot(None).is_empty()); + assert_eq!(retained_after_young_collections, 1); +} + +#[test] +fn heap_introspection_and_freezing_are_owner_local() { + let first = crate::Interpreter::without_stdlib(Default::default()); + let second = first.create_subinterpreter(); + first.enter_raw(|a| { + a.state.gc.freeze(); + let frozen = a.state.gc.get_freeze_count(); + assert!(frozen > 0); + second.enter_raw(|b| { + assert_eq!(b.state.gc.get_freeze_count(), 0); + b.state.gc.unfreeze(); + let objects = b.state.gc.get_objects(None); + assert!(!objects.is_empty()); + assert!(objects.iter().all(|obj| { + obj.gc_handle().heap().unwrap().owner == Some(b.state.interpreter_id) + })); + }); + assert_eq!(a.state.gc.get_freeze_count(), frozen); + a.state.gc.unfreeze(); + }); +} + +#[test] +fn untracked_allocations_route_callbacks_to_the_owner() { + use crate::AsObject; + let first = crate::Interpreter::without_stdlib(Default::default()); + let second = first.create_subinterpreter(); + first.enter_raw(|vm| { + let object = vm.ctx.new_bytearray(Vec::new()); + assert!(!object.as_object().is_gc_tracked()); + second.enter_raw(|other| { + let owner = + crate::vm::thread::with_vm(object.as_object(), |owner| owner.state.interpreter_id); + assert_eq!(owner, Some(vm.state.interpreter_id)); + crate::vm::thread::with_current_vm(|current| { + assert_eq!(current.state.interpreter_id, other.state.interpreter_id); + }); + }); + }); +} + +#[test] +fn collection_schedule_and_growth_guard() { + let mut schedule = GcSchedule::default(); + for expected in [0, 0, 1, 0, 0, 1, 2, 0] { + let generation = schedule.generation((100, 1, 1)); + assert_eq!(generation, expected); + schedule.start_collection(generation); + schedule.record_survivors(generation, 100); + } + schedule.start_collection(2); + schedule.record_survivors(2, 1000); + schedule.start_collection(1); + schedule.record_survivors(1, 249); + assert_eq!(schedule.generation((100, 0, 0)), 0); + schedule.record_survivors(1, 1); + assert_eq!(schedule.generation((100, 0, 0)), 2); +} + +#[test] +fn interpreter_policy_and_pressure_are_independent() { + let ctx = crate::vm::Context::genesis(); + let first = GcInterpreterState::new(ctx); + let second = GcInterpreterState::new(ctx); + assert_eq!(first.get_threshold(), (2000, 10, 10)); + assert!(first.is_enabled()); + first.disable(); + assert!(!first.is_enabled()); + assert!(second.is_enabled()); + first.enable(); + first.set_threshold(123, Some(4), Some(5)); + first.set_debug(GcDebugFlags::SAVEALL); + assert_eq!(first.get_threshold(), (123, 4, 5)); + assert_eq!(second.get_threshold(), (2000, 10, 10)); + assert_eq!(second.get_debug(), GcDebugFlags::empty()); + first.heap.allocations.store(42, Ordering::Relaxed); + first.schedule.lock().start_collection(1); + assert_eq!(first.get_count(), (42, 0, 1)); + assert_eq!(second.get_count(), (0, 0, 0)); +} + +#[test] +fn dense_graph_counts_duplicate_edges_and_external_roots() { + let _collector = gc_state().collecting.lock(); + let ctx = crate::vm::Context::genesis(); + for rooted in [false, true] { + let a = ctx.new_list(Vec::new()); + let b = ctx.new_list(vec![a.clone().into()]); + a.borrow_vec_mut() + .extend([b.clone().into(), b.clone().into()]); + let root = rooted.then(|| a.clone()); + let mut graph = unsafe { Graph::capture(vec![a.into(), b.into()]) }; + graph.analyze(); + let (live, dead) = graph.partition(); + assert_eq!(live.len(), if rooted { 2 } else { 0 }); + assert_eq!(dead.len(), if rooted { 0 } else { 2 }); + for obj in live.iter().chain(&dead) { + assert!(!obj.is_gc_collecting()); + obj.downcast_ref::() + .unwrap() + .borrow_vec_mut() + .clear(); + } + drop(root); + } +} + +#[test] +fn revalidation_preserves_newly_frozen_objects_and_their_referents() { + let _collector = gc_state().collecting.lock(); + let first = crate::Interpreter::without_stdlib(Default::default()); + let a = first.enter_raw(|vm| { + vm.state.gc.disable(); + vm.ctx.new_list(Vec::new()) + }); + let b = first.enter_raw(|vm| vm.ctx.new_list(vec![a.clone().into()])); + a.borrow_vec_mut().push(b.clone().into()); + let mut graph = unsafe { Graph::capture(vec![a.into(), b.into()]) }; + graph.analyze(); + let (live, dead) = graph.partition(); + assert!(live.is_empty()); + assert_eq!(dead.len(), 2); + first.enter_raw(|vm| vm.state.gc.heap.freeze()); + let (live, dead) = unsafe { collector::revalidate(dead) }; + assert_eq!(live.len(), 2); + assert!(dead.is_empty()); + for obj in live { + obj.downcast_ref::() + .unwrap() + .borrow_vec_mut() + .clear(); + } +} + +#[test] +fn immutable_tuple_tracking_distinguishes_mutable_children_and_placeholders() { + let ctx = crate::vm::Context::genesis(); + let tuple = ctx.new_tuple(vec![ctx.new_int(1).into()]); + assert!(!tuple.as_object().is_gc_tracked()); + assert!(tuple.as_object().gc_is_acyclic()); + let nested = ctx.new_tuple(vec![tuple.into()]); + assert!(!nested.as_object().is_gc_tracked()); + let mutable = ctx.new_tuple(vec![ctx.new_list(Vec::new()).into()]); + assert!(mutable.as_object().is_gc_tracked()); + let placeholder = crate::PyRef::new_ref_tracked( + crate::builtins::PyTuple::new_unchecked(vec![ctx.none()].into_boxed_slice()), + ctx.types.tuple_type.to_owned(), + ); + assert!(placeholder.as_object().is_gc_tracked()); + assert!(!placeholder.as_object().gc_is_acyclic()); +} + +#[test] +fn heap_membership_survives_swap_promotion_freeze_and_retirement() { + let _collector = gc_state().collecting.lock(); + let ctx = crate::vm::Context::genesis(); + let state = GcInterpreterState::new(ctx); + let weak = PyRc::downgrade(&state.heap); + // Populate only this heap's objects, without interpreter bootstrap roots. + let objects: Vec = (0..600) + .map(|_| { + let obj: PyObjectRef = ctx.new_list(Vec::new()).into(); + // This fresh, unpublished fixture has no other references. Remove + // its bootstrap membership before assigning the isolated heap. + assert_eq!(obj.strong_count(), 1); + obj.gc_handle().heap().unwrap().untrack(&obj); + obj.gc_handle().release(); + unsafe { + GcHeap::track(&state.heap, NonNull::from(obj.as_ref()), true); + } + obj + }) + .collect(); + let heap = weak.upgrade().unwrap(); + heap.promote(&objects[..200], 0); + heap.promote(&objects[..100], 1); + heap.untrack(&objects[250]); + heap.promote(&objects, 2); + // gc.garbage and gc.callbacks are private bootstrap roots in generation 0. + assert_eq!(heap.get_objects(Some(0)).len(), 2); + assert_eq!(heap.get_objects(Some(1)).len(), 0); + assert_eq!(heap.get_objects(Some(2)).len(), 599); + heap.freeze(); + assert_eq!(heap.freeze_count(), 601); + assert!(heap.get_objects(None).is_empty()); + heap.unfreeze(); + assert_eq!(heap.freeze_count(), 0); + assert_eq!(heap.get_objects(Some(2)).len(), 601); + drop(state); + assert!(heap.retired.load(Ordering::Relaxed)); + drop(heap); + assert!(weak.upgrade().is_some()); + drop(objects); + assert!(weak.upgrade().is_none()); +} + +#[cfg(feature = "threading")] +#[test] +fn pending_requests_survive_busy_collectors_and_foreign_resets() { + let state = GcInterpreterState::new(crate::vm::Context::genesis()); + let other = GcInterpreterState::new(crate::vm::Context::genesis()); + let collector = GcState::new(); + state.set_threshold(1, Some(0), Some(0)); + state.heap.allocations.store(1, Ordering::Relaxed); + collector.maybe_collect(&state); + other.heap.allocations.store(0, Ordering::Relaxed); + { + let _guard = state.collecting.lock(); + collector.collect_inner(&state, None, false); + assert!(state.scheduled.load(Ordering::Relaxed)); + assert_eq!(state.get_count(), (1, 0, 0)); + } + state.set_threshold(0, None, None); + collector.collect_inner(&state, None, false); + assert!(!state.scheduled.load(Ordering::Relaxed)); + assert_eq!(state.get_count(), (1, 0, 0)); + state.scheduled.store(true, Ordering::Relaxed); + state.disable(); + collector.collect_inner(&state, None, false); + assert!(!state.scheduled.load(Ordering::Relaxed)); +} diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 3fb6e5fdc2e..56c76d37d33 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -12,7 +12,7 @@ pub(crate) fn check_pyc_magic_number_bytes(buf: &[u8]) -> bool { buf.starts_with(&crate::version::PYC_MAGIC_NUMBER_BYTES) } -pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult { +pub(crate) fn init_importlib_base(vm: &VirtualMachine) -> PyResult { flame_guard!("init importlib"); // importlib_bootstrap needs these and it inlines checks to sys.modules before calling into @@ -29,8 +29,10 @@ pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult PyResult<()> { if is_module_initializing(module, vm)? { - let lock_unlock = vm.importlib.get_attr("_lock_unlock_module", vm)?; + let lock_unlock = vm.importlib().get_attr("_lock_unlock_module", vm)?; lock_unlock.call((name.to_owned(),), vm)?; } Ok(()) @@ -445,8 +447,8 @@ pub(crate) fn import_module_level( m } _ => { - let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; - find_and_load.call((abs_name.clone(), vm.import_func.clone()), vm)? + let find_and_load = vm.importlib().get_attr("_find_and_load", vm)?; + find_and_load.call((abs_name.clone(), vm.import_func().to_owned()), vm)? } }; @@ -466,8 +468,8 @@ pub(crate) fn import_module_level( .get_attribute_opt(&module, vm.ctx.intern_str("__path__"))? .is_some(); if has_path { - let handle_fromlist = vm.importlib.get_attr("_handle_fromlist", vm)?; - handle_fromlist.call((module, fromlist, vm.import_func.clone()), vm) + let handle_fromlist = vm.importlib().get_attr("_handle_fromlist", vm)?; + handle_fromlist.call((module, fromlist, vm.import_func().to_owned()), vm) } else { Ok(module) } @@ -488,8 +490,8 @@ pub(crate) fn import_module_level( Err(_) if level == 0 => { // For absolute imports (level 0), try importing the // parent. Matches _bootstrap.__import__ behavior. - let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; - find_and_load.call((to_return, vm.import_func.clone()), vm) + let find_and_load = vm.importlib().get_attr("_find_and_load", vm)?; + find_and_load.call((to_return, vm.import_func().to_owned()), vm) } Err(_) => { // For relative imports (level > 0), raise KeyError diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index c6813f63e3a..015c1d02b0c 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -60,6 +60,7 @@ pub mod convert; mod coroutine; pub mod datastack; mod dict_inner; +pub mod embedding; #[cfg(feature = "rustpython-compiler")] pub mod eval; @@ -115,7 +116,7 @@ pub use self::object::{ }; pub use self::vm::runtime; pub use self::vm::{ - Context, InterpFeatureFlags, Interpreter, InterpreterBuilder, InterpreterConfig, + Context, FinalizeBusy, InterpFeatureFlags, Interpreter, InterpreterBuilder, InterpreterConfig, InterpreterGil, InterpreterInfo, InterpreterWhence, MAIN_INTERPRETER_ID, Settings, VirtualMachine, }; diff --git a/crates/vm/src/macros.rs b/crates/vm/src/macros.rs index 340e5461571..138873684d2 100644 --- a/crates/vm/src/macros.rs +++ b/crates/vm/src/macros.rs @@ -81,13 +81,14 @@ macro_rules! py_namespace { /// # Examples /// /// ```rust +/// # unsafe { /// use malachite_bigint::ToBigInt; /// use num_traits::Zero; /// /// use rustpython_vm::{PyPayload, match_class}; /// use rustpython_vm::builtins::{PyFloat, PyInt}; /// -/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { +/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter_unchecked(|vm| { /// let obj = PyInt::from(0).into_pyobject(vm); /// assert_eq!( /// "int", @@ -99,18 +100,20 @@ macro_rules! py_namespace { /// ); /// # }); /// +/// # } /// ``` /// /// With a binding to the downcasted type: /// /// ```rust +/// # unsafe { /// use malachite_bigint::ToBigInt; /// use num_traits::Zero; /// /// use rustpython_vm::{PyPayload, match_class}; /// use rustpython_vm::builtins::{PyFloat, PyInt}; /// -/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { +/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter_unchecked(|vm| { /// let obj = PyInt::from(0).into_pyobject(vm); /// /// let int_value = match_class!(match obj { @@ -121,6 +124,7 @@ macro_rules! py_namespace { /// /// assert!(int_value.is_zero()); /// # }); +/// # } /// ``` #[macro_export] macro_rules! match_class { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index eb7edc93cca..1d079974e01 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -22,6 +22,7 @@ use crate::{ atomic::{Ordering, PyAtomic, Radium}, linked_list::{Link, Pointers}, lock::PyRwLock, + rc::PyRc, refcount::RefCount, }, vm::VirtualMachine, @@ -173,25 +174,30 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { return; // resurrected by __del__ } - // Only tracked objects take the trashcan recursion guard and untrack path. - // Untracked objects either own no children (int, float, str, ...) or, like + // Acyclic immutable containers still need the trashcan recursion guard: + // a long tuple chain must not consume one Rust stack frame per object. + // Other untracked objects either own no children (int, float, str, ...) or, like // non-escaped frames, are released at interpreter depth with at most one // unguarded link before their tracked children (dicts, functions, code) // re-enter guarded deallocation, so recursion stays bounded. A frame stored // in an object graph is forced to escape, becoming tracked and guarded here. // Read once and reuse for both gates below. let tracked = obj_ref.is_gc_tracked(); + let guarded = tracked || obj_ref.gc_is_acyclic(); // Trashcan: limit recursive deallocation depth to prevent stack overflow - if tracked && !unsafe { trashcan::begin(obj, default_dealloc::) } { + if guarded && !unsafe { trashcan::begin(obj, default_dealloc::) } { return; // deferred to queue } let vtable = obj_ref.0.vtable; + // A weakref's referent list also holds a raw pointer to its payload. + // Unlink under the stripe before tp_clear/Drop borrows that payload mutably. + obj_ref.detach_weakref(); + // Untrack from GC BEFORE deallocation. - // Must happen before memory is freed because intrusive list removal - // reads the object's gc_pointers (prev/next). + // Membership must be removed before its non-owning pointer can dangle. if tracked { let ptr = unsafe { NonNull::new_unchecked(obj) }; unsafe { @@ -235,7 +241,7 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { let typ = obj_ref.class(); let pushed = if T::HAS_FREELIST && typ.heaptype_ext.is_none() - && core::ptr::eq(typ, T::class(crate::vm::Context::genesis())) + && core::ptr::eq(typ, &*T::class(crate::vm::Context::genesis())) && !obj_ref.0.ref_count.is_published() { if let Some(ext) = obj_ref.0.ext_ref() { @@ -251,6 +257,7 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { drop(slot.store(None)); } } + unsafe { (*obj.cast::>()).gc_handle.release() }; unsafe { T::freelist_push(obj) } } else { false @@ -262,7 +269,7 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { } // Trashcan: decrement depth and process deferred objects at outermost level - if tracked { + if guarded { unsafe { trashcan::end() }; } } @@ -283,8 +290,8 @@ pub(super) unsafe fn try_traverse_obj(x: &PyObject, tracer_fn: &mu /// Call `try_clear` on payload to extract child references (tp_clear) pub(super) unsafe fn try_clear_obj(x: *mut PyObject, out: &mut Vec) { - let x = unsafe { &mut *(x as *mut Py) }; - x.payload.try_clear(out); + let payload = unsafe { &mut (*x.cast::>()).payload }; + payload.try_clear(out); } bitflags::bitflags! { @@ -299,8 +306,8 @@ bitflags::bitflags! { const FINALIZED = 1 << 1; /// Object is unreachable (during GC collection) const UNREACHABLE = 1 << 2; - /// Object is frozen (immutable) - const FROZEN = 1 << 3; + /// A fully initialized immutable container with permanently acyclic children. + const ACYCLIC = 1 << 3; /// Memory the object references is shared between multiple threads /// and needs special handling when freeing due to possible in-flight lock-free reads const SHARED = 1 << 4; @@ -318,47 +325,6 @@ bitflags::bitflags! { /// GC generation constants pub(crate) const GC_UNTRACKED: u8 = 0xFF; pub(crate) const GC_PERMANENT: u8 = 3; -/// Width of an interpreter's `gc_owner` tag. -/// -/// Sized to the padding the header alignment already forces, so the tag costs -/// no space on either pointer width. Running out of tags is not an error: an -/// interpreter that gets none uses [`GC_NO_OWNER`] and its objects stay -/// collectable by every interpreter, which is how they behaved before tagging. -pub(crate) type GcOwner = u16; - -/// `gc_owner` of an object that belongs to no single interpreter: everything -/// the shared context allocates, and anything allocated with no interpreter -/// current. Every interpreter collects these. -pub(crate) const GC_NO_OWNER: GcOwner = 0; - -/// `gc_refs` of an object a running collection has proved reachable. One past -/// the largest count [`PyObject::start_gc_refs`] stores, so no real count can -/// be taken for it. -pub(crate) const GC_REACHABLE: u32 = u32::MAX; - -/// Link implementation for GC intrusive linked list tracking -pub(crate) struct GcLink; - -// SAFETY: PyObject (Py) is heap-allocated and pinned in memory -// once created. gc_pointers is at a fixed offset in Py. -unsafe impl Link for GcLink { - type Handle = NonNull; - type Target = PyObject; - - fn as_raw(handle: &NonNull) -> NonNull { - *handle - } - - unsafe fn from_raw(ptr: NonNull) -> NonNull { - ptr - } - - unsafe fn pointers(target: NonNull) -> NonNull> { - let inner_ptr = target.as_ptr() as *mut Py; - unsafe { NonNull::new_unchecked(&raw mut (*inner_ptr).gc_pointers) } - } -} - /// Extension fields for objects that need dict or member slots. /// Allocated as a prefix before Py when needed (prefix allocation pattern). /// Access via `Py::ext_ref()` using negative offset from the object pointer. @@ -454,17 +420,10 @@ pub struct Py { /// GC generation index (0-2=gen, GC_PERMANENT=permanent, GC_UNTRACKED=not tracked). /// Uses PyAtomic for interior mutability (writes happen through &self under list locks). pub(super) gc_generation: PyAtomic, - /// Interpreter that tracked this object, or `GC_NO_OWNER`. Written by - /// `track_object`; read to scope a collection to one interpreter. - /// Sits in what would otherwise be padding, so it costs no space. - pub(super) gc_owner: PyAtomic, - /// The count a running collection is working with: the strong count with - /// the references held from inside the candidate set taken off, or - /// [`GC_REACHABLE`] once the object has been proved reachable. Only - /// meaningful while `gc_bits` has [`GcBits::COLLECTING`]. + /// Temporary dense index while a collection snapshots references. pub(super) gc_refs: PyAtomic, - /// Intrusive linked list pointers for GC generational tracking - pub(super) gc_pointers: Pointers, + /// Heap lifetime and membership slot; occupies the former link pointers. + pub(super) gc_handle: crate::gc_state::GcHandle, pub(super) typ: PyAtomicRef, // __class__ member @@ -489,22 +448,15 @@ pub const fn dict_member_offset() -> isize { -(core::mem::size_of::() as isize) } -// ref_count, vtable, gc_pointers (two) and typ are one word each; the gc bits, -// generation, owner and refs take eight bytes between them. A 64-bit header had -// those eight as the padding its alignment forces, so they cost it nothing; a -// 32-bit header spends a word on them. Adding to that group is free only while -// this holds. +// The heap handle uses the same two words as the former intrusive links. const _: () = assert!(SIZEOF_PYOBJECT_HEAD == 5 * core::mem::size_of::() + 8); -// `Py::drop_fields` names `payload` and `typ`; it is only complete while -// every other field stays trivially destructible. +// drop_fields explicitly destroys payload, type, and the owning heap handle. const _: () = assert!( !core::mem::needs_drop::() && !core::mem::needs_drop::<&'static PyObjVTable>() && !core::mem::needs_drop::>() && !core::mem::needs_drop::>() - && !core::mem::needs_drop::>() - && !core::mem::needs_drop::>() ); impl Py { @@ -661,7 +613,7 @@ pub(crate) fn reset_weakref_locks_after_fork() { // === WeakRefList: inline on every object (tp_weaklist) === #[repr(C)] -pub(super) struct WeakRefList { +pub(crate) struct WeakRefList { /// Head of the intrusive doubly-linked list of weakrefs. head: PyAtomic<*mut Py>, /// Cached generic weakref (no callback, exact weakref type). @@ -669,6 +621,13 @@ pub(super) struct WeakRefList { generic: PyAtomic<*mut Py>, } +// All list access is protected by the referent's stripe lock. An owner heap's +// directory contains only native list heads, never strong Python references. +#[cfg(feature = "threading")] +unsafe impl Send for WeakRefList {} +#[cfg(feature = "threading")] +unsafe impl Sync for WeakRefList {} + impl fmt::Debug for WeakRefList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("WeakRefList").finish_non_exhaustive() @@ -715,6 +674,18 @@ unsafe fn try_reuse_weakref(ptr: *mut Py) -> Option> { } impl WeakRefList { + fn owner_list(&self, obj: &PyObject) -> Option> { + if obj + .gc_handle() + .heap() + .is_some_and(|heap| heap.owner.is_some()) + { + return None; + } + let heap = crate::gc_state::current_owner(); + heap.owner.map(|_| heap.weakrefs(obj)) + } + pub(super) fn new() -> Self { Self { head: Radium::new(ptr::null_mut()), @@ -732,6 +703,30 @@ impl WeakRefList { callback: Option, dict: Option, ) -> PyRef { + let list = self.owner_list(obj); + list.as_deref().unwrap_or(self).add_local( + obj, + cls, + cls_is_weakref, + cls_is_weakproxy, + callback, + dict, + list.clone(), + ) + } + + #[allow(clippy::too_many_arguments)] + fn add_local( + &self, + obj: &PyObject, + cls: PyTypeRef, + cls_is_weakref: bool, + cls_is_weakproxy: bool, + callback: Option, + dict: Option, + shared_list: Option>, + ) -> PyRef { + let owner = crate::gc_state::current_interpreter_id(); let is_generic = cls_is_weakref && callback.is_none(); let is_generic_proxy = cls_is_weakproxy && callback.is_none(); @@ -739,9 +734,9 @@ impl WeakRefList { { let _lock = weakref_lock::lock(obj as *const PyObject as usize); let existing = if is_generic { - unsafe { try_reuse_weakref(self.generic.load(Ordering::Relaxed)) } + unsafe { try_reuse_weakref(self.find_basic_ptr(owner, BasicWeakKind::Reference)) } } else if is_generic_proxy { - unsafe { try_reuse_weakref(self.find_generic_proxy_ptr()) } + unsafe { try_reuse_weakref(self.find_basic_ptr(owner, BasicWeakKind::Proxy)) } } else { None }; @@ -754,8 +749,18 @@ impl WeakRefList { // maybe_collect → GC → WeakRefList::clear on another object that // hashes to the same stripe, which would deadlock on the spinlock. let weak_payload = PyWeak { + owner, + shared_list, + basic: if is_generic { + BasicWeakKind::Reference + } else if is_generic_proxy { + BasicWeakKind::Proxy + } else { + BasicWeakKind::Other + }, pointers: Pointers::new(), - wr_object: Radium::new(obj as *const PyObject as *mut PyObject), + // Stay unlinked if allocation or layout validation unwinds. + wr_object: Radium::new(ptr::null_mut()), callback: UnsafeCell::new(callback), hash: Radium::new(crate::common::hash::SENTINEL), }; @@ -768,29 +773,28 @@ impl WeakRefList { // while we were allocating outside the lock. If so, reuse it and // drop ours. let existing = if is_generic { - unsafe { try_reuse_weakref(self.generic.load(Ordering::Relaxed)) } + unsafe { try_reuse_weakref(self.find_basic_ptr(owner, BasicWeakKind::Reference)) } } else if is_generic_proxy { - unsafe { try_reuse_weakref(self.find_generic_proxy_ptr()) } + unsafe { try_reuse_weakref(self.find_basic_ptr(owner, BasicWeakKind::Proxy)) } } else { None }; if let Some(existing) = existing { - // Nullify wr_object so drop_inner won't unlink an - // un-inserted node (which would corrupt the list head). - weak.wr_object.store(ptr::null_mut(), Ordering::Relaxed); return existing; } // Insert into linked list under stripe lock // (insert_weakref: generic ref at head, generic proxy right after it) let node_ptr = NonNull::from(&*weak); + weak.wr_object + .store(core::ptr::from_ref(obj).cast_mut(), Ordering::Relaxed); let after = if is_generic { None } else if is_generic_proxy { - NonNull::new(self.generic.load(Ordering::Relaxed)) + NonNull::new(self.find_basic_ptr(owner, BasicWeakKind::Reference)) } else { - NonNull::new(self.find_generic_proxy_ptr()) - .or_else(|| NonNull::new(self.generic.load(Ordering::Relaxed))) + NonNull::new(self.find_basic_ptr(owner, BasicWeakKind::Proxy)) + .or_else(|| NonNull::new(self.find_basic_ptr(owner, BasicWeakKind::Reference))) }; match after { Some(after) => unsafe { self.insert_after(after, node_ptr) }, @@ -831,33 +835,24 @@ impl WeakRefList { } } - // get_basic_refs - fn find_generic_proxy_ptr(&self) -> *mut Py { - let generic_ptr = self.generic.load(Ordering::Relaxed); - let candidate_ptr = if let Some(generic_node) = NonNull::new(generic_ptr) { - unsafe { WeakLink::pointers(generic_node).as_ref().get_next() } - .map_or(ptr::null_mut(), |n| n.as_ptr()) - } else { - self.head.load(Ordering::Relaxed) - }; - match NonNull::new(candidate_ptr) { - Some(candidate) => { - let node = unsafe { candidate.as_ref() }; - let has_callback = unsafe { (&*node.payload.callback.get()).is_some() }; - let node_cls = node.class(); - // PyWeakref_CheckProxy: the basic-proxy slot is reserved for - // the canonical proxy type; subclasses and callback-less ref - // subclasses must not be mistaken for it. - let is_proxy = node_cls.is(crate::builtins::PyWeakProxy::static_type()) - || node_cls.is(crate::builtins::PyWeakCallableProxy::static_type()); - if has_callback || !is_proxy { - ptr::null_mut() - } else { - candidate_ptr - } + // get_basic_refs. The list has already been selected by owner, without + // dereferencing a Python node from another interpreter. + fn find_basic_ptr(&self, owner: Option, kind: BasicWeakKind) -> *mut Py { + if kind == BasicWeakKind::Reference { + let cached = self.generic.load(Ordering::Relaxed); + if !cached.is_null() && unsafe { &*cached }.owner == owner { + return cached; } - None => ptr::null_mut(), } + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(ptr) = current { + let node = unsafe { ptr.as_ref() }; + if node.owner == owner && node.basic == kind { + return ptr.as_ptr(); + } + current = unsafe { WeakLink::pointers(ptr).as_ref().get_next() }; + } + ptr::null_mut() } /// Clear all weakrefs and call their callbacks. @@ -878,10 +873,13 @@ impl WeakRefList { let wr = unsafe { node.as_ref() }; - // Mark weakref as dead - wr.payload - .wr_object - .store(ptr::null_mut(), Ordering::Relaxed); + // A callback-free weakref needs no pin. Never release an upgraded + // weakref under the stripe: its last reference may run __del__. + if unsafe { &*wr.payload.callback.get() }.is_some() && wr.ref_count.safe_inc() { + let wr_ref = unsafe { PyRef::from_raw(wr as *const Py) }; + let cb = unsafe { wr.payload.callback.get().replace(None) }.unwrap(); + callbacks.push((wr_ref, cb)); + } // Unlink from list unsafe { @@ -890,14 +888,11 @@ impl WeakRefList { ptrs.as_mut().set_next(None); } - // Collect callback only if we can still acquire a strong ref. - if wr.ref_count.safe_inc() { - let wr_ref = unsafe { PyRef::from_raw(wr as *const Py) }; - let cb = unsafe { wr.payload.callback.get().replace(None) }; - if let Some(cb) = cb { - callbacks.push((wr_ref, cb)); - } - } + // Publish death last: a concurrent deallocator that observes NULL + // may skip the stripe and immediately destroy this payload. + wr.payload + .wr_object + .store(ptr::null_mut(), Ordering::Release); current = next; } @@ -929,10 +924,16 @@ impl WeakRefList { let wr = unsafe { node.as_ref() }; - // Mark weakref as dead - wr.payload - .wr_object - .store(ptr::null_mut(), Ordering::Relaxed); + // A weakref in the unreachable graph must not invoke its callback. + // Keep that edge for revalidation and eventual tp_clear instead. + if !wr.as_object().is_gc_collecting() + && unsafe { &*wr.payload.callback.get() }.is_some() + && wr.ref_count.safe_inc() + { + let wr_ref = unsafe { PyRef::from_raw(wr as *const Py) }; + let cb = unsafe { wr.payload.callback.get().replace(None) }.unwrap(); + callbacks.push((wr_ref, cb)); + } // Unlink from list unsafe { @@ -941,14 +942,11 @@ impl WeakRefList { ptrs.as_mut().set_next(None); } - // Collect callback without invoking only if we can keep weakref alive. - if wr.ref_count.safe_inc() { - let wr_ref = unsafe { PyRef::from_raw(wr as *const Py) }; - let cb = unsafe { wr.payload.callback.get().replace(None) }; - if let Some(cb) = cb { - callbacks.push((wr_ref, cb)); - } - } + // As in clear(), no access to an unpinned node may follow this + // release store: drop_inner() acquires it before skipping the lock. + wr.payload + .wr_object + .store(ptr::null_mut(), Ordering::Release); current = next; } @@ -958,11 +956,15 @@ impl WeakRefList { } fn count(&self, obj: &PyObject) -> usize { + let owner = crate::gc_state::current_interpreter_id(); + let list = self.owner_list(obj); + let list = list.as_deref().unwrap_or(self); let _lock = weakref_lock::lock(obj as *const PyObject as usize); let mut count = 0usize; - let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + let mut current = NonNull::new(list.head.load(Ordering::Relaxed)); while let Some(node) = current { - if unsafe { node.as_ref() }.ref_count.get() > 0 { + let weak = unsafe { node.as_ref() }; + if weak.owner == owner && weak.ref_count.get() > 0 { count += 1; } current = unsafe { WeakLink::pointers(node).as_ref().get_next() }; @@ -971,12 +973,15 @@ impl WeakRefList { } fn get_weak_references(&self, obj: &PyObject) -> Vec> { + let owner = crate::gc_state::current_interpreter_id(); + let list = self.owner_list(obj); + let list = list.as_deref().unwrap_or(self); let _lock = weakref_lock::lock(obj as *const PyObject as usize); let mut v = Vec::new(); - let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + let mut current = NonNull::new(list.head.load(Ordering::Relaxed)); while let Some(node) = current { let wr = unsafe { node.as_ref() }; - if wr.ref_count.safe_inc() { + if wr.owner == owner && wr.ref_count.safe_inc() { v.push(unsafe { PyRef::from_raw(wr as *const Py) }); } current = unsafe { WeakLink::pointers(node).as_ref().get_next() }; @@ -1014,10 +1019,22 @@ unsafe impl Link for WeakLink { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum BasicWeakKind { + Reference, + Proxy, + Other, +} + // PyWeakReference: each weakref holds a direct pointer to its referent. -#[pyclass(name = "ReferenceType", module = "weakref")] +#[pyclass(name = "ReferenceType", module = "weakref", traverse = "manual")] #[derive(Debug)] pub struct PyWeak { + owner: Option, + // Only shared immortal referents need an out-of-line list head. Its nodes + // all belong to this owner, including during owner shutdown. + shared_list: Option>, + basic: BasicWeakKind, pointers: Pointers>, /// Direct pointer to the referent object, null when dead. /// Equivalent to wr_object in PyWeakReference. @@ -1027,6 +1044,19 @@ pub struct PyWeak { pub(crate) hash: PyAtomic, } +unsafe impl Traverse for PyWeak { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + // Collection stops this owner. A shared referent is immortal and its + // other owners cannot touch this list or consume this callback. + unsafe { &*self.callback.get() }.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec) { + self.drop_inner(); + out.extend(self.callback.get_mut().take()); + } +} + cfg_select! { feature = "threading" => { unsafe impl Send for PyWeak {} @@ -1036,6 +1066,28 @@ cfg_select! { } impl PyWeak { + pub(crate) fn is_visible(&self) -> bool { + self.owner.is_none() || self.owner == crate::gc_state::current_interpreter_id() + } + + /// A shared native base must never upgrade a private foreign subclass. + pub(crate) fn upgrade_visible(&self) -> Option { + if !self.is_visible() { + return None; + } + let object = self.upgrade()?; + // Static classes join their base's bootstrap subclass list before + // initialization publishes them as immortal definitions. Another VM + // must not observe that unfinished payload through __subclasses__. + if self.owner.is_none() + && crate::gc_state::current_interpreter_id().is_some() + && !object.is_immortal() + { + return None; + } + Some(object) + } + /// _PyWeakref_GET_REF: attempt to upgrade the weakref to a strong reference. pub(crate) fn upgrade(&self) -> Option { let obj_ptr = self.wr_object.load(Ordering::Acquire); @@ -1103,7 +1155,10 @@ impl PyWeak { let obj = unsafe { &*obj_ptr }; // Safety: if a weakref exists pointing to this object, weakref prefix must be present - let wrl = obj.0.weakref_list_ref().unwrap(); + let wrl = self + .shared_list + .as_deref() + .unwrap_or_else(|| obj.0.weakref_list_ref().unwrap()); // Compute our Py node pointer from payload address let node_ptr = unsafe { NonNull::new_unchecked(Py::from_payload_ptr(self).cast_mut()) }; @@ -1124,9 +1179,9 @@ impl PyWeak { impl Drop for PyWeak { #[inline(always)] fn drop(&mut self) { - // we do NOT have actual exclusive access! - let me: &Self = self; - me.drop_inner(); + // Deallocation detached the referent-list ingress before taking this + // mutable borrow. An unpublished payload starts out detached too. + debug_assert!(self.is_dead()); } } @@ -1251,6 +1306,7 @@ impl Py { unsafe { core::ptr::drop_in_place(&raw mut (*ptr).payload); core::ptr::drop_in_place(&raw mut (*ptr).typ); + core::ptr::drop_in_place(&raw mut (*ptr).gc_handle); } } @@ -1267,6 +1323,8 @@ impl Py { // Objects published to lock-free caches keep their memory mapped // until a QSBR grace period passes; destructors still run now. let published = (*ptr).ref_count.is_published(); + let domain = published + .then(|| crate::object::qsbr::RetireDomain::for_object(&*ptr.cast::())); if has_ext || has_weakref { // Reconstruct the same layout used in new() @@ -1315,14 +1373,14 @@ impl Py { // WeakRefList has no Drop (just raw pointers), no drop_in_place needed if published { - crate::object::qsbr::free_delayed(alloc_ptr, combined); + domain.unwrap().free(alloc_ptr, combined); } else { alloc::alloc::dealloc(alloc_ptr, combined); } } else if published { let layout = core::alloc::Layout::new::(); Self::drop_fields(ptr); - crate::object::qsbr::free_delayed(ptr as *mut u8, layout); + domain.unwrap().free(ptr as *mut u8, layout); } else { Self::drop_fields(ptr); // The fields are gone; the box is only here to free the memory @@ -1433,9 +1491,8 @@ impl Py { vtable: PyObjVTable::of::(), gc_bits: Radium::new(0), gc_generation: Radium::new(GC_UNTRACKED), - gc_owner: Radium::new(GC_NO_OWNER), gc_refs: Radium::new(0), - gc_pointers: Pointers::new(), + gc_handle: crate::gc_state::GcHandle::new(), typ: PyAtomicRef::from(typ), payload, }); @@ -1447,9 +1504,8 @@ impl Py { vtable: PyObjVTable::of::(), gc_bits: Radium::new(0), gc_generation: Radium::new(GC_UNTRACKED), - gc_owner: Radium::new(GC_NO_OWNER), gc_refs: Radium::new(0), - gc_pointers: Pointers::new(), + gc_handle: crate::gc_state::GcHandle::new(), typ: PyAtomicRef::from(typ), payload, })) @@ -1561,6 +1617,12 @@ impl ToOwned for PyObject { } impl PyObject { + pub(crate) fn detach_weakref(&self) { + if let Some(weak) = self.downcast_ref::() { + weak.drop_inner(); + } + } + /// Atomically try to create a strong reference. /// Returns `None` if the strong count is already 0 (object being destroyed). /// Uses CAS to prevent the TOCTOU race between checking strong_count and @@ -1668,7 +1730,7 @@ impl PyObjectRef { /// another downcast can be attempted without unnecessary cloning. #[inline] pub fn downcast_exact(self, vm: &VirtualMachine) -> Result, Self> { - if self.class().is(T::class(&vm.ctx)) { + if self.class().is(&T::class(&vm.ctx)) { // TODO: is this always true? assert!( self.downcastable::(), @@ -1693,19 +1755,20 @@ impl PyObject { /// Returns the first weakref in the weakref list, if any. pub(crate) fn get_weakrefs(&self) -> Option { + let owner = crate::gc_state::current_interpreter_id(); let wrl = self.weak_ref_list()?; + let list = wrl.owner_list(self); + let wrl = list.as_deref().unwrap_or(wrl); let _lock = weakref_lock::lock(self as *const Self as usize); - let head_ptr = wrl.head.load(Ordering::Relaxed); - if head_ptr.is_null() { - None - } else { - let head = unsafe { &*head_ptr }; - if head.ref_count.safe_inc() { - Some(unsafe { PyRef::from_raw(head_ptr) }.into()) - } else { - None + let mut current = NonNull::new(wrl.head.load(Ordering::Relaxed)); + while let Some(ptr) = current { + let weak = unsafe { ptr.as_ref() }; + if weak.owner == owner && weak.ref_count.safe_inc() { + return Some(unsafe { PyRef::from_raw(ptr.as_ptr()) }.into()); } + current = unsafe { WeakLink::pointers(ptr).as_ref().get_next() }; } + None } pub(crate) fn downgrade_with_weakref_typ_opt( @@ -1782,6 +1845,12 @@ impl PyObject { self.0.vtable.typeid == T::PAYLOAD_TYPE_ID } + #[doc(hidden)] + #[must_use] + pub fn supports_native_layout(&self, layout: core::any::TypeId) -> bool { + (self.0.vtable.supports_native_layout)(layout) + } + /// Force to return payload as T. /// /// # Safety @@ -1819,7 +1888,7 @@ impl PyObject { #[deprecated(note = "use downcast_ref_if_exact instead")] #[inline(always)] pub fn payload_if_exact(&self, vm: &VirtualMachine) -> Option<&T> { - if self.class().is(T::class(&vm.ctx)) { + if self.class().is(&T::class(&vm.ctx)) { #[allow(deprecated)] self.payload() } else { @@ -1892,7 +1961,7 @@ impl PyObject { #[deprecated(note = "use downcast_ref instead")] #[inline(always)] pub fn payload_if_subclass(&self, vm: &VirtualMachine) -> Option<&T> { - if self.class().fast_issubclass(T::class(&vm.ctx)) { + if self.class().fast_issubclass(&T::class(&vm.ctx)) { #[allow(deprecated)] self.payload() } else { @@ -1935,7 +2004,7 @@ impl PyObject { #[inline(always)] pub fn downcast_ref_if_exact(&self, vm: &VirtualMachine) -> Option<&Py> { self.class() - .is(T::class(&vm.ctx)) + .is(&T::class(&vm.ctx)) .then(|| unsafe { self.downcast_unchecked_ref::() }) } @@ -1990,80 +2059,46 @@ impl PyObject { } /// Set the GC generation index for this object. - /// Must only be called while holding the generation list's write lock. + /// Must only be called while holding the owning heap lock. #[inline] pub(crate) fn set_gc_generation(&self, generation: u8) { self.0.gc_generation.store(generation, Ordering::Relaxed); } - /// The interpreter whose collections consider this object. + /// The stable heap lifetime and membership slot for this object. #[inline] - pub(crate) fn gc_owner(&self) -> GcOwner { - self.0.gc_owner.load(Ordering::Relaxed) + pub(crate) fn gc_handle(&self) -> &crate::gc_state::GcHandle { + &self.0.gc_handle } - /// Set the owning interpreter. Written by `track_object` before the object - /// enters a generation list, and reset to `GC_NO_OWNER` when the owning - /// interpreter goes away. - #[inline] - pub(crate) fn set_gc_owner(&self, owner: GcOwner) { - self.0.gc_owner.store(owner, Ordering::Relaxed); + pub(crate) fn gc_is_acyclic(&self) -> bool { + GcBits::from_bits_retain(self.0.gc_bits.load(Ordering::Relaxed)).contains(GcBits::ACYCLIC) + } + + pub(crate) fn gc_is_atomic(&self) -> bool { + self.0.vtable.trace.is_none() + && self.0.ext_ref().is_none() + && self.class().heaptype_ext.is_none() } - /// Enter the running collection's candidate set, with `strong_count` as the - /// count to subtract internal references from. A count too large to hold is - /// taken as reachable outright, rather than clipped to a number the - /// subtraction could still walk down to zero. #[inline] - pub(crate) fn start_gc_refs(&self, strong_count: usize) { - let refs = if strong_count >= GC_REACHABLE as usize { - GC_REACHABLE - } else { - strong_count as u32 - }; - self.0.gc_refs.store(refs, Ordering::Relaxed); + pub(crate) fn start_gc_index(&self, index: u32) { + self.0.gc_refs.store(index, Ordering::Relaxed); self.set_gc_bit(GcBits::COLLECTING); } - /// The count the running collection is working with. #[inline] pub(crate) fn gc_refs(&self) -> u32 { self.0.gc_refs.load(Ordering::Relaxed) } - /// Whether this object is in the running collection's candidate set. #[inline] pub(crate) fn is_gc_collecting(&self) -> bool { GcBits::from_bits_retain(self.0.gc_bits.load(Ordering::Relaxed)) .contains(GcBits::COLLECTING) } - /// Take off one reference held from inside the candidate set. A count that - /// did not fit stands for more references than every subtraction together - /// could take off, so it stays where [`Self::start_gc_refs`] put it. - #[inline] - pub(crate) fn subtract_gc_ref(&self) { - let refs = self.0.gc_refs.load(Ordering::Relaxed); - if refs == GC_REACHABLE { - return; - } - self.0 - .gc_refs - .store(refs.saturating_sub(1), Ordering::Relaxed); - } - - /// Mark the object reachable, answering whether this call was the one that - /// did it. - #[inline] - pub(crate) fn mark_gc_reachable(&self) -> bool { - if self.0.gc_refs.load(Ordering::Relaxed) == GC_REACHABLE { - return false; - } - self.0.gc_refs.store(GC_REACHABLE, Ordering::Relaxed); - true - } - - /// Leave the candidate set, whatever the collection concluded. + /// Clear temporary snapshot membership before any mutator resumes. #[inline] pub(crate) fn end_gc_refs(&self) { self.0 @@ -2320,8 +2355,15 @@ impl PyObject { result } - /// Append this object's referents to `out`, for a caller that holds many - /// objects' referents in one buffer rather than one buffer each. + /// Visit this object's owned references without temporary allocations. + /// + /// # Safety + /// Same as [`Self::gc_get_referent_ptrs`]. + pub(crate) unsafe fn gc_visit_referents(&self, visit: &mut TraverseFn<'_>) { + self.0.traverse(visit); + } + + /// Append referents while their contents are stable. /// /// # Safety /// Same as [`Self::gc_get_referent_ptrs`]. @@ -2384,9 +2426,9 @@ impl PyObject { /// - The caller must guarantee exclusive access (no other references exist) /// - This is only safe during GC when the object is unreachable pub unsafe fn gc_clear(&self) -> Vec { - // SAFETY: During GC collection, this object is unreachable (gc_refs == 0), - // meaning no other code has a reference to it. The only references are - // internal cycle references which we're about to break. + // SAFETY: The collector verified that this object is unreachable and + // detached it from weakrefs and heap enumeration before clearing. Only + // internal cycle references and collector pins remain. unsafe { Self::gc_clear_raw(self as *const _ as *mut Self) } } @@ -2812,8 +2854,46 @@ impl PyRef { impl PyRef { #[inline(always)] pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option) -> Self { + Self::new_ref_inner(payload, typ, dict, false) + } + + /// Partially initialized containers (marshal placeholders) must remain + /// tracked even when their temporary contents look permanently acyclic. + pub(crate) fn new_ref_tracked(payload: T, typ: crate::builtins::PyTypeRef) -> Self { + Self::new_ref_inner(payload, typ, None, true) + } + + #[inline(always)] + fn new_ref_inner( + payload: T, + typ: crate::builtins::PyTypeRef, + dict: Option, + force_tracking: bool, + ) -> Self { let has_dict = dict.is_some(); let is_heaptype = typ.heaptype_ext.is_some(); + let owner_neutral = T::OWNER_NEUTRAL + && !has_dict + && !is_heaptype + && typ + .slots + .flags + .load() + .is_disjoint(&crate::types::PyTypeFlags::from_slice(&[ + crate::types::PyTypeFlags::HAS_DICT, + crate::types::PyTypeFlags::HAS_WEAKREF, + ])) + && typ.slots.member_count == 0 + && typ.slots.del.load().is_none(); + let acyclic = !force_tracking + && !has_dict + && !is_heaptype + && !typ + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + && typ.slots.member_count == 0 + && payload.gc_is_acyclic(&typ); // Try to reuse from freelist (no dict, no heaptype) let cached = if !has_dict && !is_heaptype { @@ -2845,13 +2925,23 @@ impl PyRef { unsafe { NonNull::new_unchecked(inner.cast::>()) } }; + if !owner_neutral { + unsafe { ptr.as_ref() } + .gc_handle + .bind_new(crate::gc_state::current_owner()); + } + // Track object if: // - HAS_TRAVERSE is true (Rust payload implements Traverse), OR // - has instance dict (user-defined class instances), OR // - heap type (all heap type instances are GC-tracked, like Py_TPFLAGS_HAVE_GC) // unless the payload opts out via NEW_REF_UNTRACKED (e.g. call frames, // which are tracked lazily only on escape). - if (::HAS_TRAVERSE || has_dict || is_heaptype) + if acyclic { + unsafe { ptr.as_ref() } + .gc_bits + .store(GcBits::ACYCLIC.bits(), Ordering::Relaxed); + } else if (::HAS_TRAVERSE || has_dict || is_heaptype) && !T::NEW_REF_UNTRACKED { // Tracks under the interpreter running now and collects if this @@ -2875,11 +2965,11 @@ where #[inline] #[must_use] pub fn into_base(self) -> PyRef { - let obj: PyObjectRef = self.into(); - match obj.downcast() { - Ok(base_ref) => base_ref, - Err(_) => unsafe { core::hint::unreachable_unchecked() }, - } + // PySubclass establishes the physical prefix; Python may have changed + // the object's class or MRO since this typed reference was constructed. + let ptr = self.ptr.cast(); + core::mem::forget(self); + PyRef { ptr } } #[inline] #[must_use] @@ -2887,12 +2977,8 @@ where where T: StaticType, { - debug_assert!(T::static_type().is_subtype(U::static_type())); let obj: PyObjectRef = self.into(); - match obj.downcast::() { - Ok(upcast_ref) => upcast_ref, - Err(_) => unsafe { core::hint::unreachable_unchecked() }, - } + obj.downcast::().expect("invalid native upcast") } } @@ -2900,7 +2986,6 @@ impl Py { /// Converts `&Py` to `&Py`. #[inline] pub fn to_base(&self) -> &Py { - debug_assert!(self.as_object().downcast_ref::().is_some()); // SAFETY: T is #[repr(transparent)] over T::Base, // so Py and Py have the same layout. unsafe { &*(self as *const Self as *const Py) } @@ -2912,9 +2997,9 @@ impl Py { where T: StaticType, { - debug_assert!(T::static_type().is_subtype(U::static_type())); - // SAFETY: T is a subtype of U, so Py can be viewed as Py. - unsafe { &*(self as *const Self as *const Py) } + self.as_object() + .downcast_ref::() + .expect("invalid native upcast") } } @@ -3085,9 +3170,8 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy { ptr::addr_of_mut!((*ptr).vtable).write(PyObjVTable::of::()); ptr::addr_of_mut!((*ptr).gc_bits).write(Radium::new(0)); ptr::addr_of_mut!((*ptr).gc_generation).write(Radium::new(GC_UNTRACKED)); - ptr::addr_of_mut!((*ptr).gc_owner).write(Radium::new(GC_NO_OWNER)); ptr::addr_of_mut!((*ptr).gc_refs).write(Radium::new(0)); - ptr::addr_of_mut!((*ptr).gc_pointers).write(Pointers::new()); + ptr::addr_of_mut!((*ptr).gc_handle).write(crate::gc_state::GcHandle::new()); ptr::addr_of_mut!((*ptr).typ).write(PyAtomicRef::from_ref_without_retag(typ)); ptr::addr_of_mut!((*ptr).payload).write(payload); } @@ -3248,6 +3332,63 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy { mod tests { use super::*; + #[test] + fn mutable_native_types_cannot_forge_payload_layouts() { + use crate::class::PyClassImpl; + + #[pyclass(interpreter_local, module = false, name = "NativeBase")] + #[derive(Debug, PyPayload)] + struct NativeBase; + #[pyclass(flags(BASETYPE))] + impl NativeBase {} + + #[pyclass(interpreter_local, module = false, name = "NativeWord", base = NativeBase)] + #[derive(Debug)] + struct NativeWord { + base: NativeBase, + word: usize, + } + #[pyclass(flags(BASETYPE))] + impl NativeWord {} + + #[pyclass(interpreter_local, module = false, name = "NativeReference", base = NativeBase, traverse)] + #[derive(Debug)] + struct NativeReference { + #[pytraverse(skip)] + base: NativeBase, + object: PyObjectRef, + } + #[pyclass(flags(BASETYPE))] + impl NativeReference {} + + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let left = NativeWord::make_class(&vm.ctx); + let right = NativeReference::make_class(&vm.ctx); + let reference = NativeReference { + base: NativeBase, + object: vm.ctx.none(), + } + .into_ref(&vm.ctx); + assert!(vm.is_none(&reference.object)); + assert_eq!(left.slots.basicsize, right.slots.basicsize); + let object = NativeWord { + base: NativeBase, + word: 42, + } + .into_pyobject(vm); + let error = object.set_attr("__class__", right.clone(), vm).unwrap_err(); + assert!(error.class().is(vm.ctx.exceptions.type_error)); + assert!(object.class().is(&left)); + assert!(object.downcast_ref::().is_none()); + + // A metaclass's MRO is a Python relationship, not a proof that a + // Rust field with this type exists in the allocation. + left.mro.write().insert(1, right); + assert!(object.downcast_ref::().is_none()); + assert_eq!(object.downcast_ref::().unwrap().word, 42); + }); + } + #[test] fn native_type_basicsize_includes_payload_padding() { use crate::class::PyClassDef; @@ -3302,9 +3443,9 @@ mod tests { } assert_eq!(core::mem::offset_of!(LayoutDerived, base), 0); - crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { - let _ = LayoutBase::make_static_type(); - let _ = LayoutDerived::make_static_type(); + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let _ = unsafe { LayoutBase::make_static_type() }; + let _ = unsafe { LayoutDerived::make_static_type() }; let value: PyObjectRef = vm.ctx.new_int(42).into(); let obj = vm.new_pyobj(LayoutDerived { base: LayoutBase { @@ -3327,7 +3468,7 @@ mod tests { fn clear_reuses_storage_and_preserves_existing_edges() { use crate::builtins::PyList; - crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { for tuple in [false, true] { for existing in [false, true] { let elements = vec![vm.ctx.none(), vm.ctx.none()]; @@ -3391,6 +3532,58 @@ mod tests { drop(obj); } + #[test] + fn shared_referents_have_disjoint_owner_lists() { + let first = crate::Interpreter::without_stdlib(Default::default()); + let second = first.create_subinterpreter(); + first.enter_raw(|a| { + let target = a.ctx.types.int_type.as_object(); + let make = |vm: &VirtualMachine| { + let reference = target + .downgrade_with_weakref_typ_opt(None, vm.ctx.types.weakref_type.to_owned()) + .unwrap(); + let proxy = target + .downgrade_with_typ(None, vm.ctx.types.weakproxy_type.to_owned(), vm) + .unwrap(); + (reference, proxy) + }; + let (a_ref, a_proxy) = make(a); + second.enter_raw(|b| { + let (b_ref, b_proxy) = make(b); + let a_list = a_ref.shared_list.as_ref().unwrap(); + let b_list = b_ref.shared_list.as_ref().unwrap(); + assert!(!PyRc::ptr_eq(a_list, b_list)); + { + let _lock = weakref_lock::lock(core::ptr::from_ref(target).addr()); + for (owner, list) in [ + (a.state.interpreter_id, a_list), + (b.state.interpreter_id, b_list), + ] { + let mut node = NonNull::new(list.head.load(Ordering::Relaxed)); + while let Some(current) = node { + assert_eq!(unsafe { current.as_ref() }.owner, Some(owner)); + node = unsafe { WeakLink::pointers(current).as_ref().get_next() }; + } + } + } + let (reused_ref, reused_proxy) = make(b); + assert!(b_ref.is(&reused_ref)); + assert!(b_proxy.is(&reused_proxy)); + assert_eq!(target.weak_count(), Some(2)); + assert!( + target + .get_weak_references() + .unwrap() + .iter() + .all(|r| r.owner == Some(b.state.interpreter_id)) + ); + }); + assert_eq!(target.weak_count(), Some(2)); + assert!(a_ref.upgrade().unwrap().is(target)); + assert!(a_proxy.upgrade().unwrap().is(target)); + }); + } + /// A weakref node stays linked into its target's list until its own /// `Drop` unlinks it, and `WeakRefList::add` reads the class off every /// node it walks looking for a proxy to reuse. A node that lost its class @@ -3401,7 +3594,7 @@ mod tests { const THREADS: usize = 8; const ROUNDS: usize = 20_000; - crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let target: PyObjectRef = vm .ctx .new_class( @@ -3416,7 +3609,7 @@ mod tests { let thread_vm = vm.new_thread(); let target = target.clone(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { let proxy_type = vm.ctx.types.weakproxy_type.to_owned(); for _ in 0..ROUNDS { let proxy = target diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index 97309722a19..3acea3842fa 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -116,7 +116,7 @@ impl PyRef { ctx: &Context, f: impl FnOnce(Self) -> PyRefExact, ) -> PyRefExact { - if self.class().is(T::class(ctx)) { + if self.class().is(&T::class(ctx)) { unsafe { PyRefExact::new_unchecked(self) } } else { f(self) @@ -156,12 +156,12 @@ impl TryFromObject for PyRefExact { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let target_cls = T::class(&vm.ctx); let cls = obj.class(); - if cls.is(target_cls) { + if cls.is(&target_cls) { let obj = obj .downcast() - .map_err(|obj| vm.new_downcast_runtime_error(target_cls, &obj))?; + .map_err(|obj| vm.new_downcast_runtime_error(&target_cls, &obj))?; Ok(Self { inner: obj }) - } else if cls.fast_issubclass(target_cls) { + } else if cls.fast_issubclass(&target_cls) { Err(vm.new_type_error(format!( "Expected an exact instance of '{}', not a subclass '{}'", target_cls.name(), @@ -958,7 +958,7 @@ mod tests { #[test] fn object_cell_snapshots_survive_replacement_and_clear() { - crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let cell = PyObjectCell::from(Some(vm.ctx.new_bytes(vec![1, 2, 3]).into())); let first = cell.load_owned().unwrap(); let previous = cell.store(Some(vm.ctx.new_bytes(vec![4, 5, 6]).into())); @@ -992,7 +992,7 @@ mod tests { #[test] fn object_cell_traverses_current_reference_once_without_cloning() { - crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + crate::Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let cell = PyObjectCell::from(None); cell.traverse(&mut |_| panic!("empty cell owns no edge")); let value: PyObjectRef = vm.ctx.new_bytes(vec![1, 2, 3]).into(); diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index 4f028338d4b..c448c75ab45 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -1,4 +1,5 @@ mod core; +pub(crate) use core::WeakRefList; mod ext; mod payload; pub(crate) mod qsbr; @@ -9,5 +10,5 @@ pub use self::core::*; pub use self::ext::*; pub use self::payload::*; pub use core::SIZEOF_PYOBJECT_HEAD; -pub(crate) use core::{GC_NO_OWNER, GC_PERMANENT, GC_REACHABLE, GC_UNTRACKED, GcLink, GcOwner}; +pub(crate) use core::{GC_PERMANENT, GC_UNTRACKED}; pub use traverse::{MaybeTraverse, Traverse, TraverseFn}; diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index a5a4bf35b7f..c47acde8c4b 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -66,6 +66,16 @@ pub(crate) fn cold_downcast_type_error( pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { const PAYLOAD_TYPE_ID: core::any::TypeId = core::any::TypeId::of::(); + /// Exact instances contain no Python references, mutable state or callbacks + /// requiring an interpreter. Subclasses never inherit this exemption. + const OWNER_NEUTRAL: bool = false; + + #[doc(hidden)] + #[must_use] + fn supports_native_layout(layout: core::any::TypeId) -> bool { + layout == core::any::TypeId::of::() + } + /// # Safety /// This function should only be called if `payload_type_id` matches the type of `obj`. #[inline] @@ -79,10 +89,12 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { } let class = Self::class(&vm.ctx); - Err(cold_downcast_type_error(vm, class, obj)) + Err(cold_downcast_type_error(vm, &class, obj)) } - fn class(ctx: &Context) -> &'static Py; + /// Return this payload's class in the entered interpreter. Immutable native + /// types share their Rust definition; mutable native types own a heap class. + fn class(ctx: &Context) -> PyTypeRef; /// Whether `PyRef::new_ref` skips auto-tracking this type in the GC even /// when it would otherwise qualify (has traverse, dict, or heap type). @@ -91,6 +103,14 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { /// which is created untracked and tracked lazily only on escape. const NEW_REF_UNTRACKED: bool = false; + /// Whether a completed exact instance can never participate in a cycle. + /// Mutable/partially initialized objects and objects with hidden mutable + /// references must return false. Implementations must check the exact class; + /// subclasses can carry additional references beyond this payload. + fn gc_is_acyclic(&self, _class: &Py) -> bool { + false + } + /// Whether this type has a freelist. Types with freelists require /// immediate (non-deferred) GC untracking during dealloc to prevent /// race conditions when the object is reused. @@ -164,7 +184,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { Self: core::fmt::Debug, { let cls = Self::class(ctx); - self._into_ref(cls.to_owned(), ctx) + self._into_ref(cls, ctx) } #[inline] @@ -201,7 +221,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { Self: core::fmt::Debug, { let exact_class = Self::class(&vm.ctx); - if cls.fast_issubclass(exact_class) { + if cls.fast_issubclass(&exact_class) { if exact_class.slots.basicsize != cls.slots.basicsize { #[cold] #[inline(never)] @@ -216,7 +236,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { exact_class.name() )) } - return Err(_into_ref_size_error(vm, &cls, exact_class)); + return Err(_into_ref_size_error(vm, &cls, &exact_class)); } let dict = if eager_dict && cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { Some(vm.ctx.new_dict()) @@ -238,7 +258,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { exact_class.name() )) } - Err(_into_ref_with_type_error(vm, &cls, exact_class)) + Err(_into_ref_with_type_error(vm, &cls, &exact_class)) } } } diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs index 576cadaeaed..2d3962c8601 100644 --- a/crates/vm/src/object/qsbr.rs +++ b/crates/vm/src/object/qsbr.rs @@ -6,10 +6,12 @@ //! borrowed pointer has passed a quiescent state. Destructors run at the //! normal drop point; only the final deallocation is deferred. //! -//! Mirrors _Py_qsbr (Python/qsbr.c): a global write sequence advances on +//! Mirrors _Py_qsbr (Python/qsbr.c): a domain's write sequence advances on //! each retirement; each thread records the last sequence it observed at a //! quiescent point (eval-breaker checkpoint, attach/detach). A retired //! allocation is freed once every online thread's sequence passes its goal. +//! Each interpreter has a domain; immutable shared definitions use a separate +//! process-wide domain. A busy interpreter cannot retain another one's memory. use core::alloc::Layout; @@ -35,6 +37,7 @@ mod threading { /// Per-thread QSBR state, owned by the thread's `ThreadSlot`. pub(crate) struct QsbrSlot { + domain: Arc, /// Last write sequence observed at a quiescent point; /// `QSBR_OFFLINE` while the thread is detached. seq: AtomicU64, @@ -42,6 +45,38 @@ mod threading { pub(crate) requested: AtomicBool, } + impl QsbrSlot { + pub(crate) fn online(&self) { + self.domain.online(self); + } + + pub(crate) fn offline(&self) { + self.domain.offline(self); + self.domain.process(); + } + + pub(crate) fn checkpoint(&self) { + self.requested.store(false, Ordering::Relaxed); + self.domain.quiescent_state(self); + self.domain.process(); + } + } + + impl Drop for QsbrSlot { + fn drop(&mut self) { + // A registry scan can release the last upgraded slot while holding + // the threads lock. Do not recursively process that registry here. + self.domain.offline(self); + } + } + + #[cfg(test)] + impl QsbrSlot { + pub(crate) fn is_offline(&self) -> bool { + self.seq.load(Ordering::Acquire) == QSBR_OFFLINE + } + } + struct Retired { ptr: *mut u8, layout: Layout, @@ -52,6 +87,7 @@ mod threading { unsafe impl Send for Retired {} pub(crate) struct Qsbr { + shared: bool, /// Global write sequence (_Py_qsbr wr_seq). wr_seq: AtomicU64, /// Cached minimum observed read sequence (rd_seq). @@ -64,11 +100,19 @@ mod threading { pending: AtomicBool, } - pub(crate) static QSBR: Qsbr = Qsbr::new(); + pub(crate) fn shared() -> &'static Arc { + static SHARED: std::sync::OnceLock> = std::sync::OnceLock::new(); + SHARED.get_or_init(|| { + let mut domain = Qsbr::new(); + domain.shared = true; + Arc::new(domain) + }) + } impl Qsbr { - const fn new() -> Self { + pub(crate) const fn new() -> Self { Self { + shared: false, wr_seq: AtomicU64::new(QSBR_INITIAL), rd_seq: AtomicU64::new(QSBR_INITIAL), threads: Mutex::new(Vec::new()), @@ -77,11 +121,8 @@ mod threading { } } - /// Whether retired allocations are pending. The hot path now reads - /// the mirrored bit in the eval-breaker word instead; this stays - /// only for unit tests that exercise local, non-global instances. + /// Whether this domain has allocations awaiting a grace period. #[inline] - #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn break_pending(&self) -> bool { self.pending.load(Ordering::Relaxed) } @@ -90,7 +131,7 @@ mod threading { /// the global QSBR instance, so unit-test instances never touch /// process-global state. fn update_breaker_bit(&self, on: bool) { - if core::ptr::eq(self, &QSBR) { + if self.shared { if on { crate::signal::set_qsbr_bit(); } else { @@ -99,11 +140,12 @@ mod threading { } } - /// Register the calling thread. The returned slot is stored in the - /// thread's `ThreadSlot`; dropping it unregisters the thread. - pub(crate) fn register(&self) -> Arc { + /// Register a detached thread. It must go online before reading any + /// cache pointer. Dropping the slot unregisters the thread. + pub(crate) fn register(self: &Arc) -> Arc { let slot = Arc::new(QsbrSlot { - seq: AtomicU64::new(self.wr_seq.load(Ordering::Acquire)), + domain: self.clone(), + seq: AtomicU64::new(QSBR_OFFLINE), requested: AtomicBool::new(false), }); self.threads.lock().unwrap().push(Arc::downgrade(&slot)); @@ -132,7 +174,11 @@ mod threading { /// Mark a thread online again (_Py_qsbr_attach). pub(crate) fn online(&self, slot: &QsbrSlot) { - self.quiescent_state(slot); + // Order publication before the first borrowed cache read. A + // release store alone allows the reader to race past a scan that + // still sees OFFLINE; pair with poll_scan's SeqCst fence. + slot.seq + .store(self.wr_seq.load(Ordering::Acquire), Ordering::SeqCst); } /// Whether every online thread has passed `goal` (_Py_qsbr_poll). @@ -146,6 +192,7 @@ mod threading { /// Recompute the minimum sequence over all live online threads, /// pruning dead ones. fn poll_scan(&self) -> u64 { + core::sync::atomic::fence(Ordering::SeqCst); let mut min_seq = self.wr_seq.load(Ordering::Acquire); let mut threads = self.threads.lock().unwrap(); threads.retain(|weak| match weak.upgrade() { @@ -218,35 +265,16 @@ mod threading { } } - /// Free all retired allocations immediately. - /// - /// # Safety - /// Only sound when no other thread can be mid-read: the post-fork - /// child, or teardown after all threads exited. + /// Stabilize queue and reader registry before the syscall, in the same + /// lock order as process(). Detached native retirement also uses these + /// mutexes, so stopping Python execution alone is insufficient. #[cfg(unix)] - pub(crate) unsafe fn drain_all(&self) { - let mut queue = self.queue.lock().unwrap(); - for item in queue.drain(..) { - // SAFETY: guaranteed single-threaded by the caller. - unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; + pub(crate) fn prepare_fork(&self) -> ForkDomain<'_> { + ForkDomain { + domain: self, + queue: self.queue.lock().unwrap(), + threads: self.threads.lock().unwrap(), } - self.pending.store(false, Ordering::Release); - self.update_breaker_bit(false); - } - - /// Reset after fork: drop all registered thread entries (dead - /// parent threads' slots would otherwise stay online forever and - /// stall every future grace period) and free all retired - /// allocations. - /// - /// # Safety - /// Only sound in the single-threaded post-fork child, before the - /// surviving thread re-registers. - #[cfg(unix)] - pub(crate) unsafe fn reset_after_fork(&self) { - self.threads.lock().unwrap().clear(); - // SAFETY: single-threaded child, no concurrent reader exists. - unsafe { self.drain_all() }; } #[cfg(test)] @@ -255,15 +283,66 @@ mod threading { } } + #[cfg(unix)] + pub(crate) struct ForkDomain<'a> { + domain: &'a Qsbr, + queue: std::sync::MutexGuard<'a, Vec>, + threads: std::sync::MutexGuard<'a, Vec>>, + } + + #[cfg(unix)] + impl ForkDomain<'_> { + /// # Safety + /// Only the fork child may run this, with no borrowed cache pointer on + /// its native stack. `surviving` contains only this thread's slots, + /// including detached slots of saved/nested interpreter entries. + pub(crate) unsafe fn repair_child(&mut self, surviving: &[Arc]) { + self.threads.retain(|weak| { + surviving + .iter() + .any(|slot| core::ptr::eq(weak.as_ptr(), Arc::as_ptr(slot))) + }); + for weak in self.threads.iter() { + if let Some(slot) = weak.upgrade() { + if slot.seq.load(Ordering::Relaxed) != QSBR_OFFLINE { + self.domain.quiescent_state(&slot); + } + slot.requested.store(false, Ordering::Relaxed); + } + } + for item in self.queue.drain(..) { + unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; + } + self.domain.pending.store(false, Ordering::Release); + self.domain.update_breaker_bit(false); + } + } + + impl Drop for Qsbr { + fn drop(&mut self) { + // Every registered slot retains the domain. Its last owner can + // therefore release queued allocations without waiting for a VM. + let queue = self + .queue + .get_mut() + .unwrap_or_else(|error| error.into_inner()); + for item in queue.drain(..) { + unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; + } + } + } + #[cfg(test)] mod tests { use super::*; #[test] fn poll_requires_all_online_threads() { - let q = Qsbr::new(); + let q = Arc::new(Qsbr::new()); let a = q.register(); let b = q.register(); + a.online(); + b.online(); let goal = q.advance(); assert!(!q.poll(goal)); q.quiescent_state(&a); @@ -274,9 +353,11 @@ mod threading { #[test] fn offline_thread_does_not_delay_grace() { - let q = Qsbr::new(); + let q = Arc::new(Qsbr::new()); let a = q.register(); let b = q.register(); + a.online(); + b.online(); let goal = q.advance(); q.quiescent_state(&a); q.offline(&b); @@ -285,9 +366,11 @@ mod threading { #[test] fn dead_thread_is_pruned() { - let q = Qsbr::new(); + let q = Arc::new(Qsbr::new()); let a = q.register(); let b = q.register(); + a.online(); + b.online(); drop(b); let goal = q.advance(); q.quiescent_state(&a); @@ -296,8 +379,9 @@ mod threading { #[test] fn process_frees_only_after_grace() { - let q = Qsbr::new(); + let q = Arc::new(Qsbr::new()); let a = q.register(); + a.online(); let layout = Layout::new::(); let ptr = unsafe { alloc::alloc::alloc(layout) }; unsafe { q.free_delayed(ptr, layout) }; @@ -311,23 +395,88 @@ mod threading { assert_eq!(q.pending(), 0); assert!(!q.break_pending()); } + + #[test] + fn domains_do_not_wait_for_each_others_readers() { + let first = Arc::new(Qsbr::new()); + let second = Arc::new(Qsbr::new()); + let a = first.register(); + let b = second.register(); + a.online(); + b.online(); + let layout = Layout::new::(); + for domain in [&first, &second] { + let ptr = unsafe { alloc::alloc::alloc(layout) }; + unsafe { domain.free_delayed(ptr, layout) }; + } + b.checkpoint(); + assert_eq!(second.pending(), 0); + assert_eq!(first.pending(), 1); + a.offline(); + assert_eq!(first.pending(), 0); + } + + #[test] + fn registered_but_unattached_threads_do_not_retain_memory() { + let domain = Arc::new(Qsbr::new()); + let slot = domain.register(); + assert!(slot.is_offline()); + let layout = Layout::new::(); + let ptr = unsafe { alloc::alloc::alloc(layout) }; + unsafe { domain.free_delayed(ptr, layout) }; + domain.process(); + assert_eq!(domain.pending(), 0); + } + + #[test] + fn last_reader_keeps_its_domain_alive() { + let domain = Arc::new(Qsbr::new()); + let reader = domain.register(); + reader.online(); + let weak = Arc::downgrade(&domain); + let layout = Layout::new::(); + let ptr = unsafe { alloc::alloc::alloc(layout) }; + unsafe { domain.free_delayed(ptr, layout) }; + drop(domain); + assert!(weak.upgrade().is_some()); + reader.checkpoint(); + assert_eq!(weak.upgrade().unwrap().pending(), 0); + drop(reader); + assert!(weak.upgrade().is_none()); + } } } -/// Defer (threading) or immediately perform (non-threading) deallocation -/// of a dead published object's memory. -/// -/// # Safety -/// Same contract as [`Qsbr::free_delayed`]. -#[inline] -pub(crate) unsafe fn free_delayed(ptr: *mut u8, layout: Layout) { +/// Retain the allocation's reclamation domain before dropping its object +/// header. The current VM can be unrelated to the allocation's owner. +pub(crate) struct RetireDomain { #[cfg(feature = "threading")] - unsafe { - QSBR.free_delayed(ptr, layout) - }; - #[cfg(not(feature = "threading"))] - // No concurrent readers can exist without threads. - unsafe { - alloc::alloc::dealloc(ptr, layout) - }; + domain: alloc::sync::Arc, +} + +impl RetireDomain { + pub(crate) fn for_object(object: &crate::PyObject) -> Self { + #[cfg(not(feature = "threading"))] + let _ = object; + Self { + #[cfg(feature = "threading")] + domain: object + .gc_handle() + .heap() + .map_or_else(|| shared().clone(), |heap| heap.qsbr.clone()), + } + } + + /// # Safety + /// The allocation is dead; only pending pre-try-incref readers may touch it. + pub(crate) unsafe fn free(self, ptr: *mut u8, layout: Layout) { + #[cfg(feature = "threading")] + unsafe { + self.domain.free_delayed(ptr, layout) + }; + #[cfg(not(feature = "threading"))] + unsafe { + alloc::alloc::dealloc(ptr, layout) + }; + } } diff --git a/crates/vm/src/object/traverse_object.rs b/crates/vm/src/object/traverse_object.rs index a2db83c7e87..bf7774d77a5 100644 --- a/crates/vm/src/object/traverse_object.rs +++ b/crates/vm/src/object/traverse_object.rs @@ -13,6 +13,7 @@ use super::{Traverse, TraverseFn}; pub(in crate::object) struct PyObjVTable { pub(in crate::object) typeid: TypeId, + pub(in crate::object) supports_native_layout: fn(TypeId) -> bool, /// dealloc: handles __del__, weakref clearing, and memory free. pub(in crate::object) dealloc: unsafe fn(*mut PyObject), pub(in crate::object) debug: unsafe fn(&PyObject, &mut fmt::Formatter<'_>) -> fmt::Result, @@ -26,6 +27,7 @@ impl PyObjVTable { pub(super) const fn of() -> &'static Self { &Self { typeid: T::PAYLOAD_TYPE_ID, + supports_native_layout: T::supports_native_layout, dealloc: default_dealloc::, debug: debug_obj::, trace: const { diff --git a/crates/vm/src/ospath_fd.rs b/crates/vm/src/ospath_fd.rs index 7e4f8561ee7..c529cf398c5 100644 --- a/crates/vm/src/ospath_fd.rs +++ b/crates/vm/src/ospath_fd.rs @@ -19,7 +19,7 @@ impl PathConverter { if let Some(int) = obj.try_index_opt(vm) { if obj .class() - .is(crate::builtins::bool_::PyBool::static_type()) + .is(unsafe { crate::builtins::bool_::PyBool::static_type() }) { crate::stdlib::_warnings::warn( vm.ctx.exceptions.runtime_warning, diff --git a/crates/vm/src/protocol/buffer.rs b/crates/vm/src/protocol/buffer.rs index 500ea5992f7..c4eede0ac2c 100644 --- a/crates/vm/src/protocol/buffer.rs +++ b/crates/vm/src/protocol/buffer.rs @@ -5,7 +5,7 @@ use crate::{ Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, common::{ borrow::{BorrowedValue, BorrowedValueMut}, - lock::{MapImmutable, PyMutex, PyMutexGuard}, + lock::{PyDetachingRwLock, PyDetachingRwLockReadGuard, PyDetachingRwLockWriteGuard}, rc::PyRc, }, object::PyObjectPayload, @@ -101,7 +101,10 @@ impl BufferFlags { } } +type ShareStorage = fn(&PyBuffer) -> Option>; + pub struct BufferMethods { + pub shared_storage: Option, pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<'_, [u8]>, pub obj_bytes_mut: fn(&PyBuffer) -> BorrowedValueMut<'_, [u8]>, pub release: fn(&PyBuffer), @@ -162,6 +165,17 @@ impl Clone for PyBuffer { } impl PyBuffer { + pub(crate) fn obj_bytes_unlocked( + &self, + vm: &VirtualMachine, + ) -> PyResult> { + crate::function::UnlockedBuf::new(self.obj_bytes(), vm) + } + + pub(crate) fn shared_storage(&self) -> Option> { + self.methods.shared_storage.and_then(|share| share(self)) + } + #[must_use] pub fn new(obj: PyObjectRef, desc: BufferDescriptor, methods: &'static BufferMethods) -> Self { #[cfg(debug_assertions)] @@ -813,20 +827,41 @@ pub trait BufferResizeGuard { #[pyclass(module = false, name = "vec_buffer")] #[derive(Debug, PyPayload)] pub struct VecBuffer { - data: PyMutex>, + data: PyRc, +} + +#[derive(Debug)] +struct VecStorage(PyDetachingRwLock>); + +// SAFETY: the private vector is only resized by take(), which preserves it +// whenever an independent storage export exists. All access uses this mutex. +unsafe impl super::SharedBufferStorage for VecStorage { + fn read(&self) -> BorrowedValue<'_, [u8]> { + PyDetachingRwLockReadGuard::map(self.0.read(), |x| x.as_slice()).into() + } + + fn write(&self) -> BorrowedValueMut<'_, [u8]> { + PyDetachingRwLockWriteGuard::map(self.0.write(), |x| x.as_mut_slice()).into() + } } #[pyclass(flags(BASETYPE, DISALLOW_INSTANTIATION))] impl VecBuffer { + #[must_use] pub fn take(&self) -> Vec { - core::mem::take(&mut self.data.lock()) + let mut data = self.data.0.write(); + if PyRc::strong_count(&self.data) > 1 { + data.clone() + } else { + core::mem::take(&mut data) + } } } impl From> for VecBuffer { fn from(data: Vec) -> Self { Self { - data: PyMutex::new(data), + data: PyRc::new(VecStorage(PyDetachingRwLock::new(data))), } } } @@ -834,7 +869,7 @@ impl From> for VecBuffer { impl PyRef { #[must_use] pub fn into_pybuffer(self, readonly: bool) -> PyBuffer { - let len = self.data.lock().len(); + let len = self.data.0.read().len(); PyBuffer::new( self.into(), BufferDescriptor::simple(len, readonly), @@ -849,12 +884,15 @@ impl PyRef { } static VEC_BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| Some(buffer.obj_as::().data.clone())), obj_bytes: |buffer| { - PyMutexGuard::map_immutable(buffer.obj_as::().data.lock(), |x| x.as_slice()) - .into() + PyDetachingRwLockReadGuard::map(buffer.obj_as::().data.0.read(), |x| { + x.as_slice() + }) + .into() }, obj_bytes_mut: |buffer| { - PyMutexGuard::map(buffer.obj_as::().data.lock(), |x| { + PyDetachingRwLockWriteGuard::map(buffer.obj_as::().data.0.write(), |x| { x.as_mut_slice() }) .into() diff --git a/crates/vm/src/protocol/mod.rs b/crates/vm/src/protocol/mod.rs index 1d191daee31..11db400f651 100644 --- a/crates/vm/src/protocol/mod.rs +++ b/crates/vm/src/protocol/mod.rs @@ -5,6 +5,7 @@ mod mapping; mod number; mod object; mod sequence; +pub(crate) mod shared_buffer; pub use buffer::{ BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer, VecBuffer, @@ -19,3 +20,4 @@ pub use number::{ numeric_literal_from_str, }; pub use sequence::{PySequence, PySequenceMethods, PySequenceSlots}; +pub use shared_buffer::{SharedBuffer, SharedBufferStorage}; diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index 472f33c26f5..4322e90ae59 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -186,9 +186,9 @@ impl PyObject { vm: &VirtualMachine, ) -> PyResult<()> { vm_trace!("object.__setattr__({:?}, {}, {:?})", self, attr_name, value); - let descr = vm - .ctx - .interned_str(attr_name) + let descr = self + .class() + .interned_attr_name(attr_name, vm) .and_then(|attr_name| self.get_class_attr(attr_name)); if let Some(attr) = &descr && let Some(descriptor) = attr.class().slots().descr_set.load() @@ -256,7 +256,7 @@ impl PyObject { vm: &VirtualMachine, ) -> PyResult> { let obj_cls = self.class(); - let cls_attr_name = vm.ctx.interned_str(name_str); + let cls_attr_name = obj_cls.interned_attr_name(name_str, vm); let cls_attr = match cls_attr_name.and_then(|name| obj_cls.get_attr(name)) { Some(descr) => { let descr_cls = descr.class(); diff --git a/crates/vm/src/protocol/sequence.rs b/crates/vm/src/protocol/sequence.rs index 33b3eb54d81..3aab89cd312 100644 --- a/crates/vm/src/protocol/sequence.rs +++ b/crates/vm/src/protocol/sequence.rs @@ -516,7 +516,7 @@ mod tests { #[test] fn unsupported_inplace_operations_keep_sequence_errors() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let range = PyRange { start: vm.ctx.new_int(0), stop: vm.ctx.new_int(3), @@ -544,7 +544,7 @@ mod tests { #[test] fn conversion_and_partial_result_cleanup() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let elements: Vec = (0..20).map(|value| vm.ctx.new_int(value).into()).collect(); let list = vm.ctx.new_list(elements.clone()); @@ -589,7 +589,7 @@ mod tests { #[test] fn capacity_overflow_precedes_conversion() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let range = PyRange { start: vm.ctx.new_int(0), stop: vm.ctx.new_int(isize::MAX), diff --git a/crates/vm/src/protocol/shared_buffer.rs b/crates/vm/src/protocol/shared_buffer.rs new file mode 100644 index 00000000000..b4e2fb551b3 --- /dev/null +++ b/crates/vm/src/protocol/shared_buffer.rs @@ -0,0 +1,92 @@ +//! Cross-interpreter buffers retain native storage, never foreign Python payloads. + +use super::{BufferDescriptor, PyBuffer}; +use crate::{ + PyResult, VirtualMachine, + common::{ + borrow::{BorrowedValue, BorrowedValueMut}, + rc::PyRc, + }, + object::PyThreadingConstraint, + vm::roots::RootLease, +}; +use core::fmt; + +/// Storage with no Python references or callbacks, independently owned by an export. +/// +/// # Safety +/// The storage must remain valid, with stable length, for this export's lifetime, +/// including after its interpreter is destroyed. Reading, writing and dropping it +/// must not access Python objects. Mutable access must synchronize with every other +/// reader/writer of the same storage, including the original exporter. +pub unsafe trait SharedBufferStorage: fmt::Debug + PyThreadingConstraint { + fn read(&self) -> BorrowedValue<'_, [u8]>; + fn write(&self) -> BorrowedValueMut<'_, [u8]>; +} + +#[derive(Clone)] +pub struct SharedBuffer { + pub desc: BufferDescriptor, + pub(crate) storage: PyRc, + // Releasing a Python buffer can call __release_buffer__. Its acquisition + // remains rooted on the owner; the receiving VM sees only this opaque lease. + _lease: PyRc, +} + +impl fmt::Debug for SharedBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SharedBuffer") + .field("desc", &self.desc) + .finish_non_exhaustive() + } +} + +impl SharedBuffer { + pub(crate) fn new(buffer: PyBuffer, vm: &VirtualMachine) -> PyResult { + // Forwarding a foreign export must not build a chain of owner roots. + if let Some(mut shared) = crate::stdlib::_interpreters::clone_shared_buffer(&buffer.obj) { + shared.desc = buffer.desc.clone(); + return Ok(shared); + } + let storage = buffer.shared_storage().ok_or_else(|| { + vm.new_buffer_error("buffer exporter does not provide interpreter-independent storage") + })?; + let desc = buffer.desc.clone(); + let lease = vm + .state + .roots + .insert_buffer(buffer) + .ok_or_else(|| vm.new_runtime_error("interpreter is closing"))?; + Ok(Self { + desc, + storage, + _lease: lease, + }) + } + + pub(crate) fn local_buffer(&self, vm: &VirtualMachine) -> Option { + let mut buffer = self._lease.buffer(vm)?; + buffer.desc = self.desc.clone(); + Some(buffer) + } + + pub(crate) fn read(&self) -> BorrowedValue<'_, [u8]> { + self.storage.read() + } + pub(crate) fn write(&self) -> BorrowedValueMut<'_, [u8]> { + self.storage.write() + } +} + +#[derive(Debug)] +pub(crate) struct ImmutableBuffer(pub(crate) Box<[u8]>); + +// SAFETY: owns immutable Rust bytes and cannot retain or execute Python. +unsafe impl SharedBufferStorage for ImmutableBuffer { + fn read(&self) -> BorrowedValue<'_, [u8]> { + self.0.as_ref().into() + } + fn write(&self) -> BorrowedValueMut<'_, [u8]> { + panic!("immutable buffer is not writable") + } +} diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index ddd4d58e556..137ddc53a98 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -129,7 +129,7 @@ fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { } } - if let Some(signal_rx) = &vm.signal_rx { + if let Some(signal_rx) = vm.signal_rx.get() { for f in signal_rx.rx.try_iter() { f(vm)?; } @@ -145,6 +145,7 @@ pub(crate) fn set_triggered() { } /// Any eval-breaker bit pending? One relaxed load; checked per instruction. +#[cfg(not(target_arch = "wasm32"))] #[inline(always)] pub(crate) fn eval_breaker_pending() -> bool { !EVAL_BREAKER.is_empty() diff --git a/crates/vm/src/stdlib/_abc.rs b/crates/vm/src/stdlib/_abc.rs index 19cf47415a9..49f034fd06f 100644 --- a/crates/vm/src/stdlib/_abc.rs +++ b/crates/vm/src/stdlib/_abc.rs @@ -17,15 +17,14 @@ mod _abc { }; use core::sync::atomic::{AtomicU64, Ordering}; - // Global invalidation counter - static ABC_INVALIDATION_COUNTER: AtomicU64 = AtomicU64::new(0); - - fn get_invalidation_counter() -> u64 { - ABC_INVALIDATION_COUNTER.load(Ordering::SeqCst) + fn get_invalidation_counter(vm: &VirtualMachine) -> u64 { + vm.state.abc_invalidation_counter.load(Ordering::SeqCst) } - fn increment_invalidation_counter() { - ABC_INVALIDATION_COUNTER.fetch_add(1, Ordering::SeqCst); + fn increment_invalidation_counter(vm: &VirtualMachine) { + vm.state + .abc_invalidation_counter + .fetch_add(1, Ordering::SeqCst); } // Internal state held by ABC machinery. @@ -42,12 +41,12 @@ mod _abc { #[pyclass(with(Constructor))] impl AbcData { - fn new() -> Self { + fn new(vm: &VirtualMachine) -> Self { Self { registry: PyRwLock::new(None), cache: PyRwLock::new(None), negative_cache: PyRwLock::new(None), - negative_cache_version: AtomicU64::new(get_invalidation_counter()), + negative_cache_version: AtomicU64::new(get_invalidation_counter(vm)), } } @@ -66,9 +65,9 @@ mod _abc { fn py_new( _cls: &crate::Py, _args: Self::Args, - _vm: &VirtualMachine, + vm: &VirtualMachine, ) -> PyResult { - Ok(Self::new()) + Ok(Self::new(vm)) } } @@ -135,8 +134,8 @@ mod _abc { } #[pyfunction] - fn get_cache_token() -> u64 { - get_invalidation_counter() + fn get_cache_token(vm: &VirtualMachine) -> u64 { + get_invalidation_counter(vm) } /// Compute set of abstract method names. @@ -207,7 +206,7 @@ mod _abc { compute_abstract_methods(&cls, vm)?; // Set up inheritance registry - let data = AbcData::new(); + let data = AbcData::new(vm); cls.set_attr("_abc_impl", data.to_pyobject(vm), vm)?; if let Some(cls_type) = cls.downcast_ref::() { @@ -244,14 +243,14 @@ mod _abc { add_to_weak_set(&impl_data.registry, &subclass, vm)?; // Invalidate negative cache - increment_invalidation_counter(); + increment_invalidation_counter(vm); if let Some(cls_type) = cls.downcast_ref::() && let Some(subclass_type) = subclass.downcast_ref::() { // _abc_register propagates Py_TPFLAGS_SEQUENCE/MAPPING // recursively so MATCH_SEQUENCE/MATCH_MAPPING see ABC registration. - let collection_flags = cls_type.slots.flags.load() & PyTypeFlags::COLLECTION; + let collection_flags = cls_type.effective_flags() & PyTypeFlags::COLLECTION; if !subclass_type.is(vm.ctx.types.str_type) && !subclass_type.is(vm.ctx.types.bytes_type) && !subclass_type.is(vm.ctx.types.bytearray_type) @@ -281,7 +280,7 @@ mod _abc { let subtype: PyObjectRef = instance.class().to_owned().into(); if subtype.is(&subclass) { - let invalidation_counter = get_invalidation_counter(); + let invalidation_counter = get_invalidation_counter(vm); if impl_data.get_cache_version() == invalidation_counter && in_weak_set(&impl_data.negative_cache, &subclass, vm)? { @@ -357,7 +356,7 @@ mod _abc { } // 2. Check negative cache; may have to invalidate - let invalidation_counter = get_invalidation_counter(); + let invalidation_counter = get_invalidation_counter(vm); if impl_data.get_cache_version() < invalidation_counter { // Invalidate the negative cache // Clone set ref and drop lock before calling into VM to avoid reentrancy diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index bba073d5b0c..7eb37f9ff2a 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -15,7 +15,7 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyDict, PyList, PyModule, PyTuple, PyType, PyUtf8StrRef}, - class::{PyClassImpl, StaticType}, + class::PyClassImpl, compiler::{CompileError, ParseError}, convert::ToPyObject, }; @@ -34,6 +34,15 @@ use rustpython_codegen as codegen; pub(crate) use python::_ast::NodeAst; +#[pyclass] +pub(crate) trait AstNodeType { + fn extend_slots(slots: &mut crate::types::PyTypeSlots) { + // AST types are mutable (heap types, not IMMUTABLETYPE). + slots.flags.remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + slots.repr.store(Some(python::_ast::ast_repr)); + } +} + mod python; mod repr; mod validate; @@ -57,26 +66,26 @@ mod type_parameters; /// Return the cached singleton instance for an operator/context node type, /// or create a new instance if none exists. -fn singleton_node_to_object(vm: &VirtualMachine, node_type: &'static Py) -> PyObjectRef { +fn singleton_node_to_object( + vm: &VirtualMachine, + node_type: crate::builtins::PyTypeRef, +) -> PyObjectRef { if let Some(instance) = node_type.get_attr(vm.ctx.intern_str("_instance")) { return instance; } - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() + NodeAst.into_ref_with_type(vm, node_type).unwrap().into() } fn is_node_instance( vm: &VirtualMachine, object: &PyObject, - node_type: &'static Py, + node_type: crate::builtins::PyTypeRef, ) -> PyResult { object.is_instance(node_type.as_object(), vm) } fn is_ast_instance(vm: &VirtualMachine, object: &PyObject) -> PyResult { - let ast_type = NodeAst::make_static_type(); + let ast_type = NodeAst::make_class(&vm.ctx); object.is_instance(ast_type.as_object(), vm) } @@ -506,21 +515,29 @@ fn copy_ast_passthrough_fields( } let fields: &[&str] = - if is_node_instance(vm, target, pyast::NodeStmtFunctionDef::static_type())? - || is_node_instance(vm, target, pyast::NodeStmtAsyncFunctionDef::static_type())? - || is_node_instance(vm, target, pyast::NodeStmtAssign::static_type())? - || is_node_instance(vm, target, pyast::NodeStmtFor::static_type())? - || is_node_instance(vm, target, pyast::NodeStmtAsyncFor::static_type())? - || is_node_instance(vm, target, pyast::NodeStmtWith::static_type())? - || is_node_instance(vm, target, pyast::NodeStmtAsyncWith::static_type())? - || is_node_instance(vm, target, pyast::NodeArg::static_type())? + if is_node_instance(vm, target, pyast::NodeStmtFunctionDef::make_class(&vm.ctx))? + || is_node_instance( + vm, + target, + pyast::NodeStmtAsyncFunctionDef::make_class(&vm.ctx), + )? + || is_node_instance(vm, target, pyast::NodeStmtAssign::make_class(&vm.ctx))? + || is_node_instance(vm, target, pyast::NodeStmtFor::make_class(&vm.ctx))? + || is_node_instance(vm, target, pyast::NodeStmtAsyncFor::make_class(&vm.ctx))? + || is_node_instance(vm, target, pyast::NodeStmtWith::make_class(&vm.ctx))? + || is_node_instance(vm, target, pyast::NodeStmtAsyncWith::make_class(&vm.ctx))? + || is_node_instance(vm, target, pyast::NodeArg::make_class(&vm.ctx))? { &["type_comment"] - } else if is_node_instance(vm, target, pyast::NodeComprehension::static_type())? { + } else if is_node_instance(vm, target, pyast::NodeComprehension::make_class(&vm.ctx))? { &["is_async"] - } else if is_node_instance(vm, target, pyast::NodeExprConstant::static_type())? { + } else if is_node_instance(vm, target, pyast::NodeExprConstant::make_class(&vm.ctx))? { &["kind"] - } else if is_node_instance(vm, target, pyast::NodeExprInterpolation::static_type())? { + } else if is_node_instance( + vm, + target, + pyast::NodeExprInterpolation::make_class(&vm.ctx), + )? { &["str"] } else { &[] @@ -934,13 +951,19 @@ fn node_add_location( /// builtin compile() accepts func_type only with PyCF_ONLY_AST. /// Source-string func_type parsing is handled separately, but Python AST /// FunctionType still uses the mode check before obj-to-AST conversion. -pub(crate) fn mode_type_and_name(mode: &str) -> Option<(PyRef, &'static str)> { +pub(crate) fn mode_type_and_name( + mode: &str, + vm: &VirtualMachine, +) -> Option<(PyRef, &'static str)> { match mode { - "exec" => Some((pyast::NodeModModule::make_static_type(), "Module")), - "eval" => Some((pyast::NodeModExpression::make_static_type(), "Expression")), - "single" => Some((pyast::NodeModInteractive::make_static_type(), "Interactive")), + "exec" => Some((pyast::NodeModModule::make_class(&vm.ctx), "Module")), + "eval" => Some((pyast::NodeModExpression::make_class(&vm.ctx), "Expression")), + "single" => Some(( + pyast::NodeModInteractive::make_class(&vm.ctx), + "Interactive", + )), "func_type" => Some(( - pyast::NodeModFunctionType::make_static_type(), + pyast::NodeModFunctionType::make_class(&vm.ctx), "FunctionType", )), _ => None, @@ -1658,18 +1681,18 @@ fn apply_type_comments_to_node( object: &PyObject, ) { let cls = object.class(); - if cls.is(pyast::NodeStmtFunctionDef::static_type()) - || cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()) + if cls.is(&pyast::NodeStmtFunctionDef::make_class(&vm.ctx)) + || cls.is(&pyast::NodeStmtAsyncFunctionDef::make_class(&vm.ctx)) { set_type_comment(vm, object, function_type_comment(vm, lines, object)); if let Some(arguments) = node_optional_field(vm, object, "args") { apply_type_comments_to_arguments(vm, lines, &arguments); } - } else if cls.is(pyast::NodeStmtAssign::static_type()) - || cls.is(pyast::NodeStmtFor::static_type()) - || cls.is(pyast::NodeStmtAsyncFor::static_type()) - || cls.is(pyast::NodeStmtWith::static_type()) - || cls.is(pyast::NodeStmtAsyncWith::static_type()) + } else if cls.is(&pyast::NodeStmtAssign::make_class(&vm.ctx)) + || cls.is(&pyast::NodeStmtFor::make_class(&vm.ctx)) + || cls.is(&pyast::NodeStmtAsyncFor::make_class(&vm.ctx)) + || cls.is(&pyast::NodeStmtWith::make_class(&vm.ctx)) + || cls.is(&pyast::NodeStmtAsyncWith::make_class(&vm.ctx)) { set_type_comment(vm, object, same_line_type_comment(vm, lines, object)); } @@ -1767,7 +1790,7 @@ fn ipython_escape_command_syntax_error( /// Create an empty `arguments` AST node (no parameters). fn empty_arguments_object(vm: &VirtualMachine) -> PyObjectRef { let node = NodeAst - .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeArguments::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); for list_field in [ @@ -1937,7 +1960,7 @@ pub(crate) fn parse( }; let obj = top.ast_to_object(vm, &source_file); if let Some(lines) = &type_comment_source - && obj.class().is(pyast::NodeModModule::static_type()) + && obj.class().is(&pyast::NodeModModule::make_class(&vm.ctx)) { apply_type_comments_to_module(vm, lines, &obj); let type_ignores = type_ignores_from_source(vm, lines); @@ -1950,12 +1973,15 @@ pub(crate) fn parse( #[cfg(feature = "parser")] pub(crate) fn wrap_interactive(vm: &VirtualMachine, module_obj: &PyObject) -> PyResult { - if !module_obj.class().is(pyast::NodeModModule::static_type()) { + if !module_obj + .class() + .is(&pyast::NodeModModule::make_class(&vm.ctx)) + { return Err(vm.new_type_error("expected Module node")); } let body = get_node_field(vm, module_obj, "body", "Module")?; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeModInteractive::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeModInteractive::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("body", body, vm).unwrap(); @@ -2121,10 +2147,7 @@ fn type_ignores_from_source( continue; }; let node = NodeAst - .into_ref_with_type( - vm, - pyast::NodeTypeIgnoreTypeIgnore::static_type().to_owned(), - ) + .into_ref_with_type(vm, pyast::NodeTypeIgnoreTypeIgnore::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let lineno = idx + 1; @@ -2460,8 +2483,8 @@ pub(crate) fn rust_mod_from_object( ) -> PyResult { let is_mod = object .class() - .fast_issubclass(pyast::NodeMod::static_type()) - || object.is_instance(pyast::NodeMod::static_type().as_object(), vm)?; + .fast_issubclass(&pyast::NodeMod::make_class(&vm.ctx)) + || object.is_instance(pyast::NodeMod::make_class(&vm.ctx).as_object(), vm)?; if !is_mod && !is_ast_instance(vm, &object)? { return Err(vm.new_type_error("expected an AST")); } diff --git a/crates/vm/src/stdlib/_ast/constant.rs b/crates/vm/src/stdlib/_ast/constant.rs index 61b25f97620..ddb9a0bb818 100644 --- a/crates/vm/src/stdlib/_ast/constant.rs +++ b/crates/vm/src/stdlib/_ast/constant.rs @@ -164,7 +164,7 @@ pub(super) fn expr_constant_to_object( } = expr; let constant = ast_constant_value_to_constant_data(value); let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprConstant::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprConstant::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", constant_data_to_object(vm, constant), vm) @@ -362,7 +362,7 @@ fn first_invalid_constant_type(vm: &VirtualMachine, value_object: &PyObject) -> .map_err(|obj| { vm.new_type_error(format!( "Expected type {}, not {}", - PyTuple::static_type().name(), + PyTuple::make_class(&vm.ctx).name(), obj.class().name() )) })?; @@ -481,7 +481,7 @@ impl Node for Constant { invalid_type: _, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprConstant::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprConstant::make_class(&vm.ctx)) .unwrap(); let kind = kind .or_else(|| constant_literal_kind(&value)) @@ -567,7 +567,7 @@ impl Node for ConstantLiteral { let tuple = value_object.downcast::().map_err(|obj| { vm.new_type_error(format!( "Expected type {}, not {}", - PyTuple::static_type().name(), + PyTuple::make_class(&vm.ctx).name(), obj.class().name() )) })?; @@ -602,7 +602,7 @@ impl Node for ConstantLiteral { None => { return Err(vm.new_type_error(format!( "Expected type {}, not {}", - PyComplex::static_type().name(), + PyComplex::make_class(&vm.ctx).name(), value_object.class().name() ))); } diff --git a/crates/vm/src/stdlib/_ast/elif_else_clause.rs b/crates/vm/src/stdlib/_ast/elif_else_clause.rs index ca8187bf32f..1c490a2234d 100644 --- a/crates/vm/src/stdlib/_ast/elif_else_clause.rs +++ b/crates/vm/src/stdlib/_ast/elif_else_clause.rs @@ -23,7 +23,7 @@ pub(super) fn ast_to_object( ); }; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtIf::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtIf::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); diff --git a/crates/vm/src/stdlib/_ast/exception.rs b/crates/vm/src/stdlib/_ast/exception.rs index 4f179513d59..20039d0e5a5 100644 --- a/crates/vm/src/stdlib/_ast/exception.rs +++ b/crates/vm/src/stdlib/_ast/exception.rs @@ -6,7 +6,7 @@ fn ensure_excepthandler_node(vm: &VirtualMachine, object: &PyObject) -> PyResult || !is_node_instance( vm, object, - pyast::NodeExceptHandlerExceptHandler::static_type(), + pyast::NodeExceptHandlerExceptHandler::make_class(&vm.ctx), )? { return Err(vm.new_type_error(format!( @@ -89,7 +89,7 @@ impl Node for ast::ExceptHandlerExceptHandler { let node = NodeAst .into_ref_with_type( vm, - pyast::NodeExceptHandlerExceptHandler::static_type().to_owned(), + pyast::NodeExceptHandlerExceptHandler::make_class(&vm.ctx), ) .unwrap(); let dict = node.as_object().dict().unwrap(); diff --git a/crates/vm/src/stdlib/_ast/expression.rs b/crates/vm/src/stdlib/_ast/expression.rs index 6d5e21302fe..b1a69ec423c 100644 --- a/crates/vm/src/stdlib/_ast/expression.rs +++ b/crates/vm/src/stdlib/_ast/expression.rs @@ -94,63 +94,75 @@ impl Node for ast::Expr { Tuple, Slice, } - let kind = if is_node_instance(vm, &object, pyast::NodeExprBoolOp::static_type())? { + let kind = if is_node_instance(vm, &object, pyast::NodeExprBoolOp::make_class(&vm.ctx))? { ExprKind::BoolOp - } else if is_node_instance(vm, &object, pyast::NodeExprNamedExpr::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprNamedExpr::make_class(&vm.ctx))? { ExprKind::Named - } else if is_node_instance(vm, &object, pyast::NodeExprBinOp::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprBinOp::make_class(&vm.ctx))? { ExprKind::BinOp - } else if is_node_instance(vm, &object, pyast::NodeExprUnaryOp::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprUnaryOp::make_class(&vm.ctx))? { ExprKind::UnaryOp - } else if is_node_instance(vm, &object, pyast::NodeExprLambda::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprLambda::make_class(&vm.ctx))? { ExprKind::Lambda - } else if is_node_instance(vm, &object, pyast::NodeExprIfExp::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprIfExp::make_class(&vm.ctx))? { ExprKind::If - } else if is_node_instance(vm, &object, pyast::NodeExprDict::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprDict::make_class(&vm.ctx))? { ExprKind::Dict - } else if is_node_instance(vm, &object, pyast::NodeExprSet::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprSet::make_class(&vm.ctx))? { ExprKind::Set - } else if is_node_instance(vm, &object, pyast::NodeExprListComp::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprListComp::make_class(&vm.ctx))? { ExprKind::ListComp - } else if is_node_instance(vm, &object, pyast::NodeExprSetComp::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprSetComp::make_class(&vm.ctx))? { ExprKind::SetComp - } else if is_node_instance(vm, &object, pyast::NodeExprDictComp::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprDictComp::make_class(&vm.ctx))? { ExprKind::DictComp - } else if is_node_instance(vm, &object, pyast::NodeExprGeneratorExp::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeExprGeneratorExp::make_class(&vm.ctx), + )? { ExprKind::Generator - } else if is_node_instance(vm, &object, pyast::NodeExprAwait::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprAwait::make_class(&vm.ctx))? { ExprKind::Await - } else if is_node_instance(vm, &object, pyast::NodeExprYield::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprYield::make_class(&vm.ctx))? { ExprKind::Yield - } else if is_node_instance(vm, &object, pyast::NodeExprYieldFrom::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprYieldFrom::make_class(&vm.ctx))? { ExprKind::YieldFrom - } else if is_node_instance(vm, &object, pyast::NodeExprCompare::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprCompare::make_class(&vm.ctx))? { ExprKind::Compare - } else if is_node_instance(vm, &object, pyast::NodeExprCall::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprCall::make_class(&vm.ctx))? { ExprKind::Call - } else if is_node_instance(vm, &object, pyast::NodeExprFormattedValue::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeExprFormattedValue::make_class(&vm.ctx), + )? { ExprKind::FormattedValue - } else if is_node_instance(vm, &object, pyast::NodeExprInterpolation::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeExprInterpolation::make_class(&vm.ctx), + )? { ExprKind::Interpolation - } else if is_node_instance(vm, &object, pyast::NodeExprJoinedStr::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprJoinedStr::make_class(&vm.ctx))? { ExprKind::JoinedStr - } else if is_node_instance(vm, &object, pyast::NodeExprTemplateStr::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprTemplateStr::make_class(&vm.ctx))? { ExprKind::TemplateStr - } else if is_node_instance(vm, &object, pyast::NodeExprConstant::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprConstant::make_class(&vm.ctx))? { ExprKind::Constant - } else if is_node_instance(vm, &object, pyast::NodeExprAttribute::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprAttribute::make_class(&vm.ctx))? { ExprKind::Attribute - } else if is_node_instance(vm, &object, pyast::NodeExprSubscript::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprSubscript::make_class(&vm.ctx))? { ExprKind::Subscript - } else if is_node_instance(vm, &object, pyast::NodeExprStarred::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprStarred::make_class(&vm.ctx))? { ExprKind::Starred - } else if is_node_instance(vm, &object, pyast::NodeExprName::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprName::make_class(&vm.ctx))? { ExprKind::Name - } else if is_node_instance(vm, &object, pyast::NodeExprList::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprList::make_class(&vm.ctx))? { ExprKind::List - } else if is_node_instance(vm, &object, pyast::NodeExprTuple::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprTuple::make_class(&vm.ctx))? { ExprKind::Tuple - } else if is_node_instance(vm, &object, pyast::NodeExprSlice::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprSlice::make_class(&vm.ctx))? { ExprKind::Slice } else { return Err(vm.new_type_error(format!( @@ -372,7 +384,7 @@ impl Node for ast::ExprBoolOp { runtime_values, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprBoolOp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprBoolOp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("op", op.ast_to_object(vm, source_file), vm) @@ -420,7 +432,7 @@ impl Node for ast::ExprNamed { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprNamedExpr::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprNamedExpr::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("target", target.ast_to_object(vm, source_file), vm) @@ -471,7 +483,7 @@ impl Node for ast::ExprBinOp { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprBinOp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprBinOp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("left", left.ast_to_object(vm, source_file), vm) @@ -522,7 +534,7 @@ impl Node for ast::ExprUnaryOp { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprUnaryOp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprUnaryOp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("op", op.ast_to_object(vm, source_file), vm) @@ -570,7 +582,7 @@ impl Node for ast::ExprLambda { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprLambda::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprLambda::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let args = match parameters { @@ -620,7 +632,7 @@ impl Node for ast::ExprIf { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprIfExp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprIfExp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("test", test.ast_to_object(vm, source_file), vm) @@ -688,7 +700,7 @@ impl Node for ast::ExprDict { (keys, values) }); let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprDict::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprDict::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("keys", keys.ast_to_object(vm, source_file), vm) @@ -738,7 +750,7 @@ impl Node for ast::ExprSet { runtime_elts, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprSet::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprSet::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let elts = runtime_elts.map_or_else( @@ -783,7 +795,7 @@ impl Node for ast::ExprListComp { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprListComp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprListComp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("elt", elt.ast_to_object(vm, source_file), vm) @@ -828,7 +840,7 @@ impl Node for ast::ExprSetComp { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprSetComp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprSetComp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("elt", elt.ast_to_object(vm, source_file), vm) @@ -881,7 +893,7 @@ impl Node for ast::ExprDictComp { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprDictComp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprDictComp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("key", key.ast_to_object(vm, source_file), vm) @@ -930,7 +942,7 @@ impl Node for ast::ExprGenerator { parenthesized: _, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprGeneratorExp::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprGeneratorExp::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("elt", elt.ast_to_object(vm, source_file), vm) @@ -973,7 +985,7 @@ impl Node for ast::ExprAwait { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprAwait::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprAwait::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1015,7 +1027,7 @@ impl Node for ast::ExprYield { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprYield::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprYield::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1056,7 +1068,7 @@ impl Node for ast::ExprYieldFrom { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprYieldFrom::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprYieldFrom::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1106,7 +1118,7 @@ impl Node for ast::ExprCompare { runtime_comparators, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprCompare::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprCompare::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("left", left.ast_to_object(vm, source_file), vm) @@ -1163,7 +1175,7 @@ impl Node for ast::ExprCall { } = self; let (positional_arguments, keyword_arguments) = split_function_call_arguments(arguments); let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprCall::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprCall::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("func", func.ast_to_object(vm, source_file), vm) @@ -1224,7 +1236,7 @@ impl Node for ast::ExprAttribute { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprAttribute::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprAttribute::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1277,7 +1289,7 @@ impl Node for ast::ExprSubscript { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprSubscript::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprSubscript::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1327,7 +1339,7 @@ impl Node for ast::ExprStarred { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprStarred::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprStarred::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1375,7 +1387,7 @@ impl Node for ast::ExprName { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprName::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprName::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("id", id.ast_to_object(vm, source_file), vm) @@ -1429,7 +1441,7 @@ impl Node for ast::ExprList { runtime_elts, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprList::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprList::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let elts = runtime_elts.map_or_else( @@ -1488,7 +1500,7 @@ impl Node for ast::ExprTuple { runtime_elts, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprTuple::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprTuple::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let elts = runtime_elts.map_or_else( @@ -1544,7 +1556,7 @@ impl Node for ast::ExprSlice { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprSlice::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprSlice::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("lower", lower.ast_to_object(vm, source_file), vm) @@ -1571,9 +1583,9 @@ impl Node for ast::ExprSlice { impl Node for ast::ExprContext { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { - Self::Load => pyast::NodeExprContextLoad::static_type(), - Self::Store => pyast::NodeExprContextStore::static_type(), - Self::Del => pyast::NodeExprContextDel::static_type(), + Self::Load => pyast::NodeExprContextLoad::make_class(&vm.ctx), + Self::Store => pyast::NodeExprContextStore::make_class(&vm.ctx), + Self::Del => pyast::NodeExprContextDel::make_class(&vm.ctx), Self::Invalid => { unreachable!() } @@ -1587,11 +1599,16 @@ impl Node for ast::ExprContext { object: PyObjectRef, ) -> PyResult { Ok( - if is_node_instance(vm, &object, pyast::NodeExprContextLoad::static_type())? { + if is_node_instance(vm, &object, pyast::NodeExprContextLoad::make_class(&vm.ctx))? { Self::Load - } else if is_node_instance(vm, &object, pyast::NodeExprContextStore::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeExprContextStore::make_class(&vm.ctx), + )? { Self::Store - } else if is_node_instance(vm, &object, pyast::NodeExprContextDel::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeExprContextDel::make_class(&vm.ctx))? + { Self::Del } else { return Err(vm.new_type_error(format!( @@ -1617,7 +1634,7 @@ impl Node for ast::Comprehension { runtime_is_async, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeComprehension::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeComprehension::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("target", target.ast_to_object(vm, source_file), vm) diff --git a/crates/vm/src/stdlib/_ast/module.rs b/crates/vm/src/stdlib/_ast/module.rs index 37a6bc62849..ed8223fea6f 100644 --- a/crates/vm/src/stdlib/_ast/module.rs +++ b/crates/vm/src/stdlib/_ast/module.rs @@ -41,21 +41,26 @@ impl Node for Mod { object: PyObjectRef, ) -> PyResult { Ok( - if object.is_instance(pyast::NodeModModule::static_type().as_object(), vm)? { + if object.is_instance(pyast::NodeModModule::make_class(&vm.ctx).as_object(), vm)? { Self::Module(ModModule::ast_from_object(vm, source_file, object)?) - } else if object - .is_instance(pyast::NodeModInteractive::static_type().as_object(), vm)? - { + } else if object.is_instance( + pyast::NodeModInteractive::make_class(&vm.ctx).as_object(), + vm, + )? { Self::Interactive(ModInteractive::ast_from_object(vm, source_file, object)?) - } else if object.is_instance(pyast::NodeModExpression::static_type().as_object(), vm)? { + } else if object.is_instance( + pyast::NodeModExpression::make_class(&vm.ctx).as_object(), + vm, + )? { Self::Expression(ast::ModExpression::ast_from_object( vm, source_file, object, )?) - } else if object - .is_instance(pyast::NodeModFunctionType::static_type().as_object(), vm)? - { + } else if object.is_instance( + pyast::NodeModFunctionType::make_class(&vm.ctx).as_object(), + vm, + )? { Self::FunctionType(ModFunctionType::ast_from_object(vm, source_file, object)?) } else { return Err(vm.new_type_error(format!( @@ -86,7 +91,7 @@ impl Node for ModModule { runtime_body, } = module; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeModModule::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeModModule::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let body = runtime_body.map_or_else( @@ -139,7 +144,7 @@ impl Node for ModInteractive { runtime_body, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeModInteractive::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeModInteractive::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let body = runtime_body.map_or_else( @@ -175,7 +180,7 @@ impl Node for ast::ModExpression { range: _, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeModExpression::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeModExpression::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("body", body.ast_to_object(vm, source_file), vm) @@ -211,7 +216,7 @@ impl Node for ModFunctionType { runtime_argtypes, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeModFunctionType::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeModFunctionType::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let argtypes = runtime_argtypes.map_or_else( diff --git a/crates/vm/src/stdlib/_ast/operator.rs b/crates/vm/src/stdlib/_ast/operator.rs index e05e490bb84..0efb76f7c8d 100644 --- a/crates/vm/src/stdlib/_ast/operator.rs +++ b/crates/vm/src/stdlib/_ast/operator.rs @@ -5,8 +5,8 @@ use rustpython_compiler_core::SourceFile; impl Node for ast::BoolOp { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { - Self::And => pyast::NodeBoolOpAnd::static_type(), - Self::Or => pyast::NodeBoolOpOr::static_type(), + Self::And => pyast::NodeBoolOpAnd::make_class(&vm.ctx), + Self::Or => pyast::NodeBoolOpOr::make_class(&vm.ctx), }; singleton_node_to_object(vm, node_type) } @@ -17,9 +17,9 @@ impl Node for ast::BoolOp { object: PyObjectRef, ) -> PyResult { Ok( - if is_node_instance(vm, &object, pyast::NodeBoolOpAnd::static_type())? { + if is_node_instance(vm, &object, pyast::NodeBoolOpAnd::make_class(&vm.ctx))? { Self::And - } else if is_node_instance(vm, &object, pyast::NodeBoolOpOr::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeBoolOpOr::make_class(&vm.ctx))? { Self::Or } else { return Err(vm.new_type_error(format!( @@ -35,19 +35,19 @@ impl Node for ast::BoolOp { impl Node for ast::Operator { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { - Self::Add => pyast::NodeOperatorAdd::static_type(), - Self::Sub => pyast::NodeOperatorSub::static_type(), - Self::Mult => pyast::NodeOperatorMult::static_type(), - Self::MatMult => pyast::NodeOperatorMatMult::static_type(), - Self::Div => pyast::NodeOperatorDiv::static_type(), - Self::Mod => pyast::NodeOperatorMod::static_type(), - Self::Pow => pyast::NodeOperatorPow::static_type(), - Self::LShift => pyast::NodeOperatorLShift::static_type(), - Self::RShift => pyast::NodeOperatorRShift::static_type(), - Self::BitOr => pyast::NodeOperatorBitOr::static_type(), - Self::BitXor => pyast::NodeOperatorBitXor::static_type(), - Self::BitAnd => pyast::NodeOperatorBitAnd::static_type(), - Self::FloorDiv => pyast::NodeOperatorFloorDiv::static_type(), + Self::Add => pyast::NodeOperatorAdd::make_class(&vm.ctx), + Self::Sub => pyast::NodeOperatorSub::make_class(&vm.ctx), + Self::Mult => pyast::NodeOperatorMult::make_class(&vm.ctx), + Self::MatMult => pyast::NodeOperatorMatMult::make_class(&vm.ctx), + Self::Div => pyast::NodeOperatorDiv::make_class(&vm.ctx), + Self::Mod => pyast::NodeOperatorMod::make_class(&vm.ctx), + Self::Pow => pyast::NodeOperatorPow::make_class(&vm.ctx), + Self::LShift => pyast::NodeOperatorLShift::make_class(&vm.ctx), + Self::RShift => pyast::NodeOperatorRShift::make_class(&vm.ctx), + Self::BitOr => pyast::NodeOperatorBitOr::make_class(&vm.ctx), + Self::BitXor => pyast::NodeOperatorBitXor::make_class(&vm.ctx), + Self::BitAnd => pyast::NodeOperatorBitAnd::make_class(&vm.ctx), + Self::FloorDiv => pyast::NodeOperatorFloorDiv::make_class(&vm.ctx), }; singleton_node_to_object(vm, node_type) } @@ -58,31 +58,44 @@ impl Node for ast::Operator { object: PyObjectRef, ) -> PyResult { Ok( - if is_node_instance(vm, &object, pyast::NodeOperatorAdd::static_type())? { + if is_node_instance(vm, &object, pyast::NodeOperatorAdd::make_class(&vm.ctx))? { Self::Add - } else if is_node_instance(vm, &object, pyast::NodeOperatorSub::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorSub::make_class(&vm.ctx))? { Self::Sub - } else if is_node_instance(vm, &object, pyast::NodeOperatorMult::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorMult::make_class(&vm.ctx))? { Self::Mult - } else if is_node_instance(vm, &object, pyast::NodeOperatorMatMult::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeOperatorMatMult::make_class(&vm.ctx), + )? { Self::MatMult - } else if is_node_instance(vm, &object, pyast::NodeOperatorDiv::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorDiv::make_class(&vm.ctx))? { Self::Div - } else if is_node_instance(vm, &object, pyast::NodeOperatorMod::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorMod::make_class(&vm.ctx))? { Self::Mod - } else if is_node_instance(vm, &object, pyast::NodeOperatorPow::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorPow::make_class(&vm.ctx))? { Self::Pow - } else if is_node_instance(vm, &object, pyast::NodeOperatorLShift::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorLShift::make_class(&vm.ctx))? + { Self::LShift - } else if is_node_instance(vm, &object, pyast::NodeOperatorRShift::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorRShift::make_class(&vm.ctx))? + { Self::RShift - } else if is_node_instance(vm, &object, pyast::NodeOperatorBitOr::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorBitOr::make_class(&vm.ctx))? + { Self::BitOr - } else if is_node_instance(vm, &object, pyast::NodeOperatorBitXor::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorBitXor::make_class(&vm.ctx))? + { Self::BitXor - } else if is_node_instance(vm, &object, pyast::NodeOperatorBitAnd::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeOperatorBitAnd::make_class(&vm.ctx))? + { Self::BitAnd - } else if is_node_instance(vm, &object, pyast::NodeOperatorFloorDiv::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeOperatorFloorDiv::make_class(&vm.ctx), + )? { Self::FloorDiv } else { return Err(vm.new_type_error(format!( @@ -98,10 +111,10 @@ impl Node for ast::Operator { impl Node for ast::UnaryOp { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { - Self::Invert => pyast::NodeUnaryOpInvert::static_type(), - Self::Not => pyast::NodeUnaryOpNot::static_type(), - Self::UAdd => pyast::NodeUnaryOpUAdd::static_type(), - Self::USub => pyast::NodeUnaryOpUSub::static_type(), + Self::Invert => pyast::NodeUnaryOpInvert::make_class(&vm.ctx), + Self::Not => pyast::NodeUnaryOpNot::make_class(&vm.ctx), + Self::UAdd => pyast::NodeUnaryOpUAdd::make_class(&vm.ctx), + Self::USub => pyast::NodeUnaryOpUSub::make_class(&vm.ctx), }; singleton_node_to_object(vm, node_type) } @@ -112,13 +125,13 @@ impl Node for ast::UnaryOp { object: PyObjectRef, ) -> PyResult { Ok( - if is_node_instance(vm, &object, pyast::NodeUnaryOpInvert::static_type())? { + if is_node_instance(vm, &object, pyast::NodeUnaryOpInvert::make_class(&vm.ctx))? { Self::Invert - } else if is_node_instance(vm, &object, pyast::NodeUnaryOpNot::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeUnaryOpNot::make_class(&vm.ctx))? { Self::Not - } else if is_node_instance(vm, &object, pyast::NodeUnaryOpUAdd::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeUnaryOpUAdd::make_class(&vm.ctx))? { Self::UAdd - } else if is_node_instance(vm, &object, pyast::NodeUnaryOpUSub::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeUnaryOpUSub::make_class(&vm.ctx))? { Self::USub } else { return Err(vm.new_type_error(format!( @@ -134,16 +147,16 @@ impl Node for ast::UnaryOp { impl Node for ast::CmpOp { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let node_type = match self { - Self::Eq => pyast::NodeCmpOpEq::static_type(), - Self::NotEq => pyast::NodeCmpOpNotEq::static_type(), - Self::Lt => pyast::NodeCmpOpLt::static_type(), - Self::LtE => pyast::NodeCmpOpLtE::static_type(), - Self::Gt => pyast::NodeCmpOpGt::static_type(), - Self::GtE => pyast::NodeCmpOpGtE::static_type(), - Self::Is => pyast::NodeCmpOpIs::static_type(), - Self::IsNot => pyast::NodeCmpOpIsNot::static_type(), - Self::In => pyast::NodeCmpOpIn::static_type(), - Self::NotIn => pyast::NodeCmpOpNotIn::static_type(), + Self::Eq => pyast::NodeCmpOpEq::make_class(&vm.ctx), + Self::NotEq => pyast::NodeCmpOpNotEq::make_class(&vm.ctx), + Self::Lt => pyast::NodeCmpOpLt::make_class(&vm.ctx), + Self::LtE => pyast::NodeCmpOpLtE::make_class(&vm.ctx), + Self::Gt => pyast::NodeCmpOpGt::make_class(&vm.ctx), + Self::GtE => pyast::NodeCmpOpGtE::make_class(&vm.ctx), + Self::Is => pyast::NodeCmpOpIs::make_class(&vm.ctx), + Self::IsNot => pyast::NodeCmpOpIsNot::make_class(&vm.ctx), + Self::In => pyast::NodeCmpOpIn::make_class(&vm.ctx), + Self::NotIn => pyast::NodeCmpOpNotIn::make_class(&vm.ctx), }; singleton_node_to_object(vm, node_type) } @@ -154,25 +167,25 @@ impl Node for ast::CmpOp { object: PyObjectRef, ) -> PyResult { Ok( - if is_node_instance(vm, &object, pyast::NodeCmpOpEq::static_type())? { + if is_node_instance(vm, &object, pyast::NodeCmpOpEq::make_class(&vm.ctx))? { Self::Eq - } else if is_node_instance(vm, &object, pyast::NodeCmpOpNotEq::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpNotEq::make_class(&vm.ctx))? { Self::NotEq - } else if is_node_instance(vm, &object, pyast::NodeCmpOpLt::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpLt::make_class(&vm.ctx))? { Self::Lt - } else if is_node_instance(vm, &object, pyast::NodeCmpOpLtE::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpLtE::make_class(&vm.ctx))? { Self::LtE - } else if is_node_instance(vm, &object, pyast::NodeCmpOpGt::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpGt::make_class(&vm.ctx))? { Self::Gt - } else if is_node_instance(vm, &object, pyast::NodeCmpOpGtE::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpGtE::make_class(&vm.ctx))? { Self::GtE - } else if is_node_instance(vm, &object, pyast::NodeCmpOpIs::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpIs::make_class(&vm.ctx))? { Self::Is - } else if is_node_instance(vm, &object, pyast::NodeCmpOpIsNot::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpIsNot::make_class(&vm.ctx))? { Self::IsNot - } else if is_node_instance(vm, &object, pyast::NodeCmpOpIn::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpIn::make_class(&vm.ctx))? { Self::In - } else if is_node_instance(vm, &object, pyast::NodeCmpOpNotIn::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodeCmpOpNotIn::make_class(&vm.ctx))? { Self::NotIn } else { return Err(vm.new_type_error(format!( diff --git a/crates/vm/src/stdlib/_ast/other.rs b/crates/vm/src/stdlib/_ast/other.rs index 0da6be10c46..d1939ee71fe 100644 --- a/crates/vm/src/stdlib/_ast/other.rs +++ b/crates/vm/src/stdlib/_ast/other.rs @@ -72,7 +72,7 @@ impl Node for ast::Alias { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeAlias::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeAlias::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(vm, source_file), vm) @@ -109,7 +109,7 @@ impl Node for ast::WithItem { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeWithItem::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeWithItem::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item( diff --git a/crates/vm/src/stdlib/_ast/parameter.rs b/crates/vm/src/stdlib/_ast/parameter.rs index ea324eda48c..9be29861418 100644 --- a/crates/vm/src/stdlib/_ast/parameter.rs +++ b/crates/vm/src/stdlib/_ast/parameter.rs @@ -22,7 +22,7 @@ impl Node for ast::Parameters { let (kwonlyargs, kw_defaults) = extract_keyword_parameter_defaults(kwonlyargs.into_iter().collect()); let node = NodeAst - .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeArguments::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item( @@ -166,7 +166,7 @@ impl Node for ast::Parameter { let range = TextRange::new(name.start(), range.end()); let node = NodeAst - .into_ref_with_type(vm, pyast::NodeArg::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeArg::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("arg", name.ast_to_object(vm, source_file), vm) @@ -220,7 +220,7 @@ impl Node for ast::Keyword { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeKeyword::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeKeyword::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("arg", arg.ast_to_object(vm, source_file), vm) diff --git a/crates/vm/src/stdlib/_ast/pattern.rs b/crates/vm/src/stdlib/_ast/pattern.rs index 667bb5fcf45..a3fdefc80e7 100644 --- a/crates/vm/src/stdlib/_ast/pattern.rs +++ b/crates/vm/src/stdlib/_ast/pattern.rs @@ -13,7 +13,7 @@ impl Node for ast::MatchCase { runtime_body, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeMatchCase::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeMatchCase::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("pattern", pattern.ast_to_object(vm, source_file), vm) @@ -84,21 +84,45 @@ impl Node for ast::Pattern { As, Or, } - let kind = if is_node_instance(vm, &object, pyast::NodePatternMatchValue::static_type())? { + let kind = if is_node_instance( + vm, + &object, + pyast::NodePatternMatchValue::make_class(&vm.ctx), + )? { PatternKind::Value - } else if is_node_instance(vm, &object, pyast::NodePatternMatchSingleton::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodePatternMatchSingleton::make_class(&vm.ctx), + )? { PatternKind::Singleton - } else if is_node_instance(vm, &object, pyast::NodePatternMatchSequence::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodePatternMatchSequence::make_class(&vm.ctx), + )? { PatternKind::Sequence - } else if is_node_instance(vm, &object, pyast::NodePatternMatchMapping::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodePatternMatchMapping::make_class(&vm.ctx), + )? { PatternKind::Mapping - } else if is_node_instance(vm, &object, pyast::NodePatternMatchClass::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodePatternMatchClass::make_class(&vm.ctx), + )? { PatternKind::Class - } else if is_node_instance(vm, &object, pyast::NodePatternMatchStar::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodePatternMatchStar::make_class(&vm.ctx), + )? { PatternKind::Star - } else if is_node_instance(vm, &object, pyast::NodePatternMatchAs::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodePatternMatchAs::make_class(&vm.ctx))? { PatternKind::As - } else if is_node_instance(vm, &object, pyast::NodePatternMatchOr::static_type())? { + } else if is_node_instance(vm, &object, pyast::NodePatternMatchOr::make_class(&vm.ctx))? { PatternKind::Or } else { return Err(vm.new_type_error(format!( @@ -222,7 +246,7 @@ impl Node for ast::PatternMatchValue { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodePatternMatchValue::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchValue::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -267,10 +291,7 @@ impl Node for ast::PatternMatchSingleton { range, } = self; let node = NodeAst - .into_ref_with_type( - vm, - pyast::NodePatternMatchSingleton::static_type().to_owned(), - ) + .into_ref_with_type(vm, pyast::NodePatternMatchSingleton::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -341,10 +362,7 @@ impl Node for ast::PatternMatchSequence { runtime_patterns, } = self; let node = NodeAst - .into_ref_with_type( - vm, - pyast::NodePatternMatchSequence::static_type().to_owned(), - ) + .into_ref_with_type(vm, pyast::NodePatternMatchSequence::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let patterns = runtime_patterns.map_or_else( @@ -407,7 +425,7 @@ impl Node for ast::PatternMatchMapping { runtime_patterns, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodePatternMatchMapping::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchMapping::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let keys = runtime_keys.map_or_else( @@ -494,7 +512,7 @@ impl Node for ast::PatternMatchClass { runtime_kwd_patterns, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodePatternMatchClass::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchClass::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("cls", cls.ast_to_object(vm, source_file), vm) @@ -622,7 +640,7 @@ impl Node for ast::PatternMatchStar { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodePatternMatchStar::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchStar::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(vm, source_file), vm) @@ -669,7 +687,7 @@ impl Node for ast::PatternMatchAs { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodePatternMatchAs::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchAs::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("pattern", pattern.ast_to_object(vm, source_file), vm) @@ -716,7 +734,7 @@ impl Node for ast::PatternMatchOr { runtime_patterns, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodePatternMatchOr::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodePatternMatchOr::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let patterns = runtime_patterns.map_or_else( diff --git a/crates/vm/src/stdlib/_ast/pyast.rs b/crates/vm/src/stdlib/_ast/pyast.rs index 46bf0918d15..1e9642eacb1 100644 --- a/crates/vm/src/stdlib/_ast/pyast.rs +++ b/crates/vm/src/stdlib/_ast/pyast.rs @@ -5,12 +5,12 @@ use crate::convert::ToPyObject; macro_rules! impl_node { ( - #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + #[pyclass(interpreter_local, module = $_mod:literal, name = $_name:literal, base = $base:ty)] $vis:vis struct $name:ident, fields: [$($field:expr),* $(,)?], attributes: [$($attr:expr),* $(,)?] $(,)? ) => { - #[pyclass(module = $_mod, name = $_name, base = $base)] + #[pyclass(interpreter_local, module = $_mod, name = $_name, base = $base)] #[repr(transparent)] $vis struct $name($base); @@ -18,12 +18,12 @@ macro_rules! impl_node { }; // Without attributes ( - #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + #[pyclass(interpreter_local, module = $_mod:literal, name = $_name:literal, base = $base:ty)] $vis:vis struct $name:ident, fields: [$($field:expr),* $(,)?] $(,)? ) => { impl_node!( - #[pyclass(module = $_mod, name = $_name, base = $base)] + #[pyclass(interpreter_local, module = $_mod, name = $_name, base = $base)] $vis struct $name, fields: [$($field),*], attributes: [], @@ -31,12 +31,12 @@ macro_rules! impl_node { }; // Without fields ( - #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + #[pyclass(interpreter_local, module = $_mod:literal, name = $_name:literal, base = $base:ty)] $vis:vis struct $name:ident, attributes: [$($attr:expr),* $(,)?] $(,)? ) => { impl_node!( - #[pyclass(module = $_mod, name = $_name, base = $base)] + #[pyclass(interpreter_local, module = $_mod, name = $_name, base = $base)] $vis struct $name, fields: [], attributes: [$($attr),*], @@ -44,11 +44,11 @@ macro_rules! impl_node { }; // Without fields and attributes ( - #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + #[pyclass(interpreter_local, module = $_mod:literal, name = $_name:literal, base = $base:ty)] $vis:vis struct $name:ident $(,)? ) => { impl_node!( - #[pyclass(module = $_mod, name = $_name, base = $base)] + #[pyclass(interpreter_local, module = $_mod, name = $_name, base = $base)] $vis struct $name, fields: [], attributes: [], @@ -65,7 +65,9 @@ macro_rules! impl_base_node { impl_base_node!($name, attributes: [$($attr),*], optional_end_location: false); }; ($name:ident, attributes: [$($attr:expr),* $(,)?], optional_end_location: $optional_end_location:expr) => { - #[pyclass(flags(HAS_DICT, BASETYPE))] + impl AstNodeType for $name {} + + #[pyclass(with(AstNodeType), flags(HAS_DICT, BASETYPE))] impl $name { #[pymethod] fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -82,9 +84,7 @@ macro_rules! impl_base_node { } #[extend_class] - fn extend_class(ctx: &Context, class: &'static Py) { - // AST types are mutable (heap types, not IMMUTABLETYPE). - class.slots.flags.remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + fn extend_class(ctx: &Context, class: &Py) { class.set_attr( identifier!(ctx, _fields), ctx.empty_tuple.clone().into(), @@ -109,7 +109,9 @@ macro_rules! impl_base_node { }; // Leaf node with fields and attributes ($name:ident, fields: [$($field:expr),*], attributes: [$($attr:expr),*]) => { - #[pyclass(flags(HAS_DICT, BASETYPE))] + impl AstNodeType for $name {} + + #[pyclass(with(AstNodeType), flags(HAS_DICT, BASETYPE))] impl $name { #[pymethod] fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -126,9 +128,7 @@ macro_rules! impl_base_node { } #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - // AST types are mutable (heap types, not IMMUTABLETYPE). - class.slots.flags.remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + fn extend_class_with_fields(ctx: &Context, class: &Py) { class.set_attr( identifier!(ctx, _fields), ctx.new_tuple(vec![ @@ -169,30 +169,30 @@ macro_rules! impl_base_node { }; } -#[pyclass(module = "ast", name = "mod", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "mod", base = NodeAst)] pub(crate) struct NodeMod(NodeAst); impl_base_node!(NodeMod); impl_node!( - #[pyclass(module = "ast", name = "Module", base = NodeMod)] + #[pyclass(interpreter_local, module = "ast", name = "Module", base = NodeMod)] pub(crate) struct NodeModModule, fields: ["body", "type_ignores"], ); impl_node!( - #[pyclass(module = "ast", name = "Interactive", base = NodeMod)] + #[pyclass(interpreter_local, module = "ast", name = "Interactive", base = NodeMod)] pub(crate) struct NodeModInteractive, fields: ["body"], ); impl_node!( - #[pyclass(module = "ast", name = "Expression", base = NodeMod)] + #[pyclass(interpreter_local, module = "ast", name = "Expression", base = NodeMod)] pub(crate) struct NodeModExpression, fields: ["body"], ); -#[pyclass(module = "ast", name = "stmt", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "stmt", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeStmt(NodeAst); @@ -203,199 +203,199 @@ impl_base_node!( ); impl_node!( - #[pyclass(module = "ast", name = "FunctionType", base = NodeMod)] + #[pyclass(interpreter_local, module = "ast", name = "FunctionType", base = NodeMod)] pub(crate) struct NodeModFunctionType, fields: ["argtypes", "returns"], ); impl_node!( - #[pyclass(module = "ast", name = "FunctionDef", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "FunctionDef", base = NodeStmt)] pub(crate) struct NodeStmtFunctionDef, fields: ["name", "args", "body", "decorator_list", "returns", "type_comment", "type_params"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "AsyncFunctionDef", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "AsyncFunctionDef", base = NodeStmt)] pub(crate) struct NodeStmtAsyncFunctionDef, fields: ["name", "args", "body", "decorator_list", "returns", "type_comment", "type_params"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "ClassDef", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "ClassDef", base = NodeStmt)] pub(crate) struct NodeStmtClassDef, fields: ["name", "bases", "keywords", "body", "decorator_list", "type_params"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Return", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Return", base = NodeStmt)] pub(crate) struct NodeStmtReturn, fields: ["value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Delete", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Delete", base = NodeStmt)] pub(crate) struct NodeStmtDelete, fields: ["targets"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Assign", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Assign", base = NodeStmt)] pub(crate) struct NodeStmtAssign, fields: ["targets", "value", "type_comment"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "TypeAlias", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "TypeAlias", base = NodeStmt)] pub(crate) struct NodeStmtTypeAlias, fields: ["name", "type_params", "value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "AugAssign", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "AugAssign", base = NodeStmt)] pub(crate) struct NodeStmtAugAssign, fields: ["target", "op", "value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "AnnAssign", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "AnnAssign", base = NodeStmt)] pub(crate) struct NodeStmtAnnAssign, fields: ["target", "annotation", "value", "simple"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "For", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "For", base = NodeStmt)] pub(crate) struct NodeStmtFor, fields: ["target", "iter", "body", "orelse", "type_comment"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "AsyncFor", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "AsyncFor", base = NodeStmt)] pub(crate) struct NodeStmtAsyncFor, fields: ["target", "iter", "body", "orelse", "type_comment"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "While", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "While", base = NodeStmt)] pub(crate) struct NodeStmtWhile, fields: ["test", "body", "orelse"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "If", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "If", base = NodeStmt)] pub(crate) struct NodeStmtIf, fields: ["test", "body", "orelse"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "With", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "With", base = NodeStmt)] pub(crate) struct NodeStmtWith, fields: ["items", "body", "type_comment"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "AsyncWith", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "AsyncWith", base = NodeStmt)] pub(crate) struct NodeStmtAsyncWith, fields: ["items", "body", "type_comment"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Match", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Match", base = NodeStmt)] pub(crate) struct NodeStmtMatch, fields: ["subject", "cases"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Raise", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Raise", base = NodeStmt)] pub(crate) struct NodeStmtRaise, fields: ["exc", "cause"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Try", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Try", base = NodeStmt)] pub(crate) struct NodeStmtTry, fields: ["body", "handlers", "orelse", "finalbody"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "TryStar", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "TryStar", base = NodeStmt)] pub(crate) struct NodeStmtTryStar, fields: ["body", "handlers", "orelse", "finalbody"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Assert", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Assert", base = NodeStmt)] pub(crate) struct NodeStmtAssert, fields: ["test", "msg"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Import", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Import", base = NodeStmt)] pub(crate) struct NodeStmtImport, fields: ["names"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "ImportFrom", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "ImportFrom", base = NodeStmt)] pub(crate) struct NodeStmtImportFrom, fields: ["module", "names", "level"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Global", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Global", base = NodeStmt)] pub(crate) struct NodeStmtGlobal, fields: ["names"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Nonlocal", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Nonlocal", base = NodeStmt)] pub(crate) struct NodeStmtNonlocal, fields: ["names"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Expr", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Expr", base = NodeStmt)] pub(crate) struct NodeStmtExpr, fields: ["value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Pass", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Pass", base = NodeStmt)] pub(crate) struct NodeStmtPass, attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Break", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Break", base = NodeStmt)] pub(crate) struct NodeStmtBreak, attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); -#[pyclass(module = "ast", name = "expr", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "expr", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeExpr(NodeAst); @@ -406,171 +406,168 @@ impl_base_node!( ); impl_node!( - #[pyclass(module = "ast", name = "Continue", base = NodeStmt)] + #[pyclass(interpreter_local, module = "ast", name = "Continue", base = NodeStmt)] pub(crate) struct NodeStmtContinue, attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "BoolOp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "BoolOp", base = NodeExpr)] pub(crate) struct NodeExprBoolOp, fields: ["op", "values"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "NamedExpr", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "NamedExpr", base = NodeExpr)] pub(crate) struct NodeExprNamedExpr, fields: ["target", "value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "BinOp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "BinOp", base = NodeExpr)] pub(crate) struct NodeExprBinOp, fields: ["left", "op", "right"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "UnaryOp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "UnaryOp", base = NodeExpr)] pub(crate) struct NodeExprUnaryOp, fields: ["op", "operand"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Lambda", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Lambda", base = NodeExpr)] pub(crate) struct NodeExprLambda, fields: ["args", "body"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "IfExp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "IfExp", base = NodeExpr)] pub(crate) struct NodeExprIfExp, fields: ["test", "body", "orelse"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Dict", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Dict", base = NodeExpr)] pub(crate) struct NodeExprDict, fields: ["keys", "values"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Set", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Set", base = NodeExpr)] pub(crate) struct NodeExprSet, fields: ["elts"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "ListComp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "ListComp", base = NodeExpr)] pub(crate) struct NodeExprListComp, fields: ["elt", "generators"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "SetComp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "SetComp", base = NodeExpr)] pub(crate) struct NodeExprSetComp, fields: ["elt", "generators"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "DictComp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "DictComp", base = NodeExpr)] pub(crate) struct NodeExprDictComp, fields: ["key", "value", "generators"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "GeneratorExp", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "GeneratorExp", base = NodeExpr)] pub(crate) struct NodeExprGeneratorExp, fields: ["elt", "generators"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Await", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Await", base = NodeExpr)] pub(crate) struct NodeExprAwait, fields: ["value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Yield", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Yield", base = NodeExpr)] pub(crate) struct NodeExprYield, fields: ["value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "YieldFrom", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "YieldFrom", base = NodeExpr)] pub(crate) struct NodeExprYieldFrom, fields: ["value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Compare", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Compare", base = NodeExpr)] pub(crate) struct NodeExprCompare, fields: ["left", "ops", "comparators"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Call", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Call", base = NodeExpr)] pub(crate) struct NodeExprCall, fields: ["func", "args", "keywords"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "FormattedValue", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "FormattedValue", base = NodeExpr)] pub(crate) struct NodeExprFormattedValue, fields: ["value", "conversion", "format_spec"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "JoinedStr", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "JoinedStr", base = NodeExpr)] pub(crate) struct NodeExprJoinedStr, fields: ["values"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "TemplateStr", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "TemplateStr", base = NodeExpr)] pub(crate) struct NodeExprTemplateStr, fields: ["values"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Interpolation", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Interpolation", base = NodeExpr)] pub(crate) struct NodeExprInterpolation, fields: ["value", "str", "conversion", "format_spec"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); -#[pyclass(module = "ast", name = "Constant", base = NodeExpr)] +#[pyclass(interpreter_local, module = "ast", name = "Constant", base = NodeExpr)] #[repr(transparent)] pub(crate) struct NodeExprConstant(NodeExpr); -#[pyclass(flags(HAS_DICT, BASETYPE))] +impl AstNodeType for NodeExprConstant {} + +#[pyclass(with(AstNodeType), flags(HAS_DICT, BASETYPE))] impl NodeExprConstant { #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - // AST types are mutable (heap types, not IMMUTABLETYPE). - class - .slots - .flags - .remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + fn extend_class_with_fields(ctx: &Context, class: &Py) { class.set_attr( identifier!(ctx, _fields), ctx.new_tuple(vec![ @@ -603,245 +600,245 @@ impl NodeExprConstant { } impl_node!( - #[pyclass(module = "ast", name = "Attribute", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Attribute", base = NodeExpr)] pub(crate) struct NodeExprAttribute, fields: ["value", "attr", "ctx"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Subscript", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Subscript", base = NodeExpr)] pub(crate) struct NodeExprSubscript, fields: ["value", "slice", "ctx"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Starred", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Starred", base = NodeExpr)] pub(crate) struct NodeExprStarred, fields: ["value", "ctx"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Name", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Name", base = NodeExpr)] pub(crate) struct NodeExprName, fields: ["id", "ctx"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "List", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "List", base = NodeExpr)] pub(crate) struct NodeExprList, fields: ["elts", "ctx"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Tuple", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Tuple", base = NodeExpr)] pub(crate) struct NodeExprTuple, fields: ["elts", "ctx"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); -#[pyclass(module = "ast", name = "expr_context", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "expr_context", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeExprContext(NodeAst); impl_base_node!(NodeExprContext); impl_node!( - #[pyclass(module = "ast", name = "Slice", base = NodeExpr)] + #[pyclass(interpreter_local, module = "ast", name = "Slice", base = NodeExpr)] pub(crate) struct NodeExprSlice, fields: ["lower", "upper", "step"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "Load", base = NodeExprContext)] + #[pyclass(interpreter_local, module = "ast", name = "Load", base = NodeExprContext)] pub(crate) struct NodeExprContextLoad, ); impl_node!( - #[pyclass(module = "ast", name = "Store", base = NodeExprContext)] + #[pyclass(interpreter_local, module = "ast", name = "Store", base = NodeExprContext)] pub(crate) struct NodeExprContextStore, ); -#[pyclass(module = "ast", name = "boolop", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "boolop", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeBoolOp(NodeAst); impl_base_node!(NodeBoolOp); impl_node!( - #[pyclass(module = "ast", name = "Del", base = NodeExprContext)] + #[pyclass(interpreter_local, module = "ast", name = "Del", base = NodeExprContext)] pub(crate) struct NodeExprContextDel, ); impl_node!( - #[pyclass(module = "ast", name = "And", base = NodeBoolOp)] + #[pyclass(interpreter_local, module = "ast", name = "And", base = NodeBoolOp)] pub(crate) struct NodeBoolOpAnd, ); -#[pyclass(module = "ast", name = "operator", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "operator", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeOperator(NodeAst); impl_base_node!(NodeOperator); impl_node!( - #[pyclass(module = "ast", name = "Or", base = NodeBoolOp)] + #[pyclass(interpreter_local, module = "ast", name = "Or", base = NodeBoolOp)] pub(crate) struct NodeBoolOpOr, ); impl_node!( - #[pyclass(module = "ast", name = "Add", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "Add", base = NodeOperator)] pub(crate) struct NodeOperatorAdd, ); impl_node!( - #[pyclass(module = "ast", name = "Sub", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "Sub", base = NodeOperator)] pub(crate) struct NodeOperatorSub, ); impl_node!( - #[pyclass(module = "ast", name = "Mult", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "Mult", base = NodeOperator)] pub(crate) struct NodeOperatorMult, ); impl_node!( - #[pyclass(module = "ast", name = "MatMult", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "MatMult", base = NodeOperator)] pub(crate) struct NodeOperatorMatMult, ); impl_node!( - #[pyclass(module = "ast", name = "Div", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "Div", base = NodeOperator)] pub(crate) struct NodeOperatorDiv, ); impl_node!( - #[pyclass(module = "ast", name = "Mod", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "Mod", base = NodeOperator)] pub(crate) struct NodeOperatorMod, ); impl_node!( - #[pyclass(module = "ast", name = "Pow", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "Pow", base = NodeOperator)] pub(crate) struct NodeOperatorPow, ); impl_node!( - #[pyclass(module = "ast", name = "LShift", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "LShift", base = NodeOperator)] pub(crate) struct NodeOperatorLShift, ); impl_node!( - #[pyclass(module = "ast", name = "RShift", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "RShift", base = NodeOperator)] pub(crate) struct NodeOperatorRShift, ); impl_node!( - #[pyclass(module = "ast", name = "BitOr", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "BitOr", base = NodeOperator)] pub(crate) struct NodeOperatorBitOr, ); impl_node!( - #[pyclass(module = "ast", name = "BitXor", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "BitXor", base = NodeOperator)] pub(crate) struct NodeOperatorBitXor, ); impl_node!( - #[pyclass(module = "ast", name = "BitAnd", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "BitAnd", base = NodeOperator)] pub(crate) struct NodeOperatorBitAnd, ); -#[pyclass(module = "ast", name = "unaryop", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "unaryop", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeUnaryOp(NodeAst); impl_base_node!(NodeUnaryOp); impl_node!( - #[pyclass(module = "ast", name = "FloorDiv", base = NodeOperator)] + #[pyclass(interpreter_local, module = "ast", name = "FloorDiv", base = NodeOperator)] pub(crate) struct NodeOperatorFloorDiv, ); impl_node!( - #[pyclass(module = "ast", name = "Invert", base = NodeUnaryOp)] + #[pyclass(interpreter_local, module = "ast", name = "Invert", base = NodeUnaryOp)] pub(crate) struct NodeUnaryOpInvert, ); impl_node!( - #[pyclass(module = "ast", name = "Not", base = NodeUnaryOp)] + #[pyclass(interpreter_local, module = "ast", name = "Not", base = NodeUnaryOp)] pub(crate) struct NodeUnaryOpNot, ); impl_node!( - #[pyclass(module = "ast", name = "UAdd", base = NodeUnaryOp)] + #[pyclass(interpreter_local, module = "ast", name = "UAdd", base = NodeUnaryOp)] pub(crate) struct NodeUnaryOpUAdd, ); -#[pyclass(module = "ast", name = "cmpop", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "cmpop", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeCmpOp(NodeAst); impl_base_node!(NodeCmpOp); impl_node!( - #[pyclass(module = "ast", name = "USub", base = NodeUnaryOp)] + #[pyclass(interpreter_local, module = "ast", name = "USub", base = NodeUnaryOp)] pub(crate) struct NodeUnaryOpUSub, ); impl_node!( - #[pyclass(module = "ast", name = "Eq", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "Eq", base = NodeCmpOp)] pub(crate) struct NodeCmpOpEq, ); impl_node!( - #[pyclass(module = "ast", name = "NotEq", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "NotEq", base = NodeCmpOp)] pub(crate) struct NodeCmpOpNotEq, ); impl_node!( - #[pyclass(module = "ast", name = "Lt", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "Lt", base = NodeCmpOp)] pub(crate) struct NodeCmpOpLt, ); impl_node!( - #[pyclass(module = "ast", name = "LtE", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "LtE", base = NodeCmpOp)] pub(crate) struct NodeCmpOpLtE, ); impl_node!( - #[pyclass(module = "ast", name = "Gt", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "Gt", base = NodeCmpOp)] pub(crate) struct NodeCmpOpGt, ); impl_node!( - #[pyclass(module = "ast", name = "GtE", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "GtE", base = NodeCmpOp)] pub(crate) struct NodeCmpOpGtE, ); impl_node!( - #[pyclass(module = "ast", name = "Is", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "Is", base = NodeCmpOp)] pub(crate) struct NodeCmpOpIs, ); impl_node!( - #[pyclass(module = "ast", name = "IsNot", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "IsNot", base = NodeCmpOp)] pub(crate) struct NodeCmpOpIsNot, ); impl_node!( - #[pyclass(module = "ast", name = "In", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "In", base = NodeCmpOp)] pub(crate) struct NodeCmpOpIn, ); impl_node!( - #[pyclass(module = "ast", name = "NotIn", base = NodeCmpOp)] + #[pyclass(interpreter_local, module = "ast", name = "NotIn", base = NodeCmpOp)] pub(crate) struct NodeCmpOpNotIn, ); -#[pyclass(module = "ast", name = "excepthandler", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "excepthandler", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeExceptHandler(NodeAst); @@ -852,52 +849,52 @@ impl_base_node!( ); impl_node!( - #[pyclass(module = "ast", name = "comprehension", base = NodeAst)] + #[pyclass(interpreter_local, module = "ast", name = "comprehension", base = NodeAst)] pub(crate) struct NodeComprehension, fields: ["target", "iter", "ifs", "is_async"], ); impl_node!( - #[pyclass(module = "ast", name = "ExceptHandler", base = NodeExceptHandler)] + #[pyclass(interpreter_local, module = "ast", name = "ExceptHandler", base = NodeExceptHandler)] pub(crate) struct NodeExceptHandlerExceptHandler, fields: ["type", "name", "body"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "arguments", base = NodeAst)] + #[pyclass(interpreter_local, module = "ast", name = "arguments", base = NodeAst)] pub(crate) struct NodeArguments, fields: ["posonlyargs", "args", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults"], ); impl_node!( - #[pyclass(module = "ast", name = "arg", base = NodeAst)] + #[pyclass(interpreter_local, module = "ast", name = "arg", base = NodeAst)] pub(crate) struct NodeArg, fields: ["arg", "annotation", "type_comment"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "keyword", base = NodeAst)] + #[pyclass(interpreter_local, module = "ast", name = "keyword", base = NodeAst)] pub(crate) struct NodeKeyword, fields: ["arg", "value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "alias", base = NodeAst)] + #[pyclass(interpreter_local, module = "ast", name = "alias", base = NodeAst)] pub(crate) struct NodeAlias, fields: ["name", "asname"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "withitem", base = NodeAst)] + #[pyclass(interpreter_local, module = "ast", name = "withitem", base = NodeAst)] pub(crate) struct NodeWithItem, fields: ["context_expr", "optional_vars"], ); -#[pyclass(module = "ast", name = "pattern", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "pattern", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodePattern(NodeAst); @@ -907,74 +904,74 @@ impl_base_node!( ); impl_node!( - #[pyclass(module = "ast", name = "match_case", base = NodeAst)] + #[pyclass(interpreter_local, module = "ast", name = "match_case", base = NodeAst)] pub(crate) struct NodeMatchCase, fields: ["pattern", "guard", "body"], ); impl_node!( - #[pyclass(module = "ast", name = "MatchValue", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchValue", base = NodePattern)] pub(crate) struct NodePatternMatchValue, fields: ["value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "MatchSingleton", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchSingleton", base = NodePattern)] pub(crate) struct NodePatternMatchSingleton, fields: ["value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "MatchSequence", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchSequence", base = NodePattern)] pub(crate) struct NodePatternMatchSequence, fields: ["patterns"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "MatchMapping", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchMapping", base = NodePattern)] pub(crate) struct NodePatternMatchMapping, fields: ["keys", "patterns", "rest"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "MatchClass", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchClass", base = NodePattern)] pub(crate) struct NodePatternMatchClass, fields: ["cls", "patterns", "kwd_attrs", "kwd_patterns"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "MatchStar", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchStar", base = NodePattern)] pub(crate) struct NodePatternMatchStar, fields: ["name"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "MatchAs", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchAs", base = NodePattern)] pub(crate) struct NodePatternMatchAs, fields: ["pattern", "name"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); -#[pyclass(module = "ast", name = "type_ignore", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "type_ignore", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeTypeIgnore(NodeAst); impl_base_node!(NodeTypeIgnore); impl_node!( - #[pyclass(module = "ast", name = "MatchOr", base = NodePattern)] + #[pyclass(interpreter_local, module = "ast", name = "MatchOr", base = NodePattern)] pub(crate) struct NodePatternMatchOr, fields: ["patterns"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); -#[pyclass(module = "ast", name = "type_param", base = NodeAst)] +#[pyclass(interpreter_local, module = "ast", name = "type_param", base = NodeAst)] #[repr(transparent)] pub(crate) struct NodeTypeParam(NodeAst); @@ -984,27 +981,27 @@ impl_base_node!( ); impl_node!( - #[pyclass(module = "ast", name = "TypeIgnore", base = NodeTypeIgnore)] + #[pyclass(interpreter_local, module = "ast", name = "TypeIgnore", base = NodeTypeIgnore)] pub(crate) struct NodeTypeIgnoreTypeIgnore, fields: ["lineno", "tag"], ); impl_node!( - #[pyclass(module = "ast", name = "TypeVar", base = NodeTypeParam)] + #[pyclass(interpreter_local, module = "ast", name = "TypeVar", base = NodeTypeParam)] pub(crate) struct NodeTypeParamTypeVar, fields: ["name", "bound", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "ParamSpec", base = NodeTypeParam)] + #[pyclass(interpreter_local, module = "ast", name = "ParamSpec", base = NodeTypeParam)] pub(crate) struct NodeTypeParamParamSpec, fields: ["name", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( - #[pyclass(module = "ast", name = "TypeVarTuple", base = NodeTypeParam)] + #[pyclass(interpreter_local, module = "ast", name = "TypeVarTuple", base = NodeTypeParam)] pub(crate) struct NodeTypeParamTypeVarTuple, fields: ["name", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], @@ -1535,139 +1532,138 @@ const FIELD_TYPES: &[(&str, &[(&str, FieldType)])] = &[ pub(super) fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { extend_module!(vm, module, { - "AST" => NodeAst::make_static_type(), - "mod" => NodeMod::make_static_type(), - "Module" => NodeModModule::make_static_type(), - "Interactive" => NodeModInteractive::make_static_type(), - "Expression" => NodeModExpression::make_static_type(), - "FunctionType" => NodeModFunctionType::make_static_type(), - "stmt" => NodeStmt::make_static_type(), - "FunctionDef" => NodeStmtFunctionDef::make_static_type(), - "AsyncFunctionDef" => NodeStmtAsyncFunctionDef::make_static_type(), - "ClassDef" => NodeStmtClassDef::make_static_type(), - "Return" => NodeStmtReturn::make_static_type(), - "Delete" => NodeStmtDelete::make_static_type(), - "Assign" => NodeStmtAssign::make_static_type(), - "TypeAlias" => NodeStmtTypeAlias::make_static_type(), - "AugAssign" => NodeStmtAugAssign::make_static_type(), - "AnnAssign" => NodeStmtAnnAssign::make_static_type(), - "For" => NodeStmtFor::make_static_type(), - "AsyncFor" => NodeStmtAsyncFor::make_static_type(), - "While" => NodeStmtWhile::make_static_type(), - "If" => NodeStmtIf::make_static_type(), - "With" => NodeStmtWith::make_static_type(), - "AsyncWith" => NodeStmtAsyncWith::make_static_type(), - "Match" => NodeStmtMatch::make_static_type(), - "Raise" => NodeStmtRaise::make_static_type(), - "Try" => NodeStmtTry::make_static_type(), - "TryStar" => NodeStmtTryStar::make_static_type(), - "Assert" => NodeStmtAssert::make_static_type(), - "Import" => NodeStmtImport::make_static_type(), - "ImportFrom" => NodeStmtImportFrom::make_static_type(), - "Global" => NodeStmtGlobal::make_static_type(), - "Nonlocal" => NodeStmtNonlocal::make_static_type(), - "Expr" => NodeStmtExpr::make_static_type(), - "Pass" => NodeStmtPass::make_static_type(), - "Break" => NodeStmtBreak::make_static_type(), - "Continue" => NodeStmtContinue::make_static_type(), - "expr" => NodeExpr::make_static_type(), - "BoolOp" => NodeExprBoolOp::make_static_type(), - "NamedExpr" => NodeExprNamedExpr::make_static_type(), - "BinOp" => NodeExprBinOp::make_static_type(), - "UnaryOp" => NodeExprUnaryOp::make_static_type(), - "Lambda" => NodeExprLambda::make_static_type(), - "IfExp" => NodeExprIfExp::make_static_type(), - "Dict" => NodeExprDict::make_static_type(), - "Set" => NodeExprSet::make_static_type(), - "ListComp" => NodeExprListComp::make_static_type(), - "SetComp" => NodeExprSetComp::make_static_type(), - "DictComp" => NodeExprDictComp::make_static_type(), - "GeneratorExp" => NodeExprGeneratorExp::make_static_type(), - "Await" => NodeExprAwait::make_static_type(), - "Yield" => NodeExprYield::make_static_type(), - "YieldFrom" => NodeExprYieldFrom::make_static_type(), - "Compare" => NodeExprCompare::make_static_type(), - "Call" => NodeExprCall::make_static_type(), - "FormattedValue" => NodeExprFormattedValue::make_static_type(), - "JoinedStr" => NodeExprJoinedStr::make_static_type(), - "TemplateStr" => NodeExprTemplateStr::make_static_type(), - "Interpolation" => NodeExprInterpolation::make_static_type(), - "Constant" => NodeExprConstant::make_static_type(), - "Attribute" => NodeExprAttribute::make_static_type(), - "Subscript" => NodeExprSubscript::make_static_type(), - "Starred" => NodeExprStarred::make_static_type(), - "Name" => NodeExprName::make_static_type(), - "List" => NodeExprList::make_static_type(), - "Tuple" => NodeExprTuple::make_static_type(), - "Slice" => NodeExprSlice::make_static_type(), - "expr_context" => NodeExprContext::make_static_type(), - "Load" => NodeExprContextLoad::make_static_type(), - "Store" => NodeExprContextStore::make_static_type(), - "Del" => NodeExprContextDel::make_static_type(), - "boolop" => NodeBoolOp::make_static_type(), - "And" => NodeBoolOpAnd::make_static_type(), - "Or" => NodeBoolOpOr::make_static_type(), - "operator" => NodeOperator::make_static_type(), - "Add" => NodeOperatorAdd::make_static_type(), - "Sub" => NodeOperatorSub::make_static_type(), - "Mult" => NodeOperatorMult::make_static_type(), - "MatMult" => NodeOperatorMatMult::make_static_type(), - "Div" => NodeOperatorDiv::make_static_type(), - "Mod" => NodeOperatorMod::make_static_type(), - "Pow" => NodeOperatorPow::make_static_type(), - "LShift" => NodeOperatorLShift::make_static_type(), - "RShift" => NodeOperatorRShift::make_static_type(), - "BitOr" => NodeOperatorBitOr::make_static_type(), - "BitXor" => NodeOperatorBitXor::make_static_type(), - "BitAnd" => NodeOperatorBitAnd::make_static_type(), - "FloorDiv" => NodeOperatorFloorDiv::make_static_type(), - "unaryop" => NodeUnaryOp::make_static_type(), - "Invert" => NodeUnaryOpInvert::make_static_type(), - "Not" => NodeUnaryOpNot::make_static_type(), - "UAdd" => NodeUnaryOpUAdd::make_static_type(), - "USub" => NodeUnaryOpUSub::make_static_type(), - "cmpop" => NodeCmpOp::make_static_type(), - "Eq" => NodeCmpOpEq::make_static_type(), - "NotEq" => NodeCmpOpNotEq::make_static_type(), - "Lt" => NodeCmpOpLt::make_static_type(), - "LtE" => NodeCmpOpLtE::make_static_type(), - "Gt" => NodeCmpOpGt::make_static_type(), - "GtE" => NodeCmpOpGtE::make_static_type(), - "Is" => NodeCmpOpIs::make_static_type(), - "IsNot" => NodeCmpOpIsNot::make_static_type(), - "In" => NodeCmpOpIn::make_static_type(), - "NotIn" => NodeCmpOpNotIn::make_static_type(), - "comprehension" => NodeComprehension::make_static_type(), - "excepthandler" => NodeExceptHandler::make_static_type(), - "ExceptHandler" => NodeExceptHandlerExceptHandler::make_static_type(), - "arguments" => NodeArguments::make_static_type(), - "arg" => NodeArg::make_static_type(), - "keyword" => NodeKeyword::make_static_type(), - "alias" => NodeAlias::make_static_type(), - "withitem" => NodeWithItem::make_static_type(), - "match_case" => NodeMatchCase::make_static_type(), - "pattern" => NodePattern::make_static_type(), - "MatchValue" => NodePatternMatchValue::make_static_type(), - "MatchSingleton" => NodePatternMatchSingleton::make_static_type(), - "MatchSequence" => NodePatternMatchSequence::make_static_type(), - "MatchMapping" => NodePatternMatchMapping::make_static_type(), - "MatchClass" => NodePatternMatchClass::make_static_type(), - "MatchStar" => NodePatternMatchStar::make_static_type(), - "MatchAs" => NodePatternMatchAs::make_static_type(), - "MatchOr" => NodePatternMatchOr::make_static_type(), - "type_ignore" => NodeTypeIgnore::make_static_type(), - "TypeIgnore" => NodeTypeIgnoreTypeIgnore::make_static_type(), - "type_param" => NodeTypeParam::make_static_type(), - "TypeVar" => NodeTypeParamTypeVar::make_static_type(), - "ParamSpec" => NodeTypeParamParamSpec::make_static_type(), - "TypeVarTuple" => NodeTypeParamTypeVarTuple::make_static_type(), + "AST" => NodeAst::make_class(&vm.ctx), + "mod" => NodeMod::make_class(&vm.ctx), + "Module" => NodeModModule::make_class(&vm.ctx), + "Interactive" => NodeModInteractive::make_class(&vm.ctx), + "Expression" => NodeModExpression::make_class(&vm.ctx), + "FunctionType" => NodeModFunctionType::make_class(&vm.ctx), + "stmt" => NodeStmt::make_class(&vm.ctx), + "FunctionDef" => NodeStmtFunctionDef::make_class(&vm.ctx), + "AsyncFunctionDef" => NodeStmtAsyncFunctionDef::make_class(&vm.ctx), + "ClassDef" => NodeStmtClassDef::make_class(&vm.ctx), + "Return" => NodeStmtReturn::make_class(&vm.ctx), + "Delete" => NodeStmtDelete::make_class(&vm.ctx), + "Assign" => NodeStmtAssign::make_class(&vm.ctx), + "TypeAlias" => NodeStmtTypeAlias::make_class(&vm.ctx), + "AugAssign" => NodeStmtAugAssign::make_class(&vm.ctx), + "AnnAssign" => NodeStmtAnnAssign::make_class(&vm.ctx), + "For" => NodeStmtFor::make_class(&vm.ctx), + "AsyncFor" => NodeStmtAsyncFor::make_class(&vm.ctx), + "While" => NodeStmtWhile::make_class(&vm.ctx), + "If" => NodeStmtIf::make_class(&vm.ctx), + "With" => NodeStmtWith::make_class(&vm.ctx), + "AsyncWith" => NodeStmtAsyncWith::make_class(&vm.ctx), + "Match" => NodeStmtMatch::make_class(&vm.ctx), + "Raise" => NodeStmtRaise::make_class(&vm.ctx), + "Try" => NodeStmtTry::make_class(&vm.ctx), + "TryStar" => NodeStmtTryStar::make_class(&vm.ctx), + "Assert" => NodeStmtAssert::make_class(&vm.ctx), + "Import" => NodeStmtImport::make_class(&vm.ctx), + "ImportFrom" => NodeStmtImportFrom::make_class(&vm.ctx), + "Global" => NodeStmtGlobal::make_class(&vm.ctx), + "Nonlocal" => NodeStmtNonlocal::make_class(&vm.ctx), + "Expr" => NodeStmtExpr::make_class(&vm.ctx), + "Pass" => NodeStmtPass::make_class(&vm.ctx), + "Break" => NodeStmtBreak::make_class(&vm.ctx), + "Continue" => NodeStmtContinue::make_class(&vm.ctx), + "expr" => NodeExpr::make_class(&vm.ctx), + "BoolOp" => NodeExprBoolOp::make_class(&vm.ctx), + "NamedExpr" => NodeExprNamedExpr::make_class(&vm.ctx), + "BinOp" => NodeExprBinOp::make_class(&vm.ctx), + "UnaryOp" => NodeExprUnaryOp::make_class(&vm.ctx), + "Lambda" => NodeExprLambda::make_class(&vm.ctx), + "IfExp" => NodeExprIfExp::make_class(&vm.ctx), + "Dict" => NodeExprDict::make_class(&vm.ctx), + "Set" => NodeExprSet::make_class(&vm.ctx), + "ListComp" => NodeExprListComp::make_class(&vm.ctx), + "SetComp" => NodeExprSetComp::make_class(&vm.ctx), + "DictComp" => NodeExprDictComp::make_class(&vm.ctx), + "GeneratorExp" => NodeExprGeneratorExp::make_class(&vm.ctx), + "Await" => NodeExprAwait::make_class(&vm.ctx), + "Yield" => NodeExprYield::make_class(&vm.ctx), + "YieldFrom" => NodeExprYieldFrom::make_class(&vm.ctx), + "Compare" => NodeExprCompare::make_class(&vm.ctx), + "Call" => NodeExprCall::make_class(&vm.ctx), + "FormattedValue" => NodeExprFormattedValue::make_class(&vm.ctx), + "JoinedStr" => NodeExprJoinedStr::make_class(&vm.ctx), + "TemplateStr" => NodeExprTemplateStr::make_class(&vm.ctx), + "Interpolation" => NodeExprInterpolation::make_class(&vm.ctx), + "Constant" => NodeExprConstant::make_class(&vm.ctx), + "Attribute" => NodeExprAttribute::make_class(&vm.ctx), + "Subscript" => NodeExprSubscript::make_class(&vm.ctx), + "Starred" => NodeExprStarred::make_class(&vm.ctx), + "Name" => NodeExprName::make_class(&vm.ctx), + "List" => NodeExprList::make_class(&vm.ctx), + "Tuple" => NodeExprTuple::make_class(&vm.ctx), + "Slice" => NodeExprSlice::make_class(&vm.ctx), + "expr_context" => NodeExprContext::make_class(&vm.ctx), + "Load" => NodeExprContextLoad::make_class(&vm.ctx), + "Store" => NodeExprContextStore::make_class(&vm.ctx), + "Del" => NodeExprContextDel::make_class(&vm.ctx), + "boolop" => NodeBoolOp::make_class(&vm.ctx), + "And" => NodeBoolOpAnd::make_class(&vm.ctx), + "Or" => NodeBoolOpOr::make_class(&vm.ctx), + "operator" => NodeOperator::make_class(&vm.ctx), + "Add" => NodeOperatorAdd::make_class(&vm.ctx), + "Sub" => NodeOperatorSub::make_class(&vm.ctx), + "Mult" => NodeOperatorMult::make_class(&vm.ctx), + "MatMult" => NodeOperatorMatMult::make_class(&vm.ctx), + "Div" => NodeOperatorDiv::make_class(&vm.ctx), + "Mod" => NodeOperatorMod::make_class(&vm.ctx), + "Pow" => NodeOperatorPow::make_class(&vm.ctx), + "LShift" => NodeOperatorLShift::make_class(&vm.ctx), + "RShift" => NodeOperatorRShift::make_class(&vm.ctx), + "BitOr" => NodeOperatorBitOr::make_class(&vm.ctx), + "BitXor" => NodeOperatorBitXor::make_class(&vm.ctx), + "BitAnd" => NodeOperatorBitAnd::make_class(&vm.ctx), + "FloorDiv" => NodeOperatorFloorDiv::make_class(&vm.ctx), + "unaryop" => NodeUnaryOp::make_class(&vm.ctx), + "Invert" => NodeUnaryOpInvert::make_class(&vm.ctx), + "Not" => NodeUnaryOpNot::make_class(&vm.ctx), + "UAdd" => NodeUnaryOpUAdd::make_class(&vm.ctx), + "USub" => NodeUnaryOpUSub::make_class(&vm.ctx), + "cmpop" => NodeCmpOp::make_class(&vm.ctx), + "Eq" => NodeCmpOpEq::make_class(&vm.ctx), + "NotEq" => NodeCmpOpNotEq::make_class(&vm.ctx), + "Lt" => NodeCmpOpLt::make_class(&vm.ctx), + "LtE" => NodeCmpOpLtE::make_class(&vm.ctx), + "Gt" => NodeCmpOpGt::make_class(&vm.ctx), + "GtE" => NodeCmpOpGtE::make_class(&vm.ctx), + "Is" => NodeCmpOpIs::make_class(&vm.ctx), + "IsNot" => NodeCmpOpIsNot::make_class(&vm.ctx), + "In" => NodeCmpOpIn::make_class(&vm.ctx), + "NotIn" => NodeCmpOpNotIn::make_class(&vm.ctx), + "comprehension" => NodeComprehension::make_class(&vm.ctx), + "excepthandler" => NodeExceptHandler::make_class(&vm.ctx), + "ExceptHandler" => NodeExceptHandlerExceptHandler::make_class(&vm.ctx), + "arguments" => NodeArguments::make_class(&vm.ctx), + "arg" => NodeArg::make_class(&vm.ctx), + "keyword" => NodeKeyword::make_class(&vm.ctx), + "alias" => NodeAlias::make_class(&vm.ctx), + "withitem" => NodeWithItem::make_class(&vm.ctx), + "match_case" => NodeMatchCase::make_class(&vm.ctx), + "pattern" => NodePattern::make_class(&vm.ctx), + "MatchValue" => NodePatternMatchValue::make_class(&vm.ctx), + "MatchSingleton" => NodePatternMatchSingleton::make_class(&vm.ctx), + "MatchSequence" => NodePatternMatchSequence::make_class(&vm.ctx), + "MatchMapping" => NodePatternMatchMapping::make_class(&vm.ctx), + "MatchClass" => NodePatternMatchClass::make_class(&vm.ctx), + "MatchStar" => NodePatternMatchStar::make_class(&vm.ctx), + "MatchAs" => NodePatternMatchAs::make_class(&vm.ctx), + "MatchOr" => NodePatternMatchOr::make_class(&vm.ctx), + "type_ignore" => NodeTypeIgnore::make_class(&vm.ctx), + "TypeIgnore" => NodeTypeIgnoreTypeIgnore::make_class(&vm.ctx), + "type_param" => NodeTypeParam::make_class(&vm.ctx), + "TypeVar" => NodeTypeParamTypeVar::make_class(&vm.ctx), + "ParamSpec" => NodeTypeParamParamSpec::make_class(&vm.ctx), + "TypeVarTuple" => NodeTypeParamTypeVarTuple::make_class(&vm.ctx), }); // Populate _field_types with real Python type objects populate_field_types(vm, module); populate_singletons(vm, module); force_ast_module_name(vm, module); - populate_repr(vm, module); } fn populate_field_types(vm: &VirtualMachine, module: &Py) { @@ -1806,15 +1802,3 @@ fn force_ast_module_name(vm: &VirtualMachine, module: &Py) { type_obj.set_attr(identifier!(vm, __module__), ast_name.clone().into()); } } - -fn populate_repr(_vm: &VirtualMachine, module: &Py) { - for (_name, value) in &module.dict() { - let Some(type_obj) = value.downcast_ref::() else { - continue; - }; - type_obj - .slots - .repr - .store(Some(super::python::_ast::ast_repr)); - } -} diff --git a/crates/vm/src/stdlib/_ast/python.rs b/crates/vm/src/stdlib/_ast/python.rs index cb9027aeb99..08ad3a0b8a5 100644 --- a/crates/vm/src/stdlib/_ast/python.rs +++ b/crates/vm/src/stdlib/_ast/python.rs @@ -9,26 +9,23 @@ pub(crate) mod _ast { use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDict, PyDictRef, PySet, PyStr, PyTupleRef, PyType, PyTypeRef}, - class::{PyClassImpl, StaticType}, + class::PyClassImpl, function::{ArgIterable, FuncArgs, KwArgs, PyMethodDef, PyMethodFlags}, - stdlib::_ast::repr, + stdlib::_ast::{AstNodeType, repr}, types::{Constructor, Initializer}, warn, }; #[pyattr] - #[pyclass(module = "ast", name = "AST")] + #[pyclass(interpreter_local, module = "ast", name = "AST")] #[derive(Debug, PyPayload)] pub(crate) struct NodeAst; - #[pyclass(with(Constructor, Initializer), flags(BASETYPE, HAS_DICT))] + impl AstNodeType for NodeAst {} + + #[pyclass(with(Constructor, Initializer, AstNodeType), flags(BASETYPE, HAS_DICT))] impl NodeAst { #[extend_class] - fn extend_class(ctx: &Context, class: &'static Py) { - // AST types are mutable (heap types, not IMMUTABLETYPE). - class - .slots - .flags - .remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + fn extend_class(ctx: &Context, class: &Py) { let empty_tuple = ctx.empty_tuple.clone(); class.set_str_attr("_fields", empty_tuple.clone(), ctx); class.set_str_attr("_attributes", empty_tuple.clone(), ctx); @@ -48,7 +45,6 @@ pub(crate) mod _ast { AST_DEEPCOPY.to_proper_method(class, ctx), ctx, ); - class.slots.repr.store(Some(ast_repr)); } #[pyattr] @@ -436,7 +432,7 @@ Support for arbitrary keyword arguments is deprecated and will be removed in Pyt field_types.map(|ft| ft.downcast::()) { let expr_ctx_type: PyObjectRef = - super::super::pyast::NodeExprContext::make_static_type().into(); + super::super::pyast::NodeExprContext::make_class(&vm.ctx).into(); for field in remaining_fields.elements() { if let Some(ftype) = ft_dict.get_item_opt(&*field, vm)? { @@ -449,7 +445,7 @@ Support for arbitrary keyword arguments is deprecated and will be removed in Pyt } else if ftype.is(&expr_ctx_type) { // expr_context — default to Load() let load_type = - super::super::pyast::NodeExprContextLoad::make_static_type(); + super::super::pyast::NodeExprContextLoad::make_class(&vm.ctx); let load_instance = load_type .get_attr(vm.ctx.intern_str("_instance")) .unwrap_or_else(|| { @@ -545,20 +541,20 @@ This will become an error in Python 3.15.", ast_type.set_str_attr("_attributes", empty_tuple.clone(), ctx); ast_type.set_str_attr("__match_args__", empty_tuple, ctx); for typ in [ - super::super::pyast::NodeMod::static_type(), - super::super::pyast::NodeStmt::static_type(), - super::super::pyast::NodeExpr::static_type(), - super::super::pyast::NodeExprContext::static_type(), - super::super::pyast::NodeBoolOp::static_type(), - super::super::pyast::NodeOperator::static_type(), - super::super::pyast::NodeUnaryOp::static_type(), - super::super::pyast::NodeCmpOp::static_type(), - super::super::pyast::NodeExceptHandler::static_type(), - super::super::pyast::NodePattern::static_type(), - super::super::pyast::NodeTypeIgnore::static_type(), - super::super::pyast::NodeTypeParam::static_type(), + super::super::pyast::NodeMod::make_class(&vm.ctx), + super::super::pyast::NodeStmt::make_class(&vm.ctx), + super::super::pyast::NodeExpr::make_class(&vm.ctx), + super::super::pyast::NodeExprContext::make_class(&vm.ctx), + super::super::pyast::NodeBoolOp::make_class(&vm.ctx), + super::super::pyast::NodeOperator::make_class(&vm.ctx), + super::super::pyast::NodeUnaryOp::make_class(&vm.ctx), + super::super::pyast::NodeCmpOp::make_class(&vm.ctx), + super::super::pyast::NodeExceptHandler::make_class(&vm.ctx), + super::super::pyast::NodePattern::make_class(&vm.ctx), + super::super::pyast::NodeTypeIgnore::make_class(&vm.ctx), + super::super::pyast::NodeTypeParam::make_class(&vm.ctx), ] { - set_empty_annotations(typ); + set_empty_annotations(&typ); } const AST_REDUCE: PyMethodDef = PyMethodDef::new_const( @@ -579,49 +575,50 @@ This will become an error in Python 3.15.", "__replace__($self, /, **fields)\n--\n\nReturn a copy of the AST node with new values for the specified fields.", ), ); - let base_type = NodeAst::static_type(); + let base_type = NodeAst::make_class(&vm.ctx); ast_type.set_str_attr( "__reduce__", - AST_REDUCE.to_proper_method(base_type, ctx), + AST_REDUCE.to_proper_method(&base_type, ctx), ctx, ); ast_type.set_str_attr( "__replace__", - AST_REPLACE.to_proper_method(base_type, ctx), + AST_REPLACE.to_proper_method(&base_type, ctx), ctx, ); - ast_type.slots.repr.store(Some(ast_repr)); - - const EXPR_DOC: &str = "expr = BoolOp(boolop op, expr* values)\n\ - | NamedExpr(expr target, expr value)\n\ - | BinOp(expr left, operator op, expr right)\n\ - | UnaryOp(unaryop op, expr operand)\n\ - | Lambda(arguments args, expr body)\n\ - | IfExp(expr test, expr body, expr orelse)\n\ - | Dict(expr?* keys, expr* values)\n\ - | Set(expr* elts)\n\ - | ListComp(expr elt, comprehension* generators)\n\ - | SetComp(expr elt, comprehension* generators)\n\ - | DictComp(expr key, expr value, comprehension* generators)\n\ - | GeneratorExp(expr elt, comprehension* generators)\n\ - | Await(expr value)\n\ - | Yield(expr? value)\n\ - | YieldFrom(expr value)\n\ - | Compare(expr left, cmpop* ops, expr* comparators)\n\ - | Call(expr func, expr* args, keyword* keywords)\n\ - | FormattedValue(expr value, int conversion, expr? format_spec)\n\ - | Interpolation(expr value, constant str, int conversion, expr? format_spec)\n\ - | JoinedStr(expr* values)\n\ - | TemplateStr(expr* values)\n\ - | Constant(constant value, string? kind)\n\ - | Attribute(expr value, identifier attr, expr_context ctx)\n\ - | Subscript(expr value, expr slice, expr_context ctx)\n\ - | Starred(expr value, expr_context ctx)\n\ - | Name(identifier id, expr_context ctx)\n\ - | List(expr* elts, expr_context ctx)\n\ - | Tuple(expr* elts, expr_context ctx)\n\ - | Slice(expr? lower, expr? upper, expr? step)"; - let expr_type = super::super::pyast::NodeExpr::static_type(); + + const EXPR_DOC: &str = concat!( + "expr = BoolOp(boolop op, expr* values)", + "\n | NamedExpr(expr target, expr value)", + "\n | BinOp(expr left, operator op, expr right)", + "\n | UnaryOp(unaryop op, expr operand)", + "\n | Lambda(arguments args, expr body)", + "\n | IfExp(expr test, expr body, expr orelse)", + "\n | Dict(expr?* keys, expr* values)", + "\n | Set(expr* elts)", + "\n | ListComp(expr elt, comprehension* generators)", + "\n | SetComp(expr elt, comprehension* generators)", + "\n | DictComp(expr key, expr value, comprehension* generators)", + "\n | GeneratorExp(expr elt, comprehension* generators)", + "\n | Await(expr value)", + "\n | Yield(expr? value)", + "\n | YieldFrom(expr value)", + "\n | Compare(expr left, cmpop* ops, expr* comparators)", + "\n | Call(expr func, expr* args, keyword* keywords)", + "\n | FormattedValue(expr value, int conversion, expr? format_spec)", + "\n | Interpolation(expr value, constant str, int conversion, expr? format_spec)", + "\n | JoinedStr(expr* values)", + "\n | TemplateStr(expr* values)", + "\n | Constant(constant value, string? kind)", + "\n | Attribute(expr value, identifier attr, expr_context ctx)", + "\n | Subscript(expr value, expr slice, expr_context ctx)", + "\n | Starred(expr value, expr_context ctx)", + "\n | Name(identifier id, expr_context ctx)", + "\n | List(expr* elts, expr_context ctx)", + "\n | Tuple(expr* elts, expr_context ctx)", + "\n | Slice(expr? lower, expr? upper, expr? step)", + ); + let expr_type = super::super::pyast::NodeExpr::make_class(&vm.ctx); expr_type.set_attr( identifier!(vm.ctx, __doc__), vm.ctx.new_str(EXPR_DOC).into(), diff --git a/crates/vm/src/stdlib/_ast/repr.rs b/crates/vm/src/stdlib/_ast/repr.rs index 380c2508f57..75ed6f1db84 100644 --- a/crates/vm/src/stdlib/_ast/repr.rs +++ b/crates/vm/src/stdlib/_ast/repr.rs @@ -21,7 +21,7 @@ fn repr_ast_list(vm: &VirtualMachine, items: Vec, depth: usize) -> if idx == 1 && items.len() == 1 { break; } - let repr = if item.fast_isinstance(&NodeAst::make_static_type()) { + let repr = if item.fast_isinstance(&NodeAst::make_class(&vm.ctx)) { repr_ast_node(vm, item, depth.saturating_sub(1))? } else { item.repr(vm)?.as_wtf8().to_owned() @@ -61,7 +61,7 @@ fn repr_ast_tuple(vm: &VirtualMachine, items: Vec, depth: usize) -> if idx == 1 && items.len() == 1 { break; } - let repr = if item.fast_isinstance(&NodeAst::make_static_type()) { + let repr = if item.fast_isinstance(&NodeAst::make_class(&vm.ctx)) { repr_ast_node(vm, item, depth.saturating_sub(1))? } else { item.repr(vm)?.as_wtf8().to_owned() @@ -140,7 +140,7 @@ pub(crate) fn repr_ast_node( .downcast::() .expect("tuple type should downcast"); repr_ast_tuple(vm, tuple.as_slice().to_vec(), depth)? - } else if value.fast_isinstance(&NodeAst::make_static_type()) { + } else if value.fast_isinstance(&NodeAst::make_class(&vm.ctx)) { repr_ast_node(vm, &value, depth.saturating_sub(1))? } else { value.repr(vm)?.as_wtf8().to_owned() diff --git a/crates/vm/src/stdlib/_ast/statement.rs b/crates/vm/src/stdlib/_ast/statement.rs index cfdf3e5f530..3f7359d499c 100644 --- a/crates/vm/src/stdlib/_ast/statement.rs +++ b/crates/vm/src/stdlib/_ast/statement.rs @@ -130,68 +130,78 @@ impl Node for ast::Stmt { Break, Continue, } - let kind = if is_node_instance(vm, &object, pyast::NodeStmtFunctionDef::static_type())? { - StmtKind::FunctionDef { is_async: false } - } else if is_node_instance(vm, &object, pyast::NodeStmtAsyncFunctionDef::static_type())? { - StmtKind::FunctionDef { is_async: true } - } else if is_node_instance(vm, &object, pyast::NodeStmtClassDef::static_type())? { - StmtKind::ClassDef - } else if is_node_instance(vm, &object, pyast::NodeStmtReturn::static_type())? { - StmtKind::Return - } else if is_node_instance(vm, &object, pyast::NodeStmtDelete::static_type())? { - StmtKind::Delete - } else if is_node_instance(vm, &object, pyast::NodeStmtAssign::static_type())? { - StmtKind::Assign - } else if is_node_instance(vm, &object, pyast::NodeStmtTypeAlias::static_type())? { - StmtKind::TypeAlias - } else if is_node_instance(vm, &object, pyast::NodeStmtAugAssign::static_type())? { - StmtKind::AugAssign - } else if is_node_instance(vm, &object, pyast::NodeStmtAnnAssign::static_type())? { - StmtKind::AnnAssign - } else if is_node_instance(vm, &object, pyast::NodeStmtFor::static_type())? { - StmtKind::For { is_async: false } - } else if is_node_instance(vm, &object, pyast::NodeStmtAsyncFor::static_type())? { - StmtKind::For { is_async: true } - } else if is_node_instance(vm, &object, pyast::NodeStmtWhile::static_type())? { - StmtKind::While - } else if is_node_instance(vm, &object, pyast::NodeStmtIf::static_type())? { - StmtKind::If - } else if is_node_instance(vm, &object, pyast::NodeStmtWith::static_type())? { - StmtKind::With { is_async: false } - } else if is_node_instance(vm, &object, pyast::NodeStmtAsyncWith::static_type())? { - StmtKind::With { is_async: true } - } else if is_node_instance(vm, &object, pyast::NodeStmtMatch::static_type())? { - StmtKind::Match - } else if is_node_instance(vm, &object, pyast::NodeStmtRaise::static_type())? { - StmtKind::Raise - } else if is_node_instance(vm, &object, pyast::NodeStmtTry::static_type())? { - StmtKind::Try { is_star: false } - } else if is_node_instance(vm, &object, pyast::NodeStmtTryStar::static_type())? { - StmtKind::Try { is_star: true } - } else if is_node_instance(vm, &object, pyast::NodeStmtAssert::static_type())? { - StmtKind::Assert - } else if is_node_instance(vm, &object, pyast::NodeStmtImport::static_type())? { - StmtKind::Import - } else if is_node_instance(vm, &object, pyast::NodeStmtImportFrom::static_type())? { - StmtKind::ImportFrom - } else if is_node_instance(vm, &object, pyast::NodeStmtGlobal::static_type())? { - StmtKind::Global - } else if is_node_instance(vm, &object, pyast::NodeStmtNonlocal::static_type())? { - StmtKind::Nonlocal - } else if is_node_instance(vm, &object, pyast::NodeStmtExpr::static_type())? { - StmtKind::Expr - } else if is_node_instance(vm, &object, pyast::NodeStmtPass::static_type())? { - StmtKind::Pass - } else if is_node_instance(vm, &object, pyast::NodeStmtBreak::static_type())? { - StmtKind::Break - } else if is_node_instance(vm, &object, pyast::NodeStmtContinue::static_type())? { - StmtKind::Continue - } else { - return Err(vm.new_type_error(format!( - "expected some sort of stmt, but got {}", - object.repr(vm)? - ))); - }; + let kind = + if is_node_instance(vm, &object, pyast::NodeStmtFunctionDef::make_class(&vm.ctx))? { + StmtKind::FunctionDef { is_async: false } + } else if is_node_instance( + vm, + &object, + pyast::NodeStmtAsyncFunctionDef::make_class(&vm.ctx), + )? { + StmtKind::FunctionDef { is_async: true } + } else if is_node_instance(vm, &object, pyast::NodeStmtClassDef::make_class(&vm.ctx))? { + StmtKind::ClassDef + } else if is_node_instance(vm, &object, pyast::NodeStmtReturn::make_class(&vm.ctx))? { + StmtKind::Return + } else if is_node_instance(vm, &object, pyast::NodeStmtDelete::make_class(&vm.ctx))? { + StmtKind::Delete + } else if is_node_instance(vm, &object, pyast::NodeStmtAssign::make_class(&vm.ctx))? { + StmtKind::Assign + } else if is_node_instance(vm, &object, pyast::NodeStmtTypeAlias::make_class(&vm.ctx))? + { + StmtKind::TypeAlias + } else if is_node_instance(vm, &object, pyast::NodeStmtAugAssign::make_class(&vm.ctx))? + { + StmtKind::AugAssign + } else if is_node_instance(vm, &object, pyast::NodeStmtAnnAssign::make_class(&vm.ctx))? + { + StmtKind::AnnAssign + } else if is_node_instance(vm, &object, pyast::NodeStmtFor::make_class(&vm.ctx))? { + StmtKind::For { is_async: false } + } else if is_node_instance(vm, &object, pyast::NodeStmtAsyncFor::make_class(&vm.ctx))? { + StmtKind::For { is_async: true } + } else if is_node_instance(vm, &object, pyast::NodeStmtWhile::make_class(&vm.ctx))? { + StmtKind::While + } else if is_node_instance(vm, &object, pyast::NodeStmtIf::make_class(&vm.ctx))? { + StmtKind::If + } else if is_node_instance(vm, &object, pyast::NodeStmtWith::make_class(&vm.ctx))? { + StmtKind::With { is_async: false } + } else if is_node_instance(vm, &object, pyast::NodeStmtAsyncWith::make_class(&vm.ctx))? + { + StmtKind::With { is_async: true } + } else if is_node_instance(vm, &object, pyast::NodeStmtMatch::make_class(&vm.ctx))? { + StmtKind::Match + } else if is_node_instance(vm, &object, pyast::NodeStmtRaise::make_class(&vm.ctx))? { + StmtKind::Raise + } else if is_node_instance(vm, &object, pyast::NodeStmtTry::make_class(&vm.ctx))? { + StmtKind::Try { is_star: false } + } else if is_node_instance(vm, &object, pyast::NodeStmtTryStar::make_class(&vm.ctx))? { + StmtKind::Try { is_star: true } + } else if is_node_instance(vm, &object, pyast::NodeStmtAssert::make_class(&vm.ctx))? { + StmtKind::Assert + } else if is_node_instance(vm, &object, pyast::NodeStmtImport::make_class(&vm.ctx))? { + StmtKind::Import + } else if is_node_instance(vm, &object, pyast::NodeStmtImportFrom::make_class(&vm.ctx))? + { + StmtKind::ImportFrom + } else if is_node_instance(vm, &object, pyast::NodeStmtGlobal::make_class(&vm.ctx))? { + StmtKind::Global + } else if is_node_instance(vm, &object, pyast::NodeStmtNonlocal::make_class(&vm.ctx))? { + StmtKind::Nonlocal + } else if is_node_instance(vm, &object, pyast::NodeStmtExpr::make_class(&vm.ctx))? { + StmtKind::Expr + } else if is_node_instance(vm, &object, pyast::NodeStmtPass::make_class(&vm.ctx))? { + StmtKind::Pass + } else if is_node_instance(vm, &object, pyast::NodeStmtBreak::make_class(&vm.ctx))? { + StmtKind::Break + } else if is_node_instance(vm, &object, pyast::NodeStmtContinue::make_class(&vm.ctx))? { + StmtKind::Continue + } else { + return Err(vm.new_type_error(format!( + "expected some sort of stmt, but got {}", + object.repr(vm)? + ))); + }; let range = stmt_range_from_object(vm, source_file, &object)?; Ok(match kind { StmtKind::FunctionDef { is_async } => { @@ -408,9 +418,9 @@ impl Node for ast::StmtFunctionDef { ); let cls = if !is_async { - pyast::NodeStmtFunctionDef::static_type().to_owned() + pyast::NodeStmtFunctionDef::make_class(&vm.ctx) } else { - pyast::NodeStmtAsyncFunctionDef::static_type().to_owned() + pyast::NodeStmtAsyncFunctionDef::make_class(&vm.ctx) }; let node = NodeAst.into_ref_with_type(vm, cls).unwrap(); @@ -459,8 +469,11 @@ impl Node for ast::StmtFunctionDef { source_file: &SourceFile, _object: PyObjectRef, ) -> PyResult { - let is_async = - is_node_instance(vm, &_object, pyast::NodeStmtAsyncFunctionDef::static_type())?; + let is_async = is_node_instance( + vm, + &_object, + pyast::NodeStmtAsyncFunctionDef::make_class(&vm.ctx), + )?; let typ = if is_async { "AsyncFunctionDef" } else { @@ -522,7 +535,7 @@ impl Node for ast::StmtClassDef { let range = definition_range_from_name(source_file, name.range.start(), range.end(), "class"); let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtClassDef::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtClassDef::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(vm, source_file), vm) @@ -604,7 +617,7 @@ impl Node for ast::StmtReturn { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtReturn::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtReturn::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -648,7 +661,7 @@ impl Node for ast::StmtDelete { runtime_targets, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtDelete::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtDelete::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let targets = runtime_targets.map_or_else( @@ -705,7 +718,7 @@ impl Node for ast::StmtAssign { runtime_type_comment_bytes, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtAssign::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtAssign::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let targets = runtime_targets.map_or_else( @@ -760,7 +773,7 @@ impl Node for ast::StmtTypeAlias { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtTypeAlias::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtTypeAlias::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(vm, source_file), vm) @@ -820,7 +833,7 @@ impl Node for ast::StmtAugAssign { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtAugAssign::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtAugAssign::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("target", target.ast_to_object(vm, source_file), vm) @@ -881,7 +894,7 @@ impl Node for ast::StmtAnnAssign { runtime_simple, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtAnnAssign::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtAnnAssign::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("target", target.ast_to_object(vm, source_file), vm) @@ -960,9 +973,9 @@ impl Node for ast::StmtFor { } = self; let cls = if !is_async { - pyast::NodeStmtFor::static_type().to_owned() + pyast::NodeStmtFor::make_class(&vm.ctx) } else { - pyast::NodeStmtAsyncFor::static_type().to_owned() + pyast::NodeStmtAsyncFor::make_class(&vm.ctx) }; let node = NodeAst.into_ref_with_type(vm, cls).unwrap(); @@ -997,10 +1010,11 @@ impl Node for ast::StmtFor { _object: PyObjectRef, ) -> PyResult { debug_assert!( - is_node_instance(vm, &_object, pyast::NodeStmtFor::static_type())? - || is_node_instance(vm, &_object, pyast::NodeStmtAsyncFor::static_type())? + is_node_instance(vm, &_object, pyast::NodeStmtFor::make_class(&vm.ctx))? + || is_node_instance(vm, &_object, pyast::NodeStmtAsyncFor::make_class(&vm.ctx))? ); - let is_async = is_node_instance(vm, &_object, pyast::NodeStmtAsyncFor::static_type())?; + let is_async = + is_node_instance(vm, &_object, pyast::NodeStmtAsyncFor::make_class(&vm.ctx))?; let typ = if is_async { "AsyncFor" } else { "For" }; let range = range_from_object(vm, source_file, &_object, typ)?; stmt_for_from_object_with_range(vm, source_file, &_object, range, is_async) @@ -1043,7 +1057,7 @@ impl Node for ast::StmtWhile { runtime_orelse, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtWhile::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtWhile::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("test", test.ast_to_object(vm, source_file), vm) @@ -1145,9 +1159,9 @@ impl Node for ast::StmtWith { } = self; let cls = if !is_async { - pyast::NodeStmtWith::static_type().to_owned() + pyast::NodeStmtWith::make_class(&vm.ctx) } else { - pyast::NodeStmtAsyncWith::static_type().to_owned() + pyast::NodeStmtAsyncWith::make_class(&vm.ctx) }; let node = NodeAst.into_ref_with_type(vm, cls).unwrap(); @@ -1174,10 +1188,11 @@ impl Node for ast::StmtWith { _object: PyObjectRef, ) -> PyResult { debug_assert!( - is_node_instance(vm, &_object, pyast::NodeStmtWith::static_type())? - || is_node_instance(vm, &_object, pyast::NodeStmtAsyncWith::static_type())? + is_node_instance(vm, &_object, pyast::NodeStmtWith::make_class(&vm.ctx))? + || is_node_instance(vm, &_object, pyast::NodeStmtAsyncWith::make_class(&vm.ctx))? ); - let is_async = is_node_instance(vm, &_object, pyast::NodeStmtAsyncWith::static_type())?; + let is_async = + is_node_instance(vm, &_object, pyast::NodeStmtAsyncWith::make_class(&vm.ctx))?; let typ = if is_async { "AsyncWith" } else { "With" }; let range = range_from_object(vm, source_file, &_object, typ)?; stmt_with_from_object_with_range(vm, source_file, &_object, range, is_async) @@ -1207,7 +1222,7 @@ impl Node for ast::StmtMatch { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtMatch::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtMatch::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("subject", subject.ast_to_object(vm, source_file), vm) @@ -1254,7 +1269,7 @@ impl Node for ast::StmtRaise { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtRaise::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtRaise::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("exc", exc.ast_to_object(vm, source_file), vm) @@ -1381,11 +1396,10 @@ impl Node for ast::StmtTry { } = self; let cls = if is_star { - pyast::NodeStmtTryStar::static_type() + pyast::NodeStmtTryStar::make_class(&vm.ctx) } else { - pyast::NodeStmtTry::static_type() - } - .to_owned(); + pyast::NodeStmtTry::make_class(&vm.ctx) + }; let node = NodeAst.into_ref_with_type(vm, cls).unwrap(); let dict = node.as_object().dict().unwrap(); @@ -1417,10 +1431,10 @@ impl Node for ast::StmtTry { source_file: &SourceFile, _object: PyObjectRef, ) -> PyResult { - let is_star = is_node_instance(vm, &_object, pyast::NodeStmtTryStar::static_type())?; + let is_star = is_node_instance(vm, &_object, pyast::NodeStmtTryStar::make_class(&vm.ctx))?; debug_assert!( - is_node_instance(vm, &_object, pyast::NodeStmtTry::static_type())? - || is_node_instance(vm, &_object, pyast::NodeStmtTryStar::static_type())? + is_node_instance(vm, &_object, pyast::NodeStmtTry::make_class(&vm.ctx))? + || is_node_instance(vm, &_object, pyast::NodeStmtTryStar::make_class(&vm.ctx))? ); let typ = if is_star { "TryStar" } else { "Try" }; let range = range_from_object(vm, source_file, &_object, typ)?; @@ -1454,7 +1468,7 @@ impl Node for ast::StmtAssert { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtAssert::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtAssert::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("test", test.ast_to_object(vm, source_file), vm) @@ -1497,7 +1511,7 @@ impl Node for ast::StmtImport { is_lazy: _, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtImport::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtImport::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("names", names.ast_to_object(vm, source_file), vm) @@ -1565,7 +1579,7 @@ impl Node for ast::StmtImportFrom { runtime_level, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtImportFrom::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtImportFrom::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("module", module.ast_to_object(vm, source_file), vm) @@ -1612,7 +1626,7 @@ impl Node for ast::StmtGlobal { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtGlobal::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtGlobal::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("names", names.ast_to_object(vm, source_file), vm) @@ -1651,7 +1665,7 @@ impl Node for ast::StmtNonlocal { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtNonlocal::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtNonlocal::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("names", names.ast_to_object(vm, source_file), vm) @@ -1690,7 +1704,7 @@ impl Node for ast::StmtExpr { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtExpr::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtExpr::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1722,7 +1736,7 @@ impl Node for ast::StmtPass { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtPass::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtPass::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let location = super::text_range_to_source_range(source_file, _range); @@ -1774,7 +1788,7 @@ impl Node for ast::StmtBreak { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtBreak::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtBreak::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); node_add_location(&dict, _range, vm, source_file); @@ -1806,7 +1820,7 @@ impl Node for ast::StmtContinue { range: _range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeStmtContinue::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeStmtContinue::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); node_add_location(&dict, _range, vm, source_file); diff --git a/crates/vm/src/stdlib/_ast/string.rs b/crates/vm/src/stdlib/_ast/string.rs index 553b235c46c..bcfe9e742df 100644 --- a/crates/vm/src/stdlib/_ast/string.rs +++ b/crates/vm/src/stdlib/_ast/string.rs @@ -554,7 +554,7 @@ impl Node for JoinedStr { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprJoinedStr::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprJoinedStr::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let values = if let Some(runtime_values) = runtime_values { @@ -595,7 +595,11 @@ impl Node for JoinedStrPart { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult { - if is_node_instance(vm, &object, pyast::NodeExprFormattedValue::static_type())? { + if is_node_instance( + vm, + &object, + pyast::NodeExprFormattedValue::make_class(&vm.ctx), + )? { Ok(Self::FormattedValue(Node::ast_from_object( vm, source_file, @@ -649,7 +653,7 @@ impl Node for FormattedValue { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprFormattedValue::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprFormattedValue::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) @@ -1064,7 +1068,7 @@ impl Node for TemplateStr { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprTemplateStr::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprTemplateStr::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); let values = if let Some(runtime_values) = runtime_values { @@ -1105,7 +1109,11 @@ impl Node for TemplateStrPart { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult { - if is_node_instance(vm, &object, pyast::NodeExprInterpolation::static_type())? { + if is_node_instance( + vm, + &object, + pyast::NodeExprInterpolation::make_class(&vm.ctx), + )? { Ok(Self::Interpolation(Node::ast_from_object( vm, source_file, @@ -1166,7 +1174,7 @@ impl Node for TStringInterpolation { range, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeExprInterpolation::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeExprInterpolation::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) diff --git a/crates/vm/src/stdlib/_ast/type_ignore.rs b/crates/vm/src/stdlib/_ast/type_ignore.rs index d51e54f1c5d..77c54226255 100644 --- a/crates/vm/src/stdlib/_ast/type_ignore.rs +++ b/crates/vm/src/stdlib/_ast/type_ignore.rs @@ -21,7 +21,11 @@ impl Node for TypeIgnore { ) -> PyResult { Ok(if vm.is_none(&object) { Self::None - } else if is_node_instance(vm, &object, pyast::NodeTypeIgnoreTypeIgnore::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeTypeIgnoreTypeIgnore::make_class(&vm.ctx), + )? { Self::TypeIgnore(TypeIgnoreTypeIgnore::ast_from_object( vm, source_file, @@ -46,10 +50,7 @@ impl Node for TypeIgnoreTypeIgnore { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { let Self { lineno, tag } = self; let node = NodeAst - .into_ref_with_type( - vm, - pyast::NodeTypeIgnoreTypeIgnore::static_type().to_owned(), - ) + .into_ref_with_type(vm, pyast::NodeTypeIgnoreTypeIgnore::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("lineno", vm.ctx.new_int(lineno).into(), vm) diff --git a/crates/vm/src/stdlib/_ast/type_parameters.rs b/crates/vm/src/stdlib/_ast/type_parameters.rs index a15e564e53d..515b3f7be4c 100644 --- a/crates/vm/src/stdlib/_ast/type_parameters.rs +++ b/crates/vm/src/stdlib/_ast/type_parameters.rs @@ -85,11 +85,23 @@ impl Node for ast::TypeParam { ParamSpec, TypeVarTuple, } - let kind = if is_node_instance(vm, &object, pyast::NodeTypeParamTypeVar::static_type())? { + let kind = if is_node_instance( + vm, + &object, + pyast::NodeTypeParamTypeVar::make_class(&vm.ctx), + )? { TypeParamKind::TypeVar - } else if is_node_instance(vm, &object, pyast::NodeTypeParamParamSpec::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeTypeParamParamSpec::make_class(&vm.ctx), + )? { TypeParamKind::ParamSpec - } else if is_node_instance(vm, &object, pyast::NodeTypeParamTypeVarTuple::static_type())? { + } else if is_node_instance( + vm, + &object, + pyast::NodeTypeParamTypeVarTuple::make_class(&vm.ctx), + )? { TypeParamKind::TypeVarTuple } else { return Err(vm.new_type_error(format!( @@ -148,7 +160,7 @@ impl Node for ast::TypeParamTypeVar { default, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeTypeParamTypeVar::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeTypeParamTypeVar::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(vm, source_file), vm) @@ -197,7 +209,7 @@ impl Node for ast::TypeParamParamSpec { default, } = self; let node = NodeAst - .into_ref_with_type(vm, pyast::NodeTypeParamParamSpec::static_type().to_owned()) + .into_ref_with_type(vm, pyast::NodeTypeParamParamSpec::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(vm, source_file), vm) @@ -244,10 +256,7 @@ impl Node for ast::TypeParamTypeVarTuple { default, } = self; let node = NodeAst - .into_ref_with_type( - vm, - pyast::NodeTypeParamTypeVarTuple::static_type().to_owned(), - ) + .into_ref_with_type(vm, pyast::NodeTypeParamTypeVarTuple::make_class(&vm.ctx)) .unwrap(); let dict = node.as_object().dict().unwrap(); dict.set_item("name", name.ast_to_object(vm, source_file), vm) diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index a17c025f50c..23e2f76ae4e 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -2,8 +2,6 @@ pub(crate) use _codecs::module_def; -use crate::common::static_cell::StaticCell; - #[pymodule(with(#[cfg(windows)] _codecs_windows))] mod _codecs { use core::hint::cold_path; @@ -280,14 +278,12 @@ mod _codecs { #[pyfunction] fn readbuffer_encode(args: ReadBufferEncodeArgs, vm: &VirtualMachine) -> PyResult { - rustpython_common::static_cell!( - static FUNC: PyObjectRef; - ); + struct ReadBufferEncode; let mut forwarded = vec![args.data]; if let Some(errors) = args.errors { forwarded.push(errors); } - super::delegate_pycodecs(&FUNC, "readbuffer_encode", forwarded, vm) + super::delegate_pycodecs::("readbuffer_encode", forwarded, vm) } #[derive(FromArgs)] @@ -630,10 +626,8 @@ mod _codecs { macro_rules! delegate_pycodecs { ($name:ident, $args:ident, $vm:ident) => {{ - rustpython_common::static_cell!( - static FUNC: PyObjectRef; - ); - super::delegate_pycodecs(&FUNC, stringify!($name), $args, $vm) + struct CodecFunction; + super::delegate_pycodecs::(stringify!($name), $args, $vm) }}; } @@ -699,13 +693,12 @@ mod _codecs { } #[inline] -fn delegate_pycodecs( - cell: &'static StaticCell, +fn delegate_pycodecs( name: &'static str, args: Vec, vm: &crate::VirtualMachine, ) -> crate::PyResult { - let f = cell.get_or_try_init(|| { + let f = vm.__try_cached_native::(|| { let module = vm.import("_pycodecs", 0)?; module.get_attr(name, vm) })?; diff --git a/crates/vm/src/stdlib/_collections.rs b/crates/vm/src/stdlib/_collections.rs index 1e562f0815b..7d0a16c3df3 100644 --- a/crates/vm/src/stdlib/_collections.rs +++ b/crates/vm/src/stdlib/_collections.rs @@ -315,7 +315,7 @@ mod _collections { } #[pyclass( - flags(BASETYPE, HAS_WEAKREF), + flags(BASETYPE, HAS_WEAKREF, SEQUENCE), with( Constructor, Initializer, diff --git a/crates/vm/src/stdlib/_collections/ordered_dict.rs b/crates/vm/src/stdlib/_collections/ordered_dict.rs index 4ba9777fef5..7a8b9f9cec0 100644 --- a/crates/vm/src/stdlib/_collections/ordered_dict.rs +++ b/crates/vm/src/stdlib/_collections/ordered_dict.rs @@ -178,7 +178,7 @@ pub(crate) mod ordered_dict { impl PyOrderedDict { fn is_exact(zelf: &Py) -> bool { - zelf.class().is(Self::static_type()) + zelf.class().is(unsafe { Self::static_type() }) } fn mutated_error(vm: &VirtualMachine) -> crate::builtins::PyBaseExceptionRef { diff --git a/crates/vm/src/stdlib/_ctypes.rs b/crates/vm/src/stdlib/_ctypes.rs index 780b2bc0bd2..c711ce3dc05 100644 --- a/crates/vm/src/stdlib/_ctypes.rs +++ b/crates/vm/src/stdlib/_ctypes.rs @@ -350,7 +350,7 @@ pub(crate) mod _ctypes { // Array types if type_obj .class() - .fast_issubclass(super::array::PyCArrayType::static_type()) + .fast_issubclass(unsafe { super::array::PyCArrayType::static_type() }) && let Ok(stg) = type_obj.stg_info(vm) { return Ok(stg.size); @@ -358,19 +358,19 @@ pub(crate) mod _ctypes { // Structure types if type_obj .class() - .fast_issubclass(PyCStructType::static_type()) + .fast_issubclass(unsafe { PyCStructType::static_type() }) { return super::structure::calculate_struct_size(type_obj, vm); } // Union types if type_obj .class() - .fast_issubclass(PyCUnionType::static_type()) + .fast_issubclass(unsafe { PyCUnionType::static_type() }) { return super::union::calculate_union_size(type_obj, vm); } // Simple types - if type_obj.fast_issubclass(PyCSimple::static_type()) { + if type_obj.fast_issubclass(unsafe { PyCSimple::static_type() }) { if let Ok(type_attr) = type_obj.as_object().get_attr("_type_", vm) && let Ok(type_str) = type_attr.str(vm) { @@ -382,7 +382,7 @@ pub(crate) mod _ctypes { return Ok(rustpython_host_env::ctypes::pointer_size()); } // Pointer types - if type_obj.fast_issubclass(PyCPointer::static_type()) { + if type_obj.fast_issubclass(unsafe { PyCPointer::static_type() }) { return Ok(rustpython_host_env::ctypes::pointer_size()); } return Err(vm.new_type_error("this type has no size")); @@ -393,7 +393,7 @@ pub(crate) mod _ctypes { if let Some(cdata) = obj.downcast_ref::() { return Ok(cdata.size()); } - if obj.fast_isinstance(PyCPointer::static_type()) { + if obj.fast_isinstance(unsafe { PyCPointer::static_type() }) { return Ok(rustpython_host_env::ctypes::pointer_size()); } @@ -601,8 +601,8 @@ pub(crate) mod _ctypes { use super::CArgValue; // Check if obj is a ctypes instance - if !obj.fast_isinstance(PyCData::static_type()) - && !obj.fast_isinstance(PyCSimple::static_type()) + if !obj.fast_isinstance(unsafe { PyCData::static_type() }) + && !obj.fast_isinstance(unsafe { PyCSimple::static_type() }) { return Err(vm.new_type_error(format!( "byref() argument must be a ctypes instance, not '{}'", @@ -657,26 +657,26 @@ pub(crate) mod _ctypes { } // 3. Fallback for simple types - if obj.fast_isinstance(PyCSimple::static_type()) + if obj.fast_isinstance(unsafe { PyCSimple::static_type() }) && let Ok(stg) = obj.class().stg_info(vm) { return Ok(stg.align); } - if obj.fast_isinstance(PyCArray::static_type()) + if obj.fast_isinstance(unsafe { PyCArray::static_type() }) && let Ok(stg) = obj.class().stg_info(vm) { return Ok(stg.align); } - if obj.fast_isinstance(PyCStructure::static_type()) { + if obj.fast_isinstance(unsafe { PyCStructure::static_type() }) { // Calculate alignment from _fields_ let cls = obj.class(); return alignment(Either::A(cls.to_owned()), vm); } - if obj.fast_isinstance(PyCPointer::static_type()) { + if obj.fast_isinstance(unsafe { PyCPointer::static_type() }) { // Pointer alignment is always pointer size return Ok(core::mem::align_of::()); } - if obj.fast_isinstance(PyCUnion::static_type()) { + if obj.fast_isinstance(unsafe { PyCUnion::static_type() }) { // Calculate alignment from _fields_ let cls = obj.class(); return alignment(Either::A(cls.to_owned()), vm); @@ -1045,26 +1045,26 @@ pub(crate) mod _ctypes { __module_exec(vm, module); let ctx = &vm.ctx; - let _ = PyCSimpleType::make_static_type(); - let _ = array::PyCArrayType::make_static_type(); - let _ = pointer::PyCPointerType::make_static_type(); - let _ = structure::PyCStructType::make_static_type(); - let _ = union::PyCUnionType::make_static_type(); - let _ = function::PyCFuncPtrType::make_static_type(); - let _ = function::RawMemoryBuffer::make_static_type(); + let _ = unsafe { PyCSimpleType::make_static_type() }; + let _ = unsafe { array::PyCArrayType::make_static_type() }; + let _ = unsafe { pointer::PyCPointerType::make_static_type() }; + let _ = unsafe { structure::PyCStructType::make_static_type() }; + let _ = unsafe { union::PyCUnionType::make_static_type() }; + let _ = unsafe { function::PyCFuncPtrType::make_static_type() }; + let _ = unsafe { function::RawMemoryBuffer::make_static_type() }; extend_module!(vm, module, { - "_CData" => PyCData::make_static_type(), - "_SimpleCData" => PyCSimple::make_static_type(), - "Array" => PyCArray::make_static_type(), - "CField" => PyCField::make_static_type(), - "CFuncPtr" => function::PyCFuncPtr::make_static_type(), - "_Pointer" => PyCPointer::make_static_type(), + "_CData" => unsafe { PyCData::make_static_type() }, + "_SimpleCData" => unsafe { PyCSimple::make_static_type() }, + "Array" => unsafe { PyCArray::make_static_type() }, + "CField" => unsafe { PyCField::make_static_type() }, + "CFuncPtr" => unsafe { function::PyCFuncPtr::make_static_type() }, + "_Pointer" => unsafe { PyCPointer::make_static_type() }, "_pointer_type_cache" => ctx.new_dict(), "_array_type_cache" => ctx.new_dict(), - "Structure" => PyCStructure::make_static_type(), - "CThunkObject" => function::PyCThunk::make_static_type(), - "Union" => PyCUnion::make_static_type(), + "Structure" => unsafe { PyCStructure::make_static_type() }, + "CThunkObject" => unsafe { function::PyCThunk::make_static_type() }, + "Union" => unsafe { PyCUnion::make_static_type() }, }); Ok(()) diff --git a/crates/vm/src/stdlib/_ctypes/array.rs b/crates/vm/src/stdlib/_ctypes/array.rs index 0504688d85c..a5430016b35 100644 --- a/crates/vm/src/stdlib/_ctypes/array.rs +++ b/crates/vm/src/stdlib/_ctypes/array.rs @@ -135,11 +135,11 @@ fn create_array_type_with_name( type_name: &str, vm: &VirtualMachine, ) -> PyResult { - let metaclass = PyCArrayType::static_type().to_owned(); + let metaclass = unsafe { PyCArrayType::static_type() }.to_owned(); let name = vm.ctx.new_str(type_name); let bases = vm .ctx - .new_tuple(vec![PyCArray::static_type().to_owned().into()]); + .new_tuple(vec![unsafe { PyCArray::static_type() }.to_owned().into()]); let dict = vm.ctx.new_dict(); let args = FuncArgs::new( diff --git a/crates/vm/src/stdlib/_ctypes/base.rs b/crates/vm/src/stdlib/_ctypes/base.rs index 2b697ef5558..155a31725a7 100644 --- a/crates/vm/src/stdlib/_ctypes/base.rs +++ b/crates/vm/src/stdlib/_ctypes/base.rs @@ -362,6 +362,7 @@ pub(super) fn is_big_endian(is_swapped: bool) -> bool { /// Shared BufferMethods for all ctypes types (PyCArray, PyCSimple, PyCStructure, PyCUnion) /// All these types are #[repr(transparent)] wrappers around PyCData pub(super) static CDATA_BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: None, obj_bytes: |buffer| { rustpython_common::lock::PyRwLockReadGuard::map( buffer.obj_as::().buffer.read(), @@ -919,7 +920,7 @@ impl PyCData { if let Some(proto_type) = proto.downcast_ref::() && proto_type .class() - .fast_issubclass(super::pointer::PyCPointerType::static_type()) + .fast_issubclass(unsafe { super::pointer::PyCPointerType::static_type() }) && let Some(array) = value.downcast_ref::() { let buffer_addr = { @@ -1080,7 +1081,7 @@ impl PyCData { let proto_metaclass = proto_type.class(); // Simple types: return primitive value - if proto_metaclass.fast_issubclass(super::simple::PyCSimpleType::static_type()) { + if proto_metaclass.fast_issubclass(unsafe { super::simple::PyCSimpleType::static_type() }) { // Check for byte swapping let needs_swap = base_obj .class() @@ -1107,9 +1108,11 @@ impl PyCData { let ptr = self.buffer.read().as_ptr().wrapping_add(offset) as *mut u8; let cdata_obj = unsafe { Self::from_base_obj(ptr, size, base_obj, index) }; - if proto_metaclass.fast_issubclass(super::structure::PyCStructType::static_type()) - || proto_metaclass.fast_issubclass(super::union::PyCUnionType::static_type()) - || proto_metaclass.fast_issubclass(super::pointer::PyCPointerType::static_type()) + if proto_metaclass + .fast_issubclass(unsafe { super::structure::PyCStructType::static_type() }) + || proto_metaclass.fast_issubclass(unsafe { super::union::PyCUnionType::static_type() }) + || proto_metaclass + .fast_issubclass(unsafe { super::pointer::PyCPointerType::static_type() }) { cdata_obj.into_ref_with_type(vm, proto_type).map(Into::into) } else { @@ -2079,7 +2082,7 @@ pub(super) fn is_simple_instance(typ: &Py) -> bool { // _ctypes_simple_instance // Check if the type's metaclass is PyCSimpleType let metaclass = typ.class(); - metaclass.fast_issubclass(super::simple::PyCSimpleType::static_type()) + metaclass.fast_issubclass(unsafe { super::simple::PyCSimpleType::static_type() }) } /// Set or initialize StgInfo on a type diff --git a/crates/vm/src/stdlib/_ctypes/function.rs b/crates/vm/src/stdlib/_ctypes/function.rs index e67e3871d14..bf06d896928 100644 --- a/crates/vm/src/stdlib/_ctypes/function.rs +++ b/crates/vm/src/stdlib/_ctypes/function.rs @@ -212,10 +212,10 @@ impl ArgumentType for PyTypeRef { // Validate the argument type up front (mirrors the pre-conversion // check): pointer-like ctypes types are always acceptable; a simple // type must carry a known _type_ code; anything else is unsupported. - let type_code = if self.fast_issubclass(CArgObject::static_type()) - || self.fast_issubclass(PyCPointer::static_type()) - || self.fast_issubclass(PyCStructure::static_type()) - || self.fast_issubclass(PyCUnion::static_type()) + let type_code = if self.fast_issubclass(unsafe { CArgObject::static_type() }) + || self.fast_issubclass(unsafe { PyCPointer::static_type() }) + || self.fast_issubclass(unsafe { PyCStructure::static_type() }) + || self.fast_issubclass(unsafe { PyCUnion::static_type() }) { None } else { @@ -255,7 +255,7 @@ impl ArgumentType for PyTypeRef { } // For pointer types (POINTER(T)), we need to pass the pointer VALUE stored in buffer - if self.fast_issubclass(PyCPointer::static_type()) { + if self.fast_issubclass(unsafe { PyCPointer::static_type() }) { if let Some(pointer) = converted.downcast_ref::() { return Ok((CArgValue::pointer(pointer.get_ptr_value()), None)); } @@ -265,8 +265,8 @@ impl ArgumentType for PyTypeRef { // For structure/union types, pass the aggregate by value: snapshot the // instance bytes and build its call layout from the argtype. A byref() // result is a CArgObject and was already handled above (stays a pointer). - if self.fast_issubclass(PyCStructure::static_type()) - || self.fast_issubclass(PyCUnion::static_type()) + if self.fast_issubclass(unsafe { PyCStructure::static_type() }) + || self.fast_issubclass(unsafe { PyCUnion::static_type() }) { if let Some(cdata) = converted.downcast_ref::() { let bytes = cdata.buffer.read().to_vec(); @@ -460,6 +460,7 @@ pub(super) struct RawMemoryBuffer { } static RAW_MEMORY_BUFFER_METHODS: crate::protocol::BufferMethods = crate::protocol::BufferMethods { + shared_storage: None, obj_bytes: |buffer| { let raw = buffer.obj_as::(); let slice = unsafe { raw.memory.bytes() }; @@ -505,7 +506,10 @@ fn cast_check_pointertype(ctype: &PyObject, vm: &VirtualMachine) -> bool { use super::pointer::PyCPointerType; // PyCPointerTypeObject_Check - if ctype.class().fast_issubclass(PyCPointerType::static_type()) { + if ctype + .class() + .fast_issubclass(unsafe { PyCPointerType::static_type() }) + { return true; } @@ -577,7 +581,10 @@ pub(super) fn cast_impl( // 4. _objects reference tracking // Share _objects dict between source and result, add id(src): src - if src.class().fast_issubclass(PyCData::static_type()) { + if src + .class() + .fast_issubclass(unsafe { PyCData::static_type() }) + { // Get the source's _objects, create dict if needed let shared_objects: PyObjectRef = if let Some(src_cdata) = src.downcast_ref::() { let mut src_objects = src_cdata.objects.write(); @@ -1173,7 +1180,7 @@ fn get_buffer_addr(obj: &PyObject) -> Option { /// Create OUT buffer for a parameter type fn create_out_buffer(arg_type: &Py, vm: &VirtualMachine) -> PyResult { // For POINTER(T) types, create T instance (the pointed-to type) - if arg_type.fast_issubclass(PyCPointer::static_type()) + if arg_type.fast_issubclass(unsafe { PyCPointer::static_type() }) && let Some(stg_info) = arg_type.stg_info_opt() && let Some(ref proto) = stg_info.proto { diff --git a/crates/vm/src/stdlib/_ctypes/pointer.rs b/crates/vm/src/stdlib/_ctypes/pointer.rs index 6a8ebe660a0..07cdcd4a498 100644 --- a/crates/vm/src/stdlib/_ctypes/pointer.rs +++ b/crates/vm/src/stdlib/_ctypes/pointer.rs @@ -598,9 +598,9 @@ impl PyCPointer { // Write value at address // Handle Structure/Array types by copying their buffer if let Some(cdata) = value.downcast_ref::() - && (cdata.fast_isinstance(PyCStructure::static_type()) - || cdata.fast_isinstance(PyCArray::static_type()) - || cdata.fast_isinstance(PyCSimple::static_type())) + && (cdata.fast_isinstance(unsafe { PyCStructure::static_type() }) + || cdata.fast_isinstance(unsafe { PyCArray::static_type() }) + || cdata.fast_isinstance(unsafe { PyCSimple::static_type() })) { let src_buffer = cdata.buffer.read(); unsafe { diff --git a/crates/vm/src/stdlib/_interpchannels.rs b/crates/vm/src/stdlib/_interpchannels.rs index d9fde089fd9..7ac2f2c9ae6 100644 --- a/crates/vm/src/stdlib/_interpchannels.rs +++ b/crates/vm/src/stdlib/_interpchannels.rs @@ -5,6 +5,8 @@ #[cfg_attr(not(feature = "threading"), allow(unused_imports))] pub(crate) use _interpchannels::clear_interpreter; pub(crate) use _interpchannels::module_def; +#[cfg(all(unix, feature = "threading"))] +pub(crate) use _interpchannels::with_fork; pub(crate) use _interpchannels::{channel_id_from_parts, channel_id_parts}; #[pymodule] @@ -132,7 +134,7 @@ pub(crate) mod _interpchannels { format!("channel {cid} may not be closed if not empty (try force=True)"), ), }; - vm.new_exception_msg(class.to_owned(), msg.into()) + vm.new_exception_msg(class, msg.into()) } } @@ -142,6 +144,8 @@ pub(crate) mod _interpchannels { struct Waiting { state: Mutex, cond: Condvar, + #[cfg(all(unix, feature = "threading"))] + fork_generation: u64, } #[derive(Default)] @@ -155,10 +159,18 @@ pub(crate) mod _interpchannels { Arc::new(Self { state: Mutex::new(WaitState::default()), cond: Condvar::new(), + #[cfg(all(unix, feature = "threading"))] + fork_generation: crate::vm::fork::generation(), }) } fn release(&self, received: bool) { + #[cfg(all(unix, feature = "threading"))] + if self.fork_generation != crate::vm::fork::generation() { + // A popped item can retain a waiter outside the channel table. + // Its parent thread and condition-variable waiters are gone. + return; + } let mut state = self.state.lock(); if !state.released { state.released = true; @@ -330,6 +342,17 @@ pub(crate) mod _interpchannels { }) } + #[cfg(all(unix, feature = "threading"))] + pub(crate) fn with_fork(syscall: impl FnOnce() -> R) -> R { + let table = channels().lock(); + let _states: Vec<_> = table + .refs + .values() + .filter_map(|entry| entry.chan.as_ref().map(|chan| chan.state.lock())) + .collect(); + syscall() + } + /// `_channels_lookup`: the channel, or why it is unusable. fn channels_lookup(cid: i64) -> ChanResult> { let table = channels().lock(); @@ -366,18 +389,23 @@ pub(crate) mod _interpchannels { /// `channel_destroy`: forget the channel entirely. fn channel_destroy(cid: i64) -> ChanResult, bool)>> { - let mut table = channels().lock(); - let entry = table.refs.remove(&cid).ok_or(ChanErr::NotFound)?; - Ok(match entry.chan { - Some(chan) => drain_queue(&mut chan.state.lock()), - None => Vec::new(), - }) + let entry = channels() + .lock() + .refs + .remove(&cid) + .ok_or(ChanErr::NotFound)?; + let items = match entry.chan { + Some(chan) => core::mem::take(&mut chan.state.lock().queue), + None => VecDeque::new(), + }; + Ok(retire_items(items)) } - fn drain_queue(state: &mut ChannelState) -> Vec<(Arc, bool)> { - state - .queue - .drain(..) + // Shared buffers can run owner callbacks when released. Call only after + // unlocking both the directory and channel state. + fn retire_items(items: VecDeque) -> Vec<(Arc, bool)> { + items + .into_iter() .filter_map(|item| item.waiting.map(|w| (w, false))) .collect() } @@ -394,10 +422,12 @@ pub(crate) mod _interpchannels { return Vec::new(); } let entry = table.refs.remove(&cid).expect("just looked it up"); - match entry.chan { - Some(chan) => drain_queue(&mut chan.state.lock()), - None => Vec::new(), - } + drop(table); + let items = match entry.chan { + Some(chan) => core::mem::take(&mut chan.state.lock().queue), + None => VecDeque::new(), + }; + retire_items(items) } /// `_channels_add_id_object`. @@ -456,12 +486,13 @@ pub(crate) mod _interpchannels { state.closing = true; return Ok(Vec::new()); } - let waiters = drain_queue(&mut state); + let items = core::mem::take(&mut state.queue); state.open = false; state.ends.release_all(); drop(state); entry.chan = None; - Ok(waiters) + drop(table); + Ok(retire_items(items)) } /// `channel_release`: close one or both ends for the current interpreter. @@ -486,6 +517,7 @@ pub(crate) mod _interpchannels { .filter_map(|r| r.chan.clone()) .collect(); let mut waiters = Vec::new(); + let mut dropped = Vec::new(); for chan in chans { let mut state = chan.state.lock(); let mut i = 0; @@ -500,17 +532,19 @@ pub(crate) mod _interpchannels { if let Some(w) = item.waiting { waiters.push((w, false)); } + dropped.push(item.data); continue; } // UNBOUND_ERROR / UNBOUND_REPLACE keep the slot but throw the // data away; the item is now "unbound". - item.data = None; + dropped.push(item.data.take()); i += 1; } state.ends.clear_interpreter(interpid); state.open = state.ends.is_open(); } release_waiters_detached(waiters); + drop(dropped); } /// `resolve_unboundop`. @@ -817,7 +851,8 @@ pub(crate) mod _interpchannels { #[pystruct_sequence( name = "ChannelInfo", module = "_interpchannels", - data = "ChannelInfoData" + data = "ChannelInfoData", + interpreter_local )] struct PyChannelInfo; @@ -894,12 +929,14 @@ pub(crate) mod _interpchannels { let cid = parse_cid(parsed[0].as_deref().unwrap(), vm)?.0; let send = flag(parsed[1].as_deref(), vm)?; let chan = channels_lookup(cid).map_err(|e| e.into_py(cid, vm))?; + let interpreters = crate::vm::runtime::list_interpreters(); let state = chan.state.lock(); if send && state.closing { + drop(state); return Err(ChanErr::Closed.into_py(cid, vm)); } let mut out = Vec::new(); - for info in crate::vm::runtime::list_interpreters() { + for info in interpreters { if state.ends.find(info.id, send).is_some_and(|i| { if send { state.ends.send[i].1 @@ -907,10 +944,18 @@ pub(crate) mod _interpchannels { state.ends.recv[i].1 } }) { - out.push(vm.ctx.new_int(info.id).into()); + out.push(info.id); } } - Ok(vm.ctx.new_list(out).into()) + drop(state); + Ok(vm + .ctx + .new_list( + out.into_iter() + .map(|id| vm.ctx.new_int(id).into()) + .collect(), + ) + .into()) } /// `channel_send` up to its `closing` check, which precedes converting the @@ -982,15 +1027,18 @@ pub(crate) mod _interpchannels { if timed_out { // The send is failing now, so make sure the item won't be received. let mut state = chan.state.lock(); - if let Some(pos) = state.queue.iter().position(|it| { - it.waiting - .as_ref() - .is_some_and(|w| Arc::ptr_eq(w, &waiting)) - }) { - state.queue.remove(pos); - } + let removed = state + .queue + .iter() + .position(|it| { + it.waiting + .as_ref() + .is_some_and(|w| Arc::ptr_eq(w, &waiting)) + }) + .and_then(|pos| state.queue.remove(pos)); drop(state); finish_closing(cid); + drop(removed); if !waiting.state.lock().received { return Err(vm .new_os_subtype_error( diff --git a/crates/vm/src/stdlib/_interpqueues.rs b/crates/vm/src/stdlib/_interpqueues.rs index 1e9e3945931..60016adf214 100644 --- a/crates/vm/src/stdlib/_interpqueues.rs +++ b/crates/vm/src/stdlib/_interpqueues.rs @@ -2,6 +2,8 @@ #[cfg_attr(not(feature = "threading"), allow(unused_imports))] pub(crate) use _interpqueues::clear_interpreter; +#[cfg(all(unix, feature = "threading"))] +pub(crate) use _interpqueues::with_fork; pub(crate) use _interpqueues::{ is_external_queue, module_def, queue_from_xid, queue_id_from_object, queue_xid_decref, queue_xid_incref, @@ -53,15 +55,15 @@ pub(crate) mod _interpqueues { fn into_py(self, qid: i64, vm: &VirtualMachine) -> PyBaseExceptionRef { let (class, msg) = match self { Self::NoNextId => ( - PyQueueError::class(&vm.ctx).to_owned(), + PyQueueError::class(&vm.ctx), "ran out of queue IDs".to_owned(), ), Self::NotFound => ( - PyQueueNotFoundError::class(&vm.ctx).to_owned(), + PyQueueNotFoundError::class(&vm.ctx), format!("queue {qid} not found"), ), Self::NeverBound => ( - PyQueueError::class(&vm.ctx).to_owned(), + PyQueueError::class(&vm.ctx), format!("queue {qid} never bound"), ), }; @@ -92,54 +94,8 @@ pub(crate) mod _interpqueues { full: PyTypeRef, } - #[cfg(feature = "threading")] - fn external_types() -> &'static Mutex> { - static TYPES: OnceLock>> = OnceLock::new(); - TYPES.get_or_init(|| Mutex::new(BTreeMap::new())) - } - - #[cfg(not(feature = "threading"))] - std::thread_local! { - static EXTERNAL_TYPES: core::cell::RefCell> = - const { core::cell::RefCell::new(BTreeMap::new()) }; - } - - fn get_external_types(interpid: i64) -> Option { - #[cfg(feature = "threading")] - { - external_types().lock().get(&interpid).cloned() - } - #[cfg(not(feature = "threading"))] - { - EXTERNAL_TYPES.with(|types| types.borrow().get(&interpid).cloned()) - } - } - - fn set_external_types(interpid: i64, types: ExternalTypes) { - #[cfg(feature = "threading")] - { - external_types().lock().insert(interpid, types); - } - #[cfg(not(feature = "threading"))] - { - EXTERNAL_TYPES.with(|registered| { - registered.borrow_mut().insert(interpid, types); - }); - } - } - - #[cfg_attr(not(feature = "threading"), allow(dead_code))] - fn remove_external_types(interpid: i64) { - #[cfg(feature = "threading")] - { - external_types().lock().remove(&interpid); - } - #[cfg(not(feature = "threading"))] - { - EXTERNAL_TYPES.with(|types| { - types.borrow_mut().remove(&interpid); - }); - } + fn get_external_types(vm: &VirtualMachine) -> Option { + vm.__get_native::() } fn ensure_highlevel_module_loaded(vm: &VirtualMachine) -> PyResult<()> { @@ -147,12 +103,11 @@ pub(crate) mod _interpqueues { } fn ensure_external_types(vm: &VirtualMachine) -> PyResult { - let interpid = vm.state.interpreter_id; - if let Some(types) = get_external_types(interpid) { + if let Some(types) = get_external_types(vm) { return Ok(types); } ensure_highlevel_module_loaded(vm)?; - get_external_types(interpid) + get_external_types(vm) .ok_or_else(|| vm.new_runtime_error("queue types were not registered")) } @@ -197,6 +152,17 @@ pub(crate) mod _interpqueues { }) } + #[cfg(all(unix, feature = "threading"))] + pub(crate) fn with_fork(syscall: impl FnOnce() -> R) -> R { + let table = queues().lock(); + let _states: Vec<_> = table + .refs + .values() + .map(|entry| entry.queue.lock()) + .collect(); + syscall() + } + fn queue_lookup(qid: i64) -> QueueResult>> { queues() .lock() @@ -596,16 +562,12 @@ pub(crate) mod _interpqueues { .ok() .filter(|ty| ty.fast_issubclass(vm.ctx.exceptions.base_exception_type)) .ok_or_else(|| vm.new_type_error("expected an exception type for 'fullerror'"))?; - set_external_types( - vm.state.interpreter_id, - ExternalTypes { queue, empty, full }, - ); + vm.__replace_native::(ExternalTypes { queue, empty, full }); Ok(()) } pub(crate) fn is_external_queue(obj: &PyObject, vm: &VirtualMachine) -> bool { - get_external_types(vm.state.interpreter_id) - .is_some_and(|types| obj.class().is(&types.queue)) + get_external_types(vm).is_some_and(|types| obj.class().is(&types.queue)) } pub(crate) fn queue_id_from_object( @@ -639,7 +601,6 @@ pub(crate) mod _interpqueues { #[cfg_attr(not(feature = "threading"), allow(dead_code))] pub(crate) fn clear_interpreter(interpid: i64) { - remove_external_types(interpid); let queues: Vec<_> = queues() .lock() .refs diff --git a/crates/vm/src/stdlib/_interpreters.rs b/crates/vm/src/stdlib/_interpreters.rs index 903a14f6acc..e630f1b67fd 100644 --- a/crates/vm/src/stdlib/_interpreters.rs +++ b/crates/vm/src/stdlib/_interpreters.rs @@ -2,7 +2,9 @@ //! //! Mirrors CPython `Modules/_interpretersmodule.c`. -pub(crate) use _interpreters::{config_from_pyobject, init_xi_types, module_def}; +pub(crate) use _interpreters::{ + clone_shared_buffer, config_from_pyobject, init_xi_types, module_def, +}; #[cfg_attr(not(feature = "threading"), allow(unused_imports))] pub(crate) use _interpreters::{ interpreter_error, interpreter_not_found, not_shareable_error, xibufferview_from_buffer, @@ -67,15 +69,43 @@ pub(crate) mod _interpreters { impl PyNotShareableError {} #[pyattr] - #[pyclass(name = "CrossInterpreterBufferView", module = "_interpreters")] + #[pyclass( + name = "CrossInterpreterBufferView", + module = "_interpreters", + traverse = "manual" + )] #[derive(Debug, PyPayload)] pub(crate) struct XiBufferView { - view: PyBuffer, + view: XiBufferBacking, + } + + #[derive(Debug)] + enum XiBufferBacking { + Local(PyBuffer), + Shared(crate::protocol::SharedBuffer), + } + + unsafe impl crate::object::Traverse for XiBufferView { + fn traverse(&self, tracer: &mut crate::object::TraverseFn<'_>) { + if let XiBufferBacking::Local(buffer) = &self.view { + buffer.traverse(tracer); + } + } } static XI_BUFFER_VIEW_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| buffer.obj_as::().view.obj_bytes(), - obj_bytes_mut: |buffer| buffer.obj_as::().view.obj_bytes_mut(), + shared_storage: Some(|buffer| match &buffer.obj_as::().view { + XiBufferBacking::Local(view) => view.shared_storage(), + XiBufferBacking::Shared(view) => Some(view.storage.clone()), + }), + obj_bytes: |buffer| match &buffer.obj_as::().view { + XiBufferBacking::Local(view) => view.obj_bytes(), + XiBufferBacking::Shared(view) => view.read(), + }, + obj_bytes_mut: |buffer| match &buffer.obj_as::().view { + XiBufferBacking::Local(view) => view.obj_bytes_mut(), + XiBufferBacking::Shared(view) => view.write(), + }, release: |_buffer| {}, retain: |_buffer| {}, }; @@ -90,26 +120,44 @@ pub(crate) mod _interpreters { fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { Ok(PyBuffer::new( zelf.to_owned().into(), - zelf.view.desc.clone(), + match &zelf.view { + XiBufferBacking::Local(view) => view.desc.clone(), + XiBufferBacking::Shared(view) => view.desc.clone(), + }, &XI_BUFFER_VIEW_METHODS, )) } } /// `xibufferview_from_buffer`. - pub(crate) fn xibufferview_from_buffer(view: PyBuffer, vm: &VirtualMachine) -> PyObjectRef { + pub(crate) fn xibufferview_from_buffer( + view: crate::protocol::SharedBuffer, + vm: &VirtualMachine, + ) -> PyObjectRef { + let view = if let Some(local) = view.local_buffer(vm) { + XiBufferBacking::Local(local) + } else { + XiBufferBacking::Shared(view) + }; XiBufferView { view }.into_pyobject(vm) } + pub(crate) fn clone_shared_buffer(obj: &PyObject) -> Option { + match &obj.downcast_ref::()?.view { + XiBufferBacking::Shared(view) => Some(view.clone()), + XiBufferBacking::Local(_) => None, + } + } + /// The cross-interpreter exception types belong to the runtime rather than /// to this module, so they are created before either of the modules that /// raise them finishes importing. pub(crate) fn init_xi_types(vm: &VirtualMachine) { let module = vm.new_pyobj("concurrent.interpreters"); for class in [ - PyInterpreterError::make_static_type(), - PyInterpreterNotFoundError::make_static_type(), - PyNotShareableError::make_static_type(), + unsafe { PyInterpreterError::make_static_type() }, + unsafe { PyInterpreterNotFoundError::make_static_type() }, + unsafe { PyNotShareableError::make_static_type() }, ] { class.set_attr(crate::identifier!(vm, __module__), module.clone()); } @@ -128,15 +176,12 @@ pub(crate) mod _interpreters { vm: &VirtualMachine, msg: impl Into, ) -> PyBaseExceptionRef { - vm.new_exception_msg( - PyInterpreterError::class(&vm.ctx).to_owned(), - msg.into().into(), - ) + vm.new_exception_msg(PyInterpreterError::class(&vm.ctx), msg.into().into()) } pub(crate) fn interpreter_not_found(vm: &VirtualMachine, id: i64) -> PyBaseExceptionRef { vm.new_exception_msg( - PyInterpreterNotFoundError::class(&vm.ctx).to_owned(), + PyInterpreterNotFoundError::class(&vm.ctx), format!("unrecognized interpreter ID {id}").into(), ) } @@ -145,10 +190,7 @@ pub(crate) mod _interpreters { vm: &VirtualMachine, msg: impl Into, ) -> PyBaseExceptionRef { - vm.new_exception_msg( - PyNotShareableError::class(&vm.ctx).to_owned(), - msg.into().into(), - ) + vm.new_exception_msg(PyNotShareableError::class(&vm.ctx), msg.into().into()) } /// `_PyArg_BadArgument`. @@ -206,7 +248,8 @@ pub(crate) mod _interpreters { None => "current interpreter".to_owned(), }; let id = id.unwrap_or(vm.state.interpreter_id); - let state = runtime::lookup_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; + let state = + runtime::lookup_open_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; if reqready && !state.ready.load(Ordering::Acquire) { return Err(interpreter_error( vm, @@ -514,7 +557,7 @@ pub(crate) mod _interpreters { let mut items = Vec::new(); for info in runtime::list_interpreters() { if reqready - && !runtime::lookup_interpreter(info.id) + && !runtime::lookup_open_interpreter(info.id) .is_some_and(|s| s.ready.load(Ordering::Acquire)) { continue; @@ -560,7 +603,8 @@ pub(crate) mod _interpreters { } .parse(&args, vm)?; let id = parse_id(parsed[0].as_deref().unwrap(), vm)?; - let state = runtime::lookup_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; + let state = + runtime::lookup_open_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; Ok(vm.ctx.new_int(state.whence.as_i32()).into()) } @@ -581,7 +625,8 @@ pub(crate) mod _interpreters { Some(parse_id(id_obj, vm)?) }; let id = resolve_interp(id, restricted, false, "get the config of", vm)?; - let state = runtime::lookup_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; + let state = + runtime::lookup_open_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; Ok(config_namespace(state.config(), vm)) } @@ -972,7 +1017,8 @@ pub(crate) mod _interpreters { let restricted = flag(parsed[2].as_deref(), vm)?; let id = parse_id(parsed[0].as_deref().unwrap(), vm)?; resolve_interp(Some(id), restricted, true, "incref", vm)?; - let state = runtime::lookup_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; + let state = + runtime::lookup_open_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; if implieslink { // Decref to 0 will destroy the interpreter. state.require_idref.store(true, Ordering::Release); @@ -993,7 +1039,8 @@ pub(crate) mod _interpreters { let restricted = flag(parsed[1].as_deref(), vm)?; let id = parse_id(parsed[0].as_deref().unwrap(), vm)?; resolve_interp(Some(id), restricted, true, "decref", vm)?; - let state = runtime::lookup_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; + let state = + runtime::lookup_open_interpreter(id).ok_or_else(|| interpreter_not_found(vm, id))?; let prev = state.id_refcount.fetch_sub(1, Ordering::AcqRel); if prev == 1 && state.require_idref.load(Ordering::Acquire) { #[cfg(feature = "threading")] diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index d4cd66e1491..9ed97bde4e9 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -124,10 +124,15 @@ mod _io { }, class::{PyClassDef, StaticType}, common::lock::{ - PyMappedThreadMutexGuard, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, - PyThreadMutex, PyThreadMutexGuard, + PyDetachingRwLock, PyDetachingRwLockReadGuard, PyDetachingRwLockWriteGuard, + PyMappedThreadMutexGuard, PyRwLock, PyRwLockWriteGuard, PyThreadMutex, + PyThreadMutexGuard, }, common::wtf8::{Wtf8, Wtf8Buf}, + common::{ + borrow::{BorrowedValue, BorrowedValueMut}, + rc::PyRc, + }, convert::ToPyObject, exceptions::nul_char_error, function::{ @@ -201,7 +206,7 @@ mod _io { msg: T, vm: &VirtualMachine, ) -> PyBaseExceptionRef { - vm.new_os_subtype_error(unsupported_operation().to_owned(), None, msg.into()) + vm.new_os_subtype_error(unsupported_operation(vm), None, msg.into()) .upcast() } @@ -762,10 +767,13 @@ mod _io { ))); } let n = n as usize; - let mut bytes = b.borrow_buf_mut(); - bytes.truncate(n); - // FIXME: try to use Arc::unwrap on the bytearray to get at the inner buffer - bytes.clone().to_pyobject(vm) + // readinto may have retained an export of b. Return the + // requested prefix without resizing its backing storage. + let bytes = { + let bytes = b.borrow_buf(); + bytes[..n.min(bytes.len())].to_vec() + }; + bytes.to_pyobject(vm) } }) } else { @@ -851,9 +859,9 @@ mod _io { if data.is(buf_obj) { return Ok(l); } - let mut buf = b.borrow_buf_mut(); let data = ArgBytesLike::try_from_object(vm, data)?; - let data = data.borrow_buf(); + let data = data.borrow_buf_unlocked(vm)?; + let mut buf = b.borrow_buf_mut(); match buf.get_mut(..data.len()) { Some(slice) => { slice.copy_from_slice(&data); @@ -2019,7 +2027,7 @@ mod _io { #[pymethod] fn __reduce_ex__(zelf: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { - if zelf.class().is(Self::static_type()) { + if zelf.class().is(unsafe { Self::static_type() }) { return Err( vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())) ); @@ -4248,7 +4256,7 @@ mod _io { #[pymethod] fn __reduce_ex__(zelf: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { - if zelf.class().is(TextIOWrapper::static_type()) { + if zelf.class().is(unsafe { TextIOWrapper::static_type() }) { return Err( vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())) ); @@ -5131,20 +5139,58 @@ mod _io { #[derive(Debug)] struct BytesIO { _base: _BufferedIOBase, - buffer: PyRwLock, + storage: PyRc, + } + + #[derive(Debug)] + struct BytesIOStorage { + buffer: PyDetachingRwLock, closed: AtomicCell, exports: AtomicCell, } + impl core::ops::Deref for BytesIO { + type Target = BytesIOStorage; + fn deref(&self) -> &Self::Target { + &self.storage + } + } + + #[derive(Debug)] + struct SharedBytesIO(PyRc); + + impl Drop for SharedBytesIO { + fn drop(&mut self) { + self.0.exports.fetch_sub(1); + } + } + + // SAFETY: exports prevent resizing/closing; BufferedIO owns only Rust bytes + // and cursor state. Every view shares its original lock. + unsafe impl crate::protocol::SharedBufferStorage for SharedBytesIO { + fn read(&self) -> BorrowedValue<'_, [u8]> { + PyDetachingRwLockReadGuard::map(self.0.buffer.read(), |x| x.cursor.get_ref().as_slice()) + .into() + } + fn write(&self) -> BorrowedValueMut<'_, [u8]> { + PyDetachingRwLockWriteGuard::map(self.0.buffer.write(), |x| { + x.cursor.get_mut().as_mut_slice() + }) + .into() + } + } + impl Constructor for BytesIO { type Args = FuncArgs; fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { Ok(Self { _base: Default::default(), - buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), - closed: AtomicCell::new(false), - exports: AtomicCell::new(0), + storage: PyRc::new(BytesIOStorage { + buffer: PyDetachingRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), + closed: AtomicCell::new(false), + exports: AtomicCell::new(0), + }), }) } } @@ -5168,7 +5214,10 @@ mod _io { } impl BytesIO { - fn buffer(&self, vm: &VirtualMachine) -> PyResult> { + fn buffer( + &self, + vm: &VirtualMachine, + ) -> PyResult> { if !self.closed.load() { Ok(self.buffer.write()) } else { @@ -5236,25 +5285,14 @@ mod _io { #[pymethod] fn readinto(zelf: &Self, buffer: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult { - // Reading locks this object, and a destination that views it locks - // it too, so such a destination is filled after the read is done. - if buffer.source_object().is(zelf.as_object()) { - let mut data = vm.new_zeroed_bytes(buffer.len())?; - let ret = zelf - .buffer(vm)? - .cursor - .read(&mut data) - .map_err(|_| vm.new_value_error("Error readinto from Take"))?; - buffer.borrow_buf_mut()[..ret].copy_from_slice(&data[..ret]); - return Ok(ret); - } - let mut buf = zelf.buffer(vm)?; - let ret = buf - .cursor - .read(&mut buffer.borrow_buf_mut()) - .map_err(|_| vm.new_value_error("Error readinto from Take"))?; - - Ok(ret) + // A forwarding or XI destination can share our storage. Reading + // first also avoids ABBA between two simultaneous cross-VM copies. + let data = zelf + .buffer(vm)? + .read(Some(buffer.len())) + .unwrap_or_default(); + buffer.borrow_buf_mut()[..data.len()].copy_from_slice(&data); + Ok(data.len()) } //skip to the jth position @@ -5392,26 +5430,40 @@ mod _io { if self.closed.load() { return Err(vm.new_value_error("I/O operation on closed file.")); } - let len = self.buffer.read().cursor.get_ref().len(); + let data = self.buffer.read(); + if self.closed.load() { + drop(data); + return Err(vm.new_value_error("I/O operation on closed file.")); + } + let len = data.cursor.get_ref().len(); let buffer = PyBuffer::new( - self.into(), + self.clone().into(), BufferDescriptor::simple(len, false), &BYTES_IO_BUFFER_METHODS, ); + drop(data); let view = PyMemoryView::from_buffer(buffer, vm)?; Ok(view) } } static BYTES_IO_BUFFER_METHODS: BufferMethods = BufferMethods { + shared_storage: Some(|buffer| { + let storage = buffer.obj_as::().storage.clone(); + storage.exports.fetch_add(1); + Some(PyRc::new(SharedBytesIO(storage))) + }), obj_bytes: |buffer| { let zelf = buffer.obj_as::(); - PyRwLockReadGuard::map(zelf.buffer.read(), |x| x.cursor.get_ref().as_slice()).into() + PyDetachingRwLockReadGuard::map(zelf.buffer.read(), |x| x.cursor.get_ref().as_slice()) + .into() }, obj_bytes_mut: |buffer| { let zelf = buffer.obj_as::(); - PyRwLockWriteGuard::map(zelf.buffer.write(), |x| x.cursor.get_mut().as_mut_slice()) - .into() + PyDetachingRwLockWriteGuard::map(zelf.buffer.write(), |x| { + x.cursor.get_mut().as_mut_slice() + }) + .into() }, release: |buffer| { @@ -5424,7 +5476,7 @@ mod _io { }; impl BufferResizeGuard for BytesIO { - type Resizable<'a> = PyRwLockWriteGuard<'a, BufferedIO>; + type Resizable<'a> = PyDetachingRwLockWriteGuard<'a, BufferedIO>; fn try_resizable_opt(&self) -> Option> { let w = self.buffer.write(); @@ -5599,7 +5651,7 @@ mod _io { #[cfg(all(unix, feature = "threading", feature = "host_env"))] pub(crate) unsafe fn reinit_std_streams_after_fork(vm: &VirtualMachine) { for name in ["stdin", "stdout", "stderr"] { - let Ok(stream) = vm.sys_module.get_attr(name, vm) else { + let Some(stream) = crate::vm::fork::namespace_item(&vm.sys_module.dict(), name) else { continue; }; reinit_io_locks(&stream); @@ -5702,12 +5754,12 @@ mod _io { let file_io_class: &Py = cfg_select! { all(feature = "host_env", windows) => { if is_console { - Some(super::winconsoleio::WindowsConsoleIO::static_type()) + Some(unsafe { super::winconsoleio::WindowsConsoleIO::static_type() }) } else { - Some(super::fileio::FileIO::static_type()) + Some(unsafe { super::fileio::FileIO::static_type() }) } } - feature = "host_env" => Some(super::fileio::FileIO::static_type()), + feature = "host_env" => Some(unsafe { super::fileio::FileIO::static_type() }), _ => None, } .ok_or_else(|| { @@ -5757,11 +5809,11 @@ mod _io { } let cls = if mode.plus { - BufferedRandom::static_type() + unsafe { BufferedRandom::static_type() } } else if let FileMode::Read = mode.file { - BufferedReader::static_type() + unsafe { BufferedReader::static_type() } } else { - BufferedWriter::static_type() + unsafe { BufferedWriter::static_type() } }; let buffered = PyType::call(cls, (raw, buffering).into_args(vm), vm)?; @@ -5780,7 +5832,7 @@ mod _io { } } }; - let tio = TextIOWrapper::static_type(); + let tio = unsafe { TextIOWrapper::static_type() }; let wrapper = PyType::call( tio, ( @@ -5824,11 +5876,9 @@ mod _io { .unwrap() } - pub(super) fn unsupported_operation() -> &'static Py { - rustpython_common::static_cell! { - static CELL: PyTypeRef; - } - CELL.get_or_init(|| create_unsupported_operation(Context::genesis())) + pub(super) fn unsupported_operation(vm: &VirtualMachine) -> PyTypeRef { + struct UnsupportedOperation; + vm.__cached_native::(|| create_unsupported_operation(&vm.ctx)) } #[pyfunction] @@ -5921,7 +5971,7 @@ mod _io { #[cfg(all(feature = "host_env", windows))] super::winconsoleio::module_exec(vm, module)?; - let unsupported_operation = unsupported_operation().to_owned(); + let unsupported_operation = unsupported_operation(vm); extend_module!(vm, module, { "UnsupportedOperation" => unsupported_operation, "BlockingIOError" => vm.ctx.exceptions.blocking_io_error.to_owned(), diff --git a/crates/vm/src/stdlib/_testinternalcapi.rs b/crates/vm/src/stdlib/_testinternalcapi.rs index 58430a374cd..77b5f7566b3 100644 --- a/crates/vm/src/stdlib/_testinternalcapi.rs +++ b/crates/vm/src/stdlib/_testinternalcapi.rs @@ -31,8 +31,7 @@ mod _testinternalcapi { AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, builtins::{ - PyBytesRef, PyCode, PyDict, PyDictRef, PyStrRef, PyType, PyTypeRef, - descriptor::PyWrapper, + PyBytesRef, PyCode, PyDict, PyDictRef, PyStrRef, PyTypeRef, descriptor::PyWrapper, }, common::{hash::PyHash, lock::LazyLock}, dict_inner, @@ -393,13 +392,7 @@ mod _testinternalcapi { continue; } out.push(cls.clone().into()); - for weak in cls.subclasses.read().iter() { - if let Some(sub) = weak.upgrade() - && let Ok(sub) = sub.downcast::() - { - stack.push(sub); - } - } + stack.extend(cls.subclass_snapshot()); } out } diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 984a0865560..78c74c7d754 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -1,7 +1,7 @@ //! Implementation of the _thread module #[cfg(all(unix, feature = "threading", feature = "host_env"))] -pub(crate) use _thread::after_fork_child; +pub(crate) use _thread::{after_fork_child, repair_handles_after_fork}; pub use _thread::get_ident; @@ -23,7 +23,7 @@ pub(crate) mod _thread { builtins::{ PyBaseExceptionRef, PyDictRef, PyIntRef, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, }, - common::{lock::PyMutex, wtf8::Wtf8Buf}, + common::{lock::PyMutex, rc::PyRc, wtf8::Wtf8Buf}, convert::ToPyException, frame::FrameObjectRef, function::{ @@ -32,6 +32,7 @@ pub(crate) mod _thread { }, object::{Traverse, TraverseFn}, types::{Constructor, GetAttr, PyStructSequence, Representable, SetAttr}, + vm::roots::RootLease, }; use alloc::sync::{Arc, Weak}; @@ -152,7 +153,24 @@ pub(crate) mod _thread { #[pyclass(module = "_thread", name = "lock")] #[derive(PyPayload)] struct Lock { - mu: RawMutex, + mu: LockStorage, + } + + enum LockStorage { + Inline(RawMutex), + // Thread-exit sentinels must unlock without entering Python, including + // after their VM has closed. Ordinary locks need no extra allocation. + Shared(Arc), + } + + impl core::ops::Deref for LockStorage { + type Target = RawMutex; + fn deref(&self) -> &RawMutex { + match self { + Self::Inline(lock) => lock, + Self::Shared(lock) => lock, + } + } } impl fmt::Debug for Lock { @@ -189,7 +207,7 @@ pub(crate) mod _thread { fn _at_fork_reinit(&self, _vm: &VirtualMachine) { // Overwrite lock state to unlocked. Do NOT call unlock() here — // after fork(), unlock_slow() would try to unpark stale waiters. - unsafe { rustpython_common::lock::zero_reinit_after_fork(&self.mu) }; + unsafe { rustpython_common::lock::zero_reinit_after_fork(&*self.mu) }; } #[pymethod] @@ -211,7 +229,9 @@ pub(crate) mod _thread { type Args = (); fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { - Ok(Self { mu: RawMutex::INIT }) + Ok(Self { + mu: LockStorage::Inline(RawMutex::INIT), + }) } } @@ -591,7 +611,9 @@ pub(crate) mod _thread { #[pyfunction] const fn allocate_lock() -> Lock { - Lock { mu: RawMutex::INIT } + Lock { + mu: LockStorage::Inline(RawMutex::INIT), + } } struct StartNewThreadArgs { @@ -691,6 +713,10 @@ pub(crate) mod _thread { .state .finalizing .load(core::sync::atomic::Ordering::Acquire) + || vm + .state + .admission_closed + .load(core::sync::atomic::Ordering::Acquire) { return Err(vm.new_exception_msg( vm.ctx.exceptions.python_finalization_error.to_owned(), @@ -716,8 +742,8 @@ pub(crate) mod _thread { let thread_builder = apply_thread_stack_size(thread::Builder::new(), vm); thread_builder .spawn( - vm.new_thread() - .make_spawn_func(move |vm| run_thread(func, args, vm)), + vm.new_python_thread() + .make_spawn_func_raw(move |vm| run_thread(func, args, vm)), ) .map(|handle| thread_to_id(&handle)) .map_err(|_err| vm.new_runtime_error("can't start new thread")) @@ -740,11 +766,8 @@ pub(crate) mod _thread { // Increment thread count when thread actually starts executing vm.state.thread_count.fetch_add(1); - // Inner scope: drop `func` (and its Python refs) before the thread - // slot is torn down below. Otherwise the parameter `func` would drop - // at end-of-function, after cleanup_current_thread_frames has cleared - // CURRENT_THREAD_SLOT, and a weakref callback fired during that drop - // would panic in push_thread_frame. + // Drop `func` and its Python references before releasing sentinels and + // thread-local roots. The slot stays attached through the entry drain. { let func = func; if let Err(exc) = func.invoke(args, vm) @@ -753,16 +776,11 @@ pub(crate) mod _thread { report_unraisable_thread_exception(exc, &func, vm); } } - for lock in SENTINELS.take() { - if lock.mu.is_locked() { - unsafe { lock.mu.unlock() }; - } - } + release_sentinels(vm); // Clean up thread-local storage while VM context is still active // This ensures __del__ methods are called properly - cleanup_thread_local_data(); - // Clean up frame tracking - crate::vm::thread::cleanup_current_thread_frames(vm); + cleanup_thread_local_data(vm); + crate::vm::thread::retire_thread_slot(vm); vm.state.thread_count.fetch_sub(1); } @@ -808,11 +826,18 @@ pub(crate) mod _thread { /// Clean up thread-local data for the current thread. /// This triggers __del__ on objects stored in thread-local variables. - fn cleanup_thread_local_data() { - // Move all guards out before dropping them. A local dict's __del__ may - // re-enter thread-local access and borrow LOCAL_GUARDS again. - let guards = LOCAL_GUARDS.take(); - drop(guards); + fn cleanup_thread_local_data(vm: &VirtualMachine) { + // Move this owner's guards out before dropping them. A local dict's + // __del__ may re-enter thread-local access and borrow LOCAL_GUARDS again. + let guards = LOCAL_GUARDS.with_borrow_mut(|guards| { + guards + .extract_if(.., |guard| guard.lease.owner() == vm.state.interpreter_id) + .collect::>() + }); + for guard in guards { + guard.lease.release_now(vm); + } + vm.state.roots.drain_pending(); } #[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] @@ -835,13 +860,44 @@ pub(crate) mod _thread { Err(vm.new_system_exit(vec![].into())) } - thread_local!(static SENTINELS: RefCell>> = const { RefCell::new(Vec::new()) }); + thread_local!(static SENTINELS: RefCell> = const { RefCell::new(Vec::new()) }); + + struct Sentinel { + owner: i64, + lock: Arc, + } + + impl Drop for Sentinel { + fn drop(&mut self) { + if self.lock.is_locked() { + unsafe { self.lock.unlock() }; + } + } + } + + fn release_sentinels(vm: &VirtualMachine) { + let sentinels = SENTINELS.with_borrow_mut(|sentinels| { + sentinels + .extract_if(.., |sentinel| sentinel.owner == vm.state.interpreter_id) + .collect::>() + }); + drop(sentinels); + vm.state.roots.drain_pending(); + } #[pyfunction] fn _set_sentinel(vm: &VirtualMachine) -> PyRef { - let lock = Lock { mu: RawMutex::INIT }.into_ref(&vm.ctx); - SENTINELS.with_borrow_mut(|sentinels| sentinels.push(lock.clone())); - lock + let mu = Arc::new(RawMutex::INIT); + SENTINELS.with_borrow_mut(|sentinels| { + sentinels.push(Sentinel { + owner: vm.state.interpreter_id, + lock: mu.clone(), + }) + }); + Lock { + mu: LockStorage::Shared(mu), + } + .into_ref(&vm.ctx) } #[derive(FromArgs)] @@ -1012,14 +1068,15 @@ pub(crate) mod _thread { #[pystruct_sequence( name = "_ExceptHookArgs", module = "_thread", - data = "ExceptHookArgsData" + data = "ExceptHookArgsData", + interpreter_local )] struct PyExceptHookArgs; #[pyclass(with(PyStructSequence))] impl PyExceptHookArgs { #[extend_class] - fn extend_pyclass(ctx: &crate::vm::Context, class: &'static Py) { + fn extend_pyclass(ctx: &crate::vm::Context, class: &Py) { // (type, (sequence, dict)). Set before the trait installs its default. const EXCEPT_HOOK_ARGS_REDUCE: crate::function::PyMethodDef = crate::function::PyMethodDef::new_const( @@ -1123,7 +1180,14 @@ pub(crate) mod _thread { // Thread-local storage for cleanup guards // When a thread terminates, the guard is dropped, which triggers cleanup thread_local! { - static LOCAL_GUARDS: RefCell> = const { RefCell::new(Vec::new()) }; + static LOCAL_GUARDS: RefCell> = const { RefCell::new(Vec::new()) }; + } + + // TLS holds no Python references and only queues owner-local retirement. + struct LocalGuardKey { + local: Weak, + thread_id: u64, + lease: PyRc, } // Guard that removes thread-local data when dropped @@ -1243,19 +1307,25 @@ pub(crate) mod _thread { // Register cleanup guard only if we inserted a new entry if need_guard { - let guard = LocalGuard { - local: Arc::downgrade(&self.inner), + let local = Arc::downgrade(&self.inner); + if let Some(lease) = vm.state.roots.insert_resource(LocalGuard { + local: local.clone(), thread_id, - }; - LOCAL_GUARDS.with(|guards| { - guards.borrow_mut().push(guard); - }); + }) { + LOCAL_GUARDS.with_borrow_mut(|guards| { + guards.push(LocalGuardKey { + local, + thread_id, + lease, + }); + }); + } } (dict, need_guard) } - fn remove_current_dict(&self) { + fn remove_current_dict(&self, vm: &VirtualMachine) { let thread_id = current_thread_id(); let guard = LOCAL_GUARDS.with(|guards| { let mut guards = guards.borrow_mut(); @@ -1269,7 +1339,7 @@ pub(crate) mod _thread { }); if let Some(guard) = guard { - drop(guard); + guard.lease.release_now(vm); } else { let removed = self.inner.state.lock().dicts.remove(&thread_id); drop(removed); @@ -1287,7 +1357,7 @@ pub(crate) mod _thread { }; let init_args = zelf.inner.state.lock().init_args.clone(); if let Err(err) = init(zelf.as_object(), init_args, vm) { - zelf.remove_current_dict(); + zelf.remove_current_dict(vm); return Err(err); } @@ -1478,83 +1548,51 @@ pub(crate) mod _thread { // Reinitialize frame slot for current thread. // Locks are already reinit'd, so lock() is safe. crate::vm::thread::reinit_frame_slot_after_fork(vm); + } - // Clean up thread handles. All VmState locks were reinit'd to unlocked, - // so lock() won't deadlock. Per-thread Arc> - // locks are also reinit'd below before use. - { - let mut handles = vm.state.thread_handles.lock(); - handles.retain(|(inner_weak, done_event_weak): &HandleEntry| { - let Some(inner) = inner_weak.upgrade() else { - return false; - }; - let Some(done_event) = done_event_weak.upgrade() else { - return false; - }; - - // Reinit this per-handle lock in case a dead thread held it - reinit_parking_lot_mutex(&inner); - let mut inner_guard = inner.lock(); - - if inner_guard.ident == current_ident { - return true; - } - if inner_guard.state == ThreadHandleState::NotStarted { - return true; - } - - inner_guard.state = ThreadHandleState::Done; - // The OS thread did not survive the fork. Dropping JoinHandle - // would pthread_detach a copied thread descriptor. - if let Some(handle) = inner_guard.join_handle.take() { - core::mem::forget(handle); - } - drop(inner_guard); - - // Reinit and set the done event. Do not notify: no other - // thread exists in the child, and notify_all would unpark - // waiters that did not survive the fork. - let (lock, cvar) = &*done_event; - reinit_parking_lot_mutex(lock); - reinit_parking_lot_condvar(cvar); - *lock.lock() = true; - - true - }); - } - - // Clean up shutdown_handles. - { - let mut handles = vm.state.shutdown_handles.lock(); - handles.retain(|(inner_weak, done_event_weak): &ShutdownEntry| { - let Some(inner) = inner_weak.upgrade() else { - return false; - }; - let Some(done_event) = done_event_weak.upgrade() else { - return false; - }; - - reinit_parking_lot_mutex(&inner); - let mut inner_guard = inner.lock(); - - if inner_guard.ident == current_ident { - return true; - } - if inner_guard.state == ThreadHandleState::NotStarted { - return true; - } - - inner_guard.state = ThreadHandleState::Done; - drop(inner_guard); - - let (lock, cvar) = &*done_event; - reinit_parking_lot_mutex(lock); - reinit_parking_lot_condvar(cvar); - *lock.lock() = true; - - false - }); - } + /// Repair native handles before any child callback can create a thread or + /// join a parent thread. Every interpreter must complete this phase first. + #[cfg(all(unix, feature = "threading", feature = "host_env"))] + pub(crate) fn repair_handles_after_fork(state: &crate::vm::PyGlobalState) { + let current_ident = get_ident(); + state.main_thread_ident.store(current_ident); + let repair = |inner_weak: &alloc::sync::Weak>, + done_event_weak: &alloc::sync::Weak<( + parking_lot::Mutex, + parking_lot::Condvar, + )>| { + let (Some(inner), Some(done_event)) = (inner_weak.upgrade(), done_event_weak.upgrade()) + else { + return false; + }; + reinit_parking_lot_mutex(&inner); + let mut inner = inner.lock(); + if inner.ident != current_ident && inner.state != ThreadHandleState::NotStarted { + inner.state = ThreadHandleState::Done; + } + // Copied JoinHandles no longer identify joinable native threads in + // this process, including the thread that called fork. + if let Some(handle) = inner.join_handle.take() { + core::mem::forget(handle); + } + inner.joining = false; + inner.joined = inner.state == ThreadHandleState::Done; + let done = inner.joined; + drop(inner); + let (lock, cvar) = &*done_event; + reinit_parking_lot_mutex(lock); + reinit_parking_lot_condvar(cvar); + *lock.lock() = done; + true + }; + state + .thread_handles + .lock() + .retain(|(inner, event)| repair(inner, event)); + state + .shutdown_handles + .lock() + .retain(|(inner, event)| repair(inner, event)); } /// Take a thread handle's completion mutex, detaching first. @@ -1999,7 +2037,7 @@ pub(crate) mod _thread { let handle = if let Some(handle_obj) = handle_obj { if vm.is_none(&handle_obj) { None - } else if !handle_obj.class().is(thread_handle_type) { + } else if !handle_obj.class().is(&thread_handle_type) { return Err(vm.new_type_error("'handle' must be a _ThreadHandle").into()); } else { Some( @@ -2051,6 +2089,10 @@ pub(crate) mod _thread { .state .finalizing .load(core::sync::atomic::Ordering::Acquire) + || vm + .state + .admission_closed + .load(core::sync::atomic::Ordering::Acquire) { return Err(vm.new_exception_msg( vm.ctx.exceptions.python_finalization_error.to_owned(), @@ -2103,7 +2145,7 @@ pub(crate) mod _thread { let thread_builder = apply_thread_stack_size(thread::Builder::new(), vm); let join_handle = thread_builder - .spawn(vm.new_thread().make_spawn_func(move |vm| { + .spawn(vm.new_python_thread().make_spawn_func_raw(move |vm| { // Publish ident for the parent starter thread. { inner_clone.lock().ident = get_ident(); @@ -2141,17 +2183,12 @@ pub(crate) mod _thread { inner_for_cleanup.lock().state = ThreadHandleState::Done; // Handle sentinels - for lock in SENTINELS.take() { - if lock.mu.is_locked() { - unsafe { lock.mu.unlock() }; - } - } + release_sentinels(vm); // Clean up thread-local data while VM context is still active - cleanup_thread_local_data(); + cleanup_thread_local_data(vm); - // Clean up frame tracking - crate::vm::thread::cleanup_current_thread_frames(vm); + crate::vm::thread::retire_thread_slot(vm); vm_state.thread_count.fetch_sub(1); @@ -2170,12 +2207,8 @@ pub(crate) mod _thread { // Increment thread count when thread actually starts executing vm_state.thread_count.fetch_add(1); - // Inner scope: drop `func` (and its Python refs) before the - // outer scopeguard::defer tears down the thread slot. As a - // `move` closure capture, `func` would otherwise drop after - // all locals (including the scopeguard `_guard`), and a - // weakref callback fired during that drop would panic in - // push_thread_frame. + // Drop `func` and its Python references before the outer + // cleanup guard releases thread-local roots and signals done. { let func = func; // Run the function @@ -2256,7 +2289,7 @@ pub(crate) mod _thread { #[test] #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] fn default_python_thread_stack_size_debug() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { assert_eq!(vm.state.stacksize.load(), 0); let builder = apply_thread_stack_size(thread::Builder::new(), vm); let stack_size = builder @@ -2280,7 +2313,7 @@ pub(crate) mod _thread { fn explicit_python_thread_stack_size_is_a_floor_debug() { const REQUESTED: usize = 256 * 1024; - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { vm.state.stacksize.store(REQUESTED); let builder = apply_thread_stack_size(thread::Builder::new(), vm); let stack_size = builder diff --git a/crates/vm/src/stdlib/_warnings.rs b/crates/vm/src/stdlib/_warnings.rs index 8a8f92b77da..3dd98f34901 100644 --- a/crates/vm/src/stdlib/_warnings.rs +++ b/crates/vm/src/stdlib/_warnings.rs @@ -28,23 +28,23 @@ mod _warnings { #[pyattr] fn filters(vm: &VirtualMachine) -> PyListRef { - vm.state.warnings.filters.to_owned() + vm.state.warnings.filters() } #[pyattr] fn _defaultaction(vm: &VirtualMachine) -> PyStrRef { - vm.state.warnings.default_action.to_owned() + vm.state.warnings.default_action() } #[pyattr] fn _onceregistry(vm: &VirtualMachine) -> PyDictRef { - vm.state.warnings.once_registry.to_owned() + vm.state.warnings.once_registry() } #[pyattr] fn _warnings_context(vm: &VirtualMachine) -> PyObjectRef { - if let Some(ctx) = vm.state.warnings.context_var.get() { - return ctx.clone(); + if let Some(ctx) = vm.state.warnings.context_var() { + return ctx; } // _warnings is initialized before _contextvars may be importable. // Retry until ContextVar can be created; do not cache a None fallback. @@ -54,10 +54,7 @@ mod _warnings { .and_then(|m| m.get_attr("ContextVar", vm).ok()) .and_then(|cv_cls| cv_cls.call(("_warnings_context",), vm).ok()); match created { - Some(cv) => match vm.state.warnings.context_var.set(cv.clone()) { - Ok(()) => cv, - Err(_) => vm.state.warnings.context_var.get().cloned().unwrap_or(cv), - }, + Some(cv) => vm.state.warnings.init_context_var(cv), None => vm.ctx.none(), } } diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index fbced8423a7..16561ad6d5b 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -234,7 +234,7 @@ mod builtins { return Err(vm.new_value_error(msg)); }; - let ast_type = _ast::NodeAst::make_static_type().as_object().to_owned(); + let ast_type = _ast::NodeAst::make_class(&vm.ctx).as_object().to_owned(); if args.source.is_instance(&ast_type, vm)? { let explicit_future_annotations = future_features.contains(bytecode::CodeFlags::FUTURE_ANNOTATIONS); @@ -242,7 +242,7 @@ mod builtins { // compile(ast_node, ..., PyCF_ONLY_AST) returns the AST after validation if is_ast_only { - let (expected_type, expected_name) = _ast::mode_type_and_name(mode_str) + let (expected_type, expected_name) = _ast::mode_type_and_name(mode_str, vm) .ok_or_else(|| { vm.new_value_error( "compile() mode must be 'exec', 'eval', 'single' or 'func_type'", @@ -279,7 +279,7 @@ mod builtins { } #[cfg(feature = "rustpython-codegen")] { - let (expected_type, expected_name) = _ast::mode_type_and_name(mode_str) + let (expected_type, expected_name) = _ast::mode_type_and_name(mode_str, vm) .ok_or_else(|| { vm.new_value_error("compile() mode must be 'exec', 'eval' or 'single'") })?; @@ -1450,14 +1450,14 @@ mod builtins { pub fn init_module(vm: &VirtualMachine, module: &Py) { let ctx = &vm.ctx; - let _ = crate::protocol::VecBuffer::make_static_type(); + let _ = unsafe { crate::protocol::VecBuffer::make_static_type() }; module.__init_methods(vm).unwrap(); builtins::module_exec(vm, module).unwrap(); let debug_mode: bool = vm.state.config.settings.optimize == 0; // Create dynamic ExceptionGroup with multiple inheritance (BaseExceptionGroup + Exception) - let exception_group = crate::exception_group::exception_group(); + let exception_group = crate::exception_group::exception_group(vm); extend_module!(vm, module, { "__debug__" => ctx.new_bool(debug_mode), @@ -1499,7 +1499,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { // Exceptions: "BaseException" => ctx.exceptions.base_exception_type.to_owned(), "BaseExceptionGroup" => ctx.exceptions.base_exception_group.to_owned(), - "ExceptionGroup" => exception_group.to_owned(), + "ExceptionGroup" => exception_group, "SystemExit" => ctx.exceptions.system_exit.to_owned(), "KeyboardInterrupt" => ctx.exceptions.keyboard_interrupt.to_owned(), "GeneratorExit" => ctx.exceptions.generator_exit.to_owned(), diff --git a/crates/vm/src/stdlib/gc.rs b/crates/vm/src/stdlib/gc.rs index 92679b98503..2930175c74a 100644 --- a/crates/vm/src/stdlib/gc.rs +++ b/crates/vm/src/stdlib/gc.rs @@ -1,4 +1,4 @@ -pub(crate) use gc::module_def; +pub(crate) use gc::{invoke_callbacks, module_def}; #[pymodule] mod gc { @@ -50,26 +50,7 @@ mod gc { return Err(vm.new_value_error("invalid generation")); } - // Invoke callbacks with "start" phase - invoke_callbacks(vm, "start", generation_num as usize, &Default::default()); - - // Manual gc.collect() should run even if GC is disabled - let gc = &vm.state.gc; - let result = gc.collect_force(generation_num as usize); - - // Publish what the collection saved as gc.garbage (for DEBUG_SAVEALL) - { - let mut state_garbage = gc.garbage.lock(); - if !state_garbage.is_empty() { - let mut garbage_vec = gc.py_garbage.borrow_vec_mut(); - for obj in state_garbage.drain(..) { - garbage_vec.push(obj); - } - } - } - - // Invoke callbacks with "stop" phase - invoke_callbacks(vm, "stop", generation_num as usize, &result); + let result = vm.state.gc.collect_force(generation_num as usize); Ok((result.collected + result.uncollectable) as i32) } @@ -102,7 +83,7 @@ mod gc { #[pyfunction] fn get_count(vm: &VirtualMachine) -> PyObjectRef { - let (c0, c1, c2) = gc_state::gc_state().get_count(); + let (c0, c1, c2) = vm.state.gc.get_count(); vm.ctx .new_tuple(vec![ vm.ctx.new_int(c0).into(), @@ -167,6 +148,17 @@ mod gc { #[pyfunction] fn get_referents(objs: PosArgs, vm: &VirtualMachine) -> PyListRef { + #[cfg(feature = "threading")] + let result = vm + .state + .stop_the_world + .with_stopped(&vm.state, || referents(&objs)); + #[cfg(not(feature = "threading"))] + let result = referents(&objs); + vm.ctx.new_list(result) + } + + fn referents(objs: &PosArgs) -> Vec { let mut result = Vec::new(); for obj in objs.iter() { @@ -174,11 +166,22 @@ mod gc { result.extend(obj.gc_get_referents()); } - vm.ctx.new_list(result) + result } #[pyfunction] fn get_referrers(objs: PosArgs, vm: &VirtualMachine) -> PyListRef { + #[cfg(feature = "threading")] + let result = vm + .state + .stop_the_world + .with_stopped(&vm.state, || referrers(&objs, vm)); + #[cfg(not(feature = "threading"))] + let result = referrers(&objs, vm); + vm.ctx.new_list(result) + } + + fn referrers(objs: &PosArgs, vm: &VirtualMachine) -> Vec { use std::collections::HashSet; // Build a set of target object pointers for fast lookup @@ -229,7 +232,7 @@ mod gc { } } - vm.ctx.new_list(result) + result } #[pyfunction] @@ -254,30 +257,32 @@ mod gc { } #[pyfunction] - fn get_freeze_count() -> usize { - gc_state::gc_state().get_freeze_count() + fn get_freeze_count(vm: &VirtualMachine) -> usize { + vm.state.gc.get_freeze_count() } // gc.garbage - list of uncollectable objects #[pyattr] fn garbage(vm: &VirtualMachine) -> PyListRef { - vm.state.gc.py_garbage.clone() + vm.state.gc.garbage_list().expect("GC state is closed") } // gc.callbacks - list of callbacks to be invoked #[pyattr] fn callbacks(vm: &VirtualMachine) -> PyListRef { - vm.state.gc.py_callbacks.clone() + vm.state.gc.callbacks_list().expect("GC state is closed") } /// Helper function to invoke GC callbacks - fn invoke_callbacks( + pub(crate) fn invoke_callbacks( vm: &VirtualMachine, phase: &str, generation: usize, result: &gc_state::CollectResult, ) { - let callbacks_list = &vm.state.gc.py_callbacks; + let Some(callbacks_list) = vm.state.gc.callbacks_list() else { + return; + }; let callbacks: Vec = callbacks_list.borrow_vec().to_vec(); if callbacks.is_empty() { return; diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 371e3ee68c2..59a0de36fad 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -39,7 +39,7 @@ mod decl { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { let args = - crate::types::drop_kwargs_if_init_overridden(&cls, Self::class(&vm.ctx), args); + crate::types::drop_kwargs_if_init_overridden(&cls, &Self::class(&vm.ctx), args); if !args.kwargs.is_empty() { return Err( vm.new_type_error(format!("{}() takes no keyword arguments", Self::NAME)) @@ -796,7 +796,7 @@ mod decl { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { let args = - crate::types::drop_kwargs_if_init_overridden(&cls, Self::class(&vm.ctx), args); + crate::types::drop_kwargs_if_init_overridden(&cls, &Self::class(&vm.ctx), args); let (iter, start, stop, step) = match args.args.len() { 0 | 1 => { return Err(vm.new_arity_type_error(Self::NAME, 2..=4, args.args.len())); @@ -1103,7 +1103,7 @@ mod decl { impl PyItertoolsTee { fn from_iter(iterator: PyIter, vm: &VirtualMachine) -> PyResult { let class = Self::class(&vm.ctx); - if iterator.class().is(class) { + if iterator.class().is(&class) { return vm.call_special_method(&iterator, identifier!(vm, __copy__), ()); } Ok(Self { @@ -1111,7 +1111,7 @@ mod decl { index: AtomicCell::new(0), advancing: AtomicBool::new(false), } - .into_ref_with_type(vm, class.to_owned())? + .into_ref_with_type(vm, class)? .into()) } } diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index 2624e9c0edd..0c79c809f28 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -35,7 +35,7 @@ mod decl { &self, f: impl FnOnce(DumpableValue<'_, Self>) -> R, ) -> Result { - if self.is(PyStopIteration::static_type()) { + if self.is(unsafe { PyStopIteration::static_type() }) { return Ok(f(DumpableValue::StopIter)); } @@ -43,7 +43,7 @@ mod decl { PyNone => f(DumpableValue::None), PyEllipsis => f(DumpableValue::Ellipsis), ref pyint @ PyInt => { - if self.class().is(PyBool::static_type()) { + if self.class().is(unsafe { PyBool::static_type() }) { f(DumpableValue::Boolean(!pyint.as_bigint().is_zero())) } else { f(DumpableValue::Integer(pyint.as_bigint())) @@ -229,8 +229,8 @@ mod decl { // Singletons: no FLAG_REF needed let is_singleton = vm.is_none(obj) - || obj.class().is(PyBool::static_type()) - || obj.is(PyStopIteration::static_type()) + || obj.class().is(unsafe { PyBool::static_type() }) + || obj.is(unsafe { PyStopIteration::static_type() }) || obj.downcast_ref::().is_some(); // FLAG_REF: check if already written, otherwise reserve slot @@ -260,9 +260,9 @@ mod decl { if vm.is_none(obj) { buf.write_u8(b'N'); - } else if obj.is(PyStopIteration::static_type()) { + } else if obj.is(unsafe { PyStopIteration::static_type() }) { buf.write_u8(b'S'); - } else if obj.class().is(PyBool::static_type()) { + } else if obj.class().is(unsafe { PyBool::static_type() }) { let val = obj .downcast_ref::() .is_some_and(|i| !i.as_bigint().is_zero()); @@ -577,7 +577,13 @@ mod decl { len: usize, ) -> Result, marshal::MarshalError> { let elements = self.placeholder_elements(len)?; - Ok(Some(PyTuple::new_ref(elements, &self.vm.ctx).into())) + Ok(Some( + crate::PyRef::new_ref_tracked( + PyTuple::new_unchecked(elements.into_boxed_slice()), + self.vm.ctx.types.tuple_type.to_owned(), + ) + .into(), + )) } fn set_tuple_item( &self, @@ -874,18 +880,18 @@ mod decl { fn check_exact_type(obj: &PyObject, vm: &VirtualMachine) -> PyResult<()> { let cls = obj.class(); // bool is a subclass of int but is marshallable - if cls.is(PyBool::static_type()) { + if cls.is(unsafe { PyBool::static_type() }) { return Ok(()); } for base in [ - PyInt::static_type(), - PyFloat::static_type(), - PyComplex::static_type(), - PyTuple::static_type(), - PyList::static_type(), - PyDict::static_type(), - PySet::static_type(), - PyFrozenSet::static_type(), + unsafe { PyInt::static_type() }, + unsafe { PyFloat::static_type() }, + unsafe { PyComplex::static_type() }, + unsafe { PyTuple::static_type() }, + unsafe { PyList::static_type() }, + unsafe { PyDict::static_type() }, + unsafe { PySet::static_type() }, + unsafe { PyFrozenSet::static_type() }, ] { if cls.fast_issubclass(base) && !cls.is(base) { return Err(vm.new_value_error("unmarshallable object")); diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs index 73b816c579c..275e67b0494 100644 --- a/crates/vm/src/stdlib/mod.rs +++ b/crates/vm/src/stdlib/mod.rs @@ -23,7 +23,7 @@ mod _weakref; pub mod atexit; pub mod builtins; pub mod errno; -mod gc; +pub(crate) mod gc; mod itertools; mod marshal; pub mod time; diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 527643822b8..97899122011 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -157,7 +157,7 @@ pub(crate) fn warn_if_bool_fd(obj: &PyObject, vm: &VirtualMachine) -> PyResult<( use crate::class::StaticType; if obj .class() - .is(crate::builtins::bool_::PyBool::static_type()) + .is(unsafe { crate::builtins::bool_::PyBool::static_type() }) { crate::stdlib::_warnings::warn( vm.ctx.exceptions.runtime_warning, @@ -1428,7 +1428,12 @@ pub(super) mod _os { } #[pyattr] - #[pystruct_sequence(name = "stat_result", module = "os", data = "StatResultData")] + #[pystruct_sequence( + name = "stat_result", + module = "os", + data = "StatResultData", + interpreter_local + )] struct PyStatResult; #[pyclass(with(PyStructSequence))] @@ -1956,12 +1961,22 @@ pub(super) mod _os { #[cfg(all(windows, not(target_os = "redox")))] #[pyattr] - #[pystruct_sequence(name = "times_result", module = "nt", data = "TimesResultData")] + #[pystruct_sequence( + name = "times_result", + module = "nt", + data = "TimesResultData", + interpreter_local + )] struct PyTimesResult; #[cfg(all(unix, not(windows), not(target_os = "redox")))] #[pyattr] - #[pystruct_sequence(name = "times_result", module = "posix", data = "TimesResultData")] + #[pystruct_sequence( + name = "times_result", + module = "posix", + data = "TimesResultData", + interpreter_local + )] struct PyTimesResult; #[cfg(all(any(unix, windows), not(target_os = "redox")))] @@ -2170,7 +2185,12 @@ pub(super) mod _os { } #[pyattr] - #[pystruct_sequence(name = "terminal_size", module = "os", data = "TerminalSizeData")] + #[pystruct_sequence( + name = "terminal_size", + module = "os", + data = "TerminalSizeData", + interpreter_local + )] pub(crate) struct PyTerminalSize; #[pyclass(with(PyStructSequence))] @@ -2188,12 +2208,22 @@ pub(super) mod _os { #[cfg(windows)] #[pyattr] - #[pystruct_sequence(name = "uname_result", module = "nt", data = "UnameResultData")] + #[pystruct_sequence( + name = "uname_result", + module = "nt", + data = "UnameResultData", + interpreter_local + )] pub(crate) struct PyUnameResult; #[cfg(not(windows))] #[pyattr] - #[pystruct_sequence(name = "uname_result", module = "posix", data = "UnameResultData")] + #[pystruct_sequence( + name = "uname_result", + module = "posix", + data = "UnameResultData", + interpreter_local + )] pub(crate) struct PyUnameResult; #[pyclass(with(PyStructSequence))] diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 1f123812fc2..b82a3ab0b10 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -686,9 +686,6 @@ pub mod module { #[cfg(feature = "threading")] crate::stdlib::_imp::acquire_imp_lock_for_fork(vm); - - #[cfg(feature = "threading")] - vm.state.stop_the_world.stop_the_world(&vm.state); } fn py_os_after_fork_child(vm: &VirtualMachine) { @@ -708,9 +705,8 @@ pub mod module { #[cfg(feature = "threading")] reinit_locks_after_fork(vm); - // The collector stops every interpreter, so interpreters other than the - // forking one must be repaired too; otherwise the child's first - // collection waits for threads that did not survive the fork. + // Fork quiesces every interpreter. Repair their inherited thread state + // before nested or saved native entries can resume in the child. #[cfg(all(unix, feature = "threading"))] reinit_other_interpreters_after_fork(vm); @@ -741,15 +737,21 @@ pub mod module { crate::object::reset_weakref_locks_after_fork(); // Repair any type-cache entries left mid-update at fork time. - unsafe { crate::builtins::type_::type_cache_after_fork() }; + unsafe { vm.state.type_cache.after_fork() }; - // Reset QSBR: dead parent threads' slots would stall reclamation - // forever, and retired memory can be freed immediately in the - // single-threaded child. + // Repair every native handle before releasing vanished thread roots: + // their native destructors may themselves release or join handles. #[cfg(feature = "threading")] - unsafe { - crate::object::qsbr::QSBR.reset_after_fork() - }; + for state in crate::vm::runtime::live_interpreter_states() { + crate::stdlib::_thread::repair_handles_after_fork(&state); + } + + #[cfg(feature = "threading")] + for state in crate::vm::runtime::live_interpreter_states() { + if state.interpreter_id != vm.state.interpreter_id { + crate::vm::thread::retain_frame_slot_after_fork(&state); + } + } // Phase 3: Clean up thread state. Locks are now reinit'd so we can // acquire them normally instead of using try_lock(). @@ -768,7 +770,14 @@ pub mod module { vm.signal_handlers .get_or_init(crate::signal::SignalHandlers::default); - // Phase 4: Run Python-level at-fork callbacks. + #[cfg(feature = "threading")] + crate::vm::fork::finish_child(vm); + + // Phase 4: Run Python-level at-fork callbacks in their owning VM. + run_after_fork_callbacks(vm); + } + + pub(crate) fn run_after_fork_callbacks(vm: &VirtualMachine) { let after_forkers_child: Vec = vm.state.after_forkers_child.lock().clone(); run_at_forkers(after_forkers_child, false, vm); } @@ -790,7 +799,7 @@ pub mod module { reinit_mutex_after_fork(&vm.state.atexit_funcs); reinit_mutex_after_fork(&vm.state.global_trace_func); reinit_mutex_after_fork(&vm.state.global_profile_func); - reinit_mutex_after_fork(&vm.state.type_mutex); + crate::builtins::PyType::repair_type_lock_after_fork(&vm.state); reinit_mutex_after_fork(&vm.state.monitoring); // PyGlobalState parking_lot::Mutex locks @@ -803,11 +812,14 @@ pub mod module { // Codec registry RwLock vm.state.codec_registry.reinit_after_fork(); + vm.state.warnings.reinit_after_fork(); // GC state (multiple Mutex + RwLock), shared lists and this // interpreter's own policy state. crate::gc_state::gc_state().reinit_after_fork(); vm.state.gc.reinit_after_fork(); + vm.state.reinit_native_cache_after_fork(); + vm.state.roots.reinit_after_fork(); // Import lock (RawReentrantMutex) crate::stdlib::_imp::reinit_imp_lock_after_fork(); @@ -819,8 +831,7 @@ pub mod module { /// Only the forking thread survives, so each other interpreter is left with /// slots for threads that no longer exist (still ATTACHED if they were /// running bytecode) and possibly locks or stop-the-world flags held by - /// them. Since a collection stops all interpreters, that state would hang - /// the child's first collection. + /// them. Those slots must not delay this owner's next collection. /// /// # Safety /// Must only be called after `fork()` in the child, when no other threads exist. @@ -840,32 +851,25 @@ pub mod module { reinit_mutex_after_fork(&state.atexit_funcs); reinit_mutex_after_fork(&state.global_trace_func); reinit_mutex_after_fork(&state.global_profile_func); - reinit_mutex_after_fork(&state.type_mutex); + crate::builtins::PyType::repair_type_lock_after_fork(&state); reinit_mutex_after_fork(&state.monitoring); reinit_mutex_after_fork(&state.thread_frames); reinit_mutex_after_fork(&state.thread_handles); reinit_mutex_after_fork(&state.shutdown_handles); state.codec_registry.reinit_after_fork(); + state.warnings.reinit_after_fork(); state.gc.reinit_after_fork(); + state.reinit_native_cache_after_fork(); + state.roots.reinit_after_fork(); + state.type_cache.after_fork(); } state.stop_the_world.reset_after_fork(); - - // Every thread registered here belongs to the parent, including any - // slot the forking thread itself registered before the fork. - state.thread_frames.lock().clear(); - state.thread_handles.lock().clear(); - state.shutdown_handles.lock().clear(); } - - crate::vm::thread::purge_other_interpreter_slots_after_fork(vm.state.interpreter_id); } fn py_os_after_fork_parent(vm: &VirtualMachine) { - #[cfg(feature = "threading")] - vm.state.stop_the_world.start_the_world(&vm.state); - #[cfg(feature = "threading")] crate::stdlib::_imp::release_imp_lock_after_fork_parent(); @@ -947,6 +951,12 @@ pub mod module { vm.audit("os.fork", || ())?; py_os_before_fork(vm); + #[cfg(feature = "threading")] + let pid = crate::vm::fork::with_fork(vm, || { + let pid = rustpython_host_env::posix::fork(); + (matches!(pid, Ok(0)), pid) + }); + #[cfg(not(feature = "threading"))] let pid = rustpython_host_env::posix::fork(); match pid { @@ -992,6 +1002,12 @@ pub mod module { vm.audit("os.forkpty", || ())?; py_os_before_fork(vm); + #[cfg(feature = "threading")] + let result = crate::vm::fork::with_fork(vm, || { + let result = rustpython_host_env::posix::forkpty(); + (matches!(result, Ok((0, _))), result) + }); + #[cfg(not(feature = "threading"))] let result = rustpython_host_env::posix::forkpty(); match result { @@ -2891,7 +2907,7 @@ mod posix_sched { } #[extend_class] - fn extend_pyclass(ctx: &crate::vm::Context, class: &'static Py) { + fn extend_pyclass(ctx: &crate::vm::Context, class: &Py) { // Override __reduce__ to return (type, (sched_priority,)) // instead of the generic structseq (type, ((sched_priority,),)). // The trait's extend_class checks contains_key before setting default. @@ -2922,7 +2938,7 @@ mod posix_sched { builtins::{PyInt, PyTuple}, class::StaticType, }; - if !obj.fast_isinstance(PySchedParam::static_type()) { + if !obj.fast_isinstance(unsafe { PySchedParam::static_type() }) { return Err(vm.new_type_error("must have a sched_param object")); } let tuple = obj.downcast_ref::().unwrap(); diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index a77fbb10fd3..c3b74b2c6e7 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -1302,7 +1302,10 @@ pub mod sys { fn setprofile(function: PyObjectRef, vm: &VirtualMachine) { #[cfg(feature = "threading")] if let Some(slot) = crate::vm::thread::current_thread_slot() { - *slot.profile_func.lock() = function.clone(); + if slot.is_closed() { + return; + } + drop(slot.replace_profile(function.clone())); } vm.profile_func.replace(function); update_use_tracing(vm); @@ -1329,7 +1332,10 @@ pub mod sys { fn settrace(function: PyObjectRef, vm: &VirtualMachine) { #[cfg(feature = "threading")] if let Some(slot) = crate::vm::thread::current_thread_slot() { - *slot.trace_func.lock() = function.clone(); + if slot.is_closed() { + return; + } + drop(slot.replace_trace(function.clone())); } vm.trace_func.replace(function); update_use_tracing(vm); @@ -1344,17 +1350,17 @@ pub mod sys { #[pyfunction] fn _settraceallthreads(function: PyObjectRef, vm: &VirtualMachine) { let func = (!vm.is_none(&function)).then(|| function.clone()); - *vm.state.global_trace_func.lock() = func; + let old = core::mem::replace(&mut *vm.state.global_trace_func.lock(), func); + drop(old); #[cfg(feature = "threading")] { let registry = vm.state.thread_frames.lock(); - #[expect( - clippy::iter_over_hash_type, - reason = "every thread slot, order is irrelevant" - )] - for slot in registry.values() { - *slot.trace_func.lock() = function.clone(); - } + let retired: Vec<_> = registry + .values() + .filter_map(|slot| slot.replace_trace(function.clone())) + .collect(); + drop(registry); + drop(retired); } vm.trace_func.replace(function); update_use_tracing(vm); @@ -1363,17 +1369,17 @@ pub mod sys { #[pyfunction] fn _setprofileallthreads(function: PyObjectRef, vm: &VirtualMachine) { let func = (!vm.is_none(&function)).then(|| function.clone()); - *vm.state.global_profile_func.lock() = func; + let old = core::mem::replace(&mut *vm.state.global_profile_func.lock(), func); + drop(old); #[cfg(feature = "threading")] { let registry = vm.state.thread_frames.lock(); - #[expect( - clippy::iter_over_hash_type, - reason = "every thread slot, order is irrelevant" - )] - for slot in registry.values() { - *slot.profile_func.lock() = function.clone(); - } + let retired: Vec<_> = registry + .values() + .filter_map(|slot| slot.replace_profile(function.clone())) + .collect(); + drop(registry); + drop(retired); } vm.profile_func.replace(function); update_use_tracing(vm); diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index 70b6479a5d3..8db53184bf2 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -13,10 +13,10 @@ mod decl { #![allow(unreachable_pub)] #[cfg(any(unix, windows))] - use crate::builtins::PyBaseExceptionRef; + use crate::builtins::{PyBaseExceptionRef, PyStr}; use crate::{ AsObject, Py, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyStr, PyStrRef, PyTypeRef}, + builtins::{PyStrRef, PyTypeRef}, class::PyClassDef, function::{Either, FuncArgs, OptionalArg, OptionalOption}, types::{PyStructSequence, PyStructSequenceData, struct_sequence_new}, @@ -841,7 +841,12 @@ mod decl { } #[pyattr] - #[pystruct_sequence(name = "struct_time", module = "time", data = "StructTimeData")] + #[pystruct_sequence( + name = "struct_time", + module = "time", + data = "StructTimeData", + interpreter_local + )] pub struct PyStructTime; #[pyclass(with(PyStructSequence))] @@ -1330,11 +1335,11 @@ mod platform { fn global_frequency(vm: &VirtualMachine) -> PyResult { rustpython_common::static_cell! { - static FREQUENCY: PyResult; + static FREQUENCY: i64; }; FREQUENCY - .get_or_init(|| win_perf_counter_frequency(vm)) - .clone() + .get_or_try_init(|| win_perf_counter_frequency(vm)) + .copied() } pub(super) fn get_perf_time(vm: &VirtualMachine) -> PyResult { diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 03eb703f55b..2f7ecbcccd0 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -137,6 +137,10 @@ pub struct PyTypeSlots { /// Full `tp_basicsize`: object header plus payload. `0` before type /// creation means "inherit the base size". pub basicsize: usize, + /// Native payload identity, independent of Python MRO and slot sizes. + pub(crate) native_layout_id: Option, + /// Constructor captured before mutable Python slots are installed. + pub(crate) native_new: Option, pub itemsize: usize, // tp_itemsize // Methods to implement standard operations @@ -856,15 +860,7 @@ impl PyType { /// Recursively update subclasses' slots /// recurse_down_subclasses fn update_subclasses(&self, name: &'static PyStrInterned, ctx: &Context) { - let subclasses = self.subclasses.read(); - for weak_ref in subclasses.iter() { - let Some(subclass) = weak_ref.upgrade() else { - continue; - }; - let Some(subclass) = subclass.downcast_ref::() else { - continue; - }; - + for subclass in self.subclass_snapshot() { // Skip if subclass has its own definition for this attribute if subclass.attributes.contains(name) { continue; @@ -1042,31 +1038,27 @@ impl PyType { } } SlotAccessor::TpNew => { - // __new__ is a staticmethod, not a PyWrapper descriptor, so - // lookup_slot_in_mro cannot classify it. Resolve __new__ - // through the MRO dicts instead: a Python-level definition - // needs the dynamic new_wrapper, while a native type's - // builtin __new__ entry (or no entry at all) means the slot - // is inherited from the solid base, matching update_one_slot's - // tp_new special case over the tp_base-inherited value. - let needs_wrapper = if ADD && self.attributes.contains(name) { - true - } else { - // mro[0] is self, so skip it - self.mro.read()[1..] - .iter() - .find(|cls| cls.attributes.contains(name)) - .is_some_and(|cls| { - cls.slots.new.load().map(|f| fn_addr(f)) - == Some(fn_addr(new_wrapper as NewFunc)) - }) - }; - if needs_wrapper { - self.slots.new.store(Some(new_wrapper)); - self.slots.vectorcall.store(None); - } else { - let inherited = self.base.deref().and_then(|base| base.slots.new.load()); - self.slots.new.store(inherited); + // tp_new_wrapper descriptors preserve the constructor already + // inherited by the type. Direct descriptor calls still validate + // their original native constructor in call_slot_new. + let mro = self.mro.read().clone(); + let method = mro.iter().find_map(|class| class.attributes.get(name)); + match method { + Some(method) => { + let native = method + .downcast_ref::() + .is_some_and(|method| { + core::ptr::eq(method.func.value, ctx.slot_new_wrapper) + }); + if !native { + self.slots.new.store(Some(new_wrapper)); + self.slots.vectorcall.store(None); + } + } + None => { + let inherited = self.base.deref().and_then(|base| base.slots.new.load()); + self.slots.new.store(inherited); + } } } SlotAccessor::TpDel => update_main_slot!(del, del_wrapper, Del), @@ -1863,7 +1855,7 @@ impl PyType { if attr.class().is(ctx.types.wrapper_descriptor_type) { attr.downcast_ref::().and_then(|wrapper| { // Only extract slot if for_class is a subclass of wrapper.typ - if is_subclass_of(for_class_mro, wrapper.typ) { + if is_subclass_of(for_class_mro, &wrapper.typ) { extract(&wrapper.wrapped) } else { None @@ -1938,7 +1930,7 @@ pub trait Constructor: PyPayload + core::fmt::Debug { // The name is the type the slot was written for, not the subclass being // constructed, so a subclass reports what its base declares. let args = if Self::DROP_KWARGS_WHEN_INIT_OVERRIDDEN { - drop_kwargs_if_init_overridden(&cls, Self::class(&vm.ctx), args) + drop_kwargs_if_init_overridden(&cls, &Self::class(&vm.ctx), args) } else { args }; diff --git a/crates/vm/src/types/structseq.rs b/crates/vm/src/types/structseq.rs index 09822495075..8c68b7dda91 100644 --- a/crates/vm/src/types/structseq.rs +++ b/crates/vm/src/types/structseq.rs @@ -275,9 +275,9 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { fn from_data(data: Self::Data, vm: &VirtualMachine) -> PyTupleRef { let tuple = ::into_tuple(data, vm); - let typ = Self::static_type(); + let typ = Self::make_class(&vm.ctx); tuple - .into_ref_with_type(vm, typ.to_owned()) + .into_ref_with_type(vm, typ) .expect("Every PyStructSequence must be a valid tuple. This is a RustPython bug.") } @@ -385,7 +385,7 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { } #[extend_class] - fn extend_pyclass(ctx: &Context, class: &'static Py) { + fn extend_pyclass(ctx: &Context, class: &Py) { // Getters for named visible fields (indices 0 to REQUIRED_FIELD_NAMES.len() - 1) for (i, &name) in Self::Data::REQUIRED_FIELD_NAMES.iter().enumerate() { class.set_attr( @@ -438,28 +438,6 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { ctx.new_int(n_unnamed_fields).into(), ); - // Override as_sequence and as_mapping slots to use visible length - class - .slots - .as_sequence - .copy_from(&STRUCT_SEQUENCE_AS_SEQUENCE); - class - .slots - .as_mapping - .copy_from(&STRUCT_SEQUENCE_AS_MAPPING); - - // Override iter slot to return only visible elements - class.slots.iter.store(Some(struct_sequence_iter)); - - // Override hash slot to hash only visible elements - class.slots.hash.store(Some(struct_sequence_hash)); - - // Override richcompare slot to compare only visible elements - class - .slots - .richcompare - .store(Some(struct_sequence_richcompare)); - // Default __reduce__: only set if not already overridden by the impl's extend_class. // This allows struct sequences like sched_param to provide a custom __reduce__ // (equivalent to METH_COEXIST in structseq.c). @@ -470,6 +448,21 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { ); } } + + fn extend_slots(slots: &mut crate::types::PyTypeSlots) { + // Override as_sequence and as_mapping slots to use visible length + slots.as_sequence.copy_from(&STRUCT_SEQUENCE_AS_SEQUENCE); + slots.as_mapping.copy_from(&STRUCT_SEQUENCE_AS_MAPPING); + + // Override iter slot to return only visible elements + slots.iter.store(Some(struct_sequence_iter)); + + // Override hash slot to hash only visible elements + slots.hash.store(Some(struct_sequence_hash)); + + // Override richcompare slot to compare only visible elements + slots.richcompare.store(Some(struct_sequence_richcompare)); + } } /// Iterator function for struct sequences - returns only visible elements diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 0d3c5d0fb92..cbd00cdf587 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -118,10 +118,11 @@ impl TypeZoo { let object_type = object::PyBaseObject::init_manually(hierarchy.object_type); let tuple_type = tuple::PyTuple::init_manually(hierarchy.tuple_type); let weakref_type = weakref::PyWeak::init_manually(hierarchy.weakref_type); - let int_type = int::PyInt::init_builtin_type(); + let int_type = unsafe { int::PyInt::init_builtin_type() }; // builtin_function_or_method and builtin_method share the same type (CPython behavior) - let builtin_function_or_method_type = builtin_func::PyNativeFunction::init_builtin_type(); + let builtin_function_or_method_type = + unsafe { builtin_func::PyNativeFunction::init_builtin_type() }; let types = Self { type_type, @@ -130,94 +131,111 @@ impl TypeZoo { int_type, // types exposed as builtins - bool_type: bool_::PyBool::init_builtin_type(), - bytearray_type: bytearray::PyByteArray::init_builtin_type(), - bytes_type: bytes::PyBytes::init_builtin_type(), - classmethod_type: classmethod::PyClassMethod::init_builtin_type(), - complex_type: complex::PyComplex::init_builtin_type(), - dict_type: dict::PyDict::init_builtin_type(), - enumerate_type: enumerate::PyEnumerate::init_builtin_type(), - float_type: float::PyFloat::init_builtin_type(), - frozenset_type: set::PyFrozenSet::init_builtin_type(), - filter_type: filter::PyFilter::init_builtin_type(), - list_type: list::PyList::init_builtin_type(), - map_type: map::PyMap::init_builtin_type(), - memoryview_type: memory::PyMemoryView::init_builtin_type(), - property_type: property::PyProperty::init_builtin_type(), - range_type: range::PyRange::init_builtin_type(), - set_type: set::PySet::init_builtin_type(), - slice_type: slice::PySlice::init_builtin_type(), - staticmethod_type: staticmethod::PyStaticMethod::init_builtin_type(), - str_type: pystr::PyStr::init_builtin_type(), - super_type: super_::PySuper::init_builtin_type(), + bool_type: unsafe { bool_::PyBool::init_builtin_type() }, + bytearray_type: unsafe { bytearray::PyByteArray::init_builtin_type() }, + bytes_type: unsafe { bytes::PyBytes::init_builtin_type() }, + classmethod_type: unsafe { classmethod::PyClassMethod::init_builtin_type() }, + complex_type: unsafe { complex::PyComplex::init_builtin_type() }, + dict_type: unsafe { dict::PyDict::init_builtin_type() }, + enumerate_type: unsafe { enumerate::PyEnumerate::init_builtin_type() }, + float_type: unsafe { float::PyFloat::init_builtin_type() }, + frozenset_type: unsafe { set::PyFrozenSet::init_builtin_type() }, + filter_type: unsafe { filter::PyFilter::init_builtin_type() }, + list_type: unsafe { list::PyList::init_builtin_type() }, + map_type: unsafe { map::PyMap::init_builtin_type() }, + memoryview_type: unsafe { memory::PyMemoryView::init_builtin_type() }, + property_type: unsafe { property::PyProperty::init_builtin_type() }, + range_type: unsafe { range::PyRange::init_builtin_type() }, + set_type: unsafe { set::PySet::init_builtin_type() }, + slice_type: unsafe { slice::PySlice::init_builtin_type() }, + staticmethod_type: unsafe { staticmethod::PyStaticMethod::init_builtin_type() }, + str_type: unsafe { pystr::PyStr::init_builtin_type() }, + super_type: unsafe { super_::PySuper::init_builtin_type() }, tuple_type, - zip_type: zip::PyZip::init_builtin_type(), + zip_type: unsafe { zip::PyZip::init_builtin_type() }, // hidden internal types. is this really need to be cached here? - async_generator: asyncgenerator::PyAsyncGen::init_builtin_type(), - async_generator_asend: asyncgenerator::PyAsyncGenASend::init_builtin_type(), - async_generator_athrow: asyncgenerator::PyAsyncGenAThrow::init_builtin_type(), - async_generator_wrapped_value: - asyncgenerator::PyAsyncGenWrappedValue::init_builtin_type(), - anext_awaitable: asyncgenerator::PyAnextAwaitable::init_builtin_type(), - bound_method_type: function::PyBoundMethod::init_builtin_type(), + async_generator: unsafe { asyncgenerator::PyAsyncGen::init_builtin_type() }, + async_generator_asend: unsafe { asyncgenerator::PyAsyncGenASend::init_builtin_type() }, + async_generator_athrow: unsafe { + asyncgenerator::PyAsyncGenAThrow::init_builtin_type() + }, + async_generator_wrapped_value: unsafe { + asyncgenerator::PyAsyncGenWrappedValue::init_builtin_type() + }, + anext_awaitable: unsafe { asyncgenerator::PyAnextAwaitable::init_builtin_type() }, + bound_method_type: unsafe { function::PyBoundMethod::init_builtin_type() }, builtin_function_or_method_type, builtin_method_type: builtin_function_or_method_type, - bytearray_iterator_type: bytearray::PyByteArrayIterator::init_builtin_type(), - bytes_iterator_type: bytes::PyBytesIterator::init_builtin_type(), - callable_iterator: iter::PyCallableIterator::init_builtin_type(), - capsule_type: capsule::PyCapsule::init_builtin_type(), - cell_type: function::PyCell::init_builtin_type(), - code_type: code::PyCode::init_builtin_type(), - coroutine_type: coroutine::PyCoroutine::init_builtin_type(), - coroutine_wrapper_type: coroutine::PyCoroutineWrapper::init_builtin_type(), - dict_keys_type: dict::PyDictKeys::init_builtin_type(), - dict_values_type: dict::PyDictValues::init_builtin_type(), - dict_items_type: dict::PyDictItems::init_builtin_type(), - dict_keyiterator_type: dict::PyDictKeyIterator::init_builtin_type(), - dict_reversekeyiterator_type: dict::PyDictReverseKeyIterator::init_builtin_type(), - dict_valueiterator_type: dict::PyDictValueIterator::init_builtin_type(), - dict_reversevalueiterator_type: dict::PyDictReverseValueIterator::init_builtin_type(), - dict_itemiterator_type: dict::PyDictItemIterator::init_builtin_type(), - dict_reverseitemiterator_type: dict::PyDictReverseItemIterator::init_builtin_type(), - ellipsis_type: slice::PyEllipsis::init_builtin_type(), - frame_type: crate::frame::FrameObject::init_builtin_type(), - frame_locals_proxy_type: frame_locals_proxy::FrameLocalsProxy::init_builtin_type(), - function_type: function::PyFunction::init_builtin_type(), - generator_type: generator::PyGenerator::init_builtin_type(), - getset_type: getset::PyGetSet::init_builtin_type(), - iter_type: iter::PySequenceIterator::init_builtin_type(), - reverse_iter_type: enumerate::PyReverseSequenceIterator::init_builtin_type(), - list_iterator_type: list::PyListIterator::init_builtin_type(), - list_reverseiterator_type: list::PyListReverseIterator::init_builtin_type(), - mappingproxy_type: mappingproxy::PyMappingProxy::init_builtin_type(), - memoryviewiterator_type: memory::PyMemoryViewIterator::init_builtin_type(), - module_type: module::PyModule::init_builtin_type(), - namespace_type: namespace::PyNamespace::init_builtin_type(), - range_iterator_type: range::PyRangeIterator::init_builtin_type(), - long_range_iterator_type: range::PyLongRangeIterator::init_builtin_type(), - set_iterator_type: set::PySetIterator::init_builtin_type(), - str_iterator_type: pystr::PyStrIterator::init_builtin_type(), - traceback_type: traceback::PyTraceback::init_builtin_type(), - tuple_iterator_type: tuple::PyTupleIterator::init_builtin_type(), - weakproxy_type: weakproxy::PyWeakProxy::init_builtin_type(), - weakcallableproxy_type: weakproxy::PyWeakCallableProxy::init_builtin_type(), - method_descriptor_type: descriptor::PyMethodDescriptor::init_builtin_type(), - classmethod_descriptor_type: descriptor::PyClassMethodDescriptor::init_builtin_type(), - none_type: singletons::PyNone::init_builtin_type(), - typing_no_default_type: crate::stdlib::_typing::NoDefault::init_builtin_type(), - not_implemented_type: singletons::PyNotImplemented::init_builtin_type(), - generic_alias_type: genericalias::PyGenericAlias::init_builtin_type(), - generic_alias_iterator_type: genericalias::PyGenericAliasIterator::init_builtin_type(), - union_type: union_::PyUnion::init_builtin_type(), - interpolation_type: interpolation::PyInterpolation::init_builtin_type(), - template_type: template::PyTemplate::init_builtin_type(), - template_iter_type: template::PyTemplateIter::init_builtin_type(), - member_descriptor_type: descriptor::PyMemberDescriptor::init_builtin_type(), - wrapper_descriptor_type: descriptor::PyWrapper::init_builtin_type(), - method_wrapper_type: descriptor::PyMethodWrapper::init_builtin_type(), + bytearray_iterator_type: unsafe { bytearray::PyByteArrayIterator::init_builtin_type() }, + bytes_iterator_type: unsafe { bytes::PyBytesIterator::init_builtin_type() }, + callable_iterator: unsafe { iter::PyCallableIterator::init_builtin_type() }, + capsule_type: unsafe { capsule::PyCapsule::init_builtin_type() }, + cell_type: unsafe { function::PyCell::init_builtin_type() }, + code_type: unsafe { code::PyCode::init_builtin_type() }, + coroutine_type: unsafe { coroutine::PyCoroutine::init_builtin_type() }, + coroutine_wrapper_type: unsafe { coroutine::PyCoroutineWrapper::init_builtin_type() }, + dict_keys_type: unsafe { dict::PyDictKeys::init_builtin_type() }, + dict_values_type: unsafe { dict::PyDictValues::init_builtin_type() }, + dict_items_type: unsafe { dict::PyDictItems::init_builtin_type() }, + dict_keyiterator_type: unsafe { dict::PyDictKeyIterator::init_builtin_type() }, + dict_reversekeyiterator_type: unsafe { + dict::PyDictReverseKeyIterator::init_builtin_type() + }, + dict_valueiterator_type: unsafe { dict::PyDictValueIterator::init_builtin_type() }, + dict_reversevalueiterator_type: unsafe { + dict::PyDictReverseValueIterator::init_builtin_type() + }, + dict_itemiterator_type: unsafe { dict::PyDictItemIterator::init_builtin_type() }, + dict_reverseitemiterator_type: unsafe { + dict::PyDictReverseItemIterator::init_builtin_type() + }, + ellipsis_type: unsafe { slice::PyEllipsis::init_builtin_type() }, + frame_type: unsafe { crate::frame::FrameObject::init_builtin_type() }, + frame_locals_proxy_type: unsafe { + frame_locals_proxy::FrameLocalsProxy::init_builtin_type() + }, + function_type: unsafe { function::PyFunction::init_builtin_type() }, + generator_type: unsafe { generator::PyGenerator::init_builtin_type() }, + getset_type: unsafe { getset::PyGetSet::init_builtin_type() }, + iter_type: unsafe { iter::PySequenceIterator::init_builtin_type() }, + reverse_iter_type: unsafe { enumerate::PyReverseSequenceIterator::init_builtin_type() }, + list_iterator_type: unsafe { list::PyListIterator::init_builtin_type() }, + list_reverseiterator_type: unsafe { list::PyListReverseIterator::init_builtin_type() }, + mappingproxy_type: unsafe { mappingproxy::PyMappingProxy::init_builtin_type() }, + memoryviewiterator_type: unsafe { memory::PyMemoryViewIterator::init_builtin_type() }, + module_type: unsafe { module::PyModule::init_builtin_type() }, + namespace_type: unsafe { namespace::PyNamespace::init_builtin_type() }, + range_iterator_type: unsafe { range::PyRangeIterator::init_builtin_type() }, + long_range_iterator_type: unsafe { range::PyLongRangeIterator::init_builtin_type() }, + set_iterator_type: unsafe { set::PySetIterator::init_builtin_type() }, + str_iterator_type: unsafe { pystr::PyStrIterator::init_builtin_type() }, + traceback_type: unsafe { traceback::PyTraceback::init_builtin_type() }, + tuple_iterator_type: unsafe { tuple::PyTupleIterator::init_builtin_type() }, + weakproxy_type: unsafe { weakproxy::PyWeakProxy::init_builtin_type() }, + weakcallableproxy_type: unsafe { weakproxy::PyWeakCallableProxy::init_builtin_type() }, + method_descriptor_type: unsafe { descriptor::PyMethodDescriptor::init_builtin_type() }, + classmethod_descriptor_type: unsafe { + descriptor::PyClassMethodDescriptor::init_builtin_type() + }, + none_type: unsafe { singletons::PyNone::init_builtin_type() }, + typing_no_default_type: unsafe { + crate::stdlib::_typing::NoDefault::init_builtin_type() + }, + not_implemented_type: unsafe { singletons::PyNotImplemented::init_builtin_type() }, + generic_alias_type: unsafe { genericalias::PyGenericAlias::init_builtin_type() }, + generic_alias_iterator_type: unsafe { + genericalias::PyGenericAliasIterator::init_builtin_type() + }, + union_type: unsafe { union_::PyUnion::init_builtin_type() }, + interpolation_type: unsafe { interpolation::PyInterpolation::init_builtin_type() }, + template_type: unsafe { template::PyTemplate::init_builtin_type() }, + template_iter_type: unsafe { template::PyTemplateIter::init_builtin_type() }, + member_descriptor_type: unsafe { descriptor::PyMemberDescriptor::init_builtin_type() }, + wrapper_descriptor_type: unsafe { descriptor::PyWrapper::init_builtin_type() }, + method_wrapper_type: unsafe { descriptor::PyMethodWrapper::init_builtin_type() }, - method_def: crate::function::HeapMethodDef::init_builtin_type(), + method_def: unsafe { crate::function::HeapMethodDef::init_builtin_type() }, }; (types, hierarchy.empty_tuple) } diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 626659f19e1..8ae78b7c9ef 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -1610,14 +1610,14 @@ mod escape_warnings { ); vm.state .warnings - .filters + .filters() .borrow_vec_mut() .insert(0, error_filter.into()); vm.state.warnings.filters_mutated(); } fn first_compiler_warning(source: &str) -> String { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { install_syntax_warning_error_filter(vm); let err = vm .compile(source, compiler::Mode::Exec, "") @@ -1633,7 +1633,7 @@ mod escape_warnings { } fn compile_error_message(source: &str) -> String { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { install_syntax_warning_error_filter(vm); let err = match vm.compile(source, compiler::Mode::Exec, "") { Ok(_) => panic!("expected compile error"), @@ -1650,7 +1650,7 @@ mod escape_warnings { #[test] fn ast_only_compile_honors_barry_as_flufl() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let flags = CompilerFlags::ONLY_AST.bits(); vm.compile_string_object_with_flags( b"from __future__ import barry_as_FLUFL\n2 <> 3\n", @@ -1685,7 +1685,7 @@ mod escape_warnings { #[test] fn type_comment_preparse_honors_inherited_barry_as_flufl() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let flags = CompilerFlags::TYPE_COMMENTS.bits() | crate::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL.bits() as i32; vm.compile_string_object_with_flags( diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 99823748e83..412a41b3e98 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -7,8 +7,7 @@ use crate::{ bool_::PyBool, code::{self, PyCode}, descriptor::{ - MemberAccess, MemberKind, PyDescriptorOwned, PyMemberDef, PyMemberDescriptor, - PyMemberFlags, + MemberAccess, MemberKind, PyDescriptor, PyMemberDef, PyMemberDescriptor, PyMemberFlags, }, getset::PyGetSet, object, pystr, @@ -52,7 +51,7 @@ pub struct Context { pub(crate) ascii_char_cache: Vec>, // there should only be exact objects of str in here, no non-str objects and no subclasses pub(crate) string_pool: StringPool, - pub(crate) slot_new_wrapper: PyMethodDef, + pub(crate) slot_new_wrapper: &'static PyMethodDef, pub names: ConstName, // GC module state (callbacks and garbage lists) } @@ -283,15 +282,26 @@ declare_const_name! { // Basic objects: impl Context { + /// Access the process-wide native definitions and immutable singleton context. + /// + /// # Safety + /// Follow [`crate::embedding`]'s native ownership contract. In particular, + /// mutable objects created through this context belong to the attached VM. + #[must_use] + pub unsafe fn genesis_unchecked() -> &'static PyRc { + Self::genesis() + } + pub const INT_CACHE_POOL_RANGE: core::ops::RangeInclusive = (-5)..=256; const INT_CACHE_POOL_MIN: i32 = *Self::INT_CACHE_POOL_RANGE.start(); #[must_use] - pub fn genesis() -> &'static PyRc { + pub(crate) fn genesis() -> &'static PyRc { rustpython_common::static_cell! { static CONTEXT: PyRc; } CONTEXT.get_or_init(|| { + let _owner = crate::gc_state::AllocationScope::shared(); let ctx = PyRc::new(Self::init_genesis()); // SAFETY: ctx is heap-allocated via PyRc and will be stored in // the CONTEXT static cell, so the Context lives for 'static. @@ -312,14 +322,14 @@ impl Context { PyRef::new_ref(payload, cls.to_owned(), None) } - let none = create_object(PyNone, PyNone::static_type()); - let ellipsis = create_object(PyEllipsis, PyEllipsis::static_type()); - let not_implemented = create_object(PyNotImplemented, PyNotImplemented::static_type()); + let none = create_object(PyNone, unsafe { PyNone::static_type() }); + let ellipsis = create_object(PyEllipsis, unsafe { PyEllipsis::static_type() }); + let not_implemented = + create_object(PyNotImplemented, unsafe { PyNotImplemented::static_type() }); - let typing_no_default = create_object( - crate::stdlib::_typing::NoDefault, - crate::stdlib::_typing::NoDefault::static_type(), - ); + let typing_no_default = create_object(crate::stdlib::_typing::NoDefault, unsafe { + crate::stdlib::_typing::NoDefault::static_type() + }); let int_cache_pool = Self::INT_CACHE_POOL_RANGE .map(|v| { @@ -357,8 +367,8 @@ impl Context { let names = unsafe { ConstName::new(&string_pool, types.str_type) }; - let slot_new_wrapper = PyMethodDef::new_const( - names.__new__.as_str(), + const SLOT_NEW_WRAPPER: PyMethodDef = PyMethodDef::new_const( + "__new__", PyType::__new__, PyMethodFlags::METHOD, ItemDoc::static_text( @@ -420,7 +430,7 @@ impl Context { latin1_char_cache, ascii_char_cache, string_pool, - slot_new_wrapper, + slot_new_wrapper: &SLOT_NEW_WRAPPER, names, } } @@ -660,19 +670,7 @@ impl Context { where F: IntoPyNativeFn, { - let def = PyMethodDef { - name, - func: Box::leak(Box::new(f.into_func())), - flags, - #[cfg(feature = "doc")] - doc_off: doc.offset, - #[cfg(feature = "doc")] - doc_len: doc.len, - #[cfg(feature = "doc")] - doc_body_pending: false, - doc: doc.text, - }; - let payload = HeapMethodDef::new(def); + let payload = HeapMethodDef::with_owned_function(name, Box::new(f.into_func()), flags, doc); PyRef::new_ref(payload, self.types.method_def.to_owned(), None) } @@ -683,11 +681,11 @@ impl Context { kind: MemberKind, offset: isize, flags: PyMemberFlags, - class: &'static Py, + class: &Py, doc: ItemDoc, ) -> PyRef { let member_descriptor = PyMemberDescriptor { - common: PyDescriptorOwned { + common: PyDescriptor { typ: class.to_owned(), name: self.intern_str(name), qualname: PyRwLock::new(None), @@ -707,12 +705,12 @@ impl Context { pub fn new_readonly_tuple_member( &self, name: &str, - class: &'static Py, + class: &Py, index: usize, doc: ItemDoc, ) -> PyRef { let member_descriptor = PyMemberDescriptor { - common: PyDescriptorOwned { + common: PyDescriptor { typ: class.to_owned(), name: self.intern_str(name), qualname: PyRwLock::new(None), @@ -729,12 +727,7 @@ impl Context { member_descriptor.into_ref(self) } - pub fn new_readonly_getset( - &self, - name: &str, - class: &'static Py, - f: F, - ) -> PyRef + pub fn new_readonly_getset(&self, name: &str, class: &Py, f: F) -> PyRef where F: IntoPyGetterFunc, { @@ -745,7 +738,7 @@ impl Context { pub fn new_static_getset( &self, name: &str, - class: &'static Py, + class: &Py, g: G, s: S, ) -> PyRef diff --git a/crates/vm/src/vm/crossinterp.rs b/crates/vm/src/vm/crossinterp.rs index 2b799bf0031..4a47b489a97 100644 --- a/crates/vm/src/vm/crossinterp.rs +++ b/crates/vm/src/vm/crossinterp.rs @@ -17,7 +17,7 @@ use crate::{ BorrowedConstant, CodeFlags, Constant, Instruction, oparg::{OpArg, OpArgState}, }, - protocol::PyBuffer, + protocol::SharedBuffer, }; use core::sync::atomic::{AtomicBool, Ordering}; use num_traits::ToPrimitive; @@ -133,7 +133,7 @@ pub enum SharedValue { }, Queue(SharedQueueId), #[cfg(feature = "threading")] - Buffer(PyBuffer), + Buffer(SharedBuffer), #[cfg(not(feature = "threading"))] Buffer(Vec, BufferDescriptor), /// Marshalled code object. @@ -293,7 +293,7 @@ impl SharedValue { let view = PyMemoryView::from_object(obj, vm)?; #[cfg(feature = "threading")] { - Ok(Self::Buffer(view.clone_buffer())) + SharedBuffer::new(view.clone_buffer(), vm).map(Self::Buffer) } #[cfg(not(feature = "threading"))] { @@ -330,7 +330,7 @@ impl SharedValue { let buffer = VecBuffer::from(bytes) .into_ref(&vm.ctx) .into_pybuffer_with_descriptor(desc); - memoryview_from_buffer(buffer, vm) + memoryview_from_buffer(SharedBuffer::new(buffer, vm)?, vm) } Self::Code(data) => marshal_loads(&data, vm), Self::Function(data) => { @@ -362,7 +362,7 @@ fn marshal_loads(data: &[u8], vm: &VirtualMachine) -> PyResult { } /// `_memoryview_from_xid`. -fn memoryview_from_buffer(buffer: PyBuffer, vm: &VirtualMachine) -> PyResult { +fn memoryview_from_buffer(buffer: SharedBuffer, vm: &VirtualMachine) -> PyResult { let view = crate::stdlib::_interpreters::xibufferview_from_buffer(buffer, vm); let mv = PyMemoryView::from_object(&view, vm)?; Ok(mv.into_pyobject(vm)) @@ -971,7 +971,7 @@ where let tvm = crate::vm::runtime::owned_new_thread(id) .ok_or_else(|| crate::stdlib::_interpreters::interpreter_not_found(caller, id))?; - tvm.run(f) + tvm.run_raw(f) } #[must_use] diff --git a/crates/vm/src/vm/fork.rs b/crates/vm/src/vm/fork.rs new file mode 100644 index 00000000000..72709449ad8 --- /dev/null +++ b/crates/vm/src/vm/fork.rs @@ -0,0 +1,332 @@ +//! Process-wide quiescence used only around the native fork syscall. + +use super::{PyGlobalState, VirtualMachine, runtime}; +use crate::common::{ + lock::{PyDetachingRwLock, PyDetachingRwLockReadGuard}, + rc::PyRc, +}; +use core::{cell::RefCell, sync::atomic::Ordering}; + +static LIFECYCLE: PyDetachingRwLock<()> = PyDetachingRwLock::new(()); +static GENERATION: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +pub(crate) fn generation() -> u64 { + GENERATION.load(Ordering::Relaxed) +} + +thread_local! { + static NATIVE_PHASE: RefCell<(usize, Option>)> = + const { RefCell::new((0, None)) }; +} + +/// Bootstrap and terminal heap retirement cannot be interrupted halfway through +/// publication. Nested native destruction shares one read permit. +pub(crate) struct NativePhase(core::marker::PhantomData>); + +pub(crate) fn native_phase() -> NativePhase { + NATIVE_PHASE.with(|phase| { + if phase.borrow().0 == 0 { + let guard = LIFECYCLE.read(); + phase.borrow_mut().1 = Some(guard); + } + phase.borrow_mut().0 += 1; + }); + NativePhase(core::marker::PhantomData) +} + +impl Drop for NativePhase { + fn drop(&mut self) { + NATIVE_PHASE.with(|phase| { + let mut phase = phase.borrow_mut(); + phase.0 -= 1; + if phase.0 == 0 { + phase.1.take(); + } + }); + } +} + +struct Stopped(Vec>); + +/// Read a native namespace without hashing or invoking user attribute hooks. +/// Fork repair must finish before any Python callback may create a new thread. +pub(crate) fn namespace_item( + dict: &crate::Py, + name: &str, +) -> Option { + dict.into_iter().find_map(|(key, value)| { + key.downcast_ref::() + .is_some_and(|key| key.as_str() == name) + .then_some(value) + }) +} + +impl Drop for Stopped { + fn drop(&mut self) { + for state in self.0.iter().rev() { + state.stop_the_world.start_the_world(state); + } + } +} + +/// The closure must perform only the syscall and return whether this is its +/// child branch. Python at-fork hooks run outside all these guards. +pub(crate) fn with_fork(vm: &VirtualMachine, syscall: impl FnOnce() -> (bool, R)) -> R { + assert_eq!(NATIVE_PHASE.with(|phase| phase.borrow().0), 0); + // Declare the snapshot before the gate: its last Arc can retire a heap, + // which takes a read permit and therefore must run after releasing write. + let states; + let gate = LIFECYCLE.write(); + let gc = crate::gc_state::gc_state(); + let retired = vm.allow_threads(|| gc.lock_retired_for_fork()); + states = runtime::live_interpreter_states(); + let mut stopped = Stopped(Vec::with_capacity(states.len())); + for state in &states { + // Waiting for a foreign collector must detach the *current* owner. + // suspend_if_needed(foreign_state) cannot park its slot correctly. + vm.allow_threads(|| state.stop_the_world.stop_the_world(state)); + stopped.0.push(state.clone()); + } + let heaps = gc.heaps_for_fork(); + let readers = super::thread::surviving_qsbr_slots(); + // Native lookup/anchor cloning may run without a Python attachment. Hold + // both maps across fork rather than resetting a possibly interrupted map. + let registries = runtime::prepare_fork(); + let mut owners: Vec<_> = states + .iter() + .map(|state| state.owner_leases.prepare_fork()) + .collect(); + let result = crate::stdlib::_interpchannels::with_fork(|| { + crate::stdlib::_interpqueues::with_fork(|| { + let roots: Vec<_> = states + .iter() + .map(|state| state.roots.prepare_fork()) + .collect(); + let mut domains: Vec<_> = heaps.iter().map(|heap| heap.qsbr.prepare_fork()).collect(); + let (child, result) = syscall(); + if child { + GENERATION.fetch_add(1, Ordering::Relaxed); + for (state, owners) in states.iter().zip(&mut owners) { + // All registries were stabilized before fork. Only this thread + // remains, including any nested/saved native entry scopes. + let (count, teardowns) = unsafe { owners.repair_child() }; + state.owners.store(count, Ordering::Relaxed); + state.teardowns.store(teardowns, Ordering::Relaxed); + } + for domain in &mut domains { + unsafe { domain.repair_child(&readers) }; + } + for state in &stopped.0 { + state.stop_the_world.reset_after_fork(); + } + stopped.0.clear(); + } + drop(domains); + drop(roots); + result + }) + }); + drop(owners); + drop(registries); + drop(stopped); + drop(retired); + drop(gate); + result +} + +/// Retire runtimes that have no surviving native entry, after every internal +/// lock and thread handle has been repaired and the current owner is attached. +pub(crate) fn finish_child(current: &VirtualMachine) { + #[cfg(feature = "host_env")] + let active: Vec<_> = runtime::live_interpreter_states() + .iter() + .filter(|state| state.interpreter_id != current.state.interpreter_id) + // SAFETY: these scopes belong to this native thread and surround the + // fork call, including SavedThreadState scopes outside VM_STACK. + .filter_map(|state| unsafe { state.owner_leases.active_vm() }) + .collect(); + #[cfg(feature = "host_env")] + for ptr in &active { + let vm = unsafe { ptr.as_ref() }; + let _entry = super::thread::VmBootstrapGuard::new(vm); + unsafe { crate::stdlib::_io::reinit_std_streams_after_fork(vm) }; + } + let mut cleanup = super::owned::Cleanup::default(); + for state in runtime::live_interpreter_states() { + let orphaned = state.owners.load(Ordering::Relaxed) == 0 + && state.teardowns.load(Ordering::Relaxed) == 0; + let pending = state.owner_leases.take_abandoned_cleanups(); + cleanup.run(|| { + super::thread::native_sweep(|| { + let _owner = state.gc.allocation_scope(); + let mut steps = super::owned::Cleanup::default(); + if orphaned { + state.closed.store(true, Ordering::Release); + state.admission_closed.store(true, Ordering::Release); + steps.run(|| { + crate::stdlib::_interpchannels::clear_interpreter(state.interpreter_id) + }); + steps.run(|| { + crate::stdlib::_interpqueues::clear_interpreter(state.interpreter_id) + }); + steps.run(|| state.close_python_roots()); + steps.run(|| state.gc.unfreeze()); + steps.run(|| { + state.gc.collect_force(2); + }); + } + // Pending owners must be routed through destroy_vm even if a + // resource destructor fails; plain Box Drop bypasses teardown. + for vm in pending { + steps.run(|| super::thread::destroy_vm(vm)); + } + steps.finish(); + }) + }); + } + cleanup.run(runtime::discard_invalid_owned_after_fork); + #[cfg(feature = "host_env")] + for ptr in active { + let vm = unsafe { ptr.as_ref() }; + let _entry = super::thread::VmBootstrapGuard::new(vm); + // User at-fork callbacks remain scoped to the VM that called fork. + // Repair the inherited standard-library thread bookkeeping of saved + // owners before any user child hook can start another native thread. + let callback = namespace_item(&vm.sys_module.dict(), "modules") + .and_then(|modules| { + namespace_item( + modules.downcast_ref::()?, + "threading", + ) + }) + .and_then(|module| { + namespace_item( + &module.downcast_ref::()?.dict(), + "_after_fork", + ) + }); + if let Some(callback) = callback + && let Err(error) = callback.call((), vm) + { + vm.run_unraisable(error, Some("threading after fork".to_owned()), callback); + } + } + #[cfg(not(feature = "host_env"))] + let _ = current; + cleanup.finish(); +} + +#[cfg(test)] +#[cfg(all(feature = "host_env", feature = "rustpython-compiler"))] +mod tests { + use super::*; + use std::os::unix::process::CommandExt; + + // Fork in a dedicated test process: the ordinary test harness may be + // concurrently constructing unrelated interpreters on other native threads. + fn in_subprocess(test: impl FnOnce()) { + let thread = std::thread::current(); + let name = thread.name().unwrap(); + if rustpython_host_env::os::var("RUSTPYTHON_FORK_TEST").as_deref() == Ok(name) { + test(); + return; + } + #[expect( + clippy::disallowed_methods, + reason = "native test harness, not interpreter process access" + )] + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", name, "--no-capture"]) + .env("RUSTPYTHON_FORK_TEST", name) + .process_group(0) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + core::time::Duration::from_secs(40); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success(), "fork regression: {status}"); + return; + } + if std::time::Instant::now() >= deadline { + // Include grandchildren if post-fork repair deadlocked. + unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL) }; + let _ = child.wait(); + panic!("fork regression timed out"); + } + std::thread::sleep(core::time::Duration::from_millis(20)); + } + } + + fn fork(vm: &VirtualMachine) -> bool { + let module = vm.import("posix", 0).unwrap(); + let pid: i32 = module + .get_attr("fork", vm) + .unwrap() + .call((), vm) + .unwrap() + .try_to_value(vm) + .unwrap(); + if pid == 0 { + return true; + } + vm.allow_threads(|| { + let mut status = 0; + loop { + let waited = unsafe { libc::waitpid(pid, &mut status, 0) }; + if waited == -1 + && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted + { + continue; + } + assert_eq!(waited, pid); + break; + } + assert_eq!(status, 0); + }); + false + } + + #[test] + fn fork_preserves_saved_entries_and_retires_idle_owners() { + in_subprocess(|| { + let outer = crate::Interpreter::without_stdlib(Default::default()); + let forking = crate::Interpreter::without_stdlib(Default::default()); + let idle = crate::Interpreter::without_stdlib(Default::default()); + let idle_id = idle.id(); + let owned_id = runtime::store_owned_interpreter(outer.create_subinterpreter()); + let mut child = false; + outer.enter_raw(|outer_vm| { + let slot = super::super::thread::current_thread_slot().unwrap(); + let saved = super::super::thread::save_current_thread(); + forking.enter_raw(|vm| { + child = fork(vm); + if child { + assert!(outer_vm.owner_is_valid()); + assert_eq!(outer_vm.state.owners.load(Ordering::Acquire), 1); + assert!(runtime::lookup_open_interpreter(idle_id).is_none()); + assert!(!runtime::is_owned_interpreter(owned_id)); + assert!(matches!( + idle.enter(|vm| vm.new_int(1).map(|v| v.unbind())), + Err(crate::embedding::Error::ForkedOwner) + )); + vm.state.gc.collect_force(2); + } + }); + super::super::thread::restore_current_thread(saved); + assert!(super::super::thread::is_current_attached(outer_vm)); + assert!(PyRc::ptr_eq( + &slot, + &super::super::thread::current_thread_slot().unwrap() + )); + outer_vm.state.gc.collect_force(2); + }); + drop(idle); + drop(forking); + drop(outer); + if child { + unsafe { libc::_exit(0) }; + } + drop(runtime::take_owned_interpreter(owned_id)); + }); + } +} diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 7bc9f44ecb6..60a6e481003 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -2,6 +2,7 @@ use super::StopTheWorldState; use super::{ Context, PyConfig, PyGlobalState, VirtualMachine, + owned::OwnedVm, runtime::{self, InterpreterWhence}, setting::Settings, thread, @@ -13,7 +14,8 @@ use crate::{ use alloc::collections::BTreeMap; use core::sync::atomic::Ordering; -type InitFunc = Box; +type InitFunc = Box; +type ConfigFunc = Box; /// Exit code used when stdout/stderr flush fails during interpreter shutdown. /// Matches CPython's behavior (see cpython/Python/pylifecycle.c). @@ -30,15 +32,16 @@ const EXITCODE_FLUSH_FAILURE: u32 = 120; /// use rustpython_vm::Interpreter; /// /// let builder = Interpreter::builder(Default::default()); -/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) +/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(unsafe { builder.context() })) /// let interp = builder.build(); /// ``` pub struct InterpreterBuilder { settings: Settings, - pub ctx: PyRc, + ctx: PyRc, module_defs: Vec<&'static builtins::PyModuleDef>, frozen_modules: Vec<(&'static str, FrozenModule)>, init_hooks: Vec, + config_hooks: Vec, } /// Options for constructing a main or sub-interpreter VM. @@ -48,6 +51,7 @@ struct InitializeVmOpts<'a> { module_defs: Vec<&'static builtins::PyModuleDef>, frozen_modules: Vec<(&'static str, FrozenModule)>, init_hooks: Vec, + config_hooks: Vec, is_main: bool, whence: InterpreterWhence, /// When `Some`, reuse parent module_defs/frozen/config seeds for a subinterpreter. @@ -56,9 +60,9 @@ struct InitializeVmOpts<'a> { } /// Shared constructor for main and sub-interpreters. -fn initialize_vm(opts: InitializeVmOpts<'_>, init: F) -> (VirtualMachine, PyRc) +fn initialize_vm(opts: InitializeVmOpts<'_>, init: F) -> (OwnedVm, PyRc) where - F: FnOnce(&mut VirtualMachine), + F: FnOnce(&VirtualMachine), { let InitializeVmOpts { settings, @@ -66,6 +70,7 @@ where module_defs, frozen_modules, init_hooks, + config_hooks, is_main, whence, parent_state, @@ -74,7 +79,7 @@ where use crate::codecs::CodecsRegistry; use crate::common::lock::PyMutex; use crate::warn::WarningsState; - use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64}; + use core::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize}; use crossbeam_utils::atomic::AtomicCell; // Before any lock this interpreter's threads can contend on exists. @@ -94,7 +99,10 @@ where // First top-level interpreter wins. Later seeds are ignored. super::init_hash_secret(settings.hash_seed); let paths = getpath::init_path_config(&settings); - let config = PyConfig::new(settings, paths); + let mut config = PyConfig::new(settings, paths); + for hook in config_hooks { + hook(&mut config); + } // Build module_defs map from builtin modules + additional modules let mut all_module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef> = @@ -131,10 +139,14 @@ where }; // Per-interpreter ephemeral state (must not be shared across interpreters). + #[cfg(all(unix, feature = "threading"))] + let publication = super::fork::native_phase(); + let interpreter_id = runtime::alloc_interpreter_id(); + let gc = crate::gc_state::GcInterpreterState::new_for_interpreter(&ctx, interpreter_id); + let allocation_owner = gc.allocation_scope(); let codec_registry = CodecsRegistry::new(&ctx); let warnings = WarningsState::init_state(&ctx); - let interpreter_id = runtime::alloc_interpreter_id(); let runtime_root_id = parent_state.map_or(interpreter_id, |parent| parent.runtime_root_id); // Process main OS thread identity is process-global; subinterpreters inherit @@ -148,7 +160,12 @@ where // Create PyGlobalState (≈ PyInterpreterState) let global_state = PyRc::new(PyGlobalState { - gc: crate::gc_state::GcInterpreterState::new(&ctx), + owners: AtomicUsize::new(0), + owner_leases: super::owner_lease::OwnerLeases::default(), + teardowns: AtomicUsize::new(0), + closed: AtomicBool::new(false), + admission_closed: AtomicBool::new(false), + gc, interpreter_id, runtime_root_id, whence, @@ -162,6 +179,10 @@ where audit_hooks: PyMutex::default(), codec_registry, struct_format_cache: crate::buffer::FormatSpecCache::default(), + abc_invalidation_counter: AtomicU64::new(0), + native_cache: super::native_cache::NativeCache::default(), + roots: PyRc::new(super::roots::Roots::new(interpreter_id)), + native_types: super::native_types::NativeTypes::default(), finalizing: AtomicBool::new(false), #[cfg(feature = "threading")] finalizing_thread_ident: AtomicCell::new(0), @@ -174,7 +195,8 @@ where switch_interval: AtomicCell::new(0.005), global_trace_func: PyMutex::default(), global_profile_func: PyMutex::default(), - type_mutex: PyMutex::default(), + type_cache: crate::builtins::type_::TypeCache::default(), + type_mutex: crate::common::lock::PyDetachingRwLock::default(), #[cfg(feature = "threading")] main_thread_ident, #[cfg(feature = "threading")] @@ -197,31 +219,36 @@ where }); // Create VM with the global state - // Note: Don't clone here - init_hooks need exclusive access to mutate state - let mut vm = VirtualMachine::new(ctx, global_state); - - // Execute initialization hooks (can mutate vm.state) - for hook in init_hooks { - hook(&mut vm); - } + let vm = OwnedVm::new(VirtualMachine::new(ctx, global_state)); - // Call custom init function (can mutate vm.state) - init(&mut vm); - - // Register before `initialize()` runs any Python: it allocates GC-tracked - // objects, so a collection on another thread has to be able to stop this - // interpreter while that happens. It cannot be registered earlier — the - // hooks above take `PyRc::get_mut` on the state, which fails once the - // registry holds a weak reference to it. + // Keep the VM address stable and attached throughout user hooks as well as + // Python initialization. Unwinding hooks release their locals before this + // guard ends; OwnedVm then performs the remaining partial-state teardown. + let vm_guard = thread::VmBootstrapGuard::new(&vm); + drop(allocation_owner); + // Hooks can execute Python and must be visible to GC on other threads. runtime::register_interpreter(&vm.state); + #[cfg(all(unix, feature = "threading"))] + drop(publication); + let initialized = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| { + // Runtime hooks receive shared access after attachment. + for hook in init_hooks { + hook(&vm); + } - // `initialize()` runs Python bytecode directly (e.g. importing `codecs` - // and `encodings`) before any `enter_vm` scope exists, so attach this - // thread for the duration so type cache reads see it as ATTACHED. - let vm_guard = thread::VmBootstrapGuard::new(&vm); - vm.initialize(); - vm.state.ready.store(true, Ordering::Release); + // Run custom native initialization under the same shared entry. + init(&vm); + + // Imports in initialize() execute Python under the bootstrap guard, + // before the first ordinary enter_vm scope. + vm.initialize(); + vm.state.ready.store(true, Ordering::Release); + })); drop(vm_guard); + if let Err(error) = initialized { + drop(vm); + std::panic::resume_unwind(error); + } // Clone global_state for Interpreter after all initialization is done let global_state = vm.state.clone(); @@ -257,6 +284,7 @@ fn create_subinterpreter_from_parent( module_defs: Vec::new(), frozen_modules: Vec::new(), init_hooks: Vec::new(), + config_hooks: Vec::new(), is_main: false, whence: InterpreterWhence::Stdlib, parent_state: Some(&parent.state), @@ -266,13 +294,23 @@ fn create_subinterpreter_from_parent( ); let interp = Interpreter { global_state, vm }; // Every interpreter has a `__main__` module once it is initialized. - interp.enter(|vm| { + interp.enter_raw(|vm| { let _ = vm.ensure_main_module(); }); Ok(interp) } impl InterpreterBuilder { + /// Access the raw type context for defining native modules. + /// + /// # Safety + /// Follow the native ownership contract in [`crate::embedding`]. Do not retain + /// Python objects through this context outside their attached interpreter. + #[must_use] + pub unsafe fn context(&self) -> &Context { + &self.ctx + } + /// Create a new interpreter configuration with default settings. #[must_use] pub fn new() -> Self { @@ -282,9 +320,17 @@ impl InterpreterBuilder { module_defs: Vec::new(), frozen_modules: Vec::new(), init_hooks: Vec::new(), + config_hooks: Vec::new(), } } + /// Configure Rust-owned settings and paths before creating runtime objects. + #[must_use] + pub fn configure(mut self, configure: impl FnOnce(&mut PyConfig) + 'static) -> Self { + self.config_hooks.push(Box::new(configure)); + self + } + /// Set custom settings for the interpreter. /// /// If called multiple times, only the last settings will be used. @@ -302,13 +348,16 @@ impl InterpreterBuilder { /// /// let builder = Interpreter::builder(Default::default()); /// // Note: In practice, use module_def from your #[pymodule] - /// // let def = mymodule::module_def(&builder.ctx); + /// // let def = mymodule::module_def(unsafe { builder.context() }); /// // let interp = builder.add_native_module(def).build(); /// let interp = builder.build(); /// ``` #[must_use] - pub fn add_native_module(self, def: &'static builtins::PyModuleDef) -> Self { - self.add_native_modules(&[def]) + /// # Safety + /// Native callbacks must obey the raw ownership contract in [`crate::embedding`], + /// including owner-local access and destruction of all Python references. + pub unsafe fn add_native_module(self, def: &'static builtins::PyModuleDef) -> Self { + unsafe { self.add_native_modules(&[def]) } } /// Add multiple native module definitions. @@ -319,12 +368,15 @@ impl InterpreterBuilder { /// /// let builder = Interpreter::builder(Default::default()); /// // In practice, use module_defs from rustpython_stdlib: - /// // let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + /// // let defs = rustpython_stdlib::stdlib_module_defs(unsafe { builder.context() }); /// // let interp = builder.add_native_modules(&defs).build(); /// let interp = builder.build(); /// ``` #[must_use] - pub fn add_native_modules(mut self, defs: &[&'static builtins::PyModuleDef]) -> Self { + /// # Safety + /// Native callbacks must obey the raw ownership contract in [`crate::embedding`], + /// including owner-local access and destruction of all Python references. + pub unsafe fn add_native_modules(mut self, defs: &[&'static builtins::PyModuleDef]) -> Self { self.module_defs.extend_from_slice(defs); self } @@ -339,16 +391,18 @@ impl InterpreterBuilder { /// ``` /// use rustpython_vm::Interpreter; /// - /// let interp = Interpreter::builder(Default::default()) - /// .init_hook(|vm| { - /// // Custom initialization - /// }) + /// // SAFETY: this hook retains no Python references. + /// let interp = unsafe { Interpreter::builder(Default::default()) + /// .init_hook(|_vm| {}) } /// .build(); /// ``` #[must_use] - pub fn init_hook(mut self, init: F) -> Self + /// # Safety + /// Native callbacks must obey the raw ownership contract in [`crate::embedding`], + /// including owner-local access and destruction of all Python references. + pub unsafe fn init_hook(mut self, init: F) -> Self where - F: FnOnce(&mut VirtualMachine) + 'static, + F: FnOnce(&VirtualMachine) + 'static, { self.init_hooks.push(Box::new(init)); self @@ -388,6 +442,7 @@ impl InterpreterBuilder { module_defs: self.module_defs, frozen_modules: self.frozen_modules, init_hooks: self.init_hooks, + config_hooks: self.config_hooks, is_main: true, whence: InterpreterWhence::Runtime, parent_state: None, @@ -423,21 +478,52 @@ impl Default for InterpreterBuilder { /// Runs a simple embedded hello world program. /// ``` /// use rustpython_vm::Interpreter; -/// use rustpython_vm::compiler::Mode; /// Interpreter::without_stdlib(Default::default()).enter(|vm| { -/// let scope = vm.new_scope_with_builtins(); -/// let source = r#"print("Hello World!")"#; -/// let code_obj = vm.compile( -/// source, -/// Mode::Exec, -/// "", -/// ).map_err(|err| err.into_pyexception(vm, Some(source))).unwrap(); -/// vm.run_code_obj(code_obj, scope).unwrap(); +/// vm.exec(r#"print("Hello World!")"#).unwrap(); /// }); /// ``` pub struct Interpreter { - pub global_state: PyRc, - vm: VirtualMachine, + pub(crate) global_state: PyRc, + vm: OwnedVm, +} + +/// Finalization was not started because native workers still own this runtime. +/// Drop or join them, then call [`Self::retry`]. +pub struct FinalizeBusy { + interpreter: Interpreter, + result: crate::embedding::Result<()>, +} + +impl core::fmt::Debug for FinalizeBusy { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("FinalizeBusy") + .field("interpreter", &self.interpreter.id()) + .finish_non_exhaustive() + } +} + +impl core::fmt::Display for FinalizeBusy { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("native workers still own the interpreter") + } +} + +impl core::error::Error for FinalizeBusy {} + +impl FinalizeBusy { + #[must_use] + pub fn interpreter(&self) -> &Interpreter { + &self.interpreter + } + + pub fn retry(self) -> Result { + self.interpreter.finish_scoped(self.result) + } + + #[must_use] + pub fn into_interpreter(self) -> Interpreter { + self.interpreter + } } impl Interpreter { @@ -448,7 +534,7 @@ impl Interpreter { /// use rustpython_vm::Interpreter; /// /// let builder = Interpreter::builder(Default::default()); - /// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) + /// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(unsafe { builder.context() })) /// let interp = builder.build(); /// ``` #[must_use] @@ -462,15 +548,18 @@ impl Interpreter { /// try to build one from the source code of `InterpreterBuilder`. It will not be a one-liner but it also will not be too hard. #[must_use] pub fn without_stdlib(settings: Settings) -> Self { - Self::with_init(settings, |_| {}) + Self::builder(settings).build() } /// Create with initialize function taking mutable vm reference. /// /// Note: This is a legacy API. To add stdlib, use `Interpreter::builder()` instead. - pub fn with_init(settings: Settings, init: F) -> Self + /// # Safety + /// Native callbacks must obey the raw ownership contract in [`crate::embedding`], + /// including owner-local access and destruction of all Python references. + pub unsafe fn with_init(settings: Settings, init: F) -> Self where - F: FnOnce(&mut VirtualMachine), + F: FnOnce(&VirtualMachine), { let (vm, global_state) = initialize_vm( InitializeVmOpts { @@ -479,6 +568,7 @@ impl Interpreter { module_defs: Vec::new(), frozen_modules: Vec::new(), init_hooks: Vec::new(), + config_hooks: Vec::new(), is_main: true, whence: InterpreterWhence::Runtime, parent_state: None, @@ -574,42 +664,134 @@ impl Interpreter { &self, config: runtime::InterpreterConfig, ) -> Result { - create_subinterpreter_from_parent(&self.vm, config) + self.enter_raw(|vm| create_subinterpreter_from_parent(vm, config)) } /// Spawn a new OS-thread VM that shares this interpreter's `sys` / builtins. #[cfg(feature = "threading")] pub fn new_thread(&self) -> thread::ThreadedVirtualMachine { - self.vm.new_thread() + self.enter_raw(VirtualMachine::new_thread) } - /// Run a function with the main virtual machine and return a PyResult of the result. - /// - /// To enter vm context multiple times or to avoid buffer/exception management, this function is preferred. - /// `enter` is lightweight and it returns a python object in PyResult. - /// You can stop or continue the execution multiple times by calling `enter`. - /// - /// To finalize the vm once all desired `enter`s are called, calling `finalize` will be helpful. - /// - /// See also [`Interpreter::run`] for managed way to run the interpreter. - pub fn enter(&self, f: F) -> R + #[cfg(feature = "threading")] + pub(super) fn owner_is_valid(&self) -> bool { + self.vm.owner_is_valid() + } + + // Only the runtime directory uses an exclusively owned, idle anchor. + // Entering it while holding that directory's mutex could run Python. + #[cfg(feature = "threading")] + pub(super) fn new_thread_from_idle(&self) -> Option { + (self.owner_is_valid() && !self.vm.state.admission_closed.load(Ordering::Acquire)) + .then(|| self.vm.new_thread()) + } + + /// Internal entry for the interpreter implementation. + pub(crate) fn enter_raw(&self, f: F) -> R where F: FnOnce(&VirtualMachine) -> R, { thread::enter_vm(&self.vm, || f(&self.vm)) } + /// Enter with scoped object access. Only opaque handles and Rust values can + /// leave the callback; bound objects are tied to this individual entry. + pub fn enter(&self, f: impl for<'vm> FnOnce(crate::embedding::Vm<'vm>) -> R) -> R { + if !self.vm.owner_lease().is_valid() { + return f(crate::embedding::Vm::new(&self.vm)); + } + self.enter_raw(|vm| { + vm.state.roots.drain_pending(); + scopeguard::defer! { vm.state.roots.drain_pending(); } + f(crate::embedding::Vm::new(vm)) + }) + } + + /// Enter the native API without scoped object wrappers. + /// + /// # Safety + /// Follow the ownership and attachment contract in [`crate::embedding`]. + /// Raw references must not escape, cross interpreters, or be accessed while + /// detached. Use opaque handles for values retained by the host. + pub unsafe fn enter_unchecked(&self, f: impl FnOnce(&VirtualMachine) -> R) -> R { + self.enter_raw(f) + } + + /// Run Python with scoped access, then finalize the interpreter. + pub fn run( + self, + f: impl for<'vm> FnOnce(crate::embedding::Vm<'vm>) -> crate::embedding::Result<()>, + ) -> Result { + let result = self.enter(f); + self.finish_scoped(result) + } + + fn finish_scoped(self, result: crate::embedding::Result<()>) -> Result { + if !self.vm.owner_lease().is_valid() { + // An inherited inactive handle cannot finalize a surviving owner. + // Its Drop only releases its native VM storage. + return Ok(0); + } + // Stale post-fork handles are not counted as owners, but their native + // cleanup can still race finalization. Reserve native admission under + // the same lock that admits those cleanup entries. Python workers + // remain admissible until finalize_raw finishes joining them. + let Some(_finalization) = self + .global_state + .owner_leases + .try_finalize(&self.global_state) + else { + return Err(FinalizeBusy { + interpreter: self, + result, + }); + }; + // Local owners drop before the permit even if error conversion panics. + let interpreter = self; + let exception = interpreter.enter_raw(|vm| match result { + Ok(()) => None, + Err(crate::embedding::Error::Python(error)) => error.exception(vm), + Err(error) => Some(vm.new_runtime_error(error.to_string())), + }); + Ok(interpreter.finalize_raw(exception)) + } + + /// Finalize an interpreter after repeated scoped entries. + pub fn finalize(self) -> Result { + self.finish_scoped(Ok(())) + } + + /// Finalize with an exception returned by native code. + /// + /// # Safety + /// The exception and every object reachable from it must belong to this + /// interpreter. No raw reference may be accessed after finalization. + #[must_use] + pub unsafe fn finalize_unchecked(self, exc: Option) -> u32 { + self.finalize_raw(exc) + } + + /// Run the raw native API and finalize the interpreter. + /// + /// # Safety + /// The callback must obey [`Self::enter_unchecked`]'s contract. + pub unsafe fn run_unchecked(self, f: impl FnOnce(&VirtualMachine) -> PyResult<()>) -> u32 { + self.run_raw(f) + } + /// Run [`Interpreter::enter`] and call [`VirtualMachine::expect_pyresult`] for the result. /// /// This function is useful when you want to expect a result from the function, /// but also print useful panic information when exception raised. /// /// See also [`Interpreter::enter`] and [`VirtualMachine::expect_pyresult`] for more information. - pub fn enter_and_expect(&self, f: F, msg: &str) -> R + /// # Safety + /// The callback must obey [`Self::enter_unchecked`]'s contract. + pub unsafe fn enter_and_expect_unchecked(&self, f: F, msg: &str) -> R where F: FnOnce(&VirtualMachine) -> PyResult, { - self.enter(|vm| { + self.enter_raw(|vm| { let result = f(vm); vm.expect_pyresult(result, msg) }) @@ -623,12 +805,12 @@ impl Interpreter { /// /// See [`Interpreter::finalize`] for the finalization steps. /// See also [`Interpreter::enter`] for pure function call to obtain Python exception. - pub fn run(self, f: F) -> u32 + pub(crate) fn run_raw(self, f: F) -> u32 where F: FnOnce(&VirtualMachine) -> PyResult<()>, { - let res = self.enter(|vm| f(vm)); - self.finalize(res.err()) + let res = self.enter_raw(|vm| f(vm)); + self.finalize_raw(res.err()) } /// Finalize vm and turns an exception to exit code. @@ -645,8 +827,14 @@ impl Interpreter { /// 1. Final stdout/stderr flush. /// /// Note that calling `finalize` is not necessary by purpose though. - pub fn finalize(self, exc: Option) -> u32 { - self.enter(|vm| { + pub(crate) fn finalize_raw(self, exc: Option) -> u32 { + let finalization = self + .global_state + .owner_leases + .enter_finalization(&self.global_state); + // This local must drop before the permit, including on unwind. + let interpreter = self; + let result = interpreter.enter_raw(|vm| { let mut flush_status = vm.flush_std(); // See if any exception leaked out: @@ -681,6 +869,10 @@ impl Interpreter { #[cfg(feature = "threading")] finalize_subinterpreters(vm); + // Running non-daemon threads may start children while shutdown + // joins them. Close native admission only after that join phase. + vm.state.admission_closed.store(true, Ordering::Release); + // Remaining daemon threads hang in `_PyThreadState_HangThread` when // they next attach (or at the eval-breaker if they are already // running). Their `_ThreadHandle` stays not-done so `is_alive()` @@ -715,10 +907,32 @@ impl Interpreter { crate::stdlib::_interpchannels::clear_interpreter(interpreter_id); crate::stdlib::_interpqueues::clear_interpreter(interpreter_id); + vm.state.roots.close(); + vm.state.type_cache.close(); + vm.state.native_cache.close(); + vm.state.native_types.close(); + vm.state.gc.collect_force(2); + if vm.flush_std() < 0 && flush_status == 0 { flush_status = -1; } + vm.retire_execution_roots(); + #[cfg(feature = "threading")] + thread::retire_interpreter_thread_roots(vm); + // SAFETY: consuming finalization has ended this VM's user entries; + // no reference into its execution caches remains in this closure. + unsafe { vm.retire_cached_roots() }; + vm.state.retire_python_roots(&vm.ctx); + vm.state.gc.collect_force(2); + + // Python execution has ended. Late native destructors may release + // buffers or SQLite statements, but cannot reenter cleared services. + thread::native_teardown(&vm.state, || { + vm.state.closed.store(true, Ordering::Release); + vm.state.close_python_roots(); + }); + // Match CPython: if exit_code is 0 and stdout flush failed, exit 120 let exit_code = if exit_code == 0 && flush_status < 0 { EXITCODE_FLUSH_FAILURE @@ -729,10 +943,16 @@ impl Interpreter { // Daemon threads may still exist, so use the safe `process()`, // not `drain_all()`. #[cfg(feature = "threading")] - crate::object::qsbr::QSBR.process(); + { + vm.state.gc.qsbr().process(); + crate::object::qsbr::shared().process(); + } exit_code - }) + }); + drop(interpreter); + drop(finalization); + result } } @@ -876,7 +1096,7 @@ mod tests { #[test] fn add_py_integers() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let a: PyObjectRef = vm.ctx.new_int(33_i32).into(); let b: PyObjectRef = vm.ctx.new_int(12_i32).into(); let res = vm._add(&a, &b).unwrap(); @@ -887,7 +1107,7 @@ mod tests { #[test] fn multiply_str() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { let a = vm.new_pyobj(crate::common::ascii!("Hello ")); let b = vm.new_pyobj(4_i32); let res = vm._mul(&a, &b).unwrap(); @@ -957,41 +1177,30 @@ mod tests { } } - /// A collection snapshots the registry and then reads tracked objects with - /// the interpreters it found parked. An interpreter that registered inside - /// that window would be missing from the snapshot, so nothing would stop it - /// and its bootstrap would run under the scan; registration therefore waits - /// for the stop to end. + /// Independent heaps allow bootstrap while an unrelated owner is stopped. #[cfg(feature = "threading")] #[test] - fn registering_waits_for_an_in_flight_stop() { + fn registering_does_not_wait_for_another_interpreters_stop() { use core::time::Duration; use std::sync::mpsc; - // Stands in for a collector between its snapshot and its restart. - let admission = runtime::lock_admission_for_stop(); - + let first = Interpreter::without_stdlib(Default::default()); let (tx, rx) = mpsc::channel(); - let worker = std::thread::spawn(move || { - let interp = Interpreter::without_stdlib(Default::default()); - tx.send(interp.id()).expect("receiver is alive"); - interp + let (worker, result) = first.enter_raw(|vm| { + vm.state.stop_the_world.with_stopped(&vm.state, || { + let worker = std::thread::spawn(move || { + let interp = Interpreter::without_stdlib(Default::default()); + tx.send(interp.id()).expect("receiver is alive"); + interp + }); + let result = rx.recv_timeout(Duration::from_secs(5)); + (worker, result) + }) }); - - assert!( - matches!( - rx.recv_timeout(Duration::from_millis(200)), - Err(mpsc::RecvTimeoutError::Timeout) - ), - "an interpreter registered while a stop-the-world was in flight" - ); - - drop(admission); - let id = rx - .recv_timeout(Duration::from_secs(30)) - .expect("registration proceeds once the world restarts"); + let interp = worker.join().expect("worker did not panic"); + let id = result.expect("unrelated interpreter stop delayed bootstrap"); assert!(runtime::lookup_interpreter(id).is_some()); - drop(worker.join().expect("worker did not panic")); + drop(interp); wait_until_unregistered(id); } @@ -1015,15 +1224,16 @@ mod tests { let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - let (main_sys_ptr, main_builtins_ptr, main_ctx_ptr, main_state_ptr) = main.enter(|vm| { - ( - vm.sys_module.as_object() as *const _, - vm.builtins.as_object() as *const _, - PyRc::as_ptr(&vm.ctx), - PyRc::as_ptr(&vm.state), - ) - }); - let (sub_sys_ptr, sub_builtins_ptr, sub_ctx_ptr, sub_state_ptr) = sub.enter(|vm| { + let (main_sys_ptr, main_builtins_ptr, main_ctx_ptr, main_state_ptr) = + main.enter_raw(|vm| { + ( + vm.sys_module.as_object() as *const _, + vm.builtins.as_object() as *const _, + PyRc::as_ptr(&vm.ctx), + PyRc::as_ptr(&vm.state), + ) + }); + let (sub_sys_ptr, sub_builtins_ptr, sub_ctx_ptr, sub_state_ptr) = sub.enter_raw(|vm| { ( vm.sys_module.as_object() as *const _, vm.builtins.as_object() as *const _, @@ -1048,7 +1258,7 @@ mod tests { let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - main.enter(|vm| { + main.enter_raw(|vm| { vm.builtins .set_attr(PROBE, vm.ctx.new_int(11_i32), vm) .unwrap(); @@ -1059,7 +1269,7 @@ mod tests { .unwrap(); }); - sub.enter(|vm| { + sub.enter_raw(|vm| { assert!(vm.builtins.get_attr(PROBE, vm).is_err()); let modules = vm.sys_module.get_attr("modules", vm).unwrap(); assert!(modules.get_item(PROBE, vm).is_err()); @@ -1072,7 +1282,7 @@ mod tests { .unwrap(); }); - main.enter(|vm| { + main.enter_raw(|vm| { let builtin_probe = vm.builtins.get_attr(PROBE, vm).unwrap(); assert_eq!(*int::get_value(&builtin_probe), 11_i32.to_bigint().unwrap()); @@ -1091,7 +1301,7 @@ mod tests { #[test] fn create_subinterpreter_while_parent_entered() { let main = Interpreter::without_stdlib(Default::default()); - main.enter(|vm| { + main.enter_raw(|vm| { let before = vm.state.interpreter_id; let sub = main.create_subinterpreter(); assert_ne!(sub.id(), before); @@ -1111,14 +1321,14 @@ mod tests { let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - main.enter(|vm| { + main.enter_raw(|vm| { assert!(vm.state.is_main_interpreter()); let a: PyObjectRef = vm.ctx.new_int(1_i32).into(); let b: PyObjectRef = vm.ctx.new_int(2_i32).into(); let res = vm._add(&a, &b).unwrap(); assert_eq!(*int::get_value(&res), 3_i32.to_bigint().unwrap()); }); - sub.enter(|vm| { + sub.enter_raw(|vm| { assert!(!vm.state.is_main_interpreter()); let a: PyObjectRef = vm.ctx.new_int(10_i32).into(); let b: PyObjectRef = vm.ctx.new_int(5_i32).into(); @@ -1126,7 +1336,7 @@ mod tests { assert_eq!(*int::get_value(&res), 50_i32.to_bigint().unwrap()); }); // Re-enter main after sub. - main.enter(|vm| { + main.enter_raw(|vm| { assert!(vm.state.is_main_interpreter()); }); } @@ -1143,11 +1353,11 @@ mod tests { let counter = Arc::new(AtomicUsize::new(0)); let c1 = Arc::clone(&counter); - let h_main = main.enter(|vm| { + let h_main = main.enter_raw(|vm| { let thread_vm = vm.new_thread(); let c = Arc::clone(&c1); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { for _ in 0..100 { let a: PyObjectRef = vm.ctx.new_int(1_i32).into(); let b: PyObjectRef = vm.ctx.new_int(1_i32).into(); @@ -1160,11 +1370,11 @@ mod tests { }); let c2 = Arc::clone(&counter); - let h_sub = sub.enter(|vm| { + let h_sub = sub.enter_raw(|vm| { let thread_vm = vm.new_thread(); let c = Arc::clone(&c2); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { for _ in 0..100 { let a: PyObjectRef = vm.ctx.new_int(2_i32).into(); let b: PyObjectRef = vm.ctx.new_int(3_i32).into(); @@ -1204,10 +1414,10 @@ mod tests { let spawn_worker = |interpreter: &Interpreter| { let state = Arc::clone(&state); - interpreter.enter(|vm| { + interpreter.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { let a: PyObjectRef = vm.ctx.new_int(20_i32).into(); let b: PyObjectRef = vm.ctx.new_int(22_i32).into(); assert_eq!( @@ -1283,10 +1493,10 @@ mod tests { let main_started_worker = Arc::clone(&main_started); let sub_finished_worker = Arc::clone(&sub_finished); - let main_worker = main.enter(|vm| { + let main_worker = main.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { main_started_worker.store(true, Ordering::Release); let deadline = Instant::now() + Duration::from_secs(30); let mut operations = 0; @@ -1311,13 +1521,13 @@ mod tests { let main_started_worker = Arc::clone(&main_started); let sub_finished_worker = Arc::clone(&sub_finished); - let sub_worker = sub.enter(|vm| { + let sub_worker = sub.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { while !main_started_worker.load(Ordering::Acquire) { std::thread::yield_now(); } - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { let a: PyObjectRef = vm.ctx.new_int(6_i32).into(); let b: PyObjectRef = vm.ctx.new_int(7_i32).into(); let result = vm._mul(&a, &b).unwrap(); @@ -1346,10 +1556,10 @@ mod tests { let sub = main.create_subinterpreter(); let sub_id = sub.id(); - let handle = sub.enter(|vm| { + let handle = sub.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { assert_eq!(vm.state.interpreter_id, sub_id); assert!(!vm.state.is_main_interpreter()); }); @@ -1367,7 +1577,7 @@ mod tests { let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - sub.enter(|vm| { + sub.enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); let source = "x = 40 + 2\n"; let code = vm @@ -1409,32 +1619,32 @@ mod tests { // The scopes are what keep the classes alive; a subclass list holds // only weak references, so both must outlive every assertion below. - let main_scope = main.enter(|vm| { + let main_scope = main.enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); run(vm, &scope, "class MainOnly(int): pass\n"); scope }); - let sub_scope = sub.enter(|vm| { + let sub_scope = sub.enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); run(vm, &scope, "class SubOnly(int): pass\n"); scope }); - main.enter(|vm| { + main.enter_raw(|vm| { assert!(lists_subclass(vm, &main_scope, "MainOnly")); assert!(!lists_subclass(vm, &main_scope, "SubOnly")); // A subclass built before either interpreter existed belongs to the // shared context, so it stays visible to both. assert!(lists_subclass(vm, &main_scope, "bool")); }); - sub.enter(|vm| { + sub.enter_raw(|vm| { assert!(lists_subclass(vm, &sub_scope, "SubOnly")); assert!(!lists_subclass(vm, &sub_scope, "MainOnly")); assert!(lists_subclass(vm, &sub_scope, "bool")); }); - main.enter(|_| drop(main_scope)); - sub.enter(|_| drop(sub_scope)); + main.enter_raw(|_| drop(main_scope)); + sub.enter_raw(|_| drop(sub_scope)); } /// A cycle allocated in one interpreter is not the parent's to collect. @@ -1463,7 +1673,7 @@ mod tests { let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - let sub_scope = sub.enter(|vm| { + let sub_scope = sub.enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); run(vm, &scope, CYCLE); assert_eq!(live_nodes(vm), 2); @@ -1478,16 +1688,16 @@ mod tests { // bytecode never reaches a safepoint, and the collection this is // waiting for cannot stop it. let deadline = Instant::now() + Duration::from_secs(30); - while !main.enter(|vm| vm.state.gc.collect_force(2).candidates > 0) { + while !main.enter_raw(|vm| vm.state.gc.collect_force(2).candidates > 0) { assert!( Instant::now() < deadline, "no collection ran in the parent interpreter" ); std::thread::sleep(Duration::from_millis(5)); } - sub.enter(|vm| assert_eq!(live_nodes(vm), 2)); + sub.enter_raw(|vm| assert_eq!(live_nodes(vm), 2)); - sub.enter(|_| drop(sub_scope)); + sub.enter_raw(|_| drop(sub_scope)); } /// And it is not the parent's to enumerate either. @@ -1504,28 +1714,28 @@ mod tests { let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - let main_scope = main.enter(|vm| { + let main_scope = main.enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); run(vm, &scope, "class MainNode:\n pass\nkeep = MainNode()\n"); scope }); - let sub_scope = sub.enter(|vm| { + let sub_scope = sub.enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); run(vm, &scope, "class SubNode:\n pass\nkeep = SubNode()\n"); scope }); - main.enter(|vm| { + main.enter_raw(|vm| { assert!(tracks_class(vm, "MainNode")); assert!(!tracks_class(vm, "SubNode")); }); - sub.enter(|vm| { + sub.enter_raw(|vm| { assert!(tracks_class(vm, "SubNode")); assert!(!tracks_class(vm, "MainNode")); }); - main.enter(|_| drop(main_scope)); - sub.enter(|_| drop(sub_scope)); + main.enter_raw(|_| drop(main_scope)); + sub.enter_raw(|_| drop(sub_scope)); } /// The runtime can own a subinterpreter by id and hand it back on destroy. @@ -1580,13 +1790,13 @@ mod tests { let sub1 = main1.create_owned_subinterpreter(); let sub2 = main2.create_owned_subinterpreter(); - main1.enter(finalize_subinterpreters); + main1.enter_raw(finalize_subinterpreters); assert!(!runtime::is_owned_interpreter(sub1)); assert!(runtime::is_owned_interpreter(sub2)); let sub2 = runtime::take_owned_interpreter(sub2).expect("owned by second runtime"); - let _ = sub2.finalize(None); + let _ = sub2.finalize_raw(None); } /// A collection must stop every interpreter, not just the collecting one: @@ -1628,10 +1838,10 @@ for _ in range(40): // Subinterpreter thread: allocate cycles continuously. let stop_worker = Arc::clone(&stop); - let churner = sub.enter(|vm| { + let churner = sub.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { while !stop_worker.load(Ordering::Acquire) { run_source(vm, CHURN); } @@ -1640,7 +1850,7 @@ for _ in range(40): }); // Main interpreter: force collections while the sub keeps mutating. - main.enter(|vm| { + main.enter_raw(|vm| { run_source(vm, CHURN); let deadline = Instant::now() + Duration::from_secs(2); let mut collections = 0; @@ -1671,7 +1881,7 @@ for _ in range(40): let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - let sub_state = sub.enter(|vm| vm.state.clone()); + let sub_state = sub.enter_raw(|vm| vm.state.clone()); let progress = Arc::new(AtomicU64::new(0)); let stop = Arc::new(AtomicBool::new(false)); @@ -1680,10 +1890,10 @@ for _ in range(40): // reports progress every iteration. let progress_worker = Arc::clone(&progress); let stop_worker = Arc::clone(&stop); - let worker = sub.enter(|vm| { + let worker = sub.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { let source = "x = 1 + 1\n"; let code = vm .compile(source, Mode::Exec, "") @@ -1703,7 +1913,7 @@ for _ in range(40): std::thread::yield_now(); } - main.enter(|_vm| { + main.enter_raw(|_vm| { // Stop the *subinterpreter* from a thread whose current interpreter // is main — the cross-interpreter stop a collection performs. sub_state.stop_the_world.stop_the_world(&sub_state); @@ -1743,7 +1953,7 @@ for _ in range(40): let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - let sub_state = sub.enter(|vm| vm.state.clone()); + let sub_state = sub.enter_raw(|vm| vm.state.clone()); let progress = Arc::new(AtomicU64::new(0)); let stop = Arc::new(AtomicBool::new(false)); @@ -1751,10 +1961,10 @@ for _ in range(40): let progress_worker = Arc::clone(&progress); let stop_worker = Arc::clone(&stop); - let worker = sub.enter(|vm| { + let worker = sub.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { let source = "x = 1 + 1\n"; let code = vm .compile(source, Mode::Exec, "") @@ -1854,7 +2064,7 @@ for _ in range(40): #[test] fn eval_breaker_tripped_when_stop_requested() { let interp = Interpreter::without_stdlib(Default::default()); - interp.enter(|vm| { + interp.enter_raw(|vm| { crate::signal::clear_eval_breaker_for_test(); assert!( crate::vm::thread::set_stop_requested_for_current_thread(true), @@ -1883,7 +2093,7 @@ for _ in range(40): let main = Interpreter::without_stdlib(Default::default()); let sub = main.create_subinterpreter(); - let sub_state = sub.enter(|vm| vm.state.clone()); + let sub_state = sub.enter_raw(|vm| vm.state.clone()); let progress = Arc::new(AtomicU64::new(0)); let stop = Arc::new(AtomicBool::new(false)); @@ -1891,11 +2101,11 @@ for _ in range(40): // Worker runs the SUB nested inside an active MAIN section. let progress_worker = Arc::clone(&progress); let stop_worker = Arc::clone(&stop); - let main_vm = main.enter(|vm| vm.new_thread()); - let sub_vm = sub.enter(|vm| vm.new_thread()); + let main_vm = main.enter_raw(|vm| vm.new_thread()); + let sub_vm = sub.enter_raw(|vm| vm.new_thread()); let worker = std::thread::spawn(move || { - main_vm.run(|_main| { - sub_vm.run(|vm| { + main_vm.run_raw(|_main| { + sub_vm.run_raw(|vm| { let source = "x = 1 + 1\n"; let code = vm .compile(source, Mode::Exec, "") @@ -1951,7 +2161,7 @@ for _ in range(40): }; let interp = Interpreter::without_stdlib(Default::default()); - let state = interp.enter(|vm| vm.state.clone()); + let state = interp.enter_raw(|vm| vm.state.clone()); let lock: Arc> = Arc::new(PyDetachingRwLock::new(())); // The worker's thread id, published from inside the interpreter. No @@ -1963,10 +2173,10 @@ for _ in range(40): let worker_lock = Arc::clone(&lock); let published_ident = Arc::clone(&worker_ident); - let worker = interp.enter(|vm| { + let worker = interp.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|_vm| { + thread_vm.run_raw(|_vm| { published_ident.store(crate::stdlib::_thread::get_ident(), Ordering::Release); let _read = worker_lock.read(); }); @@ -2040,7 +2250,7 @@ for _ in range(40): }; let interp = Interpreter::without_stdlib(Default::default()); - let state = interp.enter(|vm| vm.state.clone()); + let state = interp.enter_raw(|vm| vm.state.clone()); let detached = Arc::new(AtomicBool::new(false)); let ran = Arc::new(AtomicBool::new(false)); @@ -2049,10 +2259,10 @@ for _ in range(40): let worker_detached = Arc::clone(&detached); let worker_ran = Arc::clone(&ran); let worker_go = Arc::clone(&go); - let worker = interp.enter(|vm| { + let worker = interp.enter_raw(|vm| { let thread_vm = vm.new_thread(); std::thread::spawn(move || { - thread_vm.run(|vm| { + thread_vm.run_raw(|vm| { vm.allow_threads(|| { worker_detached.store(true, Ordering::Release); // Spinning here is spinning *detached*, which is what a @@ -2133,7 +2343,7 @@ for _ in range(40): ..Settings::default() }; let first = Interpreter::without_stdlib(settings); - first.enter(|vm| { + first.enter_raw(|vm| { vm.ctx .intern_str("zz_hash_seed_regression") .to_object() @@ -2146,7 +2356,7 @@ for _ in range(40): ..Settings::default() }; let second = Interpreter::without_stdlib(settings); - second.enter(|vm| { + second.enter_raw(|vm| { let interned = vm.ctx.intern_str("zz_hash_seed_regression").to_object(); let dict = vm.ctx.new_dict(); dict.set_item(&*interned, vm.ctx.new_int(1).into(), vm) diff --git a/crates/vm/src/vm/method.rs b/crates/vm/src/vm/method.rs index 723d0d13a2e..709af340a32 100644 --- a/crates/vm/src/vm/method.rs +++ b/crates/vm/src/vm/method.rs @@ -26,8 +26,7 @@ impl PyMethod { return obj.get_attr(name, vm).map(Self::Attribute); } - // any correct method name is always interned already. - let interned_name = vm.ctx.interned_str(name); + let interned_name = cls.interned_attr_name(name, vm); let mut is_method = false; let cls_attr = match interned_name.and_then(|name| cls.get_attr(name)) { diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 445ff7474a0..7d46ed26a53 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -10,10 +10,17 @@ pub(crate) mod compile_mode; pub use compile::VmCompileError; mod context; pub mod crossinterp; +#[cfg(all(unix, feature = "threading"))] +pub(crate) mod fork; mod interpreter; mod method; +mod native_cache; +pub(crate) mod native_types; +pub(crate) mod owned; +mod owner_lease; #[cfg(feature = "rustpython-compiler")] mod python_run; +pub(crate) mod roots; pub mod runtime; mod setting; pub mod thread; @@ -52,7 +59,7 @@ use alloc::{borrow::Cow, collections::BTreeMap}; use core::ptr::NonNull; use core::{ cell::{Cell, OnceCell, RefCell}, - sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering}, }; use crossbeam_utils::atomic::AtomicCell; use std::{ @@ -61,7 +68,7 @@ use std::{ }; pub use context::Context; -pub use interpreter::{Interpreter, InterpreterBuilder}; +pub use interpreter::{FinalizeBusy, Interpreter, InterpreterBuilder}; pub(crate) use method::PyMethod; pub use runtime::{ InterpFeatureFlags, InterpreterConfig, InterpreterGil, InterpreterInfo, InterpreterWhence, @@ -83,16 +90,17 @@ pub const MAX_MEMORY_SIZE: usize = isize::MAX as usize; /// /// To construct the main VM of an interpreter, use [`Interpreter`]. pub struct VirtualMachine { + owner_lease: OnceCell>, pub builtins: PyRef, pub sys_module: PyRef, pub ctx: PyRc, /// Thread-local data stack for bump-allocating frame-local data /// (localsplus arrays for non-generator frames). datastack: core::cell::UnsafeCell, - pub wasm_id: Option, + pub wasm_id: OnceCell, exceptions: RefCell, - pub import_func: PyObjectRef, - pub(crate) importlib: PyObjectRef, + pub(crate) import_func: owned::ExecutionRoot, + pub(crate) importlib: owned::ExecutionRoot, pub profile_func: RefCell, pub trace_func: RefCell, pub use_tracing: Cell, @@ -102,10 +110,10 @@ pub struct VirtualMachine { tracing_depth: Cell, pub recursion_limit: Cell, pub(crate) signal_handlers: OnceCell, - pub(crate) signal_rx: Option, + pub(crate) signal_rx: OnceCell, pub repr_guards: RefCell>, pub state: PyRc, - pub initialized: bool, + pub initialized: Cell, recursion_depth: Cell, /// Depth of native recursion that pushes no Python frame, counted only /// where the stack pointer cannot be read. Everywhere else the native @@ -256,6 +264,20 @@ impl Default for StopTheWorldState { #[cfg(feature = "threading")] impl StopTheWorldState { + pub(crate) fn with_stopped(&self, state: &PyGlobalState, f: impl FnOnce() -> R) -> R { + let nested = self.world_stopped.load(Ordering::Acquire) + && self.requester.load(Ordering::Relaxed) == crate::stdlib::_thread::get_ident(); + if !nested { + self.stop_the_world(state); + } + scopeguard::defer! { + if !nested { + self.start_the_world(state); + } + } + f() + } + #[must_use] pub const fn new() -> Self { Self { @@ -789,11 +811,11 @@ pub(super) fn stw_trace(msg: core::fmt::Arguments<'_>) { #[derive(Clone, Debug, Default)] pub(crate) struct CallableCache { - pub len: Option, - pub isinstance: Option, - pub list_append: Option, - pub builtin_all: Option, - pub builtin_any: Option, + pub len: owned::ExecutionRoot, + pub isinstance: owned::ExecutionRoot, + pub list_append: owned::ExecutionRoot, + pub builtin_all: owned::ExecutionRoot, + pub builtin_any: owned::ExecutionRoot, } /// Per-interpreter shared state (≈ CPython `PyInterpreterState`). @@ -802,6 +824,13 @@ pub(crate) struct CallableCache { /// `PyGlobalState`. Process-wide pieces live elsewhere (`Context::genesis`, /// GC, the interpreter registry in [`runtime`]). pub struct PyGlobalState { + owners: AtomicUsize, + owner_leases: owner_lease::OwnerLeases, + // Count native destruction until its last graph access and detach finish. + // An owner leaves `owners` earlier, to elect live-runtime finalization. + teardowns: AtomicUsize, + pub(crate) closed: AtomicBool, + pub(crate) admission_closed: AtomicBool, /// Unique process-global interpreter id (main is [`MAIN_INTERPRETER_ID`]). pub interpreter_id: i64, /// Top-level interpreter whose runtime owns this interpreter. @@ -825,6 +854,10 @@ pub struct PyGlobalState { pub(crate) audit_hooks: PyMutex>, pub codec_registry: CodecsRegistry, pub struct_format_cache: crate::buffer::FormatSpecCache, + pub(crate) abc_invalidation_counter: AtomicU64, + native_cache: native_cache::NativeCache, + pub(crate) roots: PyRc, + pub(crate) native_types: native_types::NativeTypes, pub finalizing: AtomicBool, /// The thread performing finalization, which need not be the process main thread. #[cfg(feature = "threading")] @@ -841,7 +874,8 @@ pub struct PyGlobalState { /// Global profile function for all threads (set by sys._setprofileallthreads) pub global_profile_func: PyMutex>, /// Global type mutation/versioning mutex for CPython-style FT type operations. - pub type_mutex: PyMutex<()>, + pub(crate) type_cache: crate::builtins::type_::TypeCache, + pub type_mutex: crate::common::lock::PyDetachingRwLock<()>, /// Main thread identifier (pthread_self on Unix) #[cfg(feature = "threading")] pub main_thread_ident: AtomicCell, @@ -881,6 +915,56 @@ pub struct PyGlobalState { } impl PyGlobalState { + /// Retire every shared Python root after Python execution is disabled. + /// Native registry snapshots may keep this state alive on a foreign thread. + pub(super) fn close_python_roots(&self) { + let mut cleanup = owned::Cleanup::default(); + cleanup.run(|| self.roots.close()); + cleanup.run(|| self.type_cache.close()); + cleanup.run(|| self.native_cache.close()); + cleanup.run(|| self.native_types.close()); + cleanup.run(|| self.codec_registry.close()); + cleanup.run(|| self.warnings.close()); + cleanup.run(|| self.gc.close_python_roots()); + cleanup.run(|| self.retire_callbacks()); + cleanup.finish(); + } + + pub(super) fn retire_python_roots(&self, ctx: &Context) { + let mut cleanup = owned::Cleanup::default(); + cleanup.run(|| self.roots.close()); + cleanup.run(|| self.type_cache.close()); + cleanup.run(|| self.native_cache.close()); + cleanup.run(|| self.native_types.close()); + cleanup.run(|| self.retire_callbacks()); + cleanup.run(|| self.warnings.retire_roots(ctx)); + cleanup.run(|| self.gc.retire_python_roots(ctx)); + cleanup.run(|| self.codec_registry.close()); + cleanup.finish(); + } + + fn retire_callbacks(&self) { + let atexit = core::mem::take(&mut *self.atexit_funcs.lock()); + let audit = core::mem::take(&mut *self.audit_hooks.lock()); + let before_fork = core::mem::take(&mut *self.before_forkers.lock()); + let after_child = core::mem::take(&mut *self.after_forkers_child.lock()); + let after_parent = core::mem::take(&mut *self.after_forkers_parent.lock()); + let trace = self.global_trace_func.lock().take(); + let profile = self.global_profile_func.lock().take(); + let monitoring = core::mem::take(&mut *self.monitoring.lock()); + self.monitoring_events.store(0); + drop(( + atexit, + audit, + before_fork, + after_child, + after_parent, + trace, + profile, + monitoring, + )); + } + #[inline] #[must_use] pub fn is_main_interpreter(&self) -> bool { @@ -1141,21 +1225,104 @@ enum LengthHint<'a> { } impl VirtualMachine { - fn init_callable_cache(&mut self) -> PyResult<()> { - self.callable_cache.len = Some(self.builtins.get_attr("len", self)?); - self.callable_cache.isinstance = Some(self.builtins.get_attr("isinstance", self)?); + fn owner_lease(&self) -> &PyRc { + self.owner_lease.get().expect("VM has no native owner") + } + + pub(crate) fn owner_is_valid(&self) -> bool { + self.owner_lease().is_valid() + } + + /// Release per-thread roots while this execution context is still usable. + pub(super) fn retire_execution_roots(&self) { + let signals = self + .signal_handlers + .get() + .map(|handlers| handlers.replace(Default::default())); + let exceptions = self.exceptions.replace(ExceptionStack::default()); + let trace = self.trace_func.replace(self.ctx.none()); + let profile = self.profile_func.replace(self.ctx.none()); + self.use_tracing.set(false); + let firstiter = self.async_gen_firstiter.take(); + let finalizer = self.async_gen_finalizer.take(); + let running_loop = self.asyncio_running_loop.take(); + let running_task = self.asyncio_running_task.take(); + let contexts = self.context_stack.take(); + drop(( + exceptions, + signals, + trace, + profile, + firstiter, + finalizer, + running_loop, + running_task, + contexts, + )); + } + + /// Release cached values before masking callbacks or dropping VM storage. + /// + /// # Safety + /// All execution scopes borrowing cached roots must have ended. The owner + /// may publish a fresh shared cleanup entry while these cells are emptied. + pub(super) unsafe fn retire_cached_roots(&self) { + // Take every root before dropping any value: finalizers can reenter. + let roots = unsafe { + ( + self.callable_cache.len.retire(), + self.callable_cache.isinstance.retire(), + self.callable_cache.list_append.retire(), + self.callable_cache.builtin_all.retire(), + self.callable_cache.builtin_any.retire(), + self.import_func.retire(), + self.importlib.retire(), + ) + }; + drop(roots); + } + + fn init_callable_cache(&self) -> PyResult<()> { + let _ = self + .callable_cache + .len + .set(self.builtins.get_attr("len", self)?); + let _ = self + .callable_cache + .isinstance + .set(self.builtins.get_attr("isinstance", self)?); let list_append = self .ctx .types .list_type .get_attr(self.ctx.intern_str("append")) .ok_or_else(|| self.new_runtime_error("failed to cache list.append"))?; - self.callable_cache.list_append = Some(list_append); - self.callable_cache.builtin_all = Some(self.builtins.get_attr("all", self)?); - self.callable_cache.builtin_any = Some(self.builtins.get_attr("any", self)?); + let _ = self.callable_cache.list_append.set(list_append); + let _ = self + .callable_cache + .builtin_all + .set(self.builtins.get_attr("all", self)?); + let _ = self + .callable_cache + .builtin_any + .set(self.builtins.get_attr("any", self)?); Ok(()) } + #[inline] + pub fn import_func(&self) -> &PyObject { + self.import_func + .get() + .map_or_else(|| self.ctx.none.as_object(), |value| value.as_ref()) + } + + #[inline] + pub(crate) fn importlib(&self) -> &PyObject { + self.importlib + .get() + .map_or_else(|| self.ctx.none.as_object(), |value| value.as_ref()) + } + /// Bump-allocate `size` bytes from the thread data stack. /// /// # Safety @@ -1245,18 +1412,19 @@ impl VirtualMachine { let builtins = new_module(stdlib::builtins::module_def(&ctx)); let sys_module = new_module(stdlib::sys::module_def(&ctx)); - let import_func = ctx.none(); - let importlib = ctx.none(); + let import_func = owned::ExecutionRoot::new(); + let importlib = owned::ExecutionRoot::new(); let profile_func = RefCell::new(ctx.none()); let trace_func = RefCell::new(ctx.none()); let signal_handlers = OnceCell::from(SignalHandlers::default()); let vm = Self { + owner_lease: OnceCell::new(), builtins, sys_module, ctx, datastack: core::cell::UnsafeCell::new(crate::datastack::DataStack::new()), - wasm_id: None, + wasm_id: OnceCell::new(), exceptions: RefCell::default(), import_func, importlib, @@ -1267,10 +1435,10 @@ impl VirtualMachine { tracing_depth: Cell::new(0), recursion_limit: Cell::new(if cfg!(debug_assertions) { 256 } else { 1000 }), signal_handlers, - signal_rx: None, + signal_rx: OnceCell::new(), repr_guards: RefCell::default(), state, - initialized: false, + initialized: Cell::new(false), recursion_depth: Cell::new(0), #[cfg(any(miri, target_env = "musl"))] native_recursion_depth: Cell::new(0), @@ -1306,7 +1474,7 @@ impl VirtualMachine { /// set up the encodings search function /// init_importlib must be called before this call #[cfg(feature = "encodings")] - fn import_encodings(&mut self) -> PyResult<()> { + fn import_encodings(&self) -> PyResult<()> { self.import("encodings", 0).map_err(|import_err| { let rustpythonpath_env = crate::host_env::os::var("RUSTPYTHONPATH").ok(); let pythonpath_env = crate::host_env::os::var("PYTHONPATH").ok(); @@ -1343,7 +1511,7 @@ impl VirtualMachine { Ok(()) } - fn import_ascii_utf8_encodings(&mut self) -> PyResult<()> { + fn import_ascii_utf8_encodings(&self) -> PyResult<()> { // Use the Python import machinery (FrozenImporter) so modules get // proper __spec__ and __loader__ attributes. self.import("codecs", 0)?; @@ -1397,10 +1565,10 @@ impl VirtualMachine { Ok(()) } - fn initialize(&mut self) { + fn initialize(&self) { flame_guard!("init VirtualMachine"); - assert!(!self.initialized, "Double Initialize Error"); + assert!(!self.initialized.get(), "Double Initialize Error"); // Process main-thread identity is owned by the main interpreter only // (used for signal handling / `_thread._is_main_interpreter` helpers). @@ -1423,7 +1591,7 @@ impl VirtualMachine { "failed to initialize bootstrap stderr", ); - let mut essential_init = || -> PyResult { + let essential_init = || -> PyResult { import::import_builtin(self, "_typing")?; #[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] import::import_builtin(self, "_signal")?; @@ -1578,12 +1746,15 @@ impl VirtualMachine { ); } - self.initialized = true; + self.initialized.set(true); } /// Set the custom signal channel for the interpreter - pub fn set_user_signal_channel(&mut self, signal_rx: signal::UserSignalReceiver) { - self.signal_rx = Some(signal_rx); + pub fn set_user_signal_channel(&self, signal_rx: signal::UserSignalReceiver) { + assert!( + self.signal_rx.set(signal_rx).is_ok(), + "signal channel already set" + ); } /// Execute Python bytecode (`.pyc`) from an in-memory buffer. @@ -1598,8 +1769,10 @@ impl VirtualMachine { /// First, compile a Python source file into bytecode: /// /// ```sh + /// # unsafe { /// # Generate a .pyc file /// $ rustpython -m py_compile .py + /// # } /// ``` /// /// ## Running the bytecode @@ -1607,12 +1780,14 @@ impl VirtualMachine { /// Load the resulting `.pyc` file into memory and execute it using the VM: /// /// ```no_run + /// # unsafe { /// use rustpython_vm::Interpreter; - /// Interpreter::without_stdlib(Default::default()).enter(|vm| { + /// Interpreter::without_stdlib(Default::default()).enter_unchecked(|vm| { /// let bytes = std::fs::read("__pycache__/.rustpython-314.pyc").unwrap(); /// let main_scope = vm.new_scope_with_main().unwrap(); /// vm.run_pyc_bytes(&bytes, main_scope); /// }); + /// # } /// ``` pub fn run_pyc_bytes(&self, pyc_bytes: &[u8], scope: Scope) -> PyResult<()> { let code = PyCode::from_pyc(pyc_bytes, Some(""), None, None, self)?; @@ -3219,7 +3394,7 @@ impl VirtualMachine { .builtins .get_attr(identifier!(self, __import__), self) .map_err(|_| self.new_import_error("__import__ not found", module.to_owned()))?; - if !current_import.is(&self.import_func) { + if !current_import.is(self.import_func()) { // `builtins.__import__` was replaced by user code; must go // through it so overrides (test_import, test_importlib, // test_builtin) still take effect. @@ -3534,7 +3709,7 @@ impl VirtualMachine { } pub(crate) fn get_str_method(&self, obj: PyObjectRef, method_name: &str) -> Option { - let method_name = self.ctx.interned_str(method_name)?; + let method_name = obj.class().interned_attr_name(method_name, self)?; self.get_method(obj, method_name) } @@ -3548,8 +3723,14 @@ impl VirtualMachine { /// `check_signals` and never park, so `stop_the_world` waits forever. #[inline] pub(crate) fn eval_breaker_tripped(&self) -> bool { + if self.state.roots.has_pending() { + return true; + } #[cfg(feature = "threading")] - if thread::stop_requested_for_current_thread() || self.state.gc.collection_ready() { + if thread::stop_requested_for_current_thread() + || self.state.gc.collection_ready() + || self.state.gc.qsbr().break_pending() + { return true; } #[cfg(not(target_arch = "wasm32"))] @@ -3581,9 +3762,13 @@ impl VirtualMachine { #[cfg(feature = "threading")] thread::suspend_if_needed(&self.state); + self.state.roots.drain_pending(); + // Pass a QSBR checkpoint if requested (deferred memory reclamation). #[cfg(feature = "threading")] - if crate::signal::qsbr_bit_set() && thread::qsbr_break_requested() { + if (self.state.gc.qsbr().break_pending() || crate::signal::qsbr_bit_set()) + && thread::qsbr_break_requested() + { thread::qsbr_checkpoint(); } @@ -3601,7 +3786,7 @@ impl VirtualMachine { #[cfg(feature = "threading")] pub(crate) fn run_scheduled_gc(&self) { if self.state.gc.collection_ready() { - self.state.gc.collect(0); + self.state.gc.collect_automatic(); } } @@ -3907,7 +4092,7 @@ mod tests { dir = "../../../../extra_tests/snippets" )) .build() - .enter(|vm| { + .enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); let source = "from dir_module.dir_module_inner import value2"; @@ -3929,7 +4114,7 @@ mod tests { vm::Interpreter::builder(Default::default()) .build() - .enter(|vm| { + .enter_raw(|vm| { let check = |name, expected| { let module = import::import_frozen(vm, name).unwrap(); let origname: PyStrRef = module diff --git a/crates/vm/src/vm/native_cache.rs b/crates/vm/src/vm/native_cache.rs new file mode 100644 index 00000000000..8eca1f630e9 --- /dev/null +++ b/crates/vm/src/vm/native_cache.rs @@ -0,0 +1,231 @@ +//! Native attribute factories cache Python values in their owning interpreter. + +use crate::common::{lock::PyMutex, rc::PyRc}; +use core::any::{Any, TypeId}; +use std::collections::HashMap; + +#[cfg(feature = "threading")] +type Value = dyn Any + Send + Sync; +#[cfg(not(feature = "threading"))] +type Value = dyn Any; + +#[derive(Default)] +pub(super) struct NativeCache { + state: PyMutex, +} + +#[derive(Default)] +struct CacheState { + entries: HashMap>, + closed: bool, +} + +impl NativeCache { + fn get(&self) -> Option + where + T: Any + Clone + crate::object::PyThreadingConstraint, + { + let cached = self.state.lock().entries.get(&TypeId::of::()).cloned(); + cached.map(|value| { + value + .downcast_ref::() + .expect("native cache key reused with a different value type") + .clone() + }) + } + + fn replace(&self, value: T) + where + T: Any + crate::object::PyThreadingConstraint, + { + let value: PyRc = PyRc::new(value); + let previous = { + let mut state = self.state.lock(); + if state.closed { + None + } else { + state.entries.insert(TypeId::of::(), value.clone()) + } + }; + drop(previous); + } + + pub(super) fn get_or_init(&self, init: impl FnOnce() -> T) -> T + where + T: Any + Clone + crate::object::PyThreadingConstraint, + { + self.get_or_try_init::(|| Ok(init())) + .unwrap_or_else(|never| match never {}) + } + + fn get_or_try_init(&self, init: impl FnOnce() -> Result) -> Result + where + T: Any + Clone + crate::object::PyThreadingConstraint, + { + let key = TypeId::of::(); + let cached = self.state.lock().entries.get(&key).cloned(); + let value = if let Some(value) = cached { + value + } else { + // A factory may allocate, run GC, and reenter this same key from a + // finalizer. Initialize without locks and publish a single winner. + let candidate: PyRc = PyRc::new(init()?); + let winner = { + let mut state = self.state.lock(); + if state.closed { + candidate.clone() + } else { + state + .entries + .entry(key) + .or_insert_with(|| candidate.clone()) + .clone() + } + }; + // Neither losing Python values nor arbitrary T::clone code may run + // while the directory is locked. + drop(candidate); + winner + }; + Ok(value + .downcast_ref::() + .expect("native cache key reused with a different value type") + .clone()) + } + + /// The owner must be entered. Finalizers may still call native factories, + /// but cannot reinstall roots into a cache whose interpreter is retiring. + pub(super) fn close(&self) { + let roots = { + let mut state = self.state.lock(); + state.closed = true; + core::mem::take(&mut state.entries) + }; + drop(roots); + } + + #[cfg(all(unix, feature = "threading"))] + pub(super) unsafe fn reinit_after_fork(&self) { + unsafe { crate::common::lock::reinit_mutex_after_fork(&self.state) }; + } +} + +impl super::VirtualMachine { + /// Read a replaceable native cache entry without running clone under its lock. + #[doc(hidden)] + pub fn __get_native(&self) -> Option + where + T: Any + Clone + crate::object::PyThreadingConstraint, + { + self.state.native_cache.get::() + } + + /// Publish an immutable snapshot, releasing the previous value outside locks. + #[doc(hidden)] + pub fn __replace_native(&self, value: T) + where + T: Any + crate::object::PyThreadingConstraint, + { + self.state.native_cache.replace::(value); + } + + /// Used by `#[pyattr(once)]` to preserve identity within one interpreter. + /// Each factory must use its own key type and be idempotent: competing or + /// reentrant calls may construct multiple candidates before one is cached. + #[doc(hidden)] + pub fn __cached_native(&self, init: impl FnOnce() -> T) -> T + where + T: Any + Clone + crate::object::PyThreadingConstraint, + { + self.state.native_cache.get_or_init::(init) + } + + /// Fallible native factories do not cache an unsuccessful initialization. + #[doc(hidden)] + pub fn __try_cached_native( + &self, + init: impl FnOnce() -> Result, + ) -> Result + where + T: Any + Clone + crate::object::PyThreadingConstraint, + { + self.state.native_cache.get_or_try_init::(init) + } +} + +#[cfg(all(unix, feature = "threading"))] +impl super::PyGlobalState { + pub(crate) unsafe fn reinit_native_cache_after_fork(&self) { + unsafe { + self.native_cache.reinit_after_fork(); + self.native_types.reinit_after_fork(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reentry_publishes_one_identity_and_errors_are_retryable() { + struct Key; + let cache = NativeCache::default(); + assert_eq!( + cache.get_or_try_init::(|| Err("retry")), + Err("retry") + ); + let value = cache.get_or_init::(|| { + assert_eq!(cache.get_or_init::(|| "inner".to_owned()), "inner"); + "outer".to_owned() + }); + assert_eq!(value, "inner"); + cache.close(); + assert_eq!( + cache.get_or_init::(|| "transient".to_owned()), + "transient" + ); + assert!(cache.state.lock().entries.is_empty()); + } + + #[test] + fn replacement_and_close_release_snapshots() { + struct Key; + let cache = NativeCache::default(); + let first = PyRc::new(()); + let weak = PyRc::downgrade(&first); + cache.replace::(first); + let second = PyRc::new(()); + cache.replace::(second.clone()); + assert!(weak.upgrade().is_none()); + assert!(PyRc::ptr_eq( + &cache.get::>().unwrap(), + &second + )); + cache.close(); + cache.replace::(second); + assert!(cache.get::>().is_none()); + } + + #[cfg(feature = "threading")] + #[test] + fn concurrent_factories_keep_the_same_winner() { + struct Key; + let cache = NativeCache::default(); + let barrier = std::sync::Barrier::new(2); + std::thread::scope(|scope| { + let make = || { + cache.get_or_init::(|| { + barrier.wait(); + PyRc::new(()) + }) + }; + let first = scope.spawn(make); + let second = scope.spawn(make); + assert!(PyRc::ptr_eq( + &first.join().unwrap(), + &second.join().unwrap() + )); + }); + } +} diff --git a/crates/vm/src/vm/native_types.rs b/crates/vm/src/vm/native_types.rs new file mode 100644 index 00000000000..baad599af7e --- /dev/null +++ b/crates/vm/src/vm/native_types.rs @@ -0,0 +1,355 @@ +//! Interpreter-owned Python namespaces for immutable native type definitions. + +use crate::{ + Context, Py, PyRef, VirtualMachine, + builtins::{PyType, PyTypeRef, PyWeak, type_::TypeNamespace}, + class::NativeOperators, + common::{ + lock::{PyMutex, PyRwLock}, + rc::PyRc, + }, + types::PyTypeFlags, +}; +use core::{any::TypeId, cell::RefCell}; +use std::collections::HashMap; + +pub struct NativeNamespaceDefinition { + pub(crate) class: PyTypeRef, + pub(crate) operators: NativeOperators, + pub(crate) populate: fn(&Context, &Py, &NativeOperators), +} + +pub(crate) const RUNTIME_FLAGS: PyTypeFlags = PyTypeFlags::from_slice(&[ + PyTypeFlags::SEQUENCE, + PyTypeFlags::MAPPING, + PyTypeFlags::IS_ABSTRACT, +]); + +#[derive(Default)] +pub(crate) struct NativeTypes { + state: PyMutex, +} + +#[derive(Default)] +struct NativeTypeState { + namespaces: HashMap>, + classes: HashMap, + subclasses: HashMap>>>>, + runtime_flags: HashMap, + closed: bool, +} + +thread_local! { + static INITIALIZATION_OWNERS: RefCell> = const { RefCell::new(Vec::new()) }; + // Recursive construction sees its own candidate. Other threads only see + // fully initialized namespaces, published under the interpreter type lock. + static INITIALIZING: RefCell)>> = const { + RefCell::new(Vec::new()) + }; + static INITIALIZING_CLASSES: RefCell> = const { + RefCell::new(Vec::new()) + }; +} + +pub(crate) fn initializing(owner: i64) -> bool { + INITIALIZATION_OWNERS.with(|owners| owners.borrow().contains(&owner)) +} + +struct InitializationGuard(i64); + +impl InitializationGuard { + fn new(owner: i64) -> Self { + INITIALIZATION_OWNERS.with(|owners| owners.borrow_mut().push(owner)); + Self(owner) + } +} + +impl Drop for InitializationGuard { + fn drop(&mut self) { + let previous = INITIALIZATION_OWNERS.with(|owners| owners.borrow_mut().pop()); + debug_assert_eq!(previous, Some(self.0)); + } +} + +impl NativeTypes { + pub(crate) fn runtime_flags(&self, class: &PyType) -> Option { + self.state + .lock() + .runtime_flags + .get(&core::ptr::from_ref(class).addr()) + .copied() + } + + pub(crate) fn set_runtime_flags(&self, class: &PyType, mask: PyTypeFlags, flags: PyTypeFlags) { + let mut state = self.state.lock(); + let current = state + .runtime_flags + .entry(core::ptr::from_ref(class).addr()) + .or_insert_with(|| class.slots.flags.load() & RUNTIME_FLAGS); + *current = (*current & !mask) | (flags & mask); + } + + pub(crate) fn subclasses(&self, class: &PyType) -> PyRc>>> { + let key = core::ptr::from_ref(class).addr(); + let mut state = self.state.lock(); + if state.closed { + return PyRc::default(); + } + state.subclasses.entry(key).or_default().clone() + } + + pub(crate) fn class( + &self, + vm: &VirtualMachine, + allocate: impl FnOnce() -> PyTypeRef, + populate: impl FnOnce(&Py), + ) -> PyTypeRef { + let key = TypeId::of::(); + let cached = self.state.lock().classes.get(&key).cloned(); + if let Some(class) = cached { + return class; + } + let owner = vm.state.interpreter_id; + if let Some(class) = INITIALIZING_CLASSES.with(|entries| { + entries + .borrow() + .iter() + .rev() + .find_map(|(id, candidate, class)| { + (*id == owner && *candidate == key).then(|| class.clone()) + }) + }) { + return class; + } + PyType::with_type_lock(vm, || { + let cached = self.state.lock().classes.get(&key).cloned(); + if let Some(class) = cached { + return class; + } + let _allocation = InitializationGuard::new(owner); + let class = allocate(); + INITIALIZING_CLASSES + .with(|entries| entries.borrow_mut().push((owner, key, class.clone()))); + let _initializing = scopeguard::guard((), |()| { + INITIALIZING_CLASSES.with(|entries| { + let previous = entries.borrow_mut().pop().unwrap(); + debug_assert_eq!((previous.0, previous.1), (owner, key)); + drop(previous); + }); + }); + populate(&class); + let mut state = self.state.lock(); + if !state.closed { + state.classes.insert(key, class.clone()); + } + class + }) + } + + fn get(&self, key: usize) -> Option> { + self.state.lock().namespaces.get(&key).cloned() + } + + pub(crate) fn namespace( + &self, + definition: &NativeNamespaceDefinition, + vm: &VirtualMachine, + ) -> PyRc { + let key = core::ptr::from_ref(definition).addr(); + if let Some(namespace) = self.get(key) { + return namespace; + } + let owner = vm.state.interpreter_id; + if let Some(namespace) = INITIALIZING.with(|entries| { + entries + .borrow() + .iter() + .rev() + .find_map(|(id, candidate, namespace)| { + (*id == owner && *candidate == key).then(|| namespace.clone()) + }) + }) { + return namespace; + } + PyType::with_type_lock(vm, || { + if let Some(namespace) = self.get(key) { + return namespace; + } + let _allocation = InitializationGuard::new(owner); + let namespace = PyRc::new(TypeNamespace::default()); + INITIALIZING.with(|entries| { + entries.borrow_mut().push((owner, key, namespace.clone())); + }); + let _initializing = scopeguard::guard((), |()| { + INITIALIZING.with(|entries| { + let previous = entries.borrow_mut().pop().unwrap(); + debug_assert_eq!((previous.0, previous.1), (owner, key)); + drop(previous); + }); + }); + (definition.populate)(&vm.ctx, &definition.class, &definition.operators); + let mut state = self.state.lock(); + if !state.closed { + state.namespaces.insert(key, namespace.clone()); + } + namespace + }) + } + + pub(super) fn close(&self) { + let roots = { + let mut state = self.state.lock(); + state.closed = true; + ( + core::mem::take(&mut state.namespaces), + core::mem::take(&mut state.classes), + core::mem::take(&mut state.subclasses), + ) + }; + drop(roots); + } + + #[cfg(all(unix, feature = "threading"))] + pub(super) unsafe fn reinit_after_fork(&self) { + unsafe { crate::common::lock::reinit_mutex_after_fork(&self.state) }; + #[expect( + clippy::iter_over_hash_type, + reason = "independent locks have no reset order" + )] + for registry in self.state.lock().subclasses.values() { + unsafe { crate::common::lock::reinit_rwlock_after_fork(registry) }; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{AsObject, Interpreter, PyPayload, class::PyClassImpl}; + + #[pyclass(interpreter_local, module = false, name = "LocalNative")] + #[derive(Debug, PyPayload)] + struct LocalNative; + + #[pyclass] + impl LocalNative { + #[extend_class] + fn populate(ctx: &Context, class: &Py) { + assert!(Self::make_class(ctx).is(class)); + class.set_str_attr("state", ctx.new_list(Vec::new()), ctx); + } + } + + #[pyclass(module = false, name = "ColdNativeAttributes")] + #[derive(Debug, PyPayload)] + struct ColdNativeAttributes; + + #[pyclass] + impl ColdNativeAttributes { + #[extend_class] + fn populate(ctx: &Context, class: &Py) { + class.set_str_attr("cold_native_attribute", ctx.new_int(42), ctx); + } + } + + #[pyclass(module = false, name = "ColdNativeMethod")] + #[derive(Debug, PyPayload)] + struct ColdNativeMethod; + + #[pyclass] + impl ColdNativeMethod { + #[pymethod] + fn cold_native_method(_zelf: &Py) -> i32 { + 43 + } + } + + #[test] + fn lazy_native_names_are_resolved_on_first_lookup() { + Interpreter::without_stdlib(Default::default()).enter_raw(|vm| { + let _ = ColdNativeAttributes::make_class(&vm.ctx); + let object = ColdNativeAttributes.into_ref(&vm.ctx); + let name = vm.ctx.new_str("cold_native_attribute"); + assert!(vm.ctx.interned_str(&*name).is_none()); + let value = object.as_object().get_attr(&*name, vm).unwrap(); + assert_eq!(value.try_to_value::(vm).unwrap(), 42); + + let _ = ColdNativeMethod::make_class(&vm.ctx); + let object = ColdNativeMethod.into_ref(&vm.ctx); + assert!(vm.ctx.interned_str("cold_native_method").is_none()); + let value = vm + .call_method(object.as_object(), "cold_native_method", ()) + .unwrap(); + assert_eq!(value.try_to_value::(vm).unwrap(), 43); + + let missing = vm.ctx.new_str("cold_native_missing_attribute"); + assert!(object.class().lookup_ref(&missing, vm).is_none()); + assert!(vm.ctx.interned_str(&*missing).is_none()); + }); + } + + #[test] + fn local_types_preserve_owner_identity_and_exact_downcasts() { + let first = Interpreter::without_stdlib(Default::default()); + let second = first.create_subinterpreter(); + first.enter_raw(|vm| { + let class = LocalNative::make_class(&vm.ctx); + let object = LocalNative.into_ref(&vm.ctx); + assert!(object.class().is(&class)); + assert!( + object + .as_object() + .downcast_ref_if_exact::(vm) + .is_some() + ); + second.enter_raw(|other| { + let other_class = LocalNative::make_class(&other.ctx); + assert!(!class.is(&other_class)); + assert!(LocalNative::make_class(&other.ctx).is(&other_class)); + assert!(LocalNative.into_ref(&other.ctx).class().is(&other_class)); + }); + assert!(LocalNative::make_class(&vm.ctx).is(&class)); + }); + } + + #[cfg(feature = "threading")] + #[test] + fn competing_local_type_initializers_publish_one_class() { + let interpreter = Interpreter::without_stdlib(Default::default()); + let first = interpreter.new_thread(); + let second = interpreter.new_thread(); + let (first, second) = std::thread::scope(|scope| { + let first = scope.spawn(move || first.run_raw(|vm| LocalNative::make_class(&vm.ctx))); + let second = scope.spawn(move || second.run_raw(|vm| LocalNative::make_class(&vm.ctx))); + (first.join().unwrap(), second.join().unwrap()) + }); + interpreter.enter_raw(|vm| { + assert!(first.is(&second)); + assert!(first.is(&LocalNative::make_class(&vm.ctx))); + drop((first, second)); + }); + } + + #[test] + fn failed_factory_unwinds_initialization_state() { + struct Failed; + let interpreter = Interpreter::without_stdlib(Default::default()); + interpreter.enter_raw(|vm| { + let make = + || PyType::new_simple_heap("Retry", vm.ctx.types.object_type, &vm.ctx).unwrap(); + let failed = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| { + vm.state + .native_types + .class::(vm, make, |_| panic!("factory failed")); + })); + assert!(failed.is_err()); + assert!(!initializing(vm.state.interpreter_id)); + let class = vm.state.native_types.class::(vm, make, |_| {}); + let again = + vm.state + .native_types + .class::(vm, || unreachable!(), |_| unreachable!()); + assert!(class.is(&again)); + }); + } +} diff --git a/crates/vm/src/vm/owned.rs b/crates/vm/src/vm/owned.rs new file mode 100644 index 00000000000..ef36dbacfd6 --- /dev/null +++ b/crates/vm/src/vm/owned.rs @@ -0,0 +1,127 @@ +//! Explicit owners of VM thread state, independent of registry Arc snapshots. + +use super::{VirtualMachine, thread}; +use core::{ + cell::{OnceCell, UnsafeCell}, + ops::{Deref, DerefMut}, + sync::atomic::Ordering, +}; + +/// Finish releasing independent root groups even if a native destructor panics. +/// Each group must remove its roots before destroying their values. +#[derive(Default)] +pub(crate) struct Cleanup { + panic: Option>, +} + +impl Cleanup { + pub(crate) fn run(&mut self, step: impl FnOnce()) { + if let Err(error) = std::panic::catch_unwind(core::panic::AssertUnwindSafe(step)) { + if self.panic.is_none() { + self.panic = Some(error); + } else { + // Even a panic payload can have a panicking destructor. Keep + // the first error and finish retiring runtime roots regardless. + core::mem::forget(error); + } + } + } + + pub(crate) fn finish(self) { + if let Some(error) = self.panic { + std::panic::resume_unwind(error); + } + } +} + +/// Initialized once while a VM is live; emptied by its exclusive owner before +/// terminal destruction. Reads need no RefCell guard on the call fast paths. +pub(crate) struct ExecutionRoot(UnsafeCell>); + +impl Default for ExecutionRoot { + fn default() -> Self { + Self::new() + } +} + +impl ExecutionRoot { + pub(crate) const fn new() -> Self { + Self(UnsafeCell::new(OnceCell::new())) + } + + pub(crate) fn get(&self) -> Option<&T> { + // SAFETY: initialized roots are immutable throughout ordinary entry. + unsafe { &*self.0.get() }.get() + } + + pub(crate) fn set(&self, value: T) -> Result<(), T> { + // SAFETY: OnceCell initialization only needs shared access. + unsafe { &*self.0.get() }.set(value) + } + + /// # Safety + /// No reference returned by get() may remain. Only VM owner destruction or + /// consuming finalization, after all execution scopes end, may retire roots. + pub(crate) unsafe fn retire(&self) -> Option { + // The VM can remain shared in TLS: only this UnsafeCell is mutated. + unsafe { &mut *self.0.get() }.take() + } +} + +impl Clone for ExecutionRoot { + fn clone(&self) -> Self { + Self(UnsafeCell::new(unsafe { &*self.0.get() }.clone())) + } +} + +impl core::fmt::Debug for ExecutionRoot { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.get().fmt(f) + } +} + +pub(super) struct OwnedVm(Option>); + +impl OwnedVm { + pub(super) fn new(vm: VirtualMachine) -> Self { + Self::with_kind(vm, super::owner_lease::OwnerKind::Host) + } + + pub(super) fn with_kind(vm: VirtualMachine, kind: super::owner_lease::OwnerKind) -> Self { + assert!( + !vm.state.closed.load(Ordering::Acquire), + "interpreter is closed" + ); + let vm = Box::new(vm); + assert!( + vm.owner_lease + .set(vm.state.owner_leases.register_kind(kind)) + .is_ok() + ); + vm.owner_lease().publish_vm(&vm); + vm.state.owners.fetch_add(1, Ordering::Relaxed); + Self(Some(vm)) + } +} + +impl Deref for OwnedVm { + type Target = VirtualMachine; + + fn deref(&self) -> &Self::Target { + self.0.as_deref().expect("VM owner already released") + } +} + +impl DerefMut for OwnedVm { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.as_deref_mut().expect("VM owner already released") + } +} + +impl Drop for OwnedVm { + fn drop(&mut self) { + if let Some(vm) = self.0.take() { + thread::destroy_vm(vm); + } + } +} diff --git a/crates/vm/src/vm/owner_lease.rs b/crates/vm/src/vm/owner_lease.rs new file mode 100644 index 00000000000..1c0e01fa21d --- /dev/null +++ b/crates/vm/src/vm/owner_lease.rs @@ -0,0 +1,426 @@ +//! Native VM owners that survive independently of registry snapshots. + +use crate::common::{lock::PyMutex, rc::PyRc}; +use core::{ + marker::PhantomData, + sync::atomic::{AtomicBool, AtomicPtr, AtomicU8, AtomicU64, AtomicUsize, Ordering}, +}; + +#[cfg(not(feature = "threading"))] +use alloc::rc::Weak; +#[cfg(feature = "threading")] +use alloc::sync::Weak; + +const OWNED: u8 = 0; +const RELEASING: u8 = 1; +const RELEASED: u8 = 2; +const FINISHED: u8 = 3; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum OwnerKind { + Host, + #[cfg(feature = "threading")] + PythonThread, +} + +#[derive(Default)] +pub(super) struct OwnerLeases { + registry: PyMutex, +} + +#[derive(Default)] +struct Registry { + entries: Vec>, + finalizer: Option, + // A foreign stale owner must not block in Drop: the finalizer may join + // that native thread. The finalization permit drains these outside locks. + #[expect( + clippy::vec_box, + reason = "queued VM storage retains its stable address" + )] + pending: Vec>, +} + +#[derive(Clone, Copy)] +struct Finalizer { + thread: u64, + reserve_host_admission: bool, +} + +impl OwnerLeases { + #[cfg(test)] + pub(super) fn register(&self) -> PyRc { + self.register_kind(OwnerKind::Host) + } + + pub(super) fn register_kind(&self, kind: OwnerKind) -> PyRc { + self.registry.lock().register(OWNED, kind) + } + + /// # Safety + /// Use only on the current native thread during fork preparation/repair. + /// Its active entry scopes must remain alive until the pointer is last used. + #[cfg(all(unix, feature = "threading"))] + pub(super) unsafe fn active_vm(&self) -> Option> { + self.registry.lock().entries.iter().find_map(|entry| { + let lease = entry.upgrade()?; + if lease.is_valid() + && lease.active.load(Ordering::Acquire) != 0 + && lease.thread.load(Ordering::Acquire) == native_thread_id() + { + core::ptr::NonNull::new(lease.vm.load(Ordering::Acquire)) + } else { + None + } + }) + } + + pub(super) fn begin_teardown( + &self, + vm: Box, + ) -> Option<(Box, PyRc, bool)> { + let mut registry = self.registry.lock(); + let owns_vm = vm.owner_is_valid(); + if !owns_vm + && registry + .finalizer + .is_some_and(|finalizer| finalizer.thread != native_thread_id()) + { + registry.pending.push(vm); + return None; + } + let lease = if owns_vm { + let lease = vm.owner_lease().clone(); + lease.begin_teardown(); + lease + } else { + registry.register(RELEASED, OwnerKind::Host) + }; + vm.state.teardowns.fetch_add(1, Ordering::Relaxed); + Some((vm, lease, owns_vm)) + } + + pub(super) fn try_finalize(&self, state: &PyRc) -> Option { + let mut registry = self.registry.lock(); + if registry.finalizer.is_some() { + return None; + } + let mut owners = 0; + for lease in registry.entries.iter().filter_map(Weak::upgrade) { + if !lease.is_valid() || lease.kind != OwnerKind::Host { + continue; + } + match lease.phase.load(Ordering::Acquire) { + OWNED => owners += 1, + RELEASING | RELEASED => return None, + FINISHED => (), + _ => unreachable!(), + } + } + if owners != 1 { + return None; + } + registry.finalizer = Some(Finalizer { + thread: native_thread_id(), + reserve_host_admission: true, + }); + // Reserve host admission under the registry lock. Python workers + // must still be able to start children while _shutdown joins them. + Some(Finalization(state.clone())) + } + + /// Raw/CLI finalization may still have Python workers to join. It shares + /// cleanup admission with scoped finalization, without its Busy check. + pub(super) fn enter_finalization( + &self, + state: &PyRc, + ) -> Option { + let mut registry = self.registry.lock(); + if let Some(finalizer) = registry.finalizer { + assert_eq!( + finalizer.thread, + native_thread_id(), + "concurrent native finalization" + ); + return None; + } + registry.finalizer = Some(Finalizer { + thread: native_thread_id(), + reserve_host_admission: false, + }); + Some(Finalization(state.clone())) + } + + #[cfg(all(unix, feature = "threading"))] + #[expect(clippy::vec_box, reason = "VM storage retains its published address")] + pub(super) fn take_abandoned_cleanups(&self) -> Vec> { + let mut registry = self.registry.lock(); + if registry.finalizer.is_none() { + core::mem::take(&mut registry.pending) + } else { + Vec::new() + } + } + + /// The fork coordinator holds this guard across the syscall, after stopping + /// mutation of all registered runtimes. Resetting an inherited mutex alone + /// would not repair a Vec interrupted during a concurrent registration. + #[cfg(any(all(unix, feature = "threading"), test))] + pub(super) fn prepare_fork(&self) -> ForkOwners<'_> { + ForkOwners { + registry: self.registry.lock(), + } + } +} + +impl Registry { + fn register(&mut self, phase: u8, kind: OwnerKind) -> PyRc { + assert!( + phase != OWNED + || kind != OwnerKind::Host + || self + .finalizer + .is_none_or(|finalizer| !finalizer.reserve_host_admission), + "cannot create a native owner during interpreter shutdown" + ); + let lease = PyRc::new(OwnerLease { + kind, + valid: AtomicBool::new(true), + active: AtomicUsize::new(0), + thread: AtomicU64::new(0), + phase: AtomicU8::new(phase), + vm: AtomicPtr::new(core::ptr::null_mut()), + }); + self.entries.retain(|entry| entry.strong_count() != 0); + self.entries.push(PyRc::downgrade(&lease)); + lease + } +} + +pub(super) struct Finalization(PyRc); + +impl Drop for Finalization { + fn drop(&mut self) { + let mut cleanup = super::owned::Cleanup::default(); + loop { + let pending = { + let mut registry = self.0.owner_leases.registry.lock(); + if registry.pending.is_empty() { + registry.finalizer = None; + break; + } + core::mem::take(&mut registry.pending) + }; + for vm in pending { + cleanup.run(|| super::thread::destroy_vm(vm)); + } + } + if std::thread::panicking() { + // Preserve the original panic, including when a secondary panic + // payload itself has a panicking destructor. + core::mem::forget(cleanup); + } else { + cleanup.finish(); + } + } +} + +pub(super) struct OwnerLease { + kind: OwnerKind, + valid: AtomicBool, + active: AtomicUsize, + thread: AtomicU64, + phase: AtomicU8, + vm: AtomicPtr, +} + +impl OwnerLease { + pub(super) fn publish_vm(&self, vm: &super::VirtualMachine) { + self.vm + .store(core::ptr::from_ref(vm).cast_mut(), Ordering::Release); + } + + pub(super) fn retire_vm(&self) { + self.vm.store(core::ptr::null_mut(), Ordering::Release); + } + + pub(super) fn is_valid(&self) -> bool { + self.valid.load(Ordering::Acquire) + } + + pub(super) fn enter(self: &PyRc) -> OwnerEntry { + assert!(self.is_valid(), "VM owner did not survive fork"); + let ident = native_thread_id(); + if self.active.fetch_add(1, Ordering::Relaxed) == 0 { + self.thread.store(ident, Ordering::Release); + } else { + assert_eq!(self.thread.load(Ordering::Acquire), ident); + } + OwnerEntry { + lease: self.clone(), + _thread: PhantomData, + } + } + + pub(super) fn begin_teardown(&self) { + self.phase.store(RELEASING, Ordering::Release); + } + + pub(super) fn release_owner(&self) { + self.phase.store(RELEASED, Ordering::Release); + } + + pub(super) fn finish_teardown(&self) { + self.phase.store(FINISHED, Ordering::Release); + } +} + +/// Retained until attachment restoration, even after the VM Box was destroyed. +/// Saving/detaching a thread does not end its native entry scope. +pub(super) struct OwnerEntry { + lease: PyRc, + _thread: PhantomData>, +} + +impl Drop for OwnerEntry { + fn drop(&mut self) { + debug_assert_eq!( + self.lease.thread.load(Ordering::Acquire), + native_thread_id() + ); + let previous = self.lease.active.fetch_sub(1, Ordering::Release); + debug_assert!(previous != 0); + } +} + +fn native_thread_id() -> u64 { + #[cfg(feature = "threading")] + { + crate::stdlib::_thread::get_ident() + } + #[cfg(not(feature = "threading"))] + { + // Non-threaded runtimes and their registries are native-thread-local. + 1 + } +} + +#[cfg(any(all(unix, feature = "threading"), test))] +pub(super) struct ForkOwners<'a> { + registry: crate::common::lock::PyMutexGuard<'a, Registry>, +} + +#[cfg(any(all(unix, feature = "threading"), test))] +impl ForkOwners<'_> { + /// # Safety + /// Only the surviving fork thread may be executing. The guard was acquired + /// before fork, with owner publication and runtime mutations quiescent. + pub(super) unsafe fn repair_child(&mut self) -> (usize, usize) { + let ident = native_thread_id(); + let mut owners = 0; + let mut teardowns = 0; + if self + .registry + .finalizer + .is_some_and(|finalizer| finalizer.thread != ident) + { + self.registry.finalizer = None; + } + self.registry.entries.retain(|entry| { + let Some(lease) = entry.upgrade() else { + return false; + }; + if !lease.is_valid() { + return false; + } + if lease.active.load(Ordering::Acquire) == 0 + || lease.thread.load(Ordering::Acquire) != ident + { + lease.valid.store(false, Ordering::Release); + return false; + } + match lease.phase.load(Ordering::Acquire) { + OWNED => owners += 1, + RELEASING => { + owners += 1; + teardowns += 1; + } + RELEASED => teardowns += 1, + FINISHED => return false, + _ => unreachable!(), + } + true + }); + (owners, teardowns) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "threading")] + #[test] + fn scoped_admission_preserves_raw_finalization_workers() { + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let state = interpreter.enter_raw(|vm| vm.state.clone()); + { + let _raw = state.owner_leases.enter_finalization(&state); + let worker = interpreter.new_thread(); + drop(worker); + } + let _scoped = state.owner_leases.try_finalize(&state).unwrap(); + let nested = state.owner_leases.enter_finalization(&state); + assert!(nested.is_none()); + assert!( + std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| interpreter.new_thread())) + .is_err() + ); + let worker = interpreter.enter_raw(super::super::VirtualMachine::new_python_thread); + drop(worker); + } + + #[test] + fn repair_keeps_nested_entries_and_rejects_idle_owners() { + let registry = OwnerLeases::default(); + let active = registry.register(); + let idle = registry.register(); + let outer = active.enter(); + let inner = active.enter(); + // This native-only registry belongs exclusively to this test. + assert_eq!(unsafe { registry.prepare_fork().repair_child() }, (1, 0)); + assert!(active.is_valid()); + assert!(!idle.is_valid()); + drop(inner); + assert_eq!(unsafe { registry.prepare_fork().repair_child() }, (1, 0)); + drop(outer); + assert!(std::panic::catch_unwind(|| idle.enter()).is_err()); + } + + #[test] + fn repair_preserves_a_native_destructor_after_owner_release() { + let registry = OwnerLeases::default(); + let lease = registry.register(); + let entered = lease.enter(); + lease.begin_teardown(); + assert_eq!(unsafe { registry.prepare_fork().repair_child() }, (1, 1)); + lease.release_owner(); + assert_eq!(unsafe { registry.prepare_fork().repair_child() }, (0, 1)); + lease.finish_teardown(); + drop(entered); + } + + #[test] + fn repair_invalidates_a_vanished_threads_owner() { + let registry = OwnerLeases::default(); + let foreign = registry.register(); + // Model the completed atomic publication of a foreign native entry; + // there is no actual concurrent thread during simulated child repair. + foreign + .thread + .store(native_thread_id() ^ 1, Ordering::Relaxed); + foreign.active.store(1, Ordering::Relaxed); + assert_eq!(unsafe { registry.prepare_fork().repair_child() }, (0, 0)); + assert!(!foreign.is_valid()); + } +} diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index b8b83a240d1..57160b2efa7 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -227,7 +227,7 @@ mod tests { #[test] fn block_expr_return_const() { - interpreter().enter(|vm| { + interpreter().enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); let value = vm.unwrap_pyresult(vm.run_block_expr(scope, "1")); let value = vm.unwrap_pyresult(value.try_int(vm)); @@ -238,7 +238,7 @@ mod tests { #[test] fn block_expr_return_nonconst() { - interpreter().enter(|vm| { + interpreter().enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); vm.unwrap_pyresult(scope.globals.set_item("x", vm.new_pyobj(3), vm)); let value = vm.unwrap_pyresult(vm.run_block_expr(scope, "2 + x")); @@ -250,7 +250,7 @@ mod tests { #[test] fn block_expr_return_function_def() { - interpreter().enter(|vm| { + interpreter().enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); let value = vm.unwrap_pyresult(vm.run_block_expr(scope.clone(), "def f():\n return 7")); @@ -262,7 +262,7 @@ mod tests { #[test] fn block_expr_return_class_def() { - interpreter().enter(|vm| { + interpreter().enter_raw(|vm| { let scope = vm.new_scope_with_builtins(); let value = vm.unwrap_pyresult(vm.run_block_expr(scope.clone(), "class C:\n value = 11")); diff --git a/crates/vm/src/vm/roots.rs b/crates/vm/src/vm/roots.rs new file mode 100644 index 00000000000..150b9ae2680 --- /dev/null +++ b/crates/vm/src/vm/roots.rs @@ -0,0 +1,325 @@ +//! Persistent native roots released only while their interpreter is attached. + +use crate::{ + PyObjectRef, + common::{lock::PyMutex, rc::PyRc}, +}; +#[cfg(any(feature = "threading", test))] +use core::any::Any; +use core::sync::atomic::{AtomicBool, Ordering}; + +#[cfg(feature = "threading")] +type Resource = dyn Any + Send + Sync; +#[cfg(all(not(feature = "threading"), test))] +type Resource = dyn Any; + +enum Root { + Object(PyObjectRef), + Buffer(crate::protocol::PyBuffer), + #[cfg(any(feature = "threading", test))] + Resource { + _value: Box, + }, +} + +struct Slot { + generation: u64, + value: Option, +} + +#[derive(Clone, Copy)] +struct Key { + index: usize, + generation: u64, +} + +#[derive(Default)] +struct Table { + slots: Vec, + free: Vec, + pending: Vec, + closed: bool, +} + +pub(crate) struct Roots { + pub(crate) owner: i64, + table: PyMutex, + pending: AtomicBool, + draining: AtomicBool, + closed: AtomicBool, +} + +/// Contains no directly accessible Python reference. Clones share one table entry. +pub(crate) struct RootLease { + roots: PyRc, + key: Key, +} + +impl Drop for RootLease { + fn drop(&mut self) { + // Closed root tables may outlive the runtime and its fork snapshot. + // Never acquire their inherited mutex after native retirement. + if self.roots.is_closed() { + return; + } + { + let mut table = self.roots.table.lock(); + if table.closed { + return; + } + table.pending.push(self.key); + self.roots.pending.store(true, Ordering::Release); + } + // The normal path can release immediately. A foreign thread or a TLS + // destructor only queues work, without dereferencing a Python object. + super::thread::try_with_attached_vm(|vm| { + if vm.state.interpreter_id == self.roots.owner { + self.roots.drain_pending(); + } + }); + } +} + +impl RootLease { + pub(crate) fn buffer(&self, vm: &crate::VirtualMachine) -> Option { + if self.owner() != vm.state.interpreter_id { + return None; + } + assert!(super::thread::is_current_attached(vm)); + let table = self.roots.table.lock(); + let slot = table.slots.get(self.key.index)?; + if slot.generation != self.key.generation { + return None; + } + match slot.value.as_ref()? { + Root::Buffer(buffer) => Some(buffer.clone()), + _ => unreachable!("non-buffer root used as a buffer"), + } + } + + /// Retire one owner-local cleanup action immediately, even from a release + /// callback. The generation invalidates a later queued drop of this lease. + #[cfg(feature = "threading")] + pub(crate) fn release_now(&self, vm: &crate::VirtualMachine) { + assert_eq!(self.owner(), vm.state.interpreter_id); + assert!(super::thread::is_current_attached(vm)); + let retired = retire(&mut self.roots.table.lock(), self.key); + drop(retired); + } + + pub(crate) fn owner(&self) -> i64 { + self.roots.owner + } + + /// The caller must hold an attachment to the owner for the returned reference. + pub(crate) fn object(&self) -> Option { + let table = self.roots.table.lock(); + let slot = table.slots.get(self.key.index)?; + if slot.generation != self.key.generation { + return None; + } + match slot.value.as_ref()? { + Root::Object(object) => Some(object.clone()), + Root::Buffer(_) => { + unreachable!("resource used as a Python object") + } + #[cfg(any(feature = "threading", test))] + Root::Resource { .. } => unreachable!("resource used as a Python object"), + } + } +} + +impl Roots { + pub(crate) fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + + pub(crate) fn has_pending(&self) -> bool { + self.pending.load(Ordering::Relaxed) + } + + // Detached native threads can enqueue the last handle release without a + // Python attachment. Stabilize the Vec itself across fork, not just its lock. + #[cfg(all(unix, feature = "threading"))] + pub(super) fn prepare_fork(&self) -> impl Drop + '_ { + self.table.lock() + } + + #[cfg(all(unix, feature = "threading"))] + pub(crate) unsafe fn reinit_after_fork(&self) { + // The fork coordinator held and released the directory guard in both + // processes. No interrupted table mutation can survive the syscall. + self.draining.store(false, Ordering::Relaxed); + } + + pub(crate) fn new(owner: i64) -> Self { + Self { + owner, + table: PyMutex::default(), + pending: AtomicBool::new(false), + draining: AtomicBool::new(false), + closed: AtomicBool::new(false), + } + } + + fn insert(self: &PyRc, value: Root) -> Option> { + let key = { + let mut table = self.table.lock(); + if table.closed { + return None; + } + if let Some(index) = table.free.pop() { + let slot = &mut table.slots[index]; + debug_assert!(slot.value.is_none()); + slot.value = Some(value); + Key { + index, + generation: slot.generation, + } + } else { + let index = table.slots.len(); + table.slots.push(Slot { + generation: 0, + value: Some(value), + }); + Key { + index, + generation: 0, + } + } + }; + Some(PyRc::new(RootLease { + roots: self.clone(), + key, + })) + } + + pub(crate) fn insert_object(self: &PyRc, object: PyObjectRef) -> Option> { + self.insert(Root::Object(object)) + } + + pub(crate) fn insert_buffer( + self: &PyRc, + buffer: crate::protocol::PyBuffer, + ) -> Option> { + self.insert(Root::Buffer(buffer)) + } + + /// Retain an export or another native resource with an owner-local destructor. + #[cfg(any(feature = "threading", test))] + pub(crate) fn insert_resource(self: &PyRc, resource: T) -> Option> + where + T: Any + crate::object::PyThreadingConstraint, + { + self.insert(Root::Resource { + _value: Box::new(resource), + }) + } + + /// Detach roots under the directory lock; arbitrary Python runs only after unlocking. + pub(crate) fn drain_pending(&self) { + if !self.pending.load(Ordering::Acquire) || self.draining.swap(true, Ordering::Acquire) { + return; + } + scopeguard::defer! { self.draining.store(false, Ordering::Release); } + let retired = { + let mut table = self.table.lock(); + self.pending.store(false, Ordering::Relaxed); + let pending = core::mem::take(&mut table.pending); + let mut retired = Vec::with_capacity(pending.len()); + for key in pending { + if let Some(value) = retire(&mut table, key) { + retired.push(value); + } + } + retired + }; + drop(retired); + } + + /// Invalidate every external handle before running any of their destructors. + pub(crate) fn close(&self) { + let slots = { + let mut table = self.table.lock(); + table.closed = true; + self.closed.store(true, Ordering::Release); + table.pending.clear(); + self.pending.store(false, Ordering::Relaxed); + table.free.clear(); + core::mem::take(&mut table.slots) + }; + drop(slots); + } +} + +fn retire(table: &mut Table, key: Key) -> Option { + let slot = table.slots.get_mut(key.index)?; + if slot.generation != key.generation { + return None; + } + let value = slot.value.take()?; + // A wrapped generation is never reused, including stale queued releases. + if let Some(next) = slot.generation.checked_add(1) { + slot.generation = next; + table.free.push(key.index); + } + Some(value) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "threading")] + struct Callback(Box); + + #[cfg(feature = "threading")] + impl Drop for Callback { + fn drop(&mut self) { + (self.0)(); + } + } + + #[test] + fn retired_keys_cannot_release_a_reused_slot_and_close_rejects_reentry() { + let roots = PyRc::new(Roots::new(1)); + let first = roots.insert_resource(()).unwrap(); + let old_key = first.key; + drop(first); + roots.drain_pending(); + let second = roots.insert_resource(()).unwrap(); + assert_eq!(second.key.index, old_key.index); + assert_ne!(second.key.generation, old_key.generation); + roots.table.lock().pending.push(old_key); + roots.pending.store(true, Ordering::Release); + roots.drain_pending(); + assert!(roots.table.lock().slots[second.key.index].value.is_some()); + roots.close(); + assert!(roots.insert_resource(()).is_none()); + drop(second); + assert!(roots.table.lock().pending.is_empty()); + } + + #[cfg(feature = "threading")] + #[test] + fn native_resources_are_released_attached_and_outside_the_directory_lock() { + use crate::vm::thread; + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let roots = interpreter.enter_raw(|vm| vm.state.roots.clone()); + let weak = PyRc::downgrade(&roots); + let count = PyRc::new(core::sync::atomic::AtomicUsize::new(0)); + let released = count.clone(); + let owner = interpreter.id(); + let lease = roots + .insert_resource(Callback(Box::new(move || { + assert_eq!(thread::with_current_vm(|vm| vm.state.interpreter_id), owner); + assert!(weak.upgrade().unwrap().table.try_lock().is_some()); + released.fetch_add(1, Ordering::Relaxed); + }))) + .unwrap(); + std::thread::spawn(move || drop(lease)).join().unwrap(); + assert_eq!(count.load(Ordering::Relaxed), 0); + interpreter.enter(|_| {}); + assert_eq!(count.load(Ordering::Relaxed), 1); + } +} diff --git a/crates/vm/src/vm/runtime.rs b/crates/vm/src/vm/runtime.rs index 3f447fcbd3c..73466e51a93 100644 --- a/crates/vm/src/vm/runtime.rs +++ b/crates/vm/src/vm/runtime.rs @@ -296,35 +296,8 @@ pub(crate) fn alloc_interpreter_id() -> i64 { registry().next_id.fetch_add(1, Ordering::Relaxed) } -/// Gate between registering an interpreter and a collection's stop-the-world. -/// -/// A collection snapshots the registry, stops every interpreter in the -/// snapshot, and then reads tracked objects with those threads parked. An -/// interpreter that registered after the snapshot was taken would not be in it, -/// so nothing would stop it, and its bootstrap — which runs Python and mutates -/// the shared generation lists — would run underneath that scan. Registration -/// therefore waits for an in-flight stop to end; the next collection's snapshot -/// then contains the new interpreter. -fn admission() -> &'static Mutex<()> { - static ADMISSION: std::sync::OnceLock> = std::sync::OnceLock::new(); - ADMISSION.get_or_init(|| Mutex::new(())) -} - -/// Take the admission gate for the duration of a stop-the-world. -#[cfg(feature = "threading")] -pub(crate) fn lock_admission_for_stop() -> parking_lot::MutexGuard<'static, ()> { - admission().lock() -} - -/// Add the registry entry, behind the admission gate. -/// -/// Only ever called with this thread detached, because the gate is held across -/// a stop-the-world: an attached thread waiting here, or re-attaching while -/// holding the gate, would leave that stop no safepoint to complete at. Nothing -/// under the gate blocks or allocates a tracked object, so this cannot re-enter -/// the collection it waits for. +/// Add a weak entry without coordinating unrelated interpreters' collections. fn insert_registry_entry(state: &PyRc) { - let _admission = admission().lock(); let mut entries = registry().entries.lock(); // Entries are weak and an interpreter's lifetime is decided by its last // `PyRc` — which outlives the `Interpreter` handle whenever @@ -354,40 +327,50 @@ pub(crate) fn register_interpreter(state: &PyRc) { Ordering::Relaxed, ); } - // A subinterpreter is registered by a thread that is running its parent, so - // detach for the whole insert rather than only for the wait. - let detached = crate::vm::thread::try_with_current_vm(|vm| { - vm.allow_threads(|| insert_registry_entry(state)) - }); - if detached.is_none() { - insert_registry_entry(state); - } + insert_registry_entry(state); } /// Look up a live interpreter state by id. #[must_use] -pub fn lookup_interpreter(id: i64) -> Option> { +pub(crate) fn lookup_interpreter(id: i64) -> Option> { let entries = registry().entries.lock(); entries.get(&id).and_then(|e| e.state.upgrade()) } +/// Python APIs must not expose orphaned runtimes retained only for native cleanup. +pub(crate) fn lookup_open_interpreter(id: i64) -> Option> { + lookup_interpreter(id).filter(|state| !state.closed.load(Ordering::Acquire)) +} + /// List all currently registered (still-alive) interpreters. #[must_use] pub fn list_interpreters() -> Vec { - let entries = registry().entries.lock(); - let mut out: Vec = entries + #[expect( + clippy::needless_collect, + reason = "release strong state references only after unlocking" + )] + let states: Vec<_> = registry() + .entries + .lock() .iter() .filter_map(|(&id, entry)| { - // Drop dead weak refs from the listing. - if entry.state.strong_count() == 0 { - return None; - } - Some(InterpreterInfo { - id, - whence: entry.whence, + entry.state.upgrade().map(|state| { + ( + InterpreterInfo { + id, + whence: entry.whence, + }, + state, + ) }) }) .collect(); + // Release state snapshots outside the directory mutex. Terminal state + // destruction can revisit the runtime and heap directories. + let mut out: Vec<_> = states + .into_iter() + .filter_map(|(info, state)| (!state.closed.load(Ordering::Acquire)).then_some(info)) + .collect(); out.sort_by_key(|info| info.id); out } @@ -402,7 +385,7 @@ pub fn interpreter_count() -> usize { /// /// The tables are reachable from every thread, so a thread that died in the /// fork may have left one locked; the child would then deadlock the first time -/// it enumerates interpreters (which the collector now does on every stop). +/// it enumerates interpreters. /// /// # Safety /// Must only be called after `fork()` in the child process, when no other @@ -412,18 +395,29 @@ pub unsafe fn reinit_after_fork() { unsafe { crate::common::lock::reinit_mutex_after_fork(®istry().entries); crate::common::lock::reinit_mutex_after_fork(owned_interpreters()); - crate::common::lock::reinit_mutex_after_fork(admission()); + } +} + +#[cfg(all(unix, feature = "threading"))] +pub(super) struct ForkRegistries { + _entries: parking_lot::MutexGuard<'static, HashMap>, + _owned: parking_lot::MutexGuard<'static, HashMap>, +} + +#[cfg(all(unix, feature = "threading"))] +pub(super) fn prepare_fork() -> ForkRegistries { + ForkRegistries { + _entries: registry().entries.lock(), + _owned: owned_interpreters().lock(), } } /// All live interpreter states, ordered by id. /// -/// Used by the cyclic collector, which must stop every interpreter's threads -/// (not just the collecting one) because GC-tracked objects from all -/// interpreters share one object graph. Ordering is deterministic so that -/// multiple stop-the-world requesters always take exclusions in the same order. +/// Used to repair inherited runtime bookkeeping in the post-fork child. +#[cfg(all(unix, feature = "threading"))] #[must_use] -pub fn live_interpreter_states() -> Vec> { +pub(crate) fn live_interpreter_states() -> Vec> { let entries = registry().entries.lock(); let mut states: Vec<(i64, PyRc)> = entries .iter() @@ -466,6 +460,7 @@ fn owned_interpreters() -> &'static Mutex> { /// Transfer ownership of `interp` to the runtime, returning its id. #[cfg(feature = "threading")] +#[must_use] pub fn store_owned_interpreter(interp: crate::Interpreter) -> i64 { let id = interp.id(); let root_id = interp.global_state.runtime_root_id; @@ -494,6 +489,30 @@ pub fn take_owned_interpreter(id: i64) -> Option { .map(|owned| owned.interpreter) } +#[cfg(all(unix, feature = "threading"))] +pub(super) fn discard_invalid_owned_after_fork() { + let removed: Vec<_> = { + let mut owned = owned_interpreters().lock(); + #[expect( + clippy::needless_collect, + reason = "end immutable iteration before removing map entries" + )] + let ids: Vec<_> = owned + .iter() + .filter_map(|(&id, entry)| (!entry.interpreter.owner_is_valid()).then_some(id)) + .collect(); + ids.into_iter() + .map(|id| owned.remove(&id).unwrap()) + .collect() + }; + // Destruction can visit the registry, so never release these under its lock. + let mut cleanup = super::owned::Cleanup::default(); + for entry in removed { + cleanup.run(|| drop(entry)); + } + cleanup.finish(); +} + /// Create a thread-state VM for a runtime-owned interpreter. /// /// The lock is held only while cloning shared interpreter fields. @@ -503,7 +522,10 @@ pub fn owned_new_thread(id: i64) -> Option Vec { /// current one and is not running `__main__`. #[cfg(feature = "threading")] #[must_use] -pub fn destroy_owned_interpreter(id: i64) -> Option<()> { +pub(crate) fn destroy_owned_interpreter(id: i64) -> Option<()> { let interp = take_owned_interpreter(id)?; // Finalize like `Py_EndInterpreter`: flush, join non-daemons, atexit, GC. - let _ = interp.finalize(None); + let _ = interp.finalize_raw(None); Some(()) } diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index e1bc40c44b0..aa24a565e7f 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -10,11 +10,10 @@ use alloc::sync::Arc; use rustpython_common::lock::PyMutex; use crate::frame::InterpreterFrame; -#[cfg(feature = "threading")] use crate::vm::PyGlobalState; -use crate::{AsObject, PyObject, VirtualMachine}; #[cfg(all(unix, feature = "threading"))] use crate::{Py, frame::FrameObject}; +use crate::{PyObject, VirtualMachine}; #[cfg(all(unix, feature = "threading"))] use core::sync::atomic::AtomicPtr; use core::{ @@ -22,7 +21,6 @@ use core::{ ptr::NonNull, sync::atomic::{AtomicUsize, Ordering}, }; -use itertools::Itertools; #[cfg(feature = "threading")] use std::collections::HashMap; use std::thread_local; @@ -88,6 +86,9 @@ pub struct ThreadSlot { /// `tstate->c_profileobj`. Cross-thread source of truth for sys.getprofile / /// sys._setprofileallthreads. pub profile_func: PyMutex, + /// A retired slot may remain in native TLS, but must never retain Python + /// callbacks again. Writers check this while holding the callback lock. + closed: core::sync::atomic::AtomicBool, /// Thread state for stop-the-world: DETACHED / ATTACHED / SUSPENDED / SHUTTING_DOWN pub state: core::sync::atomic::AtomicI32, /// Per-thread stop request bit (eval breaker equivalent). @@ -96,11 +97,50 @@ pub struct ThreadSlot { pub thread: std::thread::Thread, /// QSBR state for deferred memory reclamation. pub(crate) qsbr: Arc, + shared_qsbr: Arc, } #[cfg(feature = "threading")] pub type CurrentFrameSlot = Arc; +#[cfg(feature = "threading")] +impl ThreadSlot { + pub(crate) fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + + pub(crate) fn replace_trace(&self, function: PyObjectRef) -> Option { + self.replace_callback(&self.trace_func, function) + } + + pub(crate) fn replace_profile(&self, function: PyObjectRef) -> Option { + self.replace_callback(&self.profile_func, function) + } + + fn replace_callback( + &self, + callback: &PyMutex, + function: PyObjectRef, + ) -> Option { + let mut callback = callback.lock(); + if self.is_closed() { + return None; + } + Some(core::mem::replace(&mut callback, function)) + } + + fn close(&self, vm: &VirtualMachine) { + self.closed.store(true, Ordering::Release); + // Swap everything before releasing any reference: destructors may call + // sys.settrace() or sys._setprofileallthreads() reentrantly. + let trace = core::mem::replace(&mut *self.trace_func.lock(), vm.ctx.none()); + let profile = core::mem::replace(&mut *self.profile_func.lock(), vm.ctx.none()); + // SAFETY: cleanup runs on the owner thread while still attached. + let exception = unsafe { self.exception.swap(None) }; + drop((trace, profile, exception)); + } +} + /// Coalesced per-thread frame-publishing state, touched on every /// `enter_iframe`/`exit_iframe`. Bundling `current_frame` together with /// the cached `top_frame`/`top_iframe` slot pointers means @@ -130,6 +170,17 @@ struct FrameSlotCache { thread_local! { pub(super) static VM_STACK: RefCell>> = Vec::with_capacity(1).into(); + // The final destruction phase retains native control state, not a VM + // pointer. Python callbacks and nested interpreter entry are unavailable. + static NATIVE_TEARDOWN: RefCell>> = const { RefCell::new(None) }; + static NATIVE_SWEEP: Cell = const { Cell::new(false) }; + + // Native payloads may own another interpreter. Delay that destruction until + // the outer teardown has restored attachment, and drain without recursion. + #[expect(clippy::vec_box, reason = "queued VM owners retain stable addresses")] + static DEFERRED_VMS: RefCell>> = const { RefCell::new(Vec::new()) }; + static DRAINING_VMS: Cell = const { Cell::new(false) }; + /// Thread state created through the GILState-style C API. /// /// This is separate from the current VM stack: it only means "attached now", @@ -144,6 +195,9 @@ thread_local! { #[cfg(feature = "threading")] static GILSTATE_VM: RefCell>> = const { RefCell::new(None) }; + #[cfg(feature = "threading")] + static GILSTATE_ENTRY: RefCell> = const { RefCell::new(None) }; + pub(crate) static COROUTINE_ORIGIN_TRACKING_DEPTH: Cell = const { Cell::new(0) }; /// Per-interpreter thread slots for this OS thread (PEP 734 multi-interpreter). @@ -181,12 +235,218 @@ thread_local! { } +fn callbacks_permitted() -> bool { + if NATIVE_SWEEP.try_with(Cell::get).unwrap_or(true) { + return false; + } + NATIVE_TEARDOWN + .try_with(|state| state.borrow().is_none()) + .unwrap_or(false) +} + +pub(crate) fn native_sweep(f: impl FnOnce() -> R) -> R { + let previous = NATIVE_SWEEP.with(|sweep| sweep.replace(true)); + scopeguard::defer! { + NATIVE_SWEEP.with(|sweep| sweep.set(previous)); + if !previous { + drain_deferred_vms(); + } + } + f() +} + +pub(crate) fn native_teardown( + state: &crate::common::rc::PyRc, + f: impl FnOnce() -> R, +) -> R { + NATIVE_TEARDOWN.with(|current| { + assert!(current.borrow().is_none(), "nested native teardown"); + *current.borrow_mut() = Some(state.clone()); + }); + scopeguard::defer! { + let state = NATIVE_TEARDOWN.with(|current| current.borrow_mut().take()); + drop(state); + } + f() +} + +pub(super) fn destroy_vm(vm: Box) { + if !callbacks_permitted() { + DEFERRED_VMS.with(|pending| pending.borrow_mut().push(vm)); + return; + } + scopeguard::defer! { drain_deferred_vms(); } + let state = vm.state.clone(); + let Some((vm, lease, owns_vm)) = state.owner_leases.begin_teardown(vm) else { + return; + }; + let teardown = scopeguard::guard(&state, |state| { + lease.finish_teardown(); + state.teardowns.fetch_sub(1, Ordering::Release); + }); + #[cfg(feature = "threading")] + let has_outer_owner = VM_STACK.with(|stack| { + stack.borrow().iter().any(|vm| { + // SAFETY: stack entries remain borrowed by their enclosing entry. + unsafe { vm.as_ref() }.state.interpreter_id == state.interpreter_id + }) + }); + let closed = state.closed.load(Ordering::Acquire) || !owns_vm; + let mut entered = VmBootstrapGuard::enter_with_owner(&vm, closed, lease.enter()); + let released = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| { + if !closed { + let mut cleanup = super::owned::Cleanup::default(); + cleanup.run(|| vm.retire_execution_roots()); + #[cfg(feature = "threading")] + if !has_outer_owner { + cleanup.run(|| retire_thread_slot(&vm)); + } + cleanup.finish(); + } + })); + let last = if owns_vm { + lease.release_owner(); + state.owners.fetch_sub(1, Ordering::AcqRel) == 1 + } else { + false + }; + let _finalization = (last && !closed) + .then(|| state.owner_leases.enter_finalization(&state)) + .flatten(); + let finalized = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| { + let mut cleanup = super::owned::Cleanup::default(); + if last && !closed { + state.admission_closed.store(true, Ordering::Release); + #[cfg(feature = "threading")] + state + .finalizing_thread_ident + .store(crate::stdlib::_thread::get_ident()); + state.finalizing.store(true, Ordering::Release); + cleanup.run(|| state.roots.close()); + if vm.initialized.get() { + cleanup.run(|| vm.finalize_modules()); + } + cleanup.run(|| { + crate::stdlib::_interpchannels::clear_interpreter(state.interpreter_id); + }); + cleanup.run(|| { + crate::stdlib::_interpqueues::clear_interpreter(state.interpreter_id); + }); + } + if !closed { + // SAFETY: OwnedVm destruction runs after this VM's execution scopes + // have ended. The cleanup entry has not borrowed its cached roots. + cleanup.run(|| unsafe { vm.retire_cached_roots() }); + if last { + #[cfg(feature = "threading")] + { + cleanup.run(|| retire_interpreter_thread_roots(&vm)); + } + cleanup.run(|| state.retire_python_roots(&vm.ctx)); + cleanup.run(|| { + state.gc.collect_force(2); + }); + } + } + cleanup.finish(); + })); + let entry = &mut entered; + let state_ref = &state; + let lease_ref = &lease; + let terminal = native_teardown(&state, move || { + // No TLS pointer may refer to a VM while its fields are being dropped. + // Pop before any native destructor can panic, including cache cleanup. + lease_ref.retire_vm(); + entry.pop_vm(); + let result = std::panic::catch_unwind(core::panic::AssertUnwindSafe(move || { + let mut cleanup = super::owned::Cleanup::default(); + if last { + state_ref.closed.store(true, Ordering::Release); + cleanup.run(|| state_ref.close_python_roots()); + #[cfg(feature = "threading")] + cleanup.run(|| retire_interpreter_thread_roots(&vm)); + } + #[cfg(feature = "threading")] + if !has_outer_owner { + cleanup.run(|| retire_thread_slot(&vm)); + } + cleanup.run(|| drop(vm)); + if last { + cleanup.run(|| state_ref.gc.unfreeze()); + cleanup.run(|| { + state_ref.gc.collect_force(2); + }); + } + cleanup.finish(); + })); + // Unwind has dropped VM storage under the callback mask. Keep the slot + // attached until then, and unregister it before restoring an outer VM. + #[cfg(feature = "threading")] + if !has_outer_owner { + unregister_current_thread_frames(state_ref); + } + result + }); + drop(entered); + drop(teardown); + if let Err(error) = released.and(finalized).and(terminal) { + std::panic::resume_unwind(error); + } +} + +fn drain_deferred_vms() { + if !callbacks_permitted() || DRAINING_VMS.with(|draining| draining.replace(true)) { + return; + } + scopeguard::defer! { DRAINING_VMS.with(|draining| draining.set(false)); } + let mut cleanup = super::owned::Cleanup::default(); + while let Some(vm) = DEFERRED_VMS.with(|pending| pending.borrow_mut().pop()) { + cleanup.run(|| destroy_vm(vm)); + } + cleanup.finish(); +} + #[must_use] pub fn current_vm_is_set() -> bool { VM_STACK.with(|vms| !vms.borrow().is_empty()) } -pub fn with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { +pub(crate) fn is_current_attached(vm: &VirtualMachine) -> bool { + top_slot_is_attached() + && try_with_current_vm(|current| core::ptr::eq(current, vm)).unwrap_or(false) +} + +pub(crate) fn try_with_attached_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option { + if !callbacks_permitted() { + return None; + } + let vm = VM_STACK + .try_with(|stack| stack.try_borrow().ok()?.last().copied()) + .ok()??; + #[cfg(feature = "threading")] + { + let attached = CURRENT_THREAD_SLOT + .try_with(|slot| { + slot.try_borrow() + .ok() + .and_then(|slot| { + slot.as_ref().map(|slot| { + slot.state.load(Ordering::Acquire) == ThreadState::Attached as i32 + }) + }) + .unwrap_or(false) + }) + .unwrap_or(false); + if !attached { + return None; + } + } + // SAFETY: the active VM stack retains this reference until the entry exits. + Some(f(unsafe { vm.as_ref() })) +} + +pub(crate) fn with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { + assert!(callbacks_permitted(), "Python entry during native teardown"); VM_STACK.with(|vms| { let vm = vms .borrow() @@ -200,20 +460,9 @@ pub fn with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { } fn set_current_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { - // Attach to this VM's interpreter, detaching the enclosing one if this is a - // switch between interpreters on the same OS thread. - #[cfg(feature = "threading")] - let switched = begin_interpreter_section(vm); - - VM_STACK.with(|vms| { - vms.borrow_mut().push(vm.into()); - scopeguard::defer! { - vms.borrow_mut().pop(); - #[cfg(feature = "threading")] - end_interpreter_section(switched); - } - f() - }) + assert!(callbacks_permitted(), "Python entry during native teardown"); + let _guard = VmBootstrapGuard::new(vm); + f() } /// Pointer to the GC state of the interpreter running on this thread. @@ -222,6 +471,21 @@ fn set_current_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { /// which is borrowed for the whole `set_current_vm` scope — so the pointer stays /// valid as long as the caller remains inside that scope. pub(crate) fn current_gc_state() -> Option> { + if NATIVE_SWEEP.try_with(Cell::get).unwrap_or(true) { + return None; + } + if let Some(gc) = NATIVE_TEARDOWN + .try_with(|state| { + state + .borrow() + .as_ref() + .map(|state| NonNull::from(&state.gc)) + }) + .ok() + .flatten() + { + return Some(gc); + } // Reached from every tracked allocation, including ones a thread-local // destructor makes while the VM stack is being torn down, so neither a // destroyed key nor an outstanding borrow may panic here. @@ -236,7 +500,10 @@ pub(crate) fn current_gc_state() -> Option(f: impl FnOnce(&VirtualMachine) -> R) -> Option { +pub(crate) fn try_with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option { + if !callbacks_permitted() { + return None; + } VM_STACK.with(|vms| { let vm = vms.borrow().last().copied()?; // SAFETY: entries in VM_STACK either borrow a VM for the dynamic @@ -248,16 +515,16 @@ pub fn try_with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { // Attach/detach is handled by `set_current_vm`, which pairs it with the // VM_STACK push so that switching interpreters mid-stack stays consistent. - set_current_vm(vm, f) + set_current_vm(vm, || { + vm.state.roots.drain_pending(); + scopeguard::defer! { vm.state.roots.drain_pending(); } + f() + }) } -/// RAII counterpart to `enter_vm`, for code that runs Python bytecode across -/// several statements interspersed with `&mut VirtualMachine` calls -/// (`VirtualMachine::initialize`), where a single closure-based `enter_vm` -/// scope cannot be expressed because the borrow checker won't let a closure -/// hold `&mut VirtualMachine` at the same time `enter_vm` reborrows it as -/// `&VirtualMachine`. Construction only needs a transient `&VirtualMachine` -/// borrow, so it can be dropped before subsequent `&mut` use. +/// RAII counterpart to `enter_vm` for bootstrap and owner teardown. The caller +/// keeps the VM at a stable address and may only publish shared access until +/// the guard is popped; in particular, publication forbids borrowing it mutably. /// /// Without this, code that runs Python bytecode before any `enter_vm` scope /// exists would leave the thread not ATTACHED, making lock-free type cache @@ -266,30 +533,85 @@ pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { pub(crate) struct VmBootstrapGuard { #[cfg(feature = "threading")] switched: bool, + pushed: bool, + saved_frame: Option<*const InterpreterFrame>, + _owner: super::owner_lease::OwnerEntry, } impl VmBootstrapGuard { pub(crate) fn new(vm: &VirtualMachine) -> Self { + Self::enter(vm, false) + } + + fn enter(vm: &VirtualMachine, cleanup: bool) -> Self { + Self::enter_with_owner(vm, cleanup, vm.owner_lease().enter()) + } + + fn enter_with_owner( + vm: &VirtualMachine, + cleanup: bool, + owner: super::owner_lease::OwnerEntry, + ) -> Self { + assert!( + callbacks_permitted(), + "Python bootstrap during native teardown" + ); + let changed_vm = VM_STACK.with(|vms| { + vms.borrow() + .last() + .is_none_or(|p| p.as_ptr().cast_const() != core::ptr::from_ref(vm)) + }); + let saved_frame = changed_vm.then(get_current_frame); #[cfg(feature = "threading")] - let switched = begin_interpreter_section(vm); + let switched = { + if cleanup { + let slot = ensure_thread_slot(vm); + // A cancelled/idle native worker can be dropped after explicit + // shutdown. Cleanup is not a new Python entry and must not hang. + let _ = slot.state.compare_exchange( + ThreadState::ShuttingDown as i32, + ThreadState::Detached as i32, + Ordering::AcqRel, + Ordering::Acquire, + ); + } + begin_interpreter_section(vm) + }; + #[cfg(not(feature = "threading"))] + let _ = cleanup; + if changed_vm { + let _ = set_current_frame(core::ptr::null()); + } VM_STACK.with(|vms| vms.borrow_mut().push(vm.into())); Self { #[cfg(feature = "threading")] switched, + pushed: true, + saved_frame, + _owner: owner, + } + } + + fn pop_vm(&mut self) { + if self.pushed { + VM_STACK.with(|vms| { + vms.borrow_mut().pop(); + }); + self.pushed = false; } } } impl Drop for VmBootstrapGuard { fn drop(&mut self) { - VM_STACK.with(|vms| { - vms.borrow_mut().pop(); - }); - + self.pop_vm(); #[cfg(feature = "threading")] end_interpreter_section(self.switched); + if let Some(frame) = self.saved_frame { + let _ = set_current_frame(frame); + } } } @@ -305,28 +627,41 @@ pub enum CurrentVmAttachState { pub struct SavedThreadState { vm_stack: Vec>, gilstate_vm: Option>, + gilstate_entry: Option, + frame: *const InterpreterFrame, } /// Detach the current native thread and preserve its VM context for restoration. #[cfg(feature = "threading")] #[must_use = "the saved thread state must be restored"] -pub fn save_current_thread() -> SavedThreadState { +pub(crate) fn save_current_thread() -> SavedThreadState { let vm_stack = VM_STACK.with(|vms| core::mem::take(&mut *vms.borrow_mut())); assert!( !vm_stack.is_empty(), "save_current_thread() called without an attached VM" ); let gilstate_vm = GILSTATE_VM.with(|gilstate_vm| gilstate_vm.borrow_mut().take()); + let gilstate_entry = GILSTATE_ENTRY.with(|entry| entry.borrow_mut().take()); detach_thread(); + // Keep the old interpreter's published frame intact for other threads, + // but do not let a subsequently attached interpreter inherit this chain. + let frame = FRAME_SLOT_CACHE.with(|cache| cache.current_frame.swap(0, Ordering::Relaxed)) + as *const InterpreterFrame; SavedThreadState { vm_stack, gilstate_vm, + gilstate_entry, + frame, } } /// Restore a VM context previously returned by [`save_current_thread`]. #[cfg(feature = "threading")] -pub fn restore_current_thread(state: SavedThreadState) { +pub(crate) fn restore_current_thread(state: SavedThreadState) { + assert!( + callbacks_permitted(), + "thread restoration during native teardown" + ); assert!( !current_vm_is_set(), "restore_current_thread() called with an attached VM" @@ -334,6 +669,8 @@ pub fn restore_current_thread(state: SavedThreadState) { let SavedThreadState { vm_stack, gilstate_vm, + gilstate_entry, + frame, } = state; let vm = vm_stack .last() @@ -348,6 +685,11 @@ pub fn restore_current_thread(state: SavedThreadState) { ); *current = gilstate_vm; }); + GILSTATE_ENTRY.with(|current| { + let mut current = current.borrow_mut(); + assert!(current.is_none()); + *current = gilstate_entry; + }); // SAFETY: borrowed VMs remain alive for the dynamic save/restore scope, // while an owned GILState VM was restored above before this dereference. @@ -359,14 +701,20 @@ pub fn restore_current_thread(state: SavedThreadState) { init_thread_slot_if_needed(vm); attach_thread(vm); VM_STACK.with(|vms| *vms.borrow_mut() = vm_stack); + let _ = set_current_frame(frame); } /// Attach the current native thread to a RustPython VM until /// `release_current_thread()` is called. +/// +/// # Safety +/// Pair a newly attached state with exactly one release on this native thread. +/// No Python reference or scoped VM token may outlive that attachment. #[cfg(feature = "threading")] -pub fn attach_current_thread( +pub unsafe fn attach_current_thread( make_vm: impl FnOnce() -> ThreadedVirtualMachine, ) -> CurrentVmAttachState { + assert!(callbacks_permitted(), "native attachment during teardown"); if current_vm_is_set() { return CurrentVmAttachState::AlreadyAttached; } @@ -374,7 +722,8 @@ pub fn attach_current_thread( GILSTATE_VM.with(|gilstate_vm| { let mut gilstate_vm = gilstate_vm.borrow_mut(); let threaded_vm = gilstate_vm.get_or_insert_with(|| Box::new(make_vm())); - let vm = &threaded_vm.vm; + let vm: &VirtualMachine = &threaded_vm.vm; + let entry = vm.owner_lease().enter(); vm.c_stack_soft_limit .set(VirtualMachine::calculate_c_stack_soft_limit()); @@ -387,27 +736,34 @@ pub fn attach_current_thread( debug_assert!(vms.borrow().is_empty()); vms.borrow_mut().push(vm.into()); }); + GILSTATE_ENTRY.with(|current| *current.borrow_mut() = Some(entry)); }); CurrentVmAttachState::Attached } +/// Release a native C-API attachment. +/// +/// # Safety +/// `state` must be the unmatched result of `attach_current_thread` on this +/// thread. All references and callbacks using that attachment must have ended. #[cfg(feature = "threading")] -pub fn release_current_thread(state: CurrentVmAttachState) { +pub unsafe fn release_current_thread(state: CurrentVmAttachState) { if state == CurrentVmAttachState::AlreadyAttached { return; } - let gilstate_vm = GILSTATE_VM.with(|gilstate_vm| gilstate_vm.borrow_mut().take()); - drop(gilstate_vm); - VM_STACK.with(|vms| { vms.borrow_mut() .pop() .expect("release_current_thread() called without an attached VM"); }); + let gilstate_vm = GILSTATE_VM.with(|gilstate_vm| gilstate_vm.borrow_mut().take()); + drop(gilstate_vm); + detach_thread(); + GILSTATE_ENTRY.with(|entry| entry.borrow_mut().take()); } /// Ensure this OS thread has a [`ThreadSlot`] registered with `vm`'s interpreter @@ -436,6 +792,7 @@ fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot { let thread_id = crate::stdlib::_thread::get_ident(); let mut registry = vm.state.thread_frames.lock(); let new_slot = Arc::new(ThreadSlot { + closed: core::sync::atomic::AtomicBool::new(false), #[cfg(unix)] top_frame: AtomicPtr::new(core::ptr::null_mut()), top_iframe: AtomicUsize::new(0), @@ -467,7 +824,8 @@ fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot { ), stop_requested: core::sync::atomic::AtomicBool::new(false), thread: std::thread::current(), - qsbr: crate::object::qsbr::QSBR.register(), + qsbr: vm.state.gc.qsbr().register(), + shared_qsbr: crate::object::qsbr::shared().register(), }); registry.insert(thread_id, new_slot.clone()); drop(registry); @@ -479,7 +837,7 @@ fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot { /// The current thread's `ThreadSlot` for the entered interpreter, if any. #[cfg(feature = "threading")] #[must_use] -pub fn current_thread_slot() -> Option { +pub(crate) fn current_thread_slot() -> Option { CURRENT_THREAD_SLOT.with(|slot| slot.borrow().clone()) } @@ -572,6 +930,12 @@ fn wait_while_suspended(slot: &ThreadSlot) -> u64 { /// `PyThread_hang_thread`: park this OS thread forever. #[cfg(feature = "threading")] fn hang_thread() -> ! { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(slot) = slot.borrow().as_ref() { + slot.qsbr.offline(); + slot.shared_qsbr.offline(); + } + }); loop { std::thread::park(); } @@ -590,7 +954,8 @@ pub fn hang_current_thread(state: &PyGlobalState) -> ! { .state .swap(ThreadState::ShuttingDown as i32, Ordering::AcqRel); if prev == ThreadState::Attached as i32 { - crate::object::qsbr::QSBR.offline(&s.qsbr); + s.qsbr.offline(); + s.shared_qsbr.offline(); } s.stop_requested.store(false, Ordering::Release); } @@ -621,12 +986,19 @@ pub fn set_other_threads_shutting_down(state: &PyGlobalState) { slot.stop_requested.store(false, Ordering::Release); slot.state .store(ThreadState::ShuttingDown as i32, Ordering::Release); + slot.qsbr.offline(); + slot.shared_qsbr.offline(); slot.thread.unpark(); } } #[cfg(feature = "threading")] fn attach_thread(vm: &VirtualMachine) { + attach_state(&vm.state); +} + +#[cfg(feature = "threading")] +fn attach_state(interpreter: &PyGlobalState) { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { super::stw_trace(format_args!("attach begin")); @@ -638,7 +1010,8 @@ fn attach_thread(vm: &VirtualMachine) { Ordering::Relaxed, ) { Ok(_) => { - crate::object::qsbr::QSBR.online(&s.qsbr); + s.qsbr.online(); + s.shared_qsbr.online(); super::stw_trace(format_args!("attach DETACHED->ATTACHED")); break; } @@ -647,7 +1020,9 @@ fn attach_thread(vm: &VirtualMachine) { // Parked by stop-the-world — wait until released to DETACHED super::stw_trace(format_args!("attach wait-suspended")); let wait_yields = wait_while_suspended(s); - vm.state.stop_the_world.add_attach_wait_yields(wait_yields); + interpreter + .stop_the_world + .add_attach_wait_yields(wait_yields); // Retry CAS } Some(ThreadState::ShuttingDown) => { @@ -671,7 +1046,7 @@ fn attach_thread(vm: &VirtualMachine) { // it. Safe against a concurrent start_the_world: suspend_if_needed decides // whether to park under the registry lock, so it never parks after the // request has been withdrawn. - suspend_if_needed(&vm.state); + suspend_if_needed(interpreter); } /// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`). @@ -686,7 +1061,8 @@ fn detach_thread() { Ordering::Acquire, ) { Ok(_) => { - crate::object::qsbr::QSBR.offline(&s.qsbr); + s.qsbr.offline(); + s.shared_qsbr.offline(); } Err(state) => { debug_assert!( @@ -748,6 +1124,10 @@ pub fn allow_threads(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { /// testing for DETACHED alone. #[cfg(feature = "threading")] pub fn attach_for_callback(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + assert!( + callbacks_permitted(), + "Python callback during native teardown" + ); let should_transition = CURRENT_THREAD_SLOT.with(|slot| { slot.borrow() .as_ref() @@ -769,6 +1149,10 @@ pub fn attach_for_callback(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { /// No-op on non-threading builds. #[cfg(not(feature = "threading"))] pub fn attach_for_callback(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + assert!( + callbacks_permitted(), + "Python callback during native teardown" + ); f() } @@ -788,6 +1172,20 @@ pub fn attach_for_callback(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R /// than on the reference behavior. #[cfg(feature = "threading")] fn wait_detached_from_interpreter(wait: &dyn Fn()) { + let terminal = NATIVE_TEARDOWN + .try_with(|state| state.borrow().clone()) + .ok() + .flatten(); + if let Some(state) = terminal { + if current_slot_is_attached() { + detach_thread(); + scopeguard::defer! { attach_state(&state); } + wait(); + } else { + wait(); + } + return; + } // Read the VM out before waiting: attaching afterwards reaches for the // same thread locals, which must not still be borrowed here. let current = VM_STACK @@ -866,6 +1264,8 @@ fn do_suspend(state: &PyGlobalState) { Some(Ok(_)) => { // Consumed this thread's stop request bit. s.stop_requested.store(false, Ordering::Release); + s.qsbr.offline(); + s.shared_qsbr.offline(); } Some(Err(state)) => match ThreadState::from_i32(state) { Some(ThreadState::Detached) => { @@ -929,6 +1329,8 @@ fn do_suspend(state: &PyGlobalState) { }, } } + s.qsbr.online(); + s.shared_qsbr.online(); s.stop_requested.store(false, Ordering::Release); super::stw_trace(format_args!("suspend resume -> ATTACHED")); }); @@ -967,9 +1369,10 @@ pub(crate) fn set_stop_requested_for_current_thread(value: bool) -> bool { #[cfg(feature = "threading")] pub(crate) fn qsbr_break_requested() -> bool { CURRENT_THREAD_SLOT.with(|slot| { - slot.borrow() - .as_ref() - .is_some_and(|s| s.qsbr.requested.load(Ordering::Relaxed)) + slot.borrow().as_ref().is_some_and(|s| { + s.qsbr.requested.load(Ordering::Relaxed) + || s.shared_qsbr.requested.load(Ordering::Relaxed) + }) }) } @@ -978,14 +1381,12 @@ pub(crate) fn qsbr_break_requested() -> bool { /// free retired allocations. #[cfg(feature = "threading")] pub(crate) fn qsbr_checkpoint() { - use crate::object::qsbr::QSBR; CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { - s.qsbr.requested.store(false, Ordering::Relaxed); - QSBR.quiescent_state(&s.qsbr); + s.qsbr.checkpoint(); + s.shared_qsbr.checkpoint(); } }); - QSBR.process(); } /// Debug check: lock-free type-cache reads are only sound on threads that @@ -1009,7 +1410,7 @@ pub(crate) fn debug_assert_current_thread_attached() { /// Only used on non-unix threading builds; unix builds publish the top frame /// through `set_current_frame` writing `ThreadSlot::top_frame`. #[cfg(all(not(unix), feature = "threading"))] -pub fn push_thread_frame(fp: FramePtr) { +pub(crate) fn push_thread_frame(fp: FramePtr) { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { s.frames.lock().push(fp); @@ -1025,7 +1426,7 @@ pub fn push_thread_frame(fp: FramePtr) { /// Pop a frame from the current thread's shared frame stack. /// Called when a frame is exited. #[cfg(all(not(unix), feature = "threading"))] -pub fn pop_thread_frame() { +pub(crate) fn pop_thread_frame() { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { s.frames.lock().pop(); @@ -1042,7 +1443,7 @@ pub fn pop_thread_frame() { /// Returns the previous pointer so it can be restored on pop. #[must_use] #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn set_current_frame(frame: *const InterpreterFrame) -> *const InterpreterFrame { +pub(crate) fn set_current_frame(frame: *const InterpreterFrame) -> *const InterpreterFrame { FRAME_SLOT_CACHE.with(|cache| { // Publish the top frame for cross-thread readers (faulthandler, // sys._current_frames). @@ -1075,9 +1476,13 @@ pub fn set_current_frame(frame: *const InterpreterFrame) -> *const InterpreterFr /// Lightweight version that only writes to TLS `current_frame`, returning /// the previous value. Does not update cross-thread top_frame (that's /// updated by `set_current_frame` for FrameObject-based calls). +/// +/// # Safety +/// A non-null frame must belong to the attached interpreter and remain live +/// until the previous pointer is restored, including during signal handlers. #[inline(always)] #[must_use] -pub fn set_current_frame_nosave(frame: *const InterpreterFrame) -> *const InterpreterFrame { +pub unsafe fn set_current_frame_nosave(frame: *const InterpreterFrame) -> *const InterpreterFrame { FRAME_SLOT_CACHE.with(|cache| cache.current_frame.swap(frame as usize, Ordering::Relaxed)) as *const InterpreterFrame } @@ -1093,14 +1498,15 @@ pub fn get_current_frame() -> *const InterpreterFrame { /// Update the current thread's exception slot atomically (no locks). /// Called from push_exception/pop_exception/set_exception. #[cfg(feature = "threading")] -pub fn update_thread_exception(exc: Option) { - CURRENT_THREAD_SLOT.with(|slot| { - if let Some(s) = slot.borrow().as_ref() { - // SAFETY: Called only from the owning thread. The old ref is dropped - // here on the owning thread, which is safe. - let _old = unsafe { s.exception.swap(exc) }; - } - }); +pub(crate) fn update_thread_exception(exc: Option) { + let slot = CURRENT_THREAD_SLOT.with(|slot| slot.borrow().clone()); + if let Some(slot) = slot + && !slot.is_closed() + { + // SAFETY: called only from the owning, attached thread. Release outside + // the TLS borrow, since a destructor may update the exception again. + drop(unsafe { slot.exception.swap(exc) }); + } } /// Collect all threads' current exceptions for sys._current_exceptions(). @@ -1116,31 +1522,70 @@ pub fn get_all_current_exceptions(vm: &VirtualMachine) -> Vec<(u64, Option = vm.state.thread_frames.lock().values().cloned().collect(); + let mut cleanup = super::owned::Cleanup::default(); + for slot in slots { + cleanup.run(|| slot.close(vm)); + } + cleanup.finish(); +} + +#[cfg(feature = "threading")] +fn unregister_current_thread_frames(state: &PyGlobalState) { + let thread_id = crate::stdlib::_thread::get_ident(); + let interp_id = state.interpreter_id; + let slot_to_clean = INTERP_THREAD_SLOTS.with(|slots| slots.borrow_mut().remove(&interp_id)); // A dying thread should not remain logically ATTACHED while its // thread-state slot is being removed. - if let Some(slot) = &slot_to_clean { - let _ = slot.state.compare_exchange( - ThreadState::Attached as i32, - ThreadState::Detached as i32, - Ordering::AcqRel, - Ordering::Acquire, - ); + if let Some(slot) = &slot_to_clean + && slot + .state + .compare_exchange( + ThreadState::Attached as i32, + ThreadState::Detached as i32, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + slot.qsbr.offline(); + slot.shared_qsbr.offline(); } // Guard against OS thread-id reuse races: only remove the registry entry // if it still points at this thread's own slot. let _removed = if let Some(slot) = &slot_to_clean { - let mut registry = vm.state.thread_frames.lock(); + let mut registry = state.thread_frames.lock(); match registry.get(&thread_id) { Some(registered) if Arc::ptr_eq(registered, slot) => registry.remove(&thread_id), _ => None, @@ -1150,13 +1595,13 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { }; if let Some(slot) = &_removed - && vm.state.stop_the_world.requested.load(Ordering::Acquire) - && thread_id != vm.state.stop_the_world.requester_ident() + && state.stop_the_world.requested.load(Ordering::Acquire) + && thread_id != state.stop_the_world.requester_ident() && slot.state.load(Ordering::Relaxed) != ThreadState::Suspended as i32 { // A non-requester thread disappeared while stop-the-world is pending. // Unblock requester countdown progress. - vm.state.stop_the_world.notify_thread_gone(); + state.stop_the_world.notify_thread_gone(); } // If CURRENT pointed at the cleaned slot, clear it (and top-frame cache). @@ -1170,6 +1615,7 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { *s.borrow_mut() = None; #[cfg(feature = "threading")] FRAME_SLOT_CACHE.with(|cache| { + cache.current_frame.store(0, Ordering::Relaxed); #[cfg(unix)] cache.top_frame.set(core::ptr::null()); cache.top_iframe.set(core::ptr::null()); @@ -1180,96 +1626,65 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { }); } -/// Reinitialize thread slot after fork. Called in child process. -/// Creates a fresh slot and registers it for the current thread, -/// preserving the current thread's frames from the signal-safe frame chain. -/// -/// Precondition: `reinit_locks_after_fork()` has already reset all -/// VmState locks to unlocked. -#[cfg(feature = "threading")] -pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { - let current_ident = crate::stdlib::_thread::get_ident(); - // On non-unix, rebuild the shared frame stack (bottom-to-top) from the - // current thread's frame chain, which walks top-to-bottom via `previous`. - #[cfg(not(unix))] - let current_frames: Vec = { - let mut current_frames = Vec::new(); - let mut cur = get_current_frame(); - while !cur.is_null() { - // SAFETY: the forking thread's chain frames are alive. - let iframe = unsafe { &*cur }; - if let Some(fo) = iframe.frame_obj() { - current_frames.push(FramePtr(unsafe { - NonNull::new_unchecked(fo as *const _ as *mut _) - })); - } - cur = iframe.previous.load(Ordering::Relaxed) as *const InterpreterFrame; +/// Snapshot this native thread's reclamation registrations before fork. +#[cfg(all(unix, feature = "threading"))] +pub(super) fn surviving_qsbr_slots() -> Vec> { + INTERP_THREAD_SLOTS.with(|slots| { + slots + .borrow() + .values() + .flat_map(|slot| [slot.qsbr.clone(), slot.shared_qsbr.clone()]) + .collect() + }) +} + +/// Remove vanished threads while preserving this native thread's slot. Its +/// frames, tracing hooks and exception may belong to a saved/nested entry. +#[cfg(all(unix, feature = "threading"))] +pub(crate) fn retain_frame_slot_after_fork(state: &PyGlobalState) { + let ident = crate::stdlib::_thread::get_ident(); + let removed = { + let mut registry = state.thread_frames.lock(); + let current = registry.remove(&ident); + let removed = core::mem::take(&mut *registry); + if let Some(slot) = current { + slot.stop_requested.store(false, Ordering::Relaxed); + slot.state + .store(ThreadState::Detached as i32, Ordering::Relaxed); + registry.insert(ident, slot); } - current_frames.reverse(); - current_frames + removed }; - #[cfg(unix)] - let top_fo_ptr = { - let top_iframe = get_current_frame(); - if top_iframe.is_null() { - core::ptr::null_mut() - } else { - match unsafe { (*top_iframe).frame_obj() } { - Some(fo) => fo as *const Py as *mut Py, - None => core::ptr::null_mut(), + let _owner = state.gc.allocation_scope(); + native_sweep(|| { + #[expect(clippy::iter_over_hash_type, reason = "independent vanished slots")] + for slot in removed.values() { + // Vanished native TLS retains copied Arc counts. Empty the roots + // explicitly; dropping this registry Arc alone cannot release them. + unsafe { + crate::common::lock::reinit_mutex_after_fork(&slot.trace_func); + crate::common::lock::reinit_mutex_after_fork(&slot.profile_func); } + slot.closed.store(true, Ordering::Release); + let ctx = crate::vm::Context::genesis(); + let trace = core::mem::replace(&mut *slot.trace_func.lock(), ctx.none()); + let profile = core::mem::replace(&mut *slot.profile_func.lock(), ctx.none()); + let exception = unsafe { slot.exception.swap(None) }; + slot.qsbr.offline(); + slot.shared_qsbr.offline(); + drop((trace, profile, exception)); } - }; - let top_iframe_ptr = get_current_frame() as usize; - let new_slot = Arc::new(ThreadSlot { - // The surviving child thread keeps executing its current frame chain. - // Only publish heavy frames for signal safety. - #[cfg(unix)] - top_frame: AtomicPtr::new(top_fo_ptr), - top_iframe: AtomicUsize::new(top_iframe_ptr), - #[cfg(not(unix))] - frames: parking_lot::Mutex::new(current_frames), - exception: crate::PyAtomicRef::from(vm.topmost_exception()), - trace_func: PyMutex::new(vm.trace_func.borrow().clone()), - profile_func: PyMutex::new(vm.profile_func.borrow().clone()), - state: core::sync::atomic::AtomicI32::new(ThreadState::Attached as i32), - stop_requested: core::sync::atomic::AtomicBool::new(false), - thread: std::thread::current(), - qsbr: crate::object::qsbr::QSBR.register(), - }); - FRAME_SLOT_CACHE.with(|cache| { - #[cfg(unix)] - cache.top_frame.set(&new_slot.top_frame); - cache.top_iframe.set(&new_slot.top_iframe); - }); - #[cfg(feature = "threading")] - CURRENT_STOP_REQUESTED.with(|c| c.set(&new_slot.stop_requested)); - - // Lock is safe: reinit_locks_after_fork() already reset it to unlocked. - let mut registry = vm.state.thread_frames.lock(); - registry.clear(); - registry.insert(current_ident, new_slot.clone()); - drop(registry); - - CURRENT_THREAD_SLOT.with(|s| { - *s.borrow_mut() = Some(new_slot.clone()); - }); - INTERP_THREAD_SLOTS.with(|slots| { - slots.borrow_mut().insert(vm.state.interpreter_id, new_slot); + drop(removed); }); } -/// Drop this thread's cached slots for every interpreter except `keep_id`. -/// -/// After `fork()` only the calling thread survives, and the other -/// interpreters' registries are cleared; a cached slot would otherwise stay -/// current for an interpreter that no longer lists it, hiding the thread from -/// that interpreter's stop-the-world. The next enter builds a fresh slot. -#[cfg(feature = "threading")] -pub fn purge_other_interpreter_slots_after_fork(keep_id: i64) { - INTERP_THREAD_SLOTS.with(|slots| { - slots.borrow_mut().retain(|&id, _| id == keep_id); - }); +/// Reattach the surviving slot after child repair. Keeping its identity also +/// preserves the cached frame pointers and its QSBR registrations. +#[cfg(all(unix, feature = "threading"))] +pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { + retain_frame_slot_after_fork(&vm.state); + init_thread_slot_if_needed(vm); + attach_thread(vm); } /// Whether the interpreter on top of `VM_STACK` is currently ATTACHED on this @@ -1298,12 +1713,18 @@ enum WithVmTarget { pub fn with_vm(obj: &PyObject, f: F) -> Option where - F: Fn(&VirtualMachine) -> R, + F: FnOnce(&VirtualMachine) -> R, { + if !callbacks_permitted() { + return None; + } let vm_owns_obj = |interp: NonNull| { // SAFETY: all references in VM_STACK should be valid let vm = unsafe { interp.as_ref() }; - obj.fast_isinstance(vm.ctx.types.object_type) + obj.gc_handle() + .heap() + .and_then(|heap| heap.owner) + .is_none_or(|owner| owner == vm.state.interpreter_id) }; // `with_vm` runs on every teardown of an object with a `__del__` slot or a // weakref callback (drop_slow_inner / try_call_finalizer / gc_state), which @@ -1339,13 +1760,7 @@ where { return Some(WithVmTarget::AlreadyCurrent(top)); } - let interp = match vms.iter().copied().exactly_one() { - Ok(x) => { - debug_assert!(vm_owns_obj(x)); - x - } - Err(mut others) => others.find(|x| vm_owns_obj(*x))?, - }; + let interp = vms.iter().rev().copied().find(|x| vm_owns_obj(*x))?; Some(WithVmTarget::NeedsSwitch(interp)) })?; match target { @@ -1369,11 +1784,35 @@ where #[must_use = "ThreadedVirtualMachine does nothing unless you move it to another thread and call .run()"] #[cfg(feature = "threading")] pub struct ThreadedVirtualMachine { - pub(super) vm: VirtualMachine, + pub(super) vm: super::owned::OwnedVm, } #[cfg(feature = "threading")] impl ThreadedVirtualMachine { + /// Enter this interpreter on the current native thread with scoped access. + pub fn run(&self, f: impl for<'vm> FnOnce(crate::embedding::Vm<'vm>) -> R) -> R { + if self.vm.state.closed.load(Ordering::Acquire) || !self.vm.owner_is_valid() { + return f(crate::embedding::Vm::new(&self.vm)); + } + self.run_raw(|vm| f(crate::embedding::Vm::new(vm))) + } + + /// Enter the raw native API on this thread. + /// + /// # Safety + /// Follow [`crate::Interpreter::enter_unchecked`]'s contract. + pub unsafe fn run_unchecked(&self, f: impl FnOnce(&VirtualMachine) -> R) -> R { + self.run_raw(f) + } + + /// Build a scoped interpreter entry for a native thread. + pub fn make_spawn_func( + self, + f: impl for<'vm> FnOnce(crate::embedding::Vm<'vm>) -> R, + ) -> impl FnOnce() -> R { + move || self.run(f) + } + /// Create a `FnOnce()` that can easily be passed to a function like [`std::thread::Builder::spawn`] /// /// # Note @@ -1382,11 +1821,11 @@ impl ThreadedVirtualMachine { /// on the thread this `FnOnce` runs in, there is a possibility that that thread will panic /// as `PyObjectRef`'s `Drop` implementation tries to run the `__del__` destructor of a /// Python object but finds that it's not in the context of any vm. - pub fn make_spawn_func(self, f: F) -> impl FnOnce() -> R + pub(crate) fn make_spawn_func_raw(self, f: F) -> impl FnOnce() -> R where F: FnOnce(&VirtualMachine) -> R, { - move || self.run(f) + move || self.run_raw(f) } /// Run a function in this thread context @@ -1397,7 +1836,7 @@ impl ThreadedVirtualMachine { /// to the parent thread and then `join()` on the `JoinHandle` (or similar), there is a possibility that /// the current thread will panic as `PyObjectRef`'s `Drop` implementation tries to run the `__del__` /// destructor of a python object but finds that it's not in the context of any vm. - pub fn run(&self, f: F) -> R + pub(crate) fn run_raw(&self, f: F) -> R where F: FnOnce(&VirtualMachine) -> R, { @@ -1425,7 +1864,7 @@ impl VirtualMachine { F: Send + 'static + FnOnce(&Self) -> R, R: Send + 'static, { - let func = self.new_thread().make_spawn_func(f); + let func = self.new_thread().make_spawn_func_raw(f); std::thread::spawn(func) } @@ -1437,15 +1876,12 @@ impl VirtualMachine { /// # Usage /// /// ``` - /// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { - /// use std::thread::Builder; - /// let handle = Builder::new() - /// .name("my thread :)".into()) - /// .spawn(vm.new_thread().make_spawn_func(|vm| vm.ctx.none())) - /// .expect("couldn't spawn thread"); - /// let returned_obj = handle.join().expect("thread panicked"); - /// assert!(vm.is_none(&returned_obj)); - /// # }) + /// let interpreter = rustpython_vm::Interpreter::without_stdlib(Default::default()); + /// let handle = std::thread::spawn( + /// interpreter.new_thread().make_spawn_func(|vm| vm.new_int(42).unwrap().unbind()) + /// ); + /// let returned = handle.join().unwrap(); + /// interpreter.enter(|vm| assert_eq!(vm.bind(&returned).unwrap().to_i64().unwrap(), 42)); /// ``` /// /// Note: this function is safe, but running the returned ThreadedVirtualMachine in the same @@ -1453,11 +1889,27 @@ impl VirtualMachine { /// specific guaranteed behavior. #[cfg(feature = "threading")] pub fn new_thread(&self) -> ThreadedVirtualMachine { + self.new_thread_with_kind(super::owner_lease::OwnerKind::Host) + } + + #[cfg(feature = "threading")] + pub(crate) fn new_python_thread(&self) -> ThreadedVirtualMachine { + self.new_thread_with_kind(super::owner_lease::OwnerKind::PythonThread) + } + + #[cfg(feature = "threading")] + fn new_thread_with_kind(&self, kind: super::owner_lease::OwnerKind) -> ThreadedVirtualMachine { + assert!( + !self.state.finalizing.load(Ordering::Acquire) + && !self.state.admission_closed.load(Ordering::Acquire), + "cannot create a worker during interpreter shutdown" + ); let global_trace = self.state.global_trace_func.lock().clone(); let global_profile = self.state.global_profile_func.lock().clone(); let use_tracing = global_trace.is_some() || global_profile.is_some(); let vm = Self { + owner_lease: core::cell::OnceCell::new(), builtins: self.builtins.clone(), sys_module: self.sys_module.clone(), ctx: self.ctx.clone(), @@ -1473,10 +1925,10 @@ impl VirtualMachine { tracing_depth: Cell::new(0), recursion_limit: self.recursion_limit.clone(), signal_handlers: core::cell::OnceCell::new(), - signal_rx: None, + signal_rx: core::cell::OnceCell::new(), repr_guards: RefCell::default(), state: self.state.clone(), - initialized: self.initialized, + initialized: self.initialized.clone(), recursion_depth: Cell::new(0), #[cfg(any(miri, target_env = "musl"))] native_recursion_depth: Cell::new(0), @@ -1492,6 +1944,258 @@ impl VirtualMachine { pending_gen_resume: core::cell::UnsafeCell::new(None), trampoline_stack: core::cell::UnsafeCell::new(Vec::new()), }; - ThreadedVirtualMachine { vm } + ThreadedVirtualMachine { + vm: super::owned::OwnedVm::with_kind(vm, kind), + } + } +} + +/// Access the raw VM currently entered on this native thread. +/// +/// # Safety +/// Follow [`crate::Interpreter::enter_unchecked`]'s contract. The callback must +/// not leak raw references, and the thread must be attached for object access. +pub unsafe fn with_current_vm_unchecked(f: impl FnOnce(&VirtualMachine) -> R) -> R { + with_current_vm(f) +} + +/// Preserve the current attachment for native code that releases it manually. +/// +/// # Safety +/// Restore on this same native thread before any enclosing VM entry returns. +/// No raw Python reference may be accessed while detached. +#[cfg(feature = "threading")] +#[must_use] +pub unsafe fn save_current_thread_unchecked() -> SavedThreadState { + save_current_thread() +} + +/// Restore a manually detached native entry. +/// +/// # Safety +/// The saved state's entries must all still be alive, on this native thread, +/// and this thread must not have entered any other VM since saving it. +#[cfg(feature = "threading")] +pub unsafe fn restore_current_thread_unchecked(state: SavedThreadState) { + restore_current_thread(state); +} + +#[cfg(test)] +#[cfg(all(feature = "threading", feature = "rustpython-compiler"))] +mod tests { + use super::*; + + // Only use for runtimes exclusively owned by this test. This exercises the + // ownership transition; the Unix fork coordinator must also stabilize locks. + fn repair_native_owners(state: &PyGlobalState) { + let (owners, teardowns) = unsafe { state.owner_leases.prepare_fork().repair_child() }; + state.owners.store(owners, Ordering::Relaxed); + state.teardowns.store(teardowns, Ordering::Relaxed); + } + + #[test] + fn inherited_idle_owner_cannot_enter_or_decrement_surviving_owners() { + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let idle = interpreter.new_thread(); + let nested = interpreter.new_thread(); + interpreter.enter_raw(|vm| { + nested.run_raw(|_| repair_native_owners(&vm.state)); + assert_eq!(vm.state.owners.load(Ordering::Acquire), 2); + assert!(matches!( + idle.run(|vm| vm.new_int(1).map(|value| value.unbind())), + Err(crate::embedding::Error::ForkedOwner) + )); + drop(idle); + assert_eq!(vm.state.owners.load(Ordering::Acquire), 2); + drop(nested); + assert_eq!(vm.state.owners.load(Ordering::Acquire), 1); + }); + assert_eq!(interpreter.finalize().unwrap(), 0); + } + + #[test] + fn saved_native_entries_survive_owner_repair() { + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let other = crate::Interpreter::without_stdlib(Default::default()); + interpreter.enter_raw(|vm| { + let saved = save_current_thread(); + other.enter_raw(|_| repair_native_owners(&vm.state)); + restore_current_thread(saved); + assert!(is_current_attached(vm)); + assert!(vm.owner_is_valid()); + assert_eq!(vm.state.owners.load(Ordering::Acquire), 1); + }); + assert_eq!(interpreter.finalize().unwrap(), 0); + } + + #[test] + fn native_destructor_panic_unregisters_before_restoring_outer_vm() { + struct PanicDrop; + impl Drop for PanicDrop { + fn drop(&mut self) { + assert!(!callbacks_permitted()); + assert!(try_with_current_vm(|_| ()).is_none()); + panic!("native destructor"); + } + } + + let outer = crate::Interpreter::without_stdlib(Default::default()); + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let state = interpreter.enter_raw(|vm| vm.state.clone()); + let worker = interpreter.new_thread(); + let probe = PanicDrop; + let (sender, receiver) = crate::signal::user_signal_channel(); + sender + .send(Box::new(move |_| { + let _ = &probe; + Ok(()) + })) + .unwrap(); + worker.vm.set_user_signal_channel(receiver); + drop(sender); + outer.enter(|vm| { + let result = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| drop(worker))); + assert!(result.is_err()); + assert!(callbacks_permitted()); + assert_eq!(state.teardowns.load(Ordering::Acquire), 0); + assert_eq!(state.owners.load(Ordering::Acquire), 1); + assert!(state.thread_frames.lock().is_empty()); + assert_eq!(vm.new_int(42).unwrap().to_i64().unwrap(), 42); + }); + assert_eq!(interpreter.finalize().unwrap(), 0); + } + + #[test] + fn finalize_waits_for_the_native_tail_of_worker_destruction() { + check_native_tail(false); + } + + #[test] + fn finalize_waits_for_an_inherited_stale_owners_native_tail() { + check_native_tail(true); + } + + fn check_native_tail(stale: bool) { + use core::time::Duration; + use std::sync::mpsc; + + struct PauseDrop { + started: mpsc::Sender<()>, + resume: std::sync::Mutex>, + } + impl Drop for PauseDrop { + fn drop(&mut self) { + assert!(!callbacks_permitted()); + self.started.send(()).unwrap(); + let resumed = self.resume.get_mut().unwrap(); + wait_detached_from_interpreter(&|| { + resumed.recv_timeout(Duration::from_secs(30)).unwrap(); + }); + } + } + + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let worker = interpreter.new_thread(); + let (started, waiting) = mpsc::channel(); + let (resume, resumed) = mpsc::channel(); + let probe = PauseDrop { + started, + resume: std::sync::Mutex::new(resumed), + }; + let (sender, receiver) = crate::signal::user_signal_channel(); + sender + .send(Box::new(move |_| { + let _ = &probe; + Ok(()) + })) + .unwrap(); + worker.vm.set_user_signal_channel(receiver); + drop(sender); + if stale { + interpreter.enter_raw(|vm| repair_native_owners(&vm.state)); + assert!(!worker.vm.owner_is_valid()); + } + let worker = std::thread::spawn(move || drop(worker)); + let release = scopeguard::guard(resume, |resume| { + let _ = resume.send(()); + }); + waiting.recv_timeout(Duration::from_secs(30)).unwrap(); + let busy = interpreter.finalize().unwrap_err(); + drop(release); + worker.join().unwrap(); + assert_eq!(busy.retry().unwrap(), 0); + } + + #[test] + fn finalization_defers_foreign_stale_drop_without_blocking_join() { + struct Probe(Arc); + impl Drop for Probe { + fn drop(&mut self) { + assert!(!callbacks_permitted()); + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let worker = interpreter.new_thread(); + let count = Arc::new(AtomicUsize::new(0)); + let probe = Probe(count.clone()); + let (sender, receiver) = crate::signal::user_signal_channel(); + sender + .send(Box::new(move |_| { + let _ = &probe; + Ok(()) + })) + .unwrap(); + worker.vm.set_user_signal_channel(receiver); + drop(sender); + let state = interpreter.enter_raw(|vm| { + repair_native_owners(&vm.state); + vm.state.clone() + }); + let permit = state.owner_leases.try_finalize(&state).unwrap(); + let (finished, joined) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + drop(worker); + finished.send(()).unwrap(); + }); + joined + .recv_timeout(core::time::Duration::from_secs(10)) + .unwrap(); + worker.join().unwrap(); + assert_eq!(count.load(Ordering::Relaxed), 0); + assert_eq!(state.teardowns.load(Ordering::Acquire), 0); + drop(interpreter); + drop(permit); + assert_eq!(count.load(Ordering::Relaxed), 1); + assert_eq!(state.owners.load(Ordering::Acquire), 0); + assert_eq!(state.teardowns.load(Ordering::Acquire), 0); + } + + #[test] + fn retired_thread_slots_release_callbacks_attached_and_cannot_be_refilled() { + let interpreter = crate::Interpreter::without_stdlib(Default::default()); + let retired = interpreter.enter_raw(|vm| { + let scope = crate::embedding::Vm::new(vm); + scope.exec("import sys\nevents = []\nclass Callback:\n def __del__(self):\n sys.settrace(lambda *args: None)\n events.append(1)\n").unwrap(); + let handle = scope.eval("Callback()").unwrap().unbind(); + let callback = unsafe { handle.to_object_unchecked(vm) }.unwrap(); + let slot = current_thread_slot().unwrap(); + drop(slot.replace_profile(callback)); + drop(handle); + cleanup_current_thread_frames(vm); + assert!(slot.is_closed()); + assert!(vm.is_none(&slot.trace_func.lock())); + assert!(vm.is_none(&slot.profile_func.lock())); + assert!(slot.exception.load_owned().is_none()); + assert!(current_thread_slot().is_none()); + assert!(slot.qsbr.is_offline()); + slot + }); + interpreter.enter(|vm| { + assert_eq!(vm.eval("len(events)").unwrap().to_i64().unwrap(), 1); + }); + // Keeping a retired slot in native TLS must not retain Python state. + std::thread::spawn(move || drop(retired)).join().unwrap(); } } diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs index 3d8e7cb6fb1..057a5ef83f1 100644 --- a/crates/vm/src/warn.rs +++ b/crates/vm/src/warn.rs @@ -1,24 +1,28 @@ use crate::{ - AsObject, Context, Py, PyAtomicRef, PyObject, PyObjectRef, PyResult, VirtualMachine, + AsObject, Context, Py, PyObject, PyObjectRef, PyResult, VirtualMachine, anystr::SplitLinesArgs, builtins::{ - PyBaseExceptionRef, PyDict, PyDictRef, PyList, PyListRef, PyStr, PyStrInterned, PyStrRef, - PyTuple, PyTupleRef, PyType, PyTypeRef, + PyBaseExceptionRef, PyDict, PyDictRef, PyListRef, PyStr, PyStrInterned, PyStrRef, PyTuple, + PyTupleRef, PyType, PyTypeRef, }, convert::TryFromObject, }; use core::sync::atomic::{AtomicUsize, Ordering}; -use rustpython_common::lock::OnceCell; +use rustpython_common::lock::PyRwLock; pub struct WarningsState { - pub filters: PyAtomicRef, - pub once_registry: PyAtomicRef, - pub default_action: PyAtomicRef, + roots: PyRwLock>, pub filters_version: AtomicUsize, - pub context_var: OnceCell, lock_count: AtomicUsize, } +struct WarningsRoots { + filters: PyListRef, + once_registry: PyDictRef, + default_action: PyStrRef, + context_var: Option, +} + impl WarningsState { fn create_default_filters(ctx: &Context) -> PyListRef { // init_filters(): non-debug default filter set. @@ -71,15 +75,96 @@ impl WarningsState { pub fn init_state(ctx: &Context) -> Self { Self { - filters: Self::create_default_filters(ctx).into(), - once_registry: ctx.new_dict().into(), - default_action: ctx.new_str("default").into(), + roots: PyRwLock::new(Some(WarningsRoots { + filters: Self::create_default_filters(ctx), + once_registry: ctx.new_dict(), + default_action: ctx.new_str("default"), + context_var: None, + })), filters_version: AtomicUsize::new(0), - context_var: OnceCell::new(), lock_count: AtomicUsize::new(0), } } + pub(crate) fn filters(&self) -> PyListRef { + self.roots + .read() + .as_ref() + .expect("warnings state closed") + .filters + .clone() + } + + pub(crate) fn once_registry(&self) -> PyDictRef { + self.roots + .read() + .as_ref() + .expect("warnings state closed") + .once_registry + .clone() + } + + pub(crate) fn default_action(&self) -> PyStrRef { + self.roots + .read() + .as_ref() + .expect("warnings state closed") + .default_action + .clone() + } + + pub(crate) fn context_var(&self) -> Option { + self.roots.read().as_ref()?.context_var.clone() + } + + pub(crate) fn init_context_var(&self, value: PyObjectRef) -> PyObjectRef { + let mut roots = self.roots.write(); + if let Some(roots) = roots.as_mut() { + roots + .context_var + .get_or_insert_with(|| value.clone()) + .clone() + } else { + value + } + } + + fn replace(&self, value: T, field: impl FnOnce(&mut WarningsRoots) -> &mut T) { + let retired = { + let mut roots = self.roots.write(); + if let Some(roots) = roots.as_mut() { + Some(core::mem::replace(field(roots), value)) + } else { + None + } + }; + drop(retired); + } + + pub(crate) fn close(&self) { + let roots = self.roots.write().take(); + drop(roots); + } + + pub(crate) fn retire_roots(&self, ctx: &Context) { + // Finalizers can still issue warnings while the old roots are released. + let replacement = WarningsRoots { + filters: ctx.new_list(Vec::new()), + once_registry: ctx.new_dict(), + default_action: ctx.new_str("default"), + context_var: None, + }; + let retired = self.roots.write().replace(replacement); + drop(retired); + } + + /// # Safety + /// Only the surviving thread may access this state after fork. + #[cfg(all(unix, feature = "threading", feature = "host_env"))] + pub(crate) unsafe fn reinit_after_fork(&self) { + unsafe { crate::common::lock::reinit_rwlock_after_fork(&self.roots) }; + } + pub fn acquire_lock(&self) { self.lock_count.fetch_add(1, Ordering::SeqCst); } @@ -148,20 +233,19 @@ fn get_warnings_filters(vm: &VirtualMachine) -> PyResult { .map_err(|_| vm.new_value_error("_warnings.filters must be a list"))?; vm.state .warnings - .filters - .swap_to_temporary_refs(filters, vm); + .replace(filters, |roots| &mut roots.filters); } - Ok(vm.state.warnings.filters.to_owned()) + Ok(vm.state.warnings.filters()) } fn get_warnings_context_filters(vm: &VirtualMachine) -> PyResult> { - let Some(ctx_var) = vm.state.warnings.context_var.get() else { + let Some(ctx_var) = vm.state.warnings.context_var() else { return Ok(None); }; - if vm.is_none(ctx_var) { + if vm.is_none(&ctx_var) { return Ok(None); } - let ctx = vm.call_method(ctx_var, "get", (vm.ctx.none(),))?; + let ctx = vm.call_method(&ctx_var, "get", (vm.ctx.none(),))?; if vm.is_none(&ctx) { return Ok(None); } @@ -176,7 +260,7 @@ fn bless_my_loader( vm: &VirtualMachine, ) -> PyResult> { let external = vm - .importlib + .importlib() .get_attr("_bootstrap_external", vm) .or_else(|_| vm.import("importlib._bootstrap_external", 0))?; if vm.is_none(&external) { @@ -247,10 +331,9 @@ fn get_default_action(vm: &VirtualMachine) -> PyResult { let action = PyStrRef::try_from_object(vm, action)?; vm.state .warnings - .default_action - .swap_to_temporary_refs(action, vm); + .replace(action, |roots| &mut roots.default_action); } - Ok(vm.state.warnings.default_action.to_owned().into()) + Ok(vm.state.warnings.default_action().into()) } /// Get the once registry from `sys.modules['warnings'].onceregistry`, @@ -266,10 +349,9 @@ fn get_once_registry(vm: &VirtualMachine) -> PyResult { let registry = PyDictRef::try_from_object(vm, registry)?; vm.state .warnings - .once_registry - .swap_to_temporary_refs(registry, vm); + .replace(registry, |roots| &mut roots.once_registry); } - Ok(vm.state.warnings.once_registry.to_owned().into()) + Ok(vm.state.warnings.once_registry().into()) } fn already_warned( diff --git a/crates/wasm/src/browser_module.rs b/crates/wasm/src/browser_module.rs index f6b20769449..da9ba5e86bc 100644 --- a/crates/wasm/src/browser_module.rs +++ b/crates/wasm/src/browser_module.rs @@ -8,7 +8,7 @@ mod _browser { Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyStrRef}, class::PyClassImpl, - convert::ToPyObject, + convert::{IntoObject, ToPyObject}, function::{ArgCallable, OptionalArg}, import::import_source, }; @@ -123,15 +123,16 @@ mod _browser { let g = f.clone(); let weak_vm = weak_vm(vm); + let func = crate::vm_class::hold_object(vm, func.into_object()); *g.borrow_mut() = Some(Closure::wrap(Box::new(move |time: f64| { let stored_vm = weak_vm .upgrade() .expect("that the vm is valid from inside of request_animation_frame"); - stored_vm.interp.enter(|vm| { - let func = func.clone(); + stored_vm.enter(|vm| { + let func = crate::vm_class::bind_object(vm, &func); let args = vec![vm.ctx.new_float(time).into()]; - let _ = func.invoke(args, vm); + let _ = func.call(args, vm); let closure = f.borrow_mut().take(); drop(closure); @@ -183,7 +184,7 @@ mod _browser { Document { doc: window().document().expect("Document missing from window"), }, - Document::make_static_type(), + unsafe { Document::make_static_type() }, None, ) } @@ -236,6 +237,8 @@ mod _browser { let window = window(); let request_prom = window.fetch_with_request(&request); + let module = module.expect_str().to_owned(); + let future = async move { let val = JsFuture::from(request_prom).await?; let response = val @@ -245,9 +248,9 @@ mod _browser { let stored_vm = &weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); - stored_vm.interp.enter(move |vm| { + stored_vm.enter(move |vm| { let resp_text = text.as_string().unwrap(); - let res = import_source(vm, module.expect_str(), &resp_text); + let res = import_source(vm, &module, &resp_text); match res { Ok(_) => Ok(JsValue::null()), Err(err) => Err(convert::py_err_to_js_err(vm, &err)), diff --git a/crates/wasm/src/convert.rs b/crates/wasm/src/convert.rs index 582039cd37e..ad463beeca3 100644 --- a/crates/wasm/src/convert.rs +++ b/crates/wasm/src/convert.rs @@ -112,26 +112,20 @@ pub fn js_err_to_py_err(vm: &VirtualMachine, js_err: &JsValue) -> PyBaseExceptio } pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { - if let Some(ref wasm_id) = vm.wasm_id + if let Some(wasm_id) = vm.wasm_id.get() && py_obj.fast_isinstance(vm.ctx.types.function_type) { let wasm_vm = WASMVirtualMachine { id: wasm_id.clone(), }; - let weak_py_obj = wasm_vm.push_held_rc(py_obj).unwrap().unwrap(); + let py_handle = crate::vm_class::hold_object(vm, py_obj); let closure = move |args: Option>, kwargs: Option| -> Result { - let py_obj = match wasm_vm.assert_valid() { - Ok(_) => weak_py_obj - .upgrade() - .expect("weak_py_obj to be valid if VM is valid"), - Err(err) => { - return Err(err); - } - }; - stored_vm_from_wasm(&wasm_vm).interp.enter(move |vm| { + wasm_vm.assert_valid()?; + stored_vm_from_wasm(&wasm_vm).enter(|vm| { + let py_obj = crate::vm_class::bind_object(vm, &py_handle); let args = match args { Some(args) => Vec::from(args) .into_iter() diff --git a/crates/wasm/src/js_module.rs b/crates/wasm/src/js_module.rs index 2ef56c85a39..faca22c8678 100644 --- a/crates/wasm/src/js_module.rs +++ b/crates/wasm/src/js_module.rs @@ -126,20 +126,12 @@ mod _js { } #[pymethod] - fn new_closure( - _zelf: &Py, - obj: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { + fn new_closure(_zelf: &Py, obj: PyObjectRef, vm: &VirtualMachine) -> JsClosure { JsClosure::new(obj, false, vm) } #[pymethod] - fn new_closure_once( - _zelf: &Py, - obj: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { + fn new_closure_once(_zelf: &Py, obj: PyObjectRef, vm: &VirtualMachine) -> JsClosure { JsClosure::new(obj, true, vm) } @@ -322,21 +314,15 @@ mod _js { #[pyclass] impl JsClosure { - fn new(obj: PyObjectRef, once: bool, vm: &VirtualMachine) -> PyResult { + fn new(obj: PyObjectRef, once: bool, vm: &VirtualMachine) -> Self { let wasm_vm = WASMVirtualMachine { - id: vm.wasm_id.clone().unwrap(), + id: vm.wasm_id.get().cloned().unwrap(), }; - let weak_py_obj = wasm_vm.push_held_rc(obj).unwrap()?; + let py_handle = crate::vm_class::hold_object(vm, obj); let f = move |this: JsValue, args: Box<[JsValue]>| { - let py_obj = match wasm_vm.assert_valid() { - Ok(_) => weak_py_obj - .upgrade() - .expect("weak_py_obj to be valid if VM is valid"), - Err(err) => { - return Err(err); - } - }; - stored_vm_from_wasm(&wasm_vm).interp.enter(move |vm| { + wasm_vm.assert_valid()?; + stored_vm_from_wasm(&wasm_vm).enter(|vm| { + let py_obj = crate::vm_class::bind_object(vm, &py_handle); let mut pyargs = vec![PyJsValue::new(this).into_pyobject(vm)]; pyargs.extend( Vec::from(args) @@ -353,11 +339,11 @@ mod _js { Closure::once(Box::new(f)) }; let wrapped = PyJsValue::new(wrap_closure(closure.as_ref())).into_ref(&vm.ctx); - Ok(Self { + Self { closure: Some((closure, wrapped)).into(), destroyed: false.into(), detached: false.into(), - }) + } } #[pygetset] @@ -521,6 +507,11 @@ mod _js { let weak_vm = weak_vm(vm); let prom = JsFuture::from(prom.clone()); + let on_fulfill = + on_fulfill.map(|f| crate::vm_class::hold_object(vm, f.into_object())); + let on_reject = + on_reject.map(|f| crate::vm_class::hold_object(vm, f.into_object())); + let ret_future = async move { let stored_vm = &weak_vm .upgrade() @@ -528,17 +519,19 @@ mod _js { let res = prom.await; match res { Ok(val) => match on_fulfill { - Some(on_fulfill) => stored_vm.interp.enter(move |vm| { + Some(on_fulfill) => stored_vm.enter(move |vm| { let val = convert::js_to_py(vm, val); - let res = on_fulfill.invoke((val,), vm); + let res = crate::vm_class::bind_object(vm, &on_fulfill) + .call((val,), vm); convert::pyresult_to_js_result(vm, res) }), None => Ok(val), }, Err(err) => match on_reject { - Some(on_reject) => stored_vm.interp.enter(move |vm| { + Some(on_reject) => stored_vm.enter(move |vm| { let err = new_js_error(vm, err); - let res = on_reject.invoke((err,), vm); + let res = crate::vm_class::bind_object(vm, &on_reject) + .call((err,), vm); convert::pyresult_to_js_result(vm, res) }), None => Err(err), @@ -656,13 +649,12 @@ mod _js { #[pyattr(name = "JSError", once)] fn js_error(vm: &VirtualMachine) -> PyTypeRef { let ctx = &vm.ctx; - let js_error = PyRef::leak( - PyType::new_simple_heap("JSError", vm.ctx.exceptions.exception_type, ctx).unwrap(), - ); + let js_error = + PyType::new_simple_heap("JSError", vm.ctx.exceptions.exception_type, ctx).unwrap(); extend_class!(ctx, js_error, { - "value" => ctx.new_readonly_getset("value", js_error, |exc: PyBaseExceptionRef| exc.get_arg(0)), + "value" => ctx.new_readonly_getset("value", &js_error, |exc: PyBaseExceptionRef| exc.get_arg(0)), }); - js_error.to_owned() + js_error } } diff --git a/crates/wasm/src/ssl.rs b/crates/wasm/src/ssl.rs index 2504c44cfce..4640235630b 100644 --- a/crates/wasm/src/ssl.rs +++ b/crates/wasm/src/ssl.rs @@ -198,42 +198,42 @@ mod _ssl { struct PySSLCertVerificationError(PySSLError); fn ssl_error(vm: &VirtualMachine, msg: impl Into) -> rustpython_vm::PyRef { - vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), None, msg.into()) + vm.new_os_subtype_error(PySSLError::class(&vm.ctx), None, msg.into()) } fn map_tls(vm: &VirtualMachine, err: TlsError) -> PyBaseExceptionRef { match err { TlsError::WantRead => vm .new_os_subtype_error( - PySSLWantReadError::class(&vm.ctx).to_owned(), + PySSLWantReadError::class(&vm.ctx), Some(SSL_ERROR_WANT_READ), "The operation did not complete (read)", ) .upcast(), TlsError::WantWrite => vm .new_os_subtype_error( - PySSLWantWriteError::class(&vm.ctx).to_owned(), + PySSLWantWriteError::class(&vm.ctx), Some(SSL_ERROR_WANT_WRITE), "The operation did not complete (write)", ) .upcast(), TlsError::ZeroReturn => vm .new_os_subtype_error( - PySSLZeroReturnError::class(&vm.ctx).to_owned(), + PySSLZeroReturnError::class(&vm.ctx), Some(SSL_ERROR_ZERO_RETURN), "TLS/SSL connection has been closed (EOF)", ) .upcast(), TlsError::Eof => vm .new_os_subtype_error( - PySSLEOFError::class(&vm.ctx).to_owned(), + PySSLEOFError::class(&vm.ctx), Some(SSL_ERROR_EOF), "EOF occurred in violation of protocol", ) .upcast(), TlsError::CertVerification(err) => vm .new_os_subtype_error( - PySSLCertVerificationError::class(&vm.ctx).to_owned(), + PySSLCertVerificationError::class(&vm.ctx), Some(SSL_ERROR_SSL), format!("{err}"), ) @@ -685,7 +685,7 @@ mod _ssl { if *context.verify_mode.read() != CERT_NONE { return Err(vm .new_os_subtype_error( - PySSLCertVerificationError::class(&vm.ctx).to_owned(), + PySSLCertVerificationError::class(&vm.ctx), Some(SSL_ERROR_SSL), "certificate verify failed: unable to get local issuer certificate", ) diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index 4b3d6f4cb77..827ad9138cb 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -8,10 +8,10 @@ use core::cell::RefCell; use js_sys::{Object, TypeError}; use ruff_text_size::Ranged; use rustpython_vm::{ - Interpreter, PyObjectRef, PyRef, PyResult, Settings, VirtualMachine, - builtins::PyWeak, + Interpreter, PyObjectRef, PyResult, Settings, VirtualMachine, bytecode::CodeFlags, compiler::{self, Mode}, + embedding::PyHandle, function::ArgMapping, scope::Scope, vm::VmCompileError, @@ -21,10 +21,7 @@ use wasm_bindgen::prelude::*; pub(crate) struct StoredVirtualMachine { pub interp: Interpreter, - pub scope: Scope, - /// you can put a Rc in here, keep it as a Weak, and it'll be held only for - /// as long as the StoredVM is alive - held_objects: RefCell>, + globals: PyHandle, future_features: RefCell, } @@ -72,6 +69,16 @@ mod _window { } impl StoredVirtualMachine { + pub(crate) fn enter(&self, f: impl FnOnce(&VirtualMachine) -> R) -> R { + // SAFETY: JS adapters bind opaque roots only inside their owning entry. + unsafe { self.interp.enter_unchecked(f) } + } + + fn scope(&self, vm: &VirtualMachine) -> Scope { + let globals = bind_object(vm, &self.globals).downcast().unwrap(); + Scope::with_builtins(None, globals, vm) + } + fn new(id: String, inject_browser_module: bool) -> Self { let mut settings = Settings::default(); settings.allow_external_library = false; @@ -80,44 +87,46 @@ impl StoredVirtualMachine { #[cfg(feature = "freeze-stdlib")] { - let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); - builder = builder - .add_native_modules(&defs) + let defs = rustpython_stdlib::stdlib_module_defs(unsafe { builder.context() }); + builder = unsafe { builder.add_native_modules(&defs) } .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB); } // Browser rustls `_ssl` overrides the rustls-free stdlib `_ssl`. // `_socket` is rustpython-stdlib's wasm shim (`socket_wasm.rs`). - let js_def = js_module::module_def(&builder.ctx); - builder = builder.add_native_module(js_def); + let js_def = js_module::module_def(unsafe { builder.context() }); + builder = unsafe { builder.add_native_module(js_def) }; #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] { install_browser_tls_provider(); - let ssl_def = crate::ssl::module_def(&builder.ctx); - builder = builder.add_native_module(ssl_def); + let ssl_def = crate::ssl::module_def(unsafe { builder.context() }); + builder = unsafe { builder.add_native_module(ssl_def) }; } if inject_browser_module { - let window_def = _window::module_def(&builder.ctx); - let browser_def = browser_module::module_def(&builder.ctx); - builder = builder - .add_native_modules(&[window_def, browser_def]) + let window_def = _window::module_def(unsafe { builder.context() }); + let browser_def = browser_module::module_def(unsafe { builder.context() }); + builder = unsafe { builder.add_native_modules(&[window_def, browser_def]) } .add_frozen_modules(rustpython_vm::py_freeze!(dir = "../Lib")); } - let interp = builder - .init_hook(move |vm| { - vm.wasm_id = Some(id); + let interp = unsafe { + builder.init_hook(move |vm| { + vm.wasm_id.set(id).unwrap(); }) - .build(); + } + .build(); - let scope = interp.enter(|vm| vm.new_scope_with_builtins()); + // SAFETY: only the opaque globals root leaves this initialization entry. + let globals = unsafe { + interp + .enter_unchecked(|vm| hold_object(vm, vm.new_scope_with_builtins().globals.into())) + }; Self { interp, - scope, - held_objects: RefCell::new(Vec::new()), + globals, future_features: RefCell::new(CodeFlags::empty()), } } @@ -152,7 +161,7 @@ thread_local! { pub fn get_vm_id(vm: &VirtualMachine) -> &str { vm.wasm_id - .as_ref() + .get() .expect("VirtualMachine inside of WASM crate should have wasm_id set") } pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc { @@ -243,7 +252,7 @@ impl WASMVirtualMachine { where F: FnOnce(&VirtualMachine, &StoredVirtualMachine) -> R, { - self.with(|stored| stored.interp.enter(|vm| f(vm, stored))) + self.with(|stored| stored.enter(|vm| f(vm, stored))) } #[must_use] @@ -251,17 +260,6 @@ impl WASMVirtualMachine { STORED_VMS.with_borrow(|vms| vms.contains_key(&self.id)) } - pub(crate) fn push_held_rc( - &self, - obj: PyObjectRef, - ) -> Result>, JsValue> { - self.with_vm(|vm, stored_vm| { - let weak = obj.downgrade(None, vm)?; - stored_vm.held_objects.borrow_mut().push(obj); - Ok(weak) - }) - } - pub fn assert_valid(&self) -> Result<(), JsValue> { if self.valid() { Ok(()) @@ -281,7 +279,8 @@ impl WASMVirtualMachine { #[wasm_bindgen(js_name = addToScope)] pub fn add_to_scope(&self, name: String, value: JsValue) -> Result<(), JsValue> { - self.with_vm(move |vm, StoredVirtualMachine { scope, .. }| { + self.with_vm(move |vm, stored| { + let scope = stored.scope(vm); let value = convert::js_to_py(vm, value); scope.globals.set_item(&name, value, vm).into_js(vm) })? @@ -420,11 +419,12 @@ impl WASMVirtualMachine { mode: Mode, source_path: Option, ) -> Result { - self.with_vm(|vm, StoredVirtualMachine { scope, .. }| { + self.with_vm(|vm, stored| { + let scope = stored.scope(vm); let source_path = source_path.unwrap_or_else(|| "".to_owned()); let code = vm.compile(source, mode, source_path.as_str()); let code = code.map_err(|err| compile_err_to_js(vm, err))?; - let result = vm.run_code_obj(code, scope.clone()); + let result = vm.run_code_obj(code, scope); convert::pyresult_to_js_result(vm, result) })? } @@ -435,7 +435,7 @@ impl WASMVirtualMachine { source_path: Option, ) -> Result { self.with_vm(|vm, stored| { - let scope = &stored.scope; + let scope = stored.scope(vm); let source_path = source_path.unwrap_or_else(|| "".to_owned()); let compile = |source: &str, mode: Mode| -> Result<_, JsValue> { let future_features = *stored.future_features.borrow(); @@ -452,7 +452,7 @@ impl WASMVirtualMachine { let future_features = *stored.future_features.borrow(); let Some(chunks) = statement_chunks(source, future_features) else { let code = compile(source, Mode::Single)?; - let result = vm.run_code_obj(code, scope.clone()); + let result = vm.run_code_obj(code, scope); return convert::pyresult_to_js_result(vm, result); }; @@ -491,3 +491,14 @@ impl WASMVirtualMachine { self.run_single(source, source_path) } } + +/// Root callback state without keeping Python references in a JS closure. +pub(crate) fn hold_object(vm: &VirtualMachine, object: PyObjectRef) -> PyHandle { + // SAFETY: all callers transfer an object from their current native entry. + unsafe { PyHandle::from_object_unchecked(object, vm) }.expect("live callback owner") +} + +pub(crate) fn bind_object(vm: &VirtualMachine, handle: &PyHandle) -> PyObjectRef { + // SAFETY: callers release the returned reference before leaving this entry. + unsafe { handle.to_object_unchecked(vm) }.expect("callback entered its live owner") +} diff --git a/example_projects/barebone/src/main.rs b/example_projects/barebone/src/main.rs index 0d9ffbefab1..9dc6b2936a2 100644 --- a/example_projects/barebone/src/main.rs +++ b/example_projects/barebone/src/main.rs @@ -3,15 +3,14 @@ use rustpython_vm::Interpreter; pub fn main() { let interp = Interpreter::without_stdlib(Default::default()); let value = interp.enter(|vm| { - let max = vm.builtins.get_attr("max", vm)?; - let value = max.call((vm.ctx.new_int(5), vm.ctx.new_int(10)), vm)?; - vm.print((vm.ctx.new_str("python print"), value.clone()))?; - Ok(value) + let builtins = vm.import("builtins")?; + let max = builtins.get_attr("max")?; + let value = max.call(&[vm.new_int(5)?, vm.new_int(10)?])?; + builtins + .get_attr("print")? + .call(&[vm.new_str("python print")?, value.clone()])?; + value.repr() }); - match value { - Ok(value) => println!("Rust repr: {:?}", value), - Err(err) => { - interp.finalize(err); - } - } + let result = value.map(|value| println!("Rust repr: {value}")); + interp.run(|_vm| result).expect("no native workers remain"); } diff --git a/example_projects/frozen_stdlib/Cargo.toml b/example_projects/frozen_stdlib/Cargo.toml index 91bb14a083b..389a4d95ac6 100644 --- a/example_projects/frozen_stdlib/Cargo.toml +++ b/example_projects/frozen_stdlib/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -rustpython = { path = "../../", default-features = false, features = ["freeze-stdlib"] } +rustpython = { path = "../../", default-features = false, features = ["freeze-stdlib", "stdio"] } rustpython-vm = { path = "../../crates/vm", default-features = false, features = ["freeze-stdlib"] } rustpython-pylib = { path = "../../crates/pylib", default-features = false, features = ["freeze-stdlib"] } diff --git a/example_projects/frozen_stdlib/src/main.rs b/example_projects/frozen_stdlib/src/main.rs index 8ff316faed8..67a8a1ed2d5 100644 --- a/example_projects/frozen_stdlib/src/main.rs +++ b/example_projects/frozen_stdlib/src/main.rs @@ -5,15 +5,15 @@ //! See also: `aheui-rust.md` for freezing your own package. use rustpython::InterpreterBuilderExt; -use rustpython_vm::{PyResult, VirtualMachine}; +use rustpython_vm::embedding::{Result, Vm}; -fn run(keyword: &str, vm: &VirtualMachine) -> PyResult<()> { - let json = vm.import("json", 0)?; - let json_loads = json.get_attr("loads", vm)?; +fn run(keyword: &str, vm: Vm<'_>) -> Result<()> { + let json = vm.import("json")?; + let json_loads = json.get_attr("loads")?; let template = r#"{"key": "value"}"#; let json_string = template.replace("value", keyword); - let dict = json_loads.call((vm.ctx.new_str(json_string),), vm)?; - vm.print((dict,))?; + let dict = json_loads.call(&[vm.new_str(&json_string)?])?; + vm.import("builtins")?.get_attr("print")?.call(&[dict])?; Ok(()) } @@ -22,7 +22,9 @@ fn interpreter_with_config() { .init_stdlib() .interpreter(); // Use interpreter.enter to reuse the same interpreter later - interpreter.run(|vm| run("rustpython::InterpreterBuilder", vm)); + interpreter + .run(|vm| run("rustpython::InterpreterBuilder", vm)) + .expect("no native workers remain"); } fn interpreter_with_vm() { @@ -30,7 +32,9 @@ fn interpreter_with_vm() { .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) .build(); // Use interpreter.enter to reuse the same interpreter later - interpreter.run(|vm| run("rustpython_vm::Interpreter::builder", vm)); + interpreter + .run(|vm| run("rustpython_vm::Interpreter::builder", vm)) + .expect("no native workers remain"); } fn main() { diff --git a/example_projects/wasm32_without_js/rustpython-without-js/src/lib.rs b/example_projects/wasm32_without_js/rustpython-without-js/src/lib.rs index 4219b698753..35c43c5a518 100644 --- a/example_projects/wasm32_without_js/rustpython-without-js/src/lib.rs +++ b/example_projects/wasm32_without_js/rustpython-without-js/src/lib.rs @@ -20,13 +20,12 @@ pub unsafe extern "C" fn eval(s: *const u8, l: usize) -> i32 { // 2. Execute Python code let interpreter = Interpreter::without_stdlib(Default::default()); let result = interpreter.enter(|vm| { - let scope = vm.new_scope_with_builtins(); - let res = match vm.run_block_expr(scope, src) { + let res = match vm.eval(src) { Ok(val) => val, Err(_) => return Err(-1), // Python execution error }; - let repr_str = match res.repr(vm) { - Ok(repr) => repr.to_string(), + let repr_str = match res.repr() { + Ok(repr) => repr, Err(_) => return Err(-1), // Failed to get string representation }; Ok(repr_str) diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index b0e13a5e60b..afb535a6f19 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -5,20 +5,21 @@ use rustpython::vm::{ pub fn main() { let builder = rustpython::Interpreter::builder(Default::default()); - let def = rust_py_module::module_def(&builder.ctx); - let interp = builder.init_stdlib().add_native_module(def).build(); + let def = rust_py_module::module_def(unsafe { builder.context() }); + // SAFETY: module callbacks use raw objects only within their native entry. + let interp = unsafe { builder.init_stdlib().add_native_module(def) }.build(); interp.enter(|vm| { - vm.insert_sys_path(vm.new_pyobj("examples")) - .expect("add path"); + vm.exec("import sys; sys.path.insert(0, 'examples')") + .unwrap(); - let module = vm.import("call_between_rust_and_python", 0).unwrap(); - let init_fn = module.get_attr("python_callback", vm).unwrap(); - init_fn.call((), vm).unwrap(); + let module = vm.import("call_between_rust_and_python").unwrap(); + let init_fn = module.get_attr("python_callback").unwrap(); + init_fn.call(&[]).unwrap(); - let take_string_fn = module.get_attr("take_string", vm).unwrap(); + let take_string_fn = module.get_attr("take_string").unwrap(); take_string_fn - .call((String::from("Rust string sent to python"),), vm) + .call(&[vm.new_str("Rust string sent to python").unwrap()]) .unwrap(); }) } diff --git a/examples/custom_tls_providers.rs b/examples/custom_tls_providers.rs index 355f75342cc..36d61ee7790 100644 --- a/examples/custom_tls_providers.rs +++ b/examples/custom_tls_providers.rs @@ -53,15 +53,12 @@ fn main() { } let builder = Interpreter::builder(Default::default()); - let defs = stdlib_module_defs(&builder.ctx); - let result = builder - .add_native_modules(&defs) + let defs = stdlib_module_defs(unsafe { builder.context() }); + let result = unsafe { builder.add_native_modules(&defs) } .add_frozen_modules(FROZEN_STDLIB) .build() - .run(|vm| { - let scope = vm.new_scope_with_builtins(); - vm.run_block_expr(scope, SCRIPT).map(|_| ()) - }); + .run(|vm| vm.exec(SCRIPT)) + .expect("no native workers remain"); assert_eq!(0, result); } diff --git a/examples/freeze/main.rs b/examples/freeze/main.rs index 7fef112df63..de9ca77d9e4 100644 --- a/examples/freeze/main.rs +++ b/examples/freeze/main.rs @@ -1,7 +1,9 @@ use rustpython_vm as vm; -fn main() -> vm::PyResult<()> { - vm::Interpreter::without_stdlib(Default::default()).enter(run) +fn main() -> std::process::ExitCode { + // SAFETY: all native scopes and object references are released inside run. + let code = unsafe { vm::Interpreter::without_stdlib(Default::default()).run_unchecked(run) }; + vm::host_env::os::exit_code(code) } #[expect(clippy::unnecessary_wraps)] diff --git a/examples/generator.rs b/examples/generator.rs index f695f094681..ccef2a541f8 100644 --- a/examples/generator.rs +++ b/examples/generator.rs @@ -1,50 +1,33 @@ +use rustpython::InterpreterBuilderExt; use rustpython_vm as vm; use std::process::ExitCode; -use vm::{ - Interpreter, PyResult, - builtins::PyIntRef, - protocol::{PyIter, PyIterReturn}, -}; +use vm::Interpreter; -fn py_main(interp: &Interpreter) -> vm::PyResult<()> { +fn py_main(interp: &Interpreter) -> vm::embedding::Result<()> { let generator = interp.enter(|vm| { - let scope = vm.new_scope_with_builtins(); - let generator = vm.run_block_expr( - scope, - r#" -def gen(): - for i in range(10): - yield i - -gen() -"#, - )?; - Ok(generator) + vm.exec("def gen():\n yield from range(10)")?; + Ok(vm.eval("gen()")?.unbind()) })?; loop { - let r = interp.enter(|vm| { - let v = match PyIter::new(generator.clone()).next(vm)? { - PyIterReturn::Return(obj) => { - PyIterReturn::Return(obj.try_into_value::(vm)?) - } - PyIterReturn::StopIteration(x) => PyIterReturn::StopIteration(x), - }; - PyResult::Ok(v) + let value = interp.enter(|vm| { + vm.bind(&generator)? + .next()? + .map(|value| value.to_i64()) + .transpose() })?; - match r { - PyIterReturn::Return(value) => println!("{value}"), - PyIterReturn::StopIteration(_) => break, + match value { + Some(value) => println!("{value}"), + None => break, } } - Ok(()) } fn main() -> ExitCode { - let builder = vm::Interpreter::builder(Default::default()); - let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); - let interp = builder.add_native_modules(&defs).build(); + let interp = vm::Interpreter::builder(Default::default()) + .init_stdlib() + .build(); let result = py_main(&interp); - vm::host_env::os::exit_code(interp.run(|_vm| result)) + vm::host_env::os::exit_code(interp.run(|_vm| result).expect("no native workers remain")) } diff --git a/examples/hello_embed.rs b/examples/hello_embed.rs index ae56ed21bf1..baa8aa0a1ee 100644 --- a/examples/hello_embed.rs +++ b/examples/hello_embed.rs @@ -1,15 +1,6 @@ use rustpython_vm as vm; -fn main() -> vm::PyResult<()> { - vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { - let scope = vm.new_scope_with_builtins(); - let source = r#"print("Hello World!")"#; - let code_obj = vm - .compile(source, vm::compiler::Mode::Exec, "") - .map_err(|err| err.into_pyexception(vm, Some(source)))?; - - vm.run_code_obj(code_obj, scope)?; - - Ok(()) - }) +fn main() -> vm::embedding::Result<()> { + vm::Interpreter::without_stdlib(Default::default()) + .enter(|vm| vm.exec(r#"print("Hello World!")"#)) } diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs index f154c20e06f..291fc406a17 100644 --- a/examples/mini_repl.rs +++ b/examples/mini_repl.rs @@ -29,8 +29,10 @@ fn on(b: bool) { ON.store(b, Ordering::Relaxed); } -fn main() -> vm::PyResult<()> { - vm::Interpreter::without_stdlib(Default::default()).enter(run) +fn main() -> std::process::ExitCode { + // SAFETY: all native scopes and object references are released inside run. + let code = unsafe { vm::Interpreter::without_stdlib(Default::default()).run_unchecked(run) }; + vm::host_env::os::exit_code(code) } fn run(vm: &vm::VirtualMachine) -> vm::PyResult<()> { diff --git a/examples/package_embed.rs b/examples/package_embed.rs index cf2593facd8..96b00b8a996 100644 --- a/examples/package_embed.rs +++ b/examples/package_embed.rs @@ -1,30 +1,23 @@ +use rustpython::InterpreterBuilderExt; use rustpython_vm as vm; use std::process::ExitCode; -use vm::{Interpreter, builtins::PyStrRef}; +use vm::Interpreter; -fn py_main(interp: &Interpreter) -> vm::PyResult { +fn py_main(interp: &Interpreter) -> vm::embedding::Result { interp.enter(|vm| { - // Add local library path - vm.insert_sys_path(vm.new_pyobj("examples")) - .expect("add examples to sys.path failed"); - let module = vm.import("package_embed", 0)?; - let name_func = module.get_attr("context", vm)?; - let result = name_func.call((), vm)?; - let result: PyStrRef = result.get_attr("name", vm)?.try_into_value(vm)?; - vm::PyResult::Ok(result) + // Add local library path. + vm.exec("import sys; sys.path.insert(0, 'examples')")?; + let module = vm.import("package_embed")?; + let result = module.get_attr("context")?.call(&[])?; + result.get_attr("name")?.to_string() }) } fn main() -> ExitCode { - // Add standard library path + // Add standard library path. let mut settings = vm::Settings::default(); settings.path_list.push("Lib".to_owned()); - let builder = vm::Interpreter::builder(settings); - let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); - let interp = builder.add_native_modules(&defs).build(); - let result = py_main(&interp); - let result = result.map(|result| { - println!("name: {result}"); - }); - vm::host_env::os::exit_code(interp.run(|_vm| result)) + let interp = vm::Interpreter::builder(settings).init_stdlib().build(); + let result = py_main(&interp).map(|name| println!("name: {name}")); + vm::host_env::os::exit_code(interp.run(|_vm| result).expect("no native workers remain")) } diff --git a/extra_tests/snippets/builtin_bytearray.py b/extra_tests/snippets/builtin_bytearray.py index d897ec94782..1f4dc7c1088 100644 --- a/extra_tests/snippets/builtin_bytearray.py +++ b/extra_tests/snippets/builtin_bytearray.py @@ -913,3 +913,12 @@ class B(bytearray): empty = bytearray() empty *= sys.maxsize assert empty == bytearray() + +# Reinitialization must preserve a live buffer's storage and extent. +exported = bytearray(b"abc") +view = memoryview(exported) +assert_raises(BufferError, lambda: exported.__init__(b"x")) +assert view.tobytes() == b"abc" +view.release() +exported.__init__(b"x") +assert exported == b"x" diff --git a/extra_tests/snippets/builtin_type.py b/extra_tests/snippets/builtin_type.py index 06b40fe41f6..5742fbe1f6f 100644 --- a/extra_tests/snippets/builtin_type.py +++ b/extra_tests/snippets/builtin_type.py @@ -743,3 +743,27 @@ class StrKey(str): warnings.simplefilter("error") with assert_raises(RuntimeWarning): type("NonStringKeyRaises", (), {6: 7}) + + +# A native __new__ descriptor preserves the inherited constructor slot, while +# a direct descriptor call must still enforce its payload safety checks. +for base in (dict, list): + + class IncompatibleNew(base): + __new__ = object.__new__ + + assert IncompatibleNew() == base() + with assert_raises(TypeError): + IncompatibleNew.__new__(IncompatibleNew) + + IncompatibleNew.__new__ = lambda cls: base.__new__(cls) + assert IncompatibleNew() == base() + IncompatibleNew.__new__ = object.__new__ + with assert_raises(TypeError): + IncompatibleNew() + + class OrdinaryBuiltinNew(base): + __new__ = len + + with assert_raises(TypeError): + OrdinaryBuiltinNew() diff --git a/extra_tests/snippets/stdlib_gc.py b/extra_tests/snippets/stdlib_gc.py index 428375cd97d..613b56e27e4 100644 --- a/extra_tests/snippets/stdlib_gc.py +++ b/extra_tests/snippets/stdlib_gc.py @@ -13,6 +13,7 @@ import gc import itertools +import sys import weakref from collections import defaultdict, deque @@ -39,8 +40,11 @@ def build(): # containers keeping their items in a field of their own assert collects(deque) +assert collects(tuple) assert collects(lambda c: defaultdict(int, {"k": c})) assert collects(lambda c: classmethod(lambda cls: c)) +assert collects(lambda c: c.append) +assert collects(lambda c: c.__iter__) # iterators: the wrapper holds an iterator, and that iterator holds the # container @@ -199,4 +203,159 @@ def __del__(self): retained_cycles_are_promoted() + +def automatic_collection_visits_older_generations(): + enabled = gc.isenabled() + thresholds = gc.get_threshold() + phases = [] + + def callback(phase, info): + phases.append((phase, info["generation"])) + # A callback must not recursively start another collection. + assert gc.collect() == 0 + + gc.disable() + try: + gc.collect() + node = Node() + node.cycle = node + ref = weakref.ref(node) + gc.collect(2) + del node + gc.callbacks.append(callback) + gc.set_threshold(50, 1, 1) + gc.enable() + held = [[i] for i in range(5000)] + gc.disable() + assert ref() is None + assert len(held) == 5000 + assert {generation for phase, generation in phases} == {0, 1, 2} + assert len(phases) % 2 == 0 + for start, stop in zip(phases[::2], phases[1::2]): + assert start[0] == "start" and stop == ("stop", start[1]) + finally: + gc.callbacks.remove(callback) + gc.set_threshold(*thresholds) + if enabled: + gc.enable() + + +automatic_collection_visits_older_generations() + + +def finalizer_can_replace_edges_and_create_weakrefs(): + refs = [] + calls = [] + + class Finalized: + def __del__(self): + refs.append(weakref.ref(self, lambda ref: calls.append(ref))) + self.cycle = [self] + + node = Finalized() + node.cycle = node + del node + # The replacement list is new and may be collected in the next cycle. + gc.collect() + gc.collect() + assert len(refs) == 1 and refs[0]() is None + assert calls == refs + + +finalizer_can_replace_edges_and_create_weakrefs() + + +def immutable_tuple_chains_and_subclass_cycles(): + chain = None + for _ in range(20000): + chain = (chain,) + gc.collect() + # Releasing an untracked immutable chain still needs bounded stack usage. + del chain + + class Tuple(tuple): + pass + + obj = Tuple((1,)) + obj.cycle = obj + node = Node() + obj.node = node + ref = weakref.ref(node) + del node, obj + gc.collect() + assert ref() is None + + +immutable_tuple_chains_and_subclass_cycles() + + +def native_descriptors_release_their_references(): + if sys.implementation.name == "rustpython": + # CPython 3.14 also retains this cycle: builtin methods have no tp_clear. + container = [] + references = sys.getrefcount(container) + method = container.append + method.__module__ = method + del method + gc.collect() + assert sys.getrefcount(container) == references + + class Slotted: + __slots__ = ("value",) + + ref = weakref.ref(Slotted) + descriptor = Slotted.value + del Slotted + gc.collect() + assert ref() is descriptor.__objclass__ + del descriptor + gc.collect() + assert ref() is None + + +native_descriptors_release_their_references() + + +def weakref_callbacks_do_not_hide_cycles(): + for factory in (weakref.ref, weakref.proxy): + marker = Node() + ref = weakref.ref(marker) + items = [marker] + + def callback(_, items=items): + pass + + items.append(factory(int, callback)) + del marker, items, callback + gc.collect() + assert ref() is None + + +weakref_callbacks_do_not_hide_cycles() + + +def only_reachable_weakrefs_run_callbacks(): + for keep_ref in (False, True): + events = [] + node = Node() + roots = [node] + node.cycle = node + + def callback(_, roots=roots): + events.append("called") + + ref = weakref.ref(node, callback) + roots.append(ref) + # A live weakref whose callback owns the target keeps that target alive. + if keep_ref: + roots.clear() + del node, roots, callback + if not keep_ref: + del ref + gc.collect() + assert events == (["called"] if keep_ref else []) + + +only_reachable_weakrefs_run_callbacks() + print("ok") diff --git a/extra_tests/snippets/stdlib_io.py b/extra_tests/snippets/stdlib_io.py index 8346ddbb62d..1b0f8d7ddac 100644 --- a/extra_tests/snippets/stdlib_io.py +++ b/extra_tests/snippets/stdlib_io.py @@ -3,6 +3,19 @@ from testutils import assert_raises + +class RetainedReadBuffer(RawIOBase): + def readinto(self, buffer): + buffer[:] = b"abcd" + self.view = memoryview(buffer) + return 2 + + +retaining = RetainedReadBuffer() +assert retaining.read(4) == b"ab" +assert retaining.view.tobytes() == b"abcd" +retaining.view.release() + fi = FileIO("README.md") assert fi.seekable() bb = BufferedReader(fi) diff --git a/extra_tests/snippets/stdlib_sqlite.py b/extra_tests/snippets/stdlib_sqlite.py index 83fc7db94f0..4eff8394798 100644 --- a/extra_tests/snippets/stdlib_sqlite.py +++ b/extra_tests/snippets/stdlib_sqlite.py @@ -70,3 +70,62 @@ def finalize(self): expected = b"t5i4 3l2b1" + data[10:] assert actual == expected, f"got {actual!r}, expected {expected!r}" blob.close() + + +def check_callback_lifetimes(): + import contextvars + import gc + import threading + import weakref + + current = contextvars.ContextVar("sqlite_callback_context", default=0) + instances = [] + errors = [] + connection = sqlite.connect(":memory:", check_same_thread=False) + + class Aggregate: + def __init__(self): + instances.append(weakref.ref(self)) + self.value = 0 + + def step(self, value): + self.value += value + + def finalize(self): + return self.value + current.get() + + def register(): + try: + connection.create_function("current_context", 0, current.get) + connection.create_aggregate("total_context", 1, Aggregate) + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=register) + worker.start() + worker.join() + assert not errors, errors + + current.set(40) + assert connection.execute("SELECT current_context()").fetchone() == (40,) + assert connection.execute("SELECT total_context(2)").fetchone() == (42,) + gc.collect() + assert len(instances) == 1 and instances[0]() is None + + class FailingAggregate(Aggregate): + def finalize(self): + raise ValueError("finalize") + + connection.create_aggregate("failing", 1, FailingAggregate) + try: + connection.execute("SELECT failing(2)").fetchone() + except sqlite.OperationalError: + pass + else: + raise AssertionError("aggregate failure did not propagate") + gc.collect() + assert len(instances) == 2 and instances[1]() is None + connection.close() + + +check_callback_lifetimes() diff --git a/extra_tests/snippets/stdlib_subinterpreters.py b/extra_tests/snippets/stdlib_subinterpreters.py index 3ca0be27411..1f3ec8940f8 100644 --- a/extra_tests/snippets/stdlib_subinterpreters.py +++ b/extra_tests/snippets/stdlib_subinterpreters.py @@ -6,6 +6,55 @@ import _interpchannels import _interpreters +# Releasing an exported buffer can reenter the channel directory. Keep deadlock +# regressions in a subprocess so a locked directory cannot stall the whole suite. +operations = ["destroy", "timeout"] +if sys.implementation.name == "rustpython": + # CPython 3.14 also deadlocks on these two release paths. + operations += ["close", "drop"] +for operation in operations: + subprocess.run( + [ + sys.executable, + "-c", + """ +import _interpchannels as channels +import _interpreters +import sys + +released = [] + +class Exporter(bytearray): + def __del__(self): + channels.list_all() + released.append(True) + +channel = channels.create(3) +operation = sys.argv[1] +if operation == 'timeout': + try: + channels.send_buffer(channel, Exporter(b'data'), timeout=0) + except TimeoutError: + pass + else: + raise AssertionError('send should time out') +else: + channels.send_buffer(channel, Exporter(b'data'), blocking=False) + assert not released + if operation == 'destroy': + channels.destroy(channel) + elif operation == 'close': + channels.close(channel, force=True) + elif operation == 'drop': + del channel +assert released == [True], released +""", + operation, + ], + check=True, + timeout=15, + ) + def roundtrip_buffer(view): cid = _interpchannels.create(3) @@ -18,6 +67,23 @@ def roundtrip_buffer(view): _interpchannels.destroy(cid) +if sys.implementation.name == "rustpython": + # CPython also retains this cycle. Returning an export to its owner lets + # RustPython expose the local exporter edge to the ordinary cycle collector. + import gc + import weakref + + class BufferOwner(bytearray): + pass + + owner = BufferOwner(b"abcd") + owner_ref = weakref.ref(owner) + owner.view = roundtrip_buffer(memoryview(owner)) + del owner + gc.collect() + assert owner_ref() is None + + data = bytearray(range(12)) for view in (memoryview(data)[2:10:2], memoryview(data).cast("I")): received = roundtrip_buffer(view) @@ -117,3 +183,231 @@ def receive(): encoding="utf-8", ) subprocess.run([sys.executable, str(script)], check=True, timeout=30) + + +# Native caches and weakref registries belong to an interpreter even when +# their Rust definitions or immutable referents are shared. +import _json +import _thread +import abc +import csv +import io +import os +import struct +import time +import weakref +from collections.abc import Sequence +from concurrent import interpreters + +# A receiving view shares writable native storage without keeping a foreign +# Python payload usable. The bytes also outlive the sending interpreter. +interp = interpreters.create() +channel = _interpchannels.create(3) +try: + interp.exec(f""" +import array, io, mmap, _interpchannels +backings = [bytearray(b'abcd'), array.array('B', b'abcd'), + io.BytesIO(b'abcd'), mmap.mmap(-1, 4)] +backings[-1][:] = b'abcd' +views = [memoryview(backings[0]), memoryview(backings[1]), + backings[2].getbuffer(), memoryview(backings[3])] +for view in views: + _interpchannels.send_buffer({int(channel)}, view, blocking=False) +""") + received_views = [_interpchannels.recv(channel)[0] for _ in range(4)] + for received in received_views: + received[1] = ord("Z") + interp.exec("assert all(view.tobytes() == b'aZcd' for view in views)") +finally: + interp.close() + _interpchannels.destroy(channel) +for received in received_views: + assert received.tobytes() == b"aZcd" + alias = roundtrip_buffer(received) + assert alias == received + alias[1:] = received[:-1] + assert received.tobytes() == b"aaZc" + alias[:] = b"aZcd" + alias.release() + received[2] = ord("Y") + assert received.tobytes() == b"aZYd" + received.release() + + +def isolation_callback(ref): + pass + + +isolation_callback._gc_owner_probe = True +owned_ref = weakref.ref(object, isolation_callback) +basic_ref = weakref.ref(object) +private_types = ( + csv.Error, + io.UnsupportedOperation, + struct.error, + ExceptionGroup, + _json.make_scanner, + _json.make_encoder, + _thread._ExceptHookArgs, + time.struct_time, + os.stat_result, +) +for typ in private_types: + typ._gc_owner_probe = [] + + +class PrivateSubclass: + _gc_owner_probe = True + + +# Even immutable native classes contain mutable Python values. In particular, +# the bound __new__ function permits arbitrary values in its __module__ field. +saved_new_module = list.__new__.__module__ +list.__new__.__module__ = ["main"] + + +interp = interpreters.create() +try: + saved_abc_token = abc.get_cache_token() + saved_int_flags = int.__flags__ + saved_int_sequence = issubclass(int, Sequence) + saved_object_flags = object.__flags__ + saved_csv_limit = csv.field_size_limit() + csv.register_dialect("_gc_main_dialect", delimiter="|") + interp.exec(""" +import csv, io, struct, weakref, _json, _thread, os, time +assert '_gc_main_dialect' not in csv.list_dialects() +csv.register_dialect('_gc_worker_dialect', delimiter=';') +csv.field_size_limit(17) +from collections.abc import Sequence +class SequenceInt(int): + pass +Sequence.register(SequenceInt) +assert SequenceInt.__flags__ & (1 << 5) +old_int_flags = int.__flags__ +Sequence.register(int) +assert int.__flags__ == old_int_flags +assert issubclass(int, Sequence) +type.__dict__['__abstractmethods__'].__set__(object, frozenset({'foreign_method'})) +assert object.__flags__ & (1 << 20) +assert list.__new__.__module__ != ['main'] +list.__new__.__module__ = ['worker'] +for typ in (csv.Error, io.UnsupportedOperation, struct.error, ExceptionGroup, + _json.make_scanner, _json.make_encoder, _thread._ExceptHookArgs, + time.struct_time, os.stat_result): + assert not hasattr(typ, '_gc_owner_probe'), typ + typ._gc_other_owner_probe = [] +assert all(not getattr(ref.__callback__, '_gc_owner_probe', False) + for ref in weakref.getweakrefs(object)) +assert all(not getattr(cls, '_gc_owner_probe', False) + for cls in object.__subclasses__()) +basic_ref = weakref.ref(object) +assert weakref.ref(object) is basic_ref +""") + assert abc.get_cache_token() == saved_abc_token + assert int.__flags__ == saved_int_flags + assert issubclass(int, Sequence) == saved_int_sequence + # CPython still changes this shared static flag across interpreters. + if sys.implementation.name == "rustpython": + assert object.__flags__ == saved_object_flags + assert csv.field_size_limit() == saved_csv_limit + assert "_gc_worker_dialect" not in csv.list_dialects() + assert weakref.ref(object) is basic_ref + assert any(ref is owned_ref for ref in weakref.getweakrefs(object)) + assert PrivateSubclass in object.__subclasses__() + assert list.__new__.__module__ == ["main"] + assert list.__dict__["__new__"] is list.__new__ + for typ in private_types: + assert not hasattr(typ, "_gc_other_owner_probe"), typ +finally: + interp.exec("type.__dict__['__abstractmethods__'].__set__(object, frozenset())") + interp.close() + list.__new__.__module__ = saved_new_module + csv.unregister_dialect("_gc_main_dialect") + for typ in private_types: + del typ._gc_owner_probe + + +# AST classes have mutable namespaces, slots and MROs; each interpreter must +# own the complete class, including references stored in _field_types. +import ast + +ast.Constant._gc_owner_probe = [] +saved_ast_bases = ast.Constant.__bases__ +saved_ast_repr = ast.Constant.__dict__.get("__repr__") +ast.Constant.__bases__ = (ast.stmt,) +ast.Constant.__repr__ = lambda self: "main constant" +interp = interpreters.create() +try: + interp.exec(""" +import ast, gc +assert not hasattr(ast.Constant, '_gc_owner_probe') +assert ast.Constant.__bases__ == (ast.expr,) +assert repr(ast.Constant(value=1)) != 'main constant' +assert ast.Constant is type(ast.parse('1').body[0].value) +assert ast.Assign._field_types['value'] is ast.expr +ast.Constant.__repr__ = lambda self: 'worker constant' +assert repr(ast.Constant(value=1)) == 'worker constant' +assert all(not hasattr(obj, '_gc_owner_probe') + for obj in gc.get_objects() if type(obj) is type) +""") + assert repr(ast.Constant(value=1)) == "main constant" + assert ast.Constant.__bases__ == (ast.stmt,) +finally: + interp.close() + ast.Constant.__bases__ = saved_ast_bases + if saved_ast_repr is None: + del ast.Constant.__repr__ + else: + ast.Constant.__repr__ = saved_ast_repr + del ast.Constant._gc_owner_probe + + +try: + import sqlite3 +except ImportError: + pass # Optional native module. +else: + sqlite3.register_converter("_GC_OWNER_PROBE", bytes) + interp = interpreters.create() + try: + interp.exec( + "import sqlite3; assert '_GC_OWNER_PROBE' not in sqlite3.converters" + ) + finally: + interp.close() + del sqlite3.converters["_GC_OWNER_PROBE"] + + +# RustPython implements these codecs by importing Python functions. Warming +# that cache must not reuse a function (and its globals) from another VM. +if sys.implementation.name == "rustpython": + import _codecs + + import _pycodecs + + original_charmap_encode = _pycodecs.charmap_encode + _pycodecs.charmap_encode = lambda *args: (b"private", 1) + try: + assert _codecs.charmap_encode("x") == (b"private", 1) + interp = interpreters.create() + try: + interp.exec( + "import _codecs; assert _codecs.charmap_encode('x') == (b'x', 1)" + ) + finally: + interp.close() + finally: + _pycodecs.charmap_encode = original_charmap_encode + + +# Entering a subinterpreter must start its own frame chain and restore the +# caller's chain afterwards. +caller_frame = sys._getframe() +interp = interpreters.create() +try: + interp.exec("import sys; assert sys._getframe().f_back is None") +finally: + interp.close() +assert sys._getframe() is caller_frame +del caller_frame diff --git a/extra_tests/snippets/stdlib_threading_gc_fork.py b/extra_tests/snippets/stdlib_threading_gc_fork.py index cd00cf00983..264aba6fc0a 100644 --- a/extra_tests/snippets/stdlib_threading_gc_fork.py +++ b/extra_tests/snippets/stdlib_threading_gc_fork.py @@ -14,6 +14,8 @@ import gc import os +import subprocess +import sys import threading import time @@ -60,3 +62,53 @@ def churn(): w.join() print("ok") + + +# The surviving native stack can hold the collector or type-mutation guard. +# Child repair must preserve it until that original stack releases it. +for source in ( + """ +gc.disable() +pid = -1 +def callback(phase, info): + global pid + if phase == 'start' and pid == -1: + pid = os.fork() +gc.callbacks.append(callback) +gc.collect() +gc.callbacks.remove(callback) +""", + """ +pid = -1 +class Meta(type): + def mro(cls): + global pid + if pid == -1: + pid = os.fork() + return super().mro() +class Sample(metaclass=Meta): + pass +Sample.value = 42 +assert Sample.value == 42 +""", +): + subprocess.run( + [ + sys.executable, + "-c", + "import gc, os\n" + + source + + """ +for _ in range(3): + node = [] + node.append(node) + del node + gc.collect() +if pid == 0: + os._exit(0) +assert os.waitpid(pid, 0)[1] == 0 +""", + ], + check=True, + timeout=30, + ) diff --git a/extra_tests/snippets/syntax_match.py b/extra_tests/snippets/syntax_match.py index 8868cb93c38..73627d14d4c 100644 --- a/extra_tests/snippets/syntax_match.py +++ b/extra_tests/snippets/syntax_match.py @@ -149,3 +149,19 @@ def test_mapping_comprehensive(): test_mapping_comprehensive() + +# Native collection patterns must not depend on ABC registration changing flags. +from collections import deque +from types import MappingProxyType + +match deque([1, 2]): + case [first, *rest]: + assert (first, rest) == (1, [2]) + case _: + assert False, "deque must match sequence patterns" + +match MappingProxyType({"a": 1, "b": 2}): + case {"a": value, **rest}: + assert (value, rest) == (1, {"b": 2}) + case _: + assert False, "mappingproxy must match mapping patterns" diff --git a/src/interpreter.rs b/src/interpreter.rs index a8b03cba0b0..3101f4fbe63 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -13,24 +13,25 @@ pub trait InterpreterBuilderExt { impl InterpreterBuilderExt for InterpreterBuilder { #[cfg(feature = "stdlib")] fn init_stdlib(self) -> Self { - let defs = rustpython_stdlib::stdlib_module_defs(&self.ctx); - let builder = self.add_native_modules(&defs); + let defs = rustpython_stdlib::stdlib_module_defs(unsafe { self.context() }); + // SAFETY: the standard library follows native owner/attachment rules. + let builder = unsafe { self.add_native_modules(&defs) }; #[cfg(all(feature = "ssl-rustls-aws-lc", not(target_arch = "wasm32")))] - let builder = builder.init_hook(install_default_tls_provider); + let builder = unsafe { builder.init_hook(install_default_tls_provider) }; cfg_select! { feature = "freeze-stdlib" => { builder .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) - .init_hook(set_frozen_stdlib_dir) + .configure(set_frozen_stdlib_dir) } - _ => builder.init_hook(setup_dynamic_stdlib), + _ => builder.configure(setup_dynamic_stdlib), } } } #[cfg(all(feature = "ssl-rustls-aws-lc", not(target_arch = "wasm32")))] -fn install_default_tls_provider(_vm: &mut crate::VirtualMachine) { +fn install_default_tls_provider(_vm: &crate::VirtualMachine) { use rustls::crypto::aws_lc_rs; use rustpython_stdlib::ssl::providers::CryptoExt; @@ -54,19 +55,13 @@ fn install_default_tls_provider(_vm: &mut crate::VirtualMachine) { /// Set stdlib_dir for frozen standard library #[cfg(all(feature = "stdlib", feature = "freeze-stdlib"))] -fn set_frozen_stdlib_dir(vm: &mut crate::VirtualMachine) { - use rustpython_vm::common::rc::PyRc; - - let state = PyRc::get_mut(&mut vm.state).unwrap(); - state.config.paths.stdlib_dir = Some(rustpython_pylib::LIB_PATH.to_owned()); +fn set_frozen_stdlib_dir(config: &mut rustpython_vm::vm::PyConfig) { + config.paths.stdlib_dir = Some(rustpython_pylib::LIB_PATH.to_owned()); } /// Setup dynamic standard library loading from filesystem #[cfg(all(feature = "stdlib", not(feature = "freeze-stdlib")))] -fn setup_dynamic_stdlib(vm: &mut crate::VirtualMachine) { - use rustpython_vm::common::rc::PyRc; - - let state = PyRc::get_mut(&mut vm.state).unwrap(); +fn setup_dynamic_stdlib(config: &mut rustpython_vm::vm::PyConfig) { let paths: Vec = collect_stdlib_paths() .into_iter() .map(|p| { @@ -87,12 +82,12 @@ fn setup_dynamic_stdlib(vm: &mut crate::VirtualMachine) { // Set stdlib_dir to the first stdlib path if available if let Some(first_path) = paths.first() { - state.config.paths.stdlib_dir = Some(first_path.clone()); + config.paths.stdlib_dir = Some(first_path.clone()); } // Insert at the beginning so stdlib comes before user paths for path in paths.into_iter().rev() { - state.config.paths.module_search_paths.insert(0, path); + config.paths.module_search_paths.insert(0, path); } } diff --git a/src/lib.rs b/src/lib.rs index ecad0a128a6..3a952914131 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,10 +11,9 @@ //! //! fn main() -> std::process::ExitCode { //! let builder = InterpreterBuilder::new().init_stdlib(); -//! // Add a native module using builder.ctx -//! let my_mod_def = my_mod::module_def(&builder.ctx); -//! let builder = builder -//! .add_native_module(my_mod_def) +//! // SAFETY: these native callbacks retain no Python references. +//! let my_mod_def = my_mod::module_def(unsafe { builder.context() }); +//! let builder = unsafe { builder.add_native_module(my_mod_def) } //! // Add a frozen module //! .add_frozen_modules(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); //! @@ -120,14 +119,17 @@ pub fn run(mut builder: InterpreterBuilder) -> ExitCode { builder = builder.settings(settings); let interp = builder.interpreter(); - let exitcode = cfg_select! { - feature = "capi" => {{ - let local_vm = interp.enter(|vm| vm.new_thread()); - rustpython_capi::init_main_interpreter(interp); - let result = local_vm.run(|vm| run_rustpython(vm, run_mode)); - rustpython_capi::get_main_interpreter().take().unwrap().finalize(result.err()) - }}, - _ => interp.run(move |vm| run_rustpython(vm, run_mode)), + // SAFETY: the CLI keeps raw references within this interpreter execution. + let exitcode = unsafe { + cfg_select! { + feature = "capi" => {{ + let local_vm = interp.new_thread(); + rustpython_capi::init_main_interpreter(interp); + let result = local_vm.run_unchecked(|vm| run_rustpython(vm, run_mode)); + rustpython_capi::get_main_interpreter().take().unwrap().finalize_unchecked(result.err()) + }}, + _ => interp.run_unchecked(move |vm| run_rustpython(vm, run_mode)), + } }; rustpython_vm::host_env::os::exit_code(exitcode) @@ -469,21 +471,24 @@ mod tests { #[test] fn run_script() { - interpreter().enter(|vm| { - vm.unwrap_pyresult((|| { - let scope = vm.new_scope_with_main()?; - // test file run - run_file(vm, scope, "extra_tests/snippets/dir_main/__main__.py")?; - - #[cfg(feature = "host_env")] - { + // SAFETY: the test releases scopes and exceptions before leaving the entry. + unsafe { + interpreter().enter_unchecked(|vm| { + vm.unwrap_pyresult((|| { let scope = vm.new_scope_with_main()?; - // test module run (directory with __main__.py) - run_file(vm, scope, "extra_tests/snippets/dir_main")?; - } - - Ok(()) - })()); - }) + // test file run + run_file(vm, scope, "extra_tests/snippets/dir_main/__main__.py")?; + + #[cfg(feature = "host_env")] + { + let scope = vm.new_scope_with_main()?; + // test module run (directory with __main__.py) + run_file(vm, scope, "extra_tests/snippets/dir_main")?; + } + + Ok(()) + })()); + }) + } } } diff --git a/tests/benchmark_scripts.rs b/tests/benchmark_scripts.rs index db7af23a011..c4bd7dec721 100644 --- a/tests/benchmark_scripts.rs +++ b/tests/benchmark_scripts.rs @@ -31,19 +31,24 @@ fn pyperformance_benchmarks_run_in_rustpython() { .unwrap_or_else(|err| panic!("failed to read {}: {err}", path.display())); let source_path = path.to_string_lossy(); - InterpreterBuilder::new() - .init_stdlib() - .interpreter() - .enter(|vm| { - let code = vm - .compile(&source, rustpython::vm::compiler::Mode::Exec, &*source_path) - .unwrap_or_else(|err| panic!("failed to compile {}: {err}", path.display())); - let scope = vm.new_scope_with_builtins(); - if let Err(err) = vm.run_code_obj(code, scope) { - vm.print_exception(&err); - panic!("failed to execute {}", path.display()); - } - }); + // SAFETY: the test retains no raw references after execution. + unsafe { + InterpreterBuilder::new() + .init_stdlib() + .interpreter() + .enter_unchecked(|vm| { + let code = vm + .compile(&source, rustpython::vm::compiler::Mode::Exec, &*source_path) + .unwrap_or_else(|err| { + panic!("failed to compile {}: {err}", path.display()) + }); + let scope = vm.new_scope_with_builtins(); + if let Err(err) = vm.run_code_obj(code, scope) { + vm.print_exception(&err); + panic!("failed to execute {}", path.display()); + } + }); + } } } @@ -73,13 +78,16 @@ assert sys.modules.get("_pickle", missing) is pickle_accelerator assert decimal.getcontext().prec == decimal_precision "#; - InterpreterBuilder::new() - .init_stdlib() - .interpreter() - .enter(|vm| { - if let Err(err) = vm.run_simple_string(source) { - vm.print_exception(&err); - panic!("pyperformance benchmarks leaked interpreter state"); - } - }); + // SAFETY: the test releases exceptions inside the native entry. + unsafe { + InterpreterBuilder::new() + .init_stdlib() + .interpreter() + .enter_unchecked(|vm| { + if let Err(err) = vm.run_simple_string(source) { + vm.print_exception(&err); + panic!("pyperformance benchmarks leaked interpreter state"); + } + }); + } }