diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index b29e39b0a8a..ee489393789 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -15,7 +15,7 @@ use crate::{ types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; -use core::cell::Cell; +use core::cell::UnsafeCell; use core::ptr::NonNull; use malachite_bigint::{BigInt, ToBigInt}; use num_complex::Complex64; @@ -36,7 +36,7 @@ impl PyFloat { } thread_local! { - static FLOAT_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; + static FLOAT_FREELIST: UnsafeCell> = const { UnsafeCell::new(crate::object::FreeList::new()) }; } impl PyPayload for PyFloat { @@ -48,34 +48,18 @@ impl PyPayload for PyFloat { ctx.types.float_type } + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_float(self.value).into() + } + #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - FLOAT_FREELIST - .try_with(|fl| { - let mut list = fl.take(); - let stored = if list.len() < Self::MAX_FREELIST { - list.push(obj); - true - } else { - false - }; - fl.set(list); - stored - }) - .unwrap_or(false) + unsafe { crate::object::FreeList::push_local(&FLOAT_FREELIST, obj) } } #[inline] unsafe fn freelist_pop(_payload: &Self) -> Option> { - FLOAT_FREELIST - .try_with(|fl| { - let mut list = fl.take(); - let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); - fl.set(list); - result - }) - .ok() - .flatten() + crate::object::FreeList::pop_local(&FLOAT_FREELIST) } } diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index c9b687168d3..19257daa987 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -20,7 +20,7 @@ use crate::{ types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; use alloc::fmt; -use core::cell::Cell; +use core::cell::UnsafeCell; use core::ops::{Neg, Not}; use core::ptr::NonNull; use malachite_bigint::{BigInt, Sign}; @@ -52,7 +52,7 @@ where // spell-checker:ignore MAXFREELIST thread_local! { - static INT_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; + static INT_FREELIST: UnsafeCell> = const { UnsafeCell::new(crate::object::FreeList::new()) }; } impl PyPayload for PyInt { @@ -70,32 +70,12 @@ impl PyPayload for PyInt { #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - INT_FREELIST - .try_with(|fl| { - let mut list = fl.take(); - let stored = if list.len() < Self::MAX_FREELIST { - list.push(obj); - true - } else { - false - }; - fl.set(list); - stored - }) - .unwrap_or(false) + unsafe { crate::object::FreeList::push_local(&INT_FREELIST, obj) } } #[inline] unsafe fn freelist_pop(_payload: &Self) -> Option> { - INT_FREELIST - .try_with(|fl| { - let mut list = fl.take(); - let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); - fl.set(list); - result - }) - .ok() - .flatten() + crate::object::FreeList::pop_local(&INT_FREELIST) } } diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 30ac9ea3a65..4316eaf100b 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -264,6 +264,29 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { unsafe { trashcan::end() }; } } +/// Dealloc for payloads that have a freelist and no `tp_clear`. An instance +/// of exactly the base class with nothing attached — no `__del__`, weakrefs, +/// dict or member slots, GC tracking or QSBR publication — needs nothing from +/// `default_dealloc` but its freelist push, so it gets there directly instead +/// of through the `__del__`/weakref/trashcan/clear machinery. Anything else +/// takes `default_dealloc`, which re-checks all of it. +pub(super) unsafe fn freelist_dealloc(obj: *mut PyObject) { + let obj_ref = unsafe { &*(obj as *const PyObject) }; + let typ = obj_ref.class(); + let plain = core::ptr::eq(typ, T::class(crate::vm::Context::genesis())) + && typ.slots.del.load().is_none() + && !typ.slots.flags.intersects( + crate::types::PyTypeFlags::HAS_DICT | crate::types::PyTypeFlags::HAS_WEAKREF, + ) + && typ.slots.member_count == 0 + && !obj_ref.is_gc_tracked() + && !obj_ref.0.ref_count.is_published(); + if plain && unsafe { T::freelist_push(obj) } { + return; + } + unsafe { default_dealloc::(obj) } +} + pub(super) unsafe fn debug_obj( x: &PyObject, f: &mut fmt::Formatter<'_>, @@ -1394,6 +1417,53 @@ impl Drop for FreeList { } } +impl FreeList { + /// Push a dead husk onto this thread's freelist in `key`, unless it is + /// full or the thread-local is already gone. + /// + /// The list is edited in place rather than moved out of a `Cell` and back, + /// which copied the whole `Vec` twice per object. Nothing between taking + /// the `&mut` and dropping it can re-enter: `Vec::push` only calls the + /// allocator, never Python code or another freelist. + /// + /// # Safety + /// Same contract as [`PyPayload::freelist_push`]. + #[inline] + pub(crate) unsafe fn push_local( + key: &'static std::thread::LocalKey>, + obj: *mut PyObject, + ) -> bool { + key.try_with(|cell| { + // SAFETY: see above; no other borrow of this cell is live. + let list = unsafe { &mut *cell.get() }; + if list.items.len() < T::MAX_FREELIST { + list.items.push(obj); + true + } else { + false + } + }) + .unwrap_or(false) + } + + /// Pop a husk from this thread's freelist in `key`, editing it in place + /// (see [`Self::push_local`]). + #[inline] + pub(crate) fn pop_local( + key: &'static std::thread::LocalKey>, + ) -> Option> { + key.try_with(|cell| { + // SAFETY: `Vec::pop` cannot re-enter; no other borrow is live. + let list = unsafe { &mut *cell.get() }; + list.items + .pop() + .map(|p| unsafe { NonNull::new_unchecked(p) }) + }) + .ok() + .flatten() + } +} + impl core::ops::Deref for FreeList { type Target = Vec<*mut PyObject>; fn deref(&self) -> &Self::Target { @@ -2746,6 +2816,41 @@ impl PyRef { Self { ptr } } + + /// `new_ref` for an instance of exactly `T::class(ctx)`, a static type + /// whose instances carry no dict. `default_dealloc` only pushes instances + /// of exactly that class onto `T`'s freelist, so a reused husk already + /// points at it: the dict and heap-type checks, the type swap and the + /// type reference clone `new_ref` does for an arbitrary class are skipped. + #[inline(always)] + pub(crate) fn new_exact_ref(payload: T, ctx: &crate::vm::Context) -> Self { + let class = T::class(ctx); + debug_assert!(class.heaptype_ext.is_none()); + debug_assert!( + !class + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + ); + let Some(cached) = (unsafe { T::freelist_pop(&payload) }) else { + return Self::new_ref(payload, class.to_owned(), None); + }; + let inner = cached.as_ptr() as *mut PyInner; + unsafe { + core::ptr::write(&mut (*inner).ref_count, RefCount::new()); + (*inner).gc_bits.store(0, Ordering::Relaxed); + core::ptr::drop_in_place(&mut (*inner).payload); + core::ptr::write(&mut (*inner).payload, payload); + debug_assert!(core::ptr::eq(&*(*inner).typ, class)); + } + let ptr = unsafe { NonNull::new_unchecked(inner.cast::>()) }; + if ::HAS_TRAVERSE && !T::NEW_REF_UNTRACKED { + unsafe { + crate::gc_state::track_new_object(ptr.cast()); + } + } + Self { ptr } + } } impl PyRef diff --git a/crates/vm/src/object/traverse_object.rs b/crates/vm/src/object/traverse_object.rs index 57542637788..c747d12da72 100644 --- a/crates/vm/src/object/traverse_object.rs +++ b/crates/vm/src/object/traverse_object.rs @@ -5,7 +5,7 @@ use crate::{ PyObject, PyObjectRef, object::{ Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, default_dealloc, - try_clear_obj, try_traverse_obj, + freelist_dealloc, try_clear_obj, try_traverse_obj, }, }; @@ -26,7 +26,13 @@ impl PyObjVTable { pub(super) const fn of() -> &'static Self { &Self { typeid: T::PAYLOAD_TYPE_ID, - dealloc: default_dealloc::, + dealloc: const { + if T::HAS_FREELIST && !T::HAS_CLEAR { + freelist_dealloc:: + } else { + default_dealloc:: + } + }, debug: debug_obj::, trace: const { if T::HAS_TRAVERSE { diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 76a4b44f68b..13325ed0691 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -469,7 +469,7 @@ impl Context { let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; return self.int_cache_pool[inner_idx].clone(); } - PyInt::from(i).into_ref(self) + PyRef::new_exact_ref(PyInt::from(i), self) } /// Borrow a cached small integer whose lifetime is tied to this context. @@ -493,7 +493,7 @@ impl Context { #[inline] pub fn new_float(&self, value: f64) -> PyRef { - PyFloat::from(value).into_ref(self) + PyRef::new_exact_ref(PyFloat::from(value), self) } #[inline] diff --git a/extra_tests/snippets/vm_freelist.py b/extra_tests/snippets/vm_freelist.py new file mode 100644 index 00000000000..e47dd094ff6 --- /dev/null +++ b/extra_tests/snippets/vm_freelist.py @@ -0,0 +1,92 @@ +import gc +import weakref + +# Objects of exactly int/float/complex/range are recycled through per-type +# freelists; everything else about deallocation must behave as before. + + +## Recycled objects carry the right value and type + + +def churn_ints(n): + total = 0 + for i in range(1000, 1000 + n): + total += i + return total + + +assert churn_ints(5000) == sum(range(1000, 6000)) +seen = [i for i in range(250, 270)] +assert seen == list(range(250, 270)) +for i in range(-5, 257): + assert i is int(str(i)), i + +floats = [x * 1.5 for x in range(2000)] +assert floats[-1] == 1999 * 1.5 +assert all(type(f) is float for f in floats) +complexes = [complex(i, -i) for i in range(2000)] +assert complexes[123] == complex(123, -123) +ranges = [range(i, i + 3) for i in range(2000)] +assert ranges[77] == range(77, 80) and list(ranges[5]) == [5, 6, 7] +del floats, complexes, ranges + + +## Subclass instances take the full path: __del__ and weakrefs still work + + +class DelInt(int): + deleted = 0 + + def __del__(self): + DelInt.deleted += 1 + + +for i in range(100): + DelInt(i + 1000) +gc.collect() +assert DelInt.deleted == 100, DelInt.deleted + + +class RefFloat(float): + pass + + +callbacks = [] +obj = RefFloat(12345.5) +ref = weakref.ref(obj, lambda r: callbacks.append(r)) +del obj +gc.collect() +assert ref() is None +assert len(callbacks) == 1 + + +class DelFloat(float): + deleted = 0 + + def __del__(self): + DelFloat.deleted += 1 + + +for i in range(50): + DelFloat(i + 0.5) +gc.collect() +assert DelFloat.deleted == 50, DelFloat.deleted + + +## Subclass husks never come back as plain ints + + +class Plain(int): + pass + + +for i in range(500): + Plain(i + 1000) +for i in range(1000, 1500): + x = i * 1 + assert type(x) is int, type(x) + assert x == i + +# bool shares int's payload; its singletons are untouched by the churn. +assert True + 0 == 1 and type(True + 0) is int +assert type(True) is bool and True is (1 == 1)