From 4c030e9e679546fefb85affbbbae9b19e0f32d5c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:26:55 +0900 Subject: [PATCH 1/7] Add object freelist infrastructure and PyFloat freelist Add freelist_push/freelist_pop hooks to PyPayload trait with default no-op implementations. Modify default_dealloc to offer dead objects to the freelist before freeing memory. Modify PyRef::new_ref to try popping from the freelist before allocating. Implement thread-local freelist for PyFloat (max 100 entries). Float objects are ideal candidates: fixed-size payload (f64), no GC tracking, no instance dict, trivial Drop. Benchmark improvement on float-heavy workloads: float_arith: ~12% faster float_list: ~17% faster --- crates/vm/src/builtins/float.rs | 33 +++++++++++++++++++++++++++++++++ crates/vm/src/object/core.rs | 33 +++++++++++++++++++++++++++++++-- crates/vm/src/object/payload.rs | 23 +++++++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 7ff47328d34..af62a2db0b3 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -15,6 +15,8 @@ use crate::{ protocol::PyNumberMethods, types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; +use core::cell::Cell; +use core::ptr::NonNull; use malachite_bigint::{BigInt, ToBigInt}; use num_complex::Complex64; use num_traits::{Signed, ToPrimitive, Zero}; @@ -32,11 +34,42 @@ impl PyFloat { } } +const PYFLOAT_MAXFREELIST: usize = 100; + +thread_local! { + static FLOAT_FREELIST: Cell> = const { Cell::new(Vec::new()) }; +} + impl PyPayload for PyFloat { #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.float_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + FLOAT_FREELIST.with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < PYFLOAT_MAXFREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + } + + #[inline] + unsafe fn freelist_pop() -> Option> { + FLOAT_FREELIST.with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + } } impl ToPyObject for f64 { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 41ddfa26b2e..5ade718ba50 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -186,8 +186,10 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { unsafe { clear_fn(obj, &mut edges) }; } - // Deallocate the object memory - drop(unsafe { Box::from_raw(obj as *mut PyInner) }); + // Try to store in freelist for reuse; otherwise deallocate. + if !unsafe { T::freelist_push(obj) } { + drop(unsafe { Box::from_raw(obj as *mut PyInner) }); + } // Drop child references - may trigger recursive destruction. // The object is already deallocated, so circular refs are broken. @@ -807,6 +809,15 @@ impl PyInner { } } +/// Drop a freelist-cached object, properly deallocating the `PyInner`. +/// +/// # Safety +/// `ptr` must point to a valid `PyInner` allocation that was stored in a freelist. +#[allow(dead_code)] +pub(crate) unsafe fn drop_freelist_object(ptr: *mut PyObject) { + drop(unsafe { Box::from_raw(ptr as *mut PyInner) }); +} + /// The `PyObjectRef` is one of the most used types. It is a reference to a /// python object. A single python object can have multiple references, and /// this reference counting is accounted for by this type. Use the `.clone()` @@ -1720,6 +1731,24 @@ impl PyRef { pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option) -> Self { let has_dict = dict.is_some(); let is_heaptype = typ.heaptype_ext.is_some(); + + // Try to reuse from freelist (exact type only, no dict, no heaptype) + if !has_dict && !is_heaptype { + if let Some(cached) = unsafe { T::freelist_pop() } { + let inner = cached.as_ptr() as *mut PyInner; + unsafe { + core::ptr::write(&mut (*inner).ref_count, RefCount::new()); + (*inner).gc_bits.store(0, Ordering::Relaxed); + core::ptr::write(&mut (*inner).payload, payload); + // typ, vtable, dict(None), weak_list, slots are preserved + } + // Drop the caller's typ since the cached object already holds one + drop(typ); + let ptr = unsafe { NonNull::new_unchecked(inner.cast::>()) }; + return Self { ptr }; + } + } + let inner = Box::into_raw(PyInner::new(payload, typ, dict)); let ptr = unsafe { NonNull::new_unchecked(inner.cast::>()) }; diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 98c61817568..ad3851f8e49 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -5,6 +5,7 @@ use crate::{ types::PyTypeFlags, vm::{Context, VirtualMachine}, }; +use core::ptr::NonNull; cfg_if::cfg_if! { if #[cfg(feature = "threading")] { @@ -46,6 +47,28 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { fn class(ctx: &Context) -> &'static Py; + /// Try to push a dead object onto this type's freelist for reuse. + /// Returns true if the object was stored (caller must NOT free the memory). + /// + /// # Safety + /// `obj` must be a valid pointer to a `PyInner` with refcount 0, + /// after `drop_slow_inner` and `tp_clear` have already run. + #[inline] + unsafe fn freelist_push(_obj: *mut PyObject) -> bool { + false + } + + /// Try to pop a pre-allocated object from this type's freelist. + /// The returned pointer still has the old payload; the caller must + /// reinitialize `ref_count`, `gc_bits`, and `payload`. + /// + /// # Safety + /// The returned pointer (if Some) points to uninitialized/stale payload. + #[inline] + unsafe fn freelist_pop() -> Option> { + None + } + #[inline] fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef where From ad496386ee2fedfebc1a41bafbcbcb844e191bb5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:34:34 +0900 Subject: [PATCH 2/7] Add PyList freelist and GC-safe freelist infrastructure Add HAS_FREELIST const to PyPayload trait. For freelist types, GC untracking is done immediately (not deferred) during dealloc to prevent race conditions when the object is reused. Restructure new_ref so freelist-popped objects also get GC tracked, enabling freelist support for GC-tracked types like PyList. Add drop_in_place for old payload before writing new one, required for types with non-trivial Drop (e.g. PyList's RwLock). Implement thread-local freelist for PyList (max 80 entries). --- crates/vm/src/builtins/float.rs | 2 ++ crates/vm/src/builtins/list.rs | 35 +++++++++++++++++++++++++++++++ crates/vm/src/object/core.rs | 37 ++++++++++++++++++++------------- crates/vm/src/object/payload.rs | 5 +++++ 4 files changed, 65 insertions(+), 14 deletions(-) diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index af62a2db0b3..30bc028abd7 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -41,6 +41,8 @@ thread_local! { } impl PyPayload for PyFloat { + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.float_type diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index 84d7a4e309c..b9545f05803 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -27,7 +27,9 @@ use crate::{ use rustpython_common::wtf8::Wtf8Buf; use alloc::fmt; +use core::cell::Cell; use core::ops::DerefMut; +use core::ptr::NonNull; #[pyclass(module = false, name = "list", unhashable = true, traverse = "manual")] #[derive(Default)] @@ -72,11 +74,44 @@ unsafe impl Traverse for PyList { } } +const PYLIST_MAXFREELIST: usize = 80; + +thread_local! { + static LIST_FREELIST: Cell> = const { Cell::new(Vec::new()) }; +} + impl PyPayload for PyList { + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.list_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + LIST_FREELIST.with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < PYLIST_MAXFREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + } + + #[inline] + unsafe fn freelist_pop() -> Option> { + LIST_FREELIST.with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + } } impl ToPyObject for Vec { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 5ade718ba50..5d6a2f50247 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -171,13 +171,19 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { // Untrack from GC BEFORE deallocation. if obj_ref.is_gc_tracked() { let ptr = unsafe { NonNull::new_unchecked(obj) }; - rustpython_common::refcount::try_defer_drop(move || { - // untrack_object only removes the pointer address from a HashSet. - // It does NOT dereference the pointer, so it's safe even after deallocation. - unsafe { - crate::gc_state::gc_state().untrack_object(ptr); - } - }); + if T::HAS_FREELIST { + // Freelist types must untrack immediately to avoid race conditions: + // a deferred untrack could remove a re-tracked entry after reuse. + unsafe { crate::gc_state::gc_state().untrack_object(ptr) }; + } else { + rustpython_common::refcount::try_defer_drop(move || { + // untrack_object only removes the pointer address from a HashSet. + // It does NOT dereference the pointer, so it's safe even after deallocation. + unsafe { + crate::gc_state::gc_state().untrack_object(ptr); + } + }); + } } // Extract child references before deallocation to break circular refs (tp_clear) @@ -1733,24 +1739,27 @@ impl PyRef { let is_heaptype = typ.heaptype_ext.is_some(); // Try to reuse from freelist (exact type only, no dict, no heaptype) - if !has_dict && !is_heaptype { + let ptr = if !has_dict && !is_heaptype { if let Some(cached) = unsafe { T::freelist_pop() } { let inner = cached.as_ptr() as *mut PyInner; unsafe { core::ptr::write(&mut (*inner).ref_count, RefCount::new()); (*inner).gc_bits.store(0, Ordering::Relaxed); + core::ptr::drop_in_place(&mut (*inner).payload); core::ptr::write(&mut (*inner).payload, payload); // typ, vtable, dict(None), weak_list, slots are preserved } // Drop the caller's typ since the cached object already holds one drop(typ); - let ptr = unsafe { NonNull::new_unchecked(inner.cast::>()) }; - return Self { ptr }; + unsafe { NonNull::new_unchecked(inner.cast::>()) } + } else { + let inner = Box::into_raw(PyInner::new(payload, typ, dict)); + unsafe { NonNull::new_unchecked(inner.cast::>()) } } - } - - let inner = Box::into_raw(PyInner::new(payload, typ, dict)); - let ptr = unsafe { NonNull::new_unchecked(inner.cast::>()) }; + } else { + let inner = Box::into_raw(PyInner::new(payload, typ, dict)); + unsafe { NonNull::new_unchecked(inner.cast::>()) } + }; // Track object if: // - HAS_TRAVERSE is true (Rust payload implements Traverse), OR diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index ad3851f8e49..86893251770 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -47,6 +47,11 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { fn class(ctx: &Context) -> &'static Py; + /// Whether this type has a freelist. Types with freelists require + /// immediate (non-deferred) GC untracking during dealloc to prevent + /// race conditions when the object is reused. + const HAS_FREELIST: bool = false; + /// Try to push a dead object onto this type's freelist for reuse. /// Returns true if the object was stored (caller must NOT free the memory). /// From c706c9ef4b60f200d14c47b82ac4364e21ded4de Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:52:56 +0900 Subject: [PATCH 3/7] Add PyDict freelist and exact-type guard for freelist push Add thread-local freelist for PyDict (max 80 entries). Add heaptype check in default_dealloc to prevent subclass instances from entering the freelist. Subclass objects have a different Python type than the base type, so reusing them would return objects with the wrong __class__. Skip PyTuple freelist: structseq types (stat_result, struct_time) are static subtypes sharing the same Rust payload, making type-safe reuse impractical with heaptype check alone. --- crates/vm/src/builtins/dict.rs | 35 +++++++++++++++++++++++++++++++++ crates/vm/src/builtins/tuple.rs | 3 +++ crates/vm/src/object/core.rs | 9 ++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 43891d3b7f7..79e1206dd5f 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -26,6 +26,8 @@ use crate::{ vm::VirtualMachine, }; use alloc::fmt; +use core::cell::Cell; +use core::ptr::NonNull; use rustpython_common::lock::PyMutex; use rustpython_common::wtf8::Wtf8Buf; @@ -60,11 +62,44 @@ impl fmt::Debug for PyDict { } } +const PYDICT_MAXFREELIST: usize = 80; + +thread_local! { + static DICT_FREELIST: Cell> = const { Cell::new(Vec::new()) }; +} + impl PyPayload for PyDict { + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.dict_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + DICT_FREELIST.with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < PYDICT_MAXFREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + } + + #[inline] + unsafe fn freelist_pop() -> Option> { + DICT_FREELIST.with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + } } impl PyDict { diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index 8ca2f74a3bf..b7ed066f1d1 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -53,6 +53,9 @@ unsafe impl Traverse for PyTuple { } } +// No freelist for PyTuple: structseq types (stat_result, struct_time, etc.) +// are static subtypes sharing the same Rust payload, making type-safe reuse +// impractical without a type-pointer comparison at push time. impl PyPayload for PyTuple { #[inline] fn class(ctx: &Context) -> &'static Py { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 5d6a2f50247..035d46c6231 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -193,7 +193,14 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { } // Try to store in freelist for reuse; otherwise deallocate. - if !unsafe { T::freelist_push(obj) } { + // Only exact types (not heaptype subclasses) go into the freelist, + // because the pop site assumes the cached typ matches the base type. + let pushed = if T::HAS_FREELIST && obj_ref.class().heaptype_ext.is_none() { + unsafe { T::freelist_push(obj) } + } else { + false + }; + if !pushed { drop(unsafe { Box::from_raw(obj as *mut PyInner) }); } From e4d4d116ce2927f54d1803f7eb033ad9d3faa553 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:57:55 +0900 Subject: [PATCH 4/7] Add PySlice freelist (max 1, matching PySlice_MAXFREELIST) --- .cspell.dict/cpython.txt | 1 + crates/vm/src/builtins/dict.rs | 5 ++--- crates/vm/src/builtins/float.rs | 5 ++--- crates/vm/src/builtins/list.rs | 5 ++--- crates/vm/src/builtins/slice.rs | 34 +++++++++++++++++++++++++++++++++ crates/vm/src/object/payload.rs | 3 +++ 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 41746996ceb..a35c7c8834e 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -70,6 +70,7 @@ finalizers firsttraceable flowgraph formatfloat +freelist freevar freevars fromlist diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 79e1206dd5f..ad6e0320659 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -62,13 +62,12 @@ impl fmt::Debug for PyDict { } } -const PYDICT_MAXFREELIST: usize = 80; - thread_local! { static DICT_FREELIST: Cell> = const { Cell::new(Vec::new()) }; } impl PyPayload for PyDict { + const MAX_FREELIST: usize = 80; const HAS_FREELIST: bool = true; #[inline] @@ -80,7 +79,7 @@ impl PyPayload for PyDict { unsafe fn freelist_push(obj: *mut PyObject) -> bool { DICT_FREELIST.with(|fl| { let mut list = fl.take(); - let stored = if list.len() < PYDICT_MAXFREELIST { + let stored = if list.len() < Self::MAX_FREELIST { list.push(obj); true } else { diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 30bc028abd7..e3e21aed205 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -34,13 +34,12 @@ impl PyFloat { } } -const PYFLOAT_MAXFREELIST: usize = 100; - thread_local! { static FLOAT_FREELIST: Cell> = const { Cell::new(Vec::new()) }; } impl PyPayload for PyFloat { + const MAX_FREELIST: usize = 100; const HAS_FREELIST: bool = true; #[inline] @@ -52,7 +51,7 @@ impl PyPayload for PyFloat { unsafe fn freelist_push(obj: *mut PyObject) -> bool { FLOAT_FREELIST.with(|fl| { let mut list = fl.take(); - let stored = if list.len() < PYFLOAT_MAXFREELIST { + let stored = if list.len() < Self::MAX_FREELIST { list.push(obj); true } else { diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index b9545f05803..a5e91693560 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -74,13 +74,12 @@ unsafe impl Traverse for PyList { } } -const PYLIST_MAXFREELIST: usize = 80; - thread_local! { static LIST_FREELIST: Cell> = const { Cell::new(Vec::new()) }; } impl PyPayload for PyList { + const MAX_FREELIST: usize = 80; const HAS_FREELIST: bool = true; #[inline] @@ -92,7 +91,7 @@ impl PyPayload for PyList { unsafe fn freelist_push(obj: *mut PyObject) -> bool { LIST_FREELIST.with(|fl| { let mut list = fl.take(); - let stored = if list.len() < PYLIST_MAXFREELIST { + let stored = if list.len() < Self::MAX_FREELIST { list.push(obj); true } else { diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index 4b3bec0530d..7402080dcbe 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -1,5 +1,7 @@ // sliceobject.{h,c} in CPython // spell-checker:ignore sliceobject +use core::cell::Cell; +use core::ptr::NonNull; use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat}; use super::{PyGenericAlias, PyStrRef, PyTupleRef, PyType, PyTypeRef}; @@ -23,11 +25,43 @@ pub struct PySlice { pub step: Option, } +thread_local! { + static SLICE_FREELIST: Cell> = const { Cell::new(Vec::new()) }; +} + impl PyPayload for PySlice { + const MAX_FREELIST: usize = 1; + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.slice_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + SLICE_FREELIST.with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + } + + #[inline] + unsafe fn freelist_pop() -> Option> { + SLICE_FREELIST.with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + } } #[pyclass(with(Comparable, Representable, Hashable))] diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 86893251770..181469a446e 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -52,6 +52,9 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { /// race conditions when the object is reused. const HAS_FREELIST: bool = false; + /// Maximum number of objects to keep in the freelist. + const MAX_FREELIST: usize = 0; + /// Try to push a dead object onto this type's freelist for reuse. /// Returns true if the object was stored (caller must NOT free the memory). /// From c0b33a68842c858ae95ccb7d81e71df39ea6c396 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 17:25:19 +0900 Subject: [PATCH 5/7] Add FreeList wrapper to drain cached objects on thread teardown Replace Cell> with Cell> which implements Drop to properly deallocate PyInner when threads exit. Also fix freelist_pop safety doc to match actual contract. --- crates/vm/src/builtins/dict.rs | 2 +- crates/vm/src/builtins/float.rs | 2 +- crates/vm/src/builtins/list.rs | 2 +- crates/vm/src/builtins/slice.rs | 2 +- crates/vm/src/object/core.rs | 49 +++++++++++++++++++++++++++++---- crates/vm/src/object/payload.rs | 4 ++- 6 files changed, 50 insertions(+), 11 deletions(-) diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index ad6e0320659..c630fc25dff 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -63,7 +63,7 @@ impl fmt::Debug for PyDict { } thread_local! { - static DICT_FREELIST: Cell> = const { Cell::new(Vec::new()) }; + static DICT_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; } impl PyPayload for PyDict { diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index e3e21aed205..f78b227fc38 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -35,7 +35,7 @@ impl PyFloat { } thread_local! { - static FLOAT_FREELIST: Cell> = const { Cell::new(Vec::new()) }; + static FLOAT_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; } impl PyPayload for PyFloat { diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index a5e91693560..ff3ed64f263 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -75,7 +75,7 @@ unsafe impl Traverse for PyList { } thread_local! { - static LIST_FREELIST: Cell> = const { Cell::new(Vec::new()) }; + static LIST_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; } impl PyPayload for PyList { diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index 7402080dcbe..abebd32283e 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -26,7 +26,7 @@ pub struct PySlice { } thread_local! { - static SLICE_FREELIST: Cell> = const { Cell::new(Vec::new()) }; + static SLICE_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; } impl PyPayload for PySlice { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 035d46c6231..a16eba96022 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -822,13 +822,50 @@ impl PyInner { } } -/// Drop a freelist-cached object, properly deallocating the `PyInner`. +/// Thread-local freelist storage that properly deallocates cached objects +/// on thread teardown. /// -/// # Safety -/// `ptr` must point to a valid `PyInner` allocation that was stored in a freelist. -#[allow(dead_code)] -pub(crate) unsafe fn drop_freelist_object(ptr: *mut PyObject) { - drop(unsafe { Box::from_raw(ptr as *mut PyInner) }); +/// Wraps a `Vec<*mut PyObject>` and implements `Drop` to convert each +/// raw pointer back into `Box>` for proper deallocation. +pub(crate) struct FreeList { + items: Vec<*mut PyObject>, + _marker: core::marker::PhantomData, +} + +impl FreeList { + pub(crate) const fn new() -> Self { + Self { + items: Vec::new(), + _marker: core::marker::PhantomData, + } + } +} + +impl Default for FreeList { + fn default() -> Self { + Self::new() + } +} + +impl Drop for FreeList { + fn drop(&mut self) { + for ptr in self.items.drain(..) { + drop(unsafe { Box::from_raw(ptr as *mut PyInner) }); + } + } +} + +impl core::ops::Deref for FreeList { + type Target = Vec<*mut PyObject>; + fn deref(&self) -> &Self::Target { + &self.items + } +} + +impl core::ops::DerefMut for FreeList { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.items + } } /// The `PyObjectRef` is one of the most used types. It is a reference to a diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 181469a446e..1af954505f7 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -71,7 +71,9 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { /// reinitialize `ref_count`, `gc_bits`, and `payload`. /// /// # Safety - /// The returned pointer (if Some) points to uninitialized/stale payload. + /// The returned pointer (if Some) must point to a valid `PyInner` + /// whose payload is still initialized from a previous allocation. The caller + /// will drop and overwrite `payload` before reuse. #[inline] unsafe fn freelist_pop() -> Option> { None From 5b78ad51c401abb808ab8c319c1e560e6dd5e0f1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 17:49:20 +0900 Subject: [PATCH 6/7] Add manual Traverse impl for PySlice with clear() and fix comment --- crates/vm/src/builtins/slice.rs | 22 +++++++++++++++++++++- crates/vm/src/object/core.rs | 3 ++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index abebd32283e..c0b68ed9e8a 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -17,7 +17,7 @@ use crate::{ use malachite_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, Zero}; -#[pyclass(module = false, name = "slice", unhashable = true, traverse)] +#[pyclass(module = false, name = "slice", unhashable = true, traverse = "manual")] #[derive(Debug)] pub struct PySlice { pub start: Option, @@ -25,6 +25,26 @@ pub struct PySlice { pub step: Option, } +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl crate::object::Traverse for PySlice { + fn traverse(&self, traverse_fn: &mut crate::object::TraverseFn<'_>) { + self.start.traverse(traverse_fn); + self.stop.traverse(traverse_fn); + self.step.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec) { + if let Some(start) = self.start.take() { + out.push(start); + } + // stop is not Option, so it will be freed when payload is dropped + // (via drop_in_place on freelist pop, or Box::from_raw on dealloc) + if let Some(step) = self.step.take() { + out.push(step); + } + } +} + thread_local! { static SLICE_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; } diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index a16eba96022..32693634683 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -1791,7 +1791,8 @@ impl PyRef { (*inner).gc_bits.store(0, Ordering::Relaxed); core::ptr::drop_in_place(&mut (*inner).payload); core::ptr::write(&mut (*inner).payload, payload); - // typ, vtable, dict(None), weak_list, slots are preserved + // typ, vtable, slots are preserved; dict is None, weak_list was + // cleared by drop_slow_inner before freelist push } // Drop the caller's typ since the cached object already holds one drop(typ); From 2266d940fc13e6a001261c48bf7c5336c6a982c3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 18:10:41 +0900 Subject: [PATCH 7/7] Deduplicate allocation fallback in PyRef::new_ref --- crates/vm/src/object/core.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 32693634683..f4e458e1484 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -1783,24 +1783,25 @@ impl PyRef { let is_heaptype = typ.heaptype_ext.is_some(); // Try to reuse from freelist (exact type only, no dict, no heaptype) - let ptr = if !has_dict && !is_heaptype { - if let Some(cached) = unsafe { T::freelist_pop() } { - let inner = cached.as_ptr() as *mut PyInner; - unsafe { - core::ptr::write(&mut (*inner).ref_count, RefCount::new()); - (*inner).gc_bits.store(0, Ordering::Relaxed); - core::ptr::drop_in_place(&mut (*inner).payload); - core::ptr::write(&mut (*inner).payload, payload); - // typ, vtable, slots are preserved; dict is None, weak_list was - // cleared by drop_slow_inner before freelist push - } - // Drop the caller's typ since the cached object already holds one - drop(typ); - unsafe { NonNull::new_unchecked(inner.cast::>()) } - } else { - let inner = Box::into_raw(PyInner::new(payload, typ, dict)); - unsafe { NonNull::new_unchecked(inner.cast::>()) } + let cached = if !has_dict && !is_heaptype { + unsafe { T::freelist_pop() } + } else { + None + }; + + let ptr = if let Some(cached) = cached { + let inner = cached.as_ptr() as *mut PyInner; + unsafe { + core::ptr::write(&mut (*inner).ref_count, RefCount::new()); + (*inner).gc_bits.store(0, Ordering::Relaxed); + core::ptr::drop_in_place(&mut (*inner).payload); + core::ptr::write(&mut (*inner).payload, payload); + // typ, vtable, slots are preserved; dict is None, weak_list was + // cleared by drop_slow_inner before freelist push } + // Drop the caller's typ since the cached object already holds one + drop(typ); + unsafe { NonNull::new_unchecked(inner.cast::>()) } } else { let inner = Box::into_raw(PyInner::new(payload, typ, dict)); unsafe { NonNull::new_unchecked(inner.cast::>()) }