Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions crates/capi/src/objimpl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ pub unsafe extern "C" fn PyObject_GC_Track(op: *mut PyObject) {
with_vm(|_vm| {
let obj = unsafe { &*op };
if !obj.is_gc_tracked() {
unsafe { gc_state::gc_state().track_object(obj.into()) };
unsafe { gc_state::gc_state().track_object(obj.into(), gc_state::current_owner()) };
}
})
}
Expand All @@ -36,29 +36,33 @@ pub unsafe extern "C" fn PyObject_GC_IsFinalized(op: *mut PyObject) -> c_int {

#[unsafe(no_mangle)]
pub extern "C" fn PyGC_Collect() -> isize {
let result = gc_state::gc_state().collect(2);
(result.collected + result.uncollectable) as isize
with_vm(|vm| {
let result = vm.state.gc.collect(2);
(result.collected + result.uncollectable) as isize
})
}

#[unsafe(no_mangle)]
pub extern "C" fn PyGC_Enable() -> c_int {
let gc = gc_state::gc_state();
let was_enabled = gc.is_enabled();
gc.enable();
was_enabled.into()
with_vm(|vm| {
let was_enabled: c_int = vm.state.gc.is_enabled().into();
vm.state.gc.enable();
was_enabled
})
}

#[unsafe(no_mangle)]
pub extern "C" fn PyGC_Disable() -> c_int {
let gc = gc_state::gc_state();
let was_enabled = gc.is_enabled();
gc.disable();
was_enabled.into()
with_vm(|vm| {
let was_enabled: c_int = vm.state.gc.is_enabled().into();
vm.state.gc.disable();
was_enabled
})
}

#[unsafe(no_mangle)]
pub extern "C" fn PyGC_IsEnabled() -> c_int {
gc_state::gc_state().is_enabled().into()
with_vm(|vm| -> c_int { vm.state.gc.is_enabled().into() })
}

#[unsafe(no_mangle)]
Expand Down
4 changes: 2 additions & 2 deletions crates/capi/src/pystate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ mod tests {
current_vm_is_set(),
"This thread did not have a vm attached"
);
vm.state.stop_the_world.stop_the_world(vm);
vm.state.stop_the_world.start_the_world(vm);
vm.state.stop_the_world.stop_the_world(&vm.state);
vm.state.stop_the_world.start_the_world(&vm.state);
});
});
});
Expand Down
33 changes: 24 additions & 9 deletions crates/stdlib/src/_queue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,20 @@ mod _queue {
}
}

fn release(&self) {
/// Take `mutex`, detaching first so that blocking on it cannot stall a
/// stop-the-world request.
///
/// A waiter holds this mutex across its `allow_threads` wait, so it can
/// still hold it when it is stopped. An attached thread blocking on it
/// would then never reach a safepoint, the stop would never complete,
/// and the holder would never be resumed to release it.
fn lock_count(&self, vm: &VirtualMachine) -> parking_lot::MutexGuard<'_, usize> {
vm.allow_threads(|| self.mutex.lock())
}

fn release(&self, vm: &VirtualMachine) {
{
let mut count = self.mutex.lock();
let mut count = self.lock_count(vm);
*count += 1;
} // lock dropped. now we can notify a waiting thread

Expand All @@ -95,7 +106,7 @@ mod _queue {
// Guard must be dropped before check_signals() below, since a
// signal handler may call back into this same queue.
{
let mut count = self.mutex.lock();
let mut count = self.lock_count(vm);

if *count > 0 {
*count -= 1;
Expand Down Expand Up @@ -151,11 +162,15 @@ mod _queue {
}

impl PySimpleQueue {
fn push(&self, item: PyObjectRef) {
#[cfg_attr(
not(feature = "threading"),
expect(unused_variables, reason = "only the semaphore needs the vm")
)]
fn push(&self, item: PyObjectRef, vm: &VirtualMachine) {
self.buf.lock().push_back(item);

#[cfg(feature = "threading")]
self.sem.release();
self.sem.release(vm);
}

/// Returns a strong reference from the head of the buffer.
Expand Down Expand Up @@ -221,14 +236,14 @@ mod _queue {
}

#[pymethod]
fn put(&self, args: PutArgs) {
fn put(&self, args: PutArgs, vm: &VirtualMachine) {
let PutArgs { item, .. } = args;
self.push(item);
self.push(item, vm);
}

#[pymethod]
fn put_nowait(&self, item: PyObjectRef) {
self.push(item);
fn put_nowait(&self, item: PyObjectRef, vm: &VirtualMachine) {
self.push(item, vm);
}

#[pymethod]
Expand Down
4 changes: 2 additions & 2 deletions crates/stdlib/src/faulthandler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,8 @@ mod decl {
use core::sync::atomic::Ordering;
let current_tid = rustpython_vm::stdlib::_thread::get_ident();
{
vm.state.stop_the_world.stop_the_world(vm);
scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); }
vm.state.stop_the_world.stop_the_world(&vm.state);
scopeguard::defer! { vm.state.stop_the_world.start_the_world(&vm.state); }
let registry = vm.state.thread_frames.lock();
#[expect(
clippy::iter_over_hash_type,
Expand Down
7 changes: 7 additions & 0 deletions crates/vm/src/builtins/bool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,20 @@ impl<'a> TryFromBorrowedObject<'a> for bool {

impl PyObjectRef {
/// Convert Python bool into Rust bool.
#[inline(always)]
pub fn try_to_bool(self, vm: &VirtualMachine) -> PyResult<bool> {
if self.is(&vm.ctx.true_value) {
return Ok(true);
} else if self.is(&vm.ctx.false_value) {
return Ok(false);
}

self.try_to_bool_slow(vm)
}

#[cold]
#[inline(never)]
fn try_to_bool_slow(self, vm: &VirtualMachine) -> PyResult<bool> {
let slots = &self.class().slots;

// 1. Try nb_bool slot first
Expand Down
24 changes: 8 additions & 16 deletions crates/vm/src/builtins/dict.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,6 @@ impl PyDict {
&self.entries
}

/// Monotonically increasing version for mutation tracking.
pub(crate) fn version(&self) -> u64 {
self.entries.version()
}

/// Returns all keys as a Vec, atomically under a single read lock.
/// Thread-safe: prevents "dictionary changed size during iteration" errors.
pub fn keys_vec(&self) -> Vec<PyObjectRef> {
Expand Down Expand Up @@ -817,18 +812,15 @@ impl Py<PyDict> {
}
}

/// Fast lookup using a cached entry index hint.
pub(crate) fn get_item_opt_hint<K: DictKey + ?Sized>(
/// Read a cached exact-dict entry after validating its key-layout stamp.
#[inline]
pub(crate) fn get_item_by_index_and_keys_version(
&self,
key: &K,
hint: u16,
vm: &VirtualMachine,
) -> PyResult<Option<PyObjectRef>> {
if self.exact_dict(vm) {
self.entries.get_hint(vm, key, usize::from(hint))
} else {
self.get_item_opt(key, vm)
}
version: u16,
index: u16,
) -> Option<PyObjectRef> {
self.entries
.get_index_if_keys_version(u32::from(version), usize::from(index))
}

/// Lookup trying a cached entry index hint first.
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/frame.rs
Original file line number Diff line number Diff line change
Expand Up @@ -897,8 +897,8 @@ impl Py<FrameObject> {
{
// Enter STW before dereferencing `prev` — the owning thread may
// return and free the stack-allocated iframe at any time.
vm.state.stop_the_world.stop_the_world(vm);
scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); }
vm.state.stop_the_world.stop_the_world(&vm.state);
scopeguard::defer! { vm.state.stop_the_world.start_the_world(&vm.state); }
let prev_ref = unsafe { &*prev };
// Fast path: already materialized.
if let Some(fo) = prev_ref.frame_obj() {
Expand Down
37 changes: 32 additions & 5 deletions crates/vm/src/builtins/function.rs
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,20 @@ impl Py<PyFunction> {
self.code.flags.contains(bytecode::CodeFlags::OPTIMIZED)
}

/// Whether this function currently has native JIT code. Adaptive Python
/// call specializations must yield to that entry point.
#[inline]
pub(crate) fn is_jitted(&self) -> bool {
#[cfg(feature = "jit")]
{
self.jitted_code.lock().is_some()
}
#[cfg(not(feature = "jit"))]
{
false
}
}

pub fn invoke_with_locals(
&self,
func_args: FuncArgs,
Expand Down Expand Up @@ -643,8 +657,8 @@ impl Py<PyFunction> {
.and_then(|()| vm.run_frame_fast(iframe));
// Release data stack memory — must happen on both success and error.
unsafe {
if let Some(base) = iframe.release_datastack_frame() {
vm.datastack_pop(base);
if let Some((base, size)) = iframe.release_datastack_frame() {
vm.datastack_pop_frame(base, size);
}
}
result
Expand All @@ -669,7 +683,10 @@ impl Py<PyFunction> {
);
// SAFETY: the frame is alive (held by `frame`) and untracked.
unsafe {
crate::gc_state::gc_state().track_object(core::ptr::NonNull::from(frame.as_object()));
crate::gc_state::gc_state().track_object(
core::ptr::NonNull::from(frame.as_object()),
crate::gc_state::current_owner(),
);
}
frame.set_generator(&obj);
obj
Expand Down Expand Up @@ -820,8 +837,8 @@ impl Py<PyFunction> {

let result = vm.run_frame_fast(iframe);
unsafe {
if let Some(base) = iframe.release_datastack_frame() {
vm.datastack_pop(base);
if let Some((base, size)) = iframe.release_datastack_frame() {
vm.datastack_pop_frame(base, size);
}
}
result
Expand Down Expand Up @@ -1616,6 +1633,16 @@ pub(crate) fn vectorcall_function(
let code: &Py<PyCode> = &zelf.code;

let has_kwargs = kwnames.is_some_and(|kw| !kw.is_empty());
if zelf.is_jitted() {
let func_args = if has_kwargs {
FuncArgs::from_vectorcall(&args, nargs, kwnames)
} else {
args.truncate(nargs);
FuncArgs::from(args)
};
return zelf.invoke(func_args, vm);
}

let is_simple = !has_kwargs
&& code.flags.contains(bytecode::CodeFlags::OPTIMIZED)
&& !code.flags.contains(bytecode::CodeFlags::VARARGS)
Expand Down
16 changes: 16 additions & 0 deletions crates/vm/src/builtins/int.rs
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,22 @@ impl PyInt {
&self.value
}

/// Extract the inline magnitude without the generic primitive-conversion path.
#[inline(always)]
pub(crate) fn try_to_i64_fast(&self) -> Option<i64> {
let bits = self.value.bits();
if bits > i64::BITS as u64 {
return None;
}
let magnitude = self.value.iter_u64_digits().next().unwrap_or(0);
let signed_magnitude = i64::try_from(magnitude).ok();
match self.value.sign() {
Sign::Minus if magnitude == 1u64 << 63 => Some(i64::MIN),
Sign::Minus => signed_magnitude.map(|value| -value),
Sign::NoSign | Sign::Plus => signed_magnitude,
}
}

/// Fast decimal string conversion, using i64 path when possible.
#[inline]
#[must_use]
Expand Down
37 changes: 35 additions & 2 deletions crates/vm/src/builtins/type.rs
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,16 @@ pub struct HeapTypeExt {
pub slots: Option<PyRef<PyTuple<PyStrRef>>>,
pub type_data: PyRwLock<Option<TypeDataSlot>>,
pub specialization_cache: TypeSpecializationCache,
/// The interpreter this type was created in, or `None` for the types the
/// shared context builds before any interpreter exists.
pub interpreter_id: Option<i64>,
}

impl HeapTypeExt {
/// The interpreter a type created right now belongs to.
fn creating_interpreter_id() -> Option<i64> {
crate::vm::thread::try_with_current_vm(|vm| vm.state.interpreter_id)
}
}

pub struct TypeSpecializationCache {
Expand Down Expand Up @@ -553,6 +563,22 @@ impl PyType {
self.modified_inner();
}

/// Whether the interpreter with `interpreter_id` can see this type.
///
/// Interpreters share the context, so a subclass of a shared type is
/// recorded on an object every interpreter reaches. Only the interpreter
/// that created it can name it, so only that one lists it.
pub fn is_visible_to_interpreter(&self, interpreter_id: i64) -> bool {
match self
.heaptype_ext
.as_ref()
.and_then(|ext| ext.interpreter_id)
{
Some(owner) => owner == interpreter_id,
None => true,
}
}

pub fn new_simple_heap(
name: &str,
base: &Py<Self>,
Expand Down Expand Up @@ -589,6 +615,7 @@ impl PyType {
slots: None,
type_data: PyRwLock::new(None),
specialization_cache: TypeSpecializationCache::new(),
interpreter_id: HeapTypeExt::creating_interpreter_id(),
};
let base = bases[0].clone();

Expand Down Expand Up @@ -1948,13 +1975,18 @@ impl PyType {
}

#[pymethod]
fn __subclasses__(&self) -> PyList {
fn __subclasses__(&self, vm: &VirtualMachine) -> PyList {
let mut subclasses = self.subclasses.write();
subclasses.retain(|x| x.upgrade().is_some());
let interpreter_id = vm.state.interpreter_id;
PyList::from(
subclasses
.iter()
.map(|x| x.upgrade().unwrap())
.filter_map(|x| x.upgrade())
.filter(|obj| {
obj.downcast_ref::<Self>()
.is_none_or(|typ| typ.is_visible_to_interpreter(interpreter_id))
})
.collect::<Vec<_>>(),
)
}
Expand Down Expand Up @@ -2368,6 +2400,7 @@ impl Constructor for PyType {
slots: heaptype_slots.clone(),
type_data: PyRwLock::new(None),
specialization_cache: TypeSpecializationCache::new(),
interpreter_id: HeapTypeExt::creating_interpreter_id(),
};
(slots, heaptype_ext)
};
Expand Down
Loading
Loading