From 5b3e2a40e2f0b98e4c56d6e637495f8415c86808 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers <7943856+bschoenmaeckers@users.noreply.github.com> Date: Sun, 4 Oct 2026 06:52:36 +0200 Subject: [PATCH 001/229] Add recursion guard to the c-api (#8796) * Add recursion guard to the c-api * Do not change with_recursion as this has side effects --- crates/capi/src/ceval.rs | 34 ++++++++++++++++++++++++++++++++-- crates/vm/src/vm/mod.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/crates/capi/src/ceval.rs b/crates/capi/src/ceval.rs index 4064b289caf..ace7896163c 100644 --- a/crates/capi/src/ceval.rs +++ b/crates/capi/src/ceval.rs @@ -1,7 +1,7 @@ use crate::pyframe::PyFrameObject; use crate::pystate::with_vm; use crate::unicodeobject::decode_fsdefault_and_size; -use crate::util::FfiPtrExt; +use crate::util::{CStrExt, FfiPtrExt}; use core::ffi::{CStr, c_char, c_int}; use rustpython_vm::builtins::{PyCode, PyDict}; use rustpython_vm::function::ArgMapping; @@ -157,10 +157,21 @@ pub unsafe extern "C" fn PyEval_GetFuncDesc(func: *mut PyObject) -> *const c_cha }) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn Py_EnterRecursiveCall(where_: *const c_char) -> c_int { + with_vm(|vm| vm.enter_recursive_call(unsafe { where_.try_as_str(vm)? })) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn Py_LeaveRecursiveCall() { + with_vm(|vm| vm.leave_recursive_call()) +} + #[cfg(test)] mod tests { + use super::{Py_EnterRecursiveCall, Py_LeaveRecursiveCall}; use alloc::ffi::CString; - use pyo3::exceptions::PyException; + use pyo3::exceptions::{PyException, PyRecursionError}; use pyo3::prelude::*; #[pyfunction] @@ -268,4 +279,23 @@ assert not hidden_leaked .unwrap(); }) } + + #[test] + fn recursion_error() { + fn recurse(py: Python<'_>) -> PyResult<()> { + unsafe { + if Py_EnterRecursiveCall(c"".as_ptr().cast()) != 0 { + return Err(PyErr::fetch(py)); + }; + let result = recurse(py); + Py_LeaveRecursiveCall(); + result + } + } + + Python::attach(|py| { + let err = recurse(py).unwrap_err(); + assert!(err.is_instance_of::(py)); + }) + } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 445ff7474a0..7cc636b37e3 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2585,6 +2585,33 @@ impl VirtualMachine { false } + /// Enter a native-recursion section equivalent to `Py_EnterRecursiveCall`. + pub fn enter_recursive_call(&self, _where: &str) -> PyResult<()> { + #[cfg(any(miri, target_env = "musl"))] + let counted_too_deep = + self.native_recursion_depth.get() >= Self::NATIVE_RECURSION_LIMIT_UNMEASURED; + #[cfg(not(any(miri, target_env = "musl")))] + let counted_too_deep = false; + + if counted_too_deep || self.check_c_stack_overflow() { + return Err( + self.new_recursion_error(format!("maximum recursion depth exceeded {_where}")) + ); + } + + #[cfg(any(miri, target_env = "musl"))] + self.native_recursion_depth.update(|d| d + 1); + + Ok(()) + } + + /// Leave a native-recursion section equivalent to + /// `Py_LeaveRecursiveCall`. + pub fn leave_recursive_call(&self) { + #[cfg(any(miri, target_env = "musl"))] + self.native_recursion_depth.update(|d| d.saturating_sub(1)); + } + /// Used to run the body of a (possibly) recursive function. It will raise a /// RecursionError if recursive functions are nested far too many times, /// preventing a stack overflow. From f39b054b9c8cbbf884f53123eef028131789990c Mon Sep 17 00:00:00 2001 From: Kwak Byoung Min <67214970+doma17@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:53:35 +0900 Subject: [PATCH 002/229] Match CPython quotechar escaping when doublequote is disabled (#8953) Let quote-only fields use the existing unquoted escaping path and enable the existing CPython regression test. Tested: test_csv, CSV snippet, 240 CPython comparisons, workspace and C-API tests, Clippy, prek Assisted-by: Codex:gpt-6.1-sol --- Lib/test/test_csv.py | 1 - crates/stdlib/src/csv.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 7e76f4bc1cb..21b02c2f05d 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -228,7 +228,6 @@ def test_write_quoting(self): self._write_test(['a','',None,1], '"a","",,"1"', quoting = csv.QUOTE_NOTNULL) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_escape(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"', escapechar='\\') diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index a0f75300eda..b07f9c644ad 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -1246,7 +1246,7 @@ mod _csv { fn field_needs_quotes(data: &[u8], dialect: &PyDialect) -> bool { data.iter().any(|&byte| { byte == dialect.delimiter - || dialect.quotechar == Some(byte) + || (dialect.doublequote && dialect.quotechar == Some(byte)) || matches!(byte, b'\r' | b'\n') }) || data_contains_lineterminator_char(data, dialect) } From 7536730ed6afb948ffa335fc113a0fcd9af0cdbf Mon Sep 17 00:00:00 2001 From: Chanho Lee Date: Sun, 4 Oct 2026 22:47:45 +0900 Subject: [PATCH 003/229] Fix store subscr list finalizer deadlock (#8956) * Release replaced list item outside the lock in STORE_SUBSCR_LIST_INT Assisted-by: Claude Code:claude-opus-5-5 * Add a snippet for finalizers reading a list during a specialized store Assisted-by: Claude Code:claude-opus-5-5 --- crates/vm/src/frame.rs | 6 ++++- extra_tests/snippets/vm_specialization.py | 30 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index cc54ad6ee18..dadc0476be8 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -6430,7 +6430,11 @@ impl ExecutingFrame<'_> { { let mut vec = list.borrow_vec_mut(); if i < vec.len() { - vec[i] = value; + // Unlock before dropping the replaced element; its + // finalizer may access this list. + let old = core::mem::replace(&mut vec[i], value); + drop(vec); + drop(old); return Ok(None); } } diff --git a/extra_tests/snippets/vm_specialization.py b/extra_tests/snippets/vm_specialization.py index b1d201e8ac1..99b4b78b640 100644 --- a/extra_tests/snippets/vm_specialization.py +++ b/extra_tests/snippets/vm_specialization.py @@ -342,3 +342,33 @@ def read(): module_attribute_custom_dict_key() + + +## STORE_SUBSCR_LIST_INT: the replaced element's finalizer can read the list + + +def store_subscr_list_int_finalizer_reads_list(): + target = [None] + seen = [] + + class Item: + def __init__(self, value): + self.value = value + + def __del__(self): + seen.append((self.value, len(target), target[0] is self)) + + def store(value): + target[0] = value + + for i in range(300): + store(Item(i)) + assert any( + op.opname == "STORE_SUBSCR_LIST_INT" + for op in dis.get_instructions(store, adaptive=True) + ) + store(None) + assert seen == [(i, 1, False) for i in range(300)] + + +store_subscr_list_int_finalizer_reads_list() From 6903fed201388d358d3f4184e5d33e7122901f83 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 4 Oct 2026 17:03:18 -0700 Subject: [PATCH 004/229] Fix disabled stream identity and annotation operand checks (#8960) Return the context's canonical None singleton when standard streams are disabled. Check the exact set type before accessing user-rebindable conditional-annotation tracking objects. Both fixes apply to the unchanged Python 3.14 target. Include the no-stdio embedding regression and the unchanged canonical regression from CPython's 3.14 maintenance backport c21e7c45, after the vulnerable 3.14.7 tag. Extracted from the larger 3.15 migration; no marshal or version changes. Assisted-by: Codex:model-version-unavailable --- Lib/test/test_type_annotations.py | 16 ++++++++++++++++ crates/vm/src/frame.rs | 8 ++++---- crates/vm/src/vm/mod.rs | 24 +++++++++++++++++++++--- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index c98b99e98e9..2b95cabf2e4 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,5 +1,6 @@ import annotationlib import inspect +import itertools import textwrap import types import unittest @@ -782,6 +783,21 @@ def test_non_name_annotations(self): class RegressionTests(unittest.TestCase): + # gh-154902 + def test_conditional_annotations_rebound(self): + # user code can rebind __conditional_annotations__ to any object + lefts = ("__conditional_annotations__", + 'globals()["__conditional_annotations__"]') + values = ("0", "{}", "[]", "''", "object()", "frozenset()") + for left, value in itertools.product(lefts, values): + with self.subTest(left=left, value=value): + code = f""" + {left} = {value} + x: int + """ + with self.assertRaises(TypeError): + run_code(code) + # gh-132479 def test_complex_comprehension_inlining(self): # Test that the various repro cases from the issue don't crash diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index dadc0476be8..90ea0d2db61 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -5587,10 +5587,10 @@ impl ExecutingFrame<'_> { Instruction::SetAdd { i } => { let item = self.pop_value(); let obj = self.nth_value(i.get(arg) - 1); - let set: &Py = unsafe { - // SAFETY: trust compiler - obj.downcast_unchecked_ref() - }; + // Annotation tracking can load a user-rebound global here. + let set = obj.downcast_ref_if_exact::(vm).ok_or_else(|| { + vm.new_type_error(format!("'{}' object is not a set", obj.class().name())) + })?; set.add(item, vm)?; Ok(None) } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 7cc636b37e3..4a74479f43d 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1514,9 +1514,7 @@ impl VirtualMachine { // No stdio: set to None (embedding use case) #[cfg(not(feature = "stdio"))] - let make_stdio = |_name: &str, _fd: i32, _write: bool| { - Ok(crate::builtins::PyNone.into_pyobject(self)) - }; + let make_stdio = |_name: &str, _fd: i32, _write: bool| Ok(self.ctx.none()); let set_stdio = |name, fd, write| { let stdio: PyObjectRef = make_stdio(name, fd, write)?; @@ -3925,6 +3923,26 @@ pub fn resolve_frozen_alias(name: &str) -> &str { mod tests { use super::*; + #[cfg(not(feature = "stdio"))] + #[test] + fn no_stdio_streams_are_none_singleton() { + Interpreter::builder(Default::default()) + .build() + .enter(|vm| { + for name in [ + "stdin", + "stdout", + "stderr", + "__stdin__", + "__stdout__", + "__stderr__", + ] { + let stream = vm.sys_module.get_attr(name, vm).unwrap(); + assert!(vm.is_none(&stream), "sys.{name} must be the None singleton"); + } + }); + } + #[test] fn nested_frozen() { use rustpython_vm as vm; From 5a61850e32e81c423254845e8d78a4bc3b332ba6 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 4 Oct 2026 17:03:52 -0700 Subject: [PATCH 005/229] Use populated entries for unpickler MEMOIZE indexes (#8959) * Use populated entries for unpickler MEMOIZE indexes A sparse BINPUT followed by MEMOIZE must use the number of occupied memo entries, not the largest index plus one. Store populated entries in a map, preserve ascending memo copies, reserve fallibly, and release displaced references after memo guards end. Keep existing memo assignment/proxy behavior and add one bounded regression for sparse BINPUT, MEMOIZE identity and copy order. No protocol version, GC traversal or unrelated pickle input changes are included. Extracted from the Python 3.15 migration onto the unchanged 3.14 target. Assisted-by: Codex:model-version-unavailable * Move sparse memo regression to standard pickle snippet Use the module-level snippet name requested in review without changing the regression assertions. Assisted-by: Codex:model-version-unavailable --- crates/stdlib/src/pickle.rs | 110 ++++++++++++++++++-------- extra_tests/snippets/stdlib_pickle.py | 11 +++ 2 files changed, 86 insertions(+), 35 deletions(-) create mode 100644 extra_tests/snippets/stdlib_pickle.py diff --git a/crates/stdlib/src/pickle.rs b/crates/stdlib/src/pickle.rs index 417a5721165..8badf9a85ad 100644 --- a/crates/stdlib/src/pickle.rs +++ b/crates/stdlib/src/pickle.rs @@ -18,6 +18,7 @@ mod _pickle { protocol::{PyBuffer, PyIter, PyIterReturn}, types::{AsBuffer, Constructor, Initializer, Representable}, }; + use alloc::collections::TryReserveError; use core::sync::atomic::{AtomicI32, Ordering}; use malachite_bigint::BigInt; use num_traits::{ToPrimitive, Zero}; @@ -407,17 +408,26 @@ mod _pickle { impl UnpicklerMemoProxy { #[pymethod] fn clear(zelf: &Py) { - zelf.unpickler.memo.write().clear(); + let old_memo = core::mem::take(&mut *zelf.unpickler.memo.write()); + drop(old_memo); } #[pymethod] fn copy(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let mut snapshot = Vec::new(); + let copied = { + let memo = zelf.unpickler.memo.read(); + snapshot.try_reserve_exact(memo.len()).map(|()| { + snapshot.extend(memo.iter().map(|(&index, obj)| (index, obj.clone()))); + }) + }; + copied.map_err(|_| vm.no_memory_error())?; + // CPython inserts present memo indices in ascending order. + snapshot.sort_unstable_by_key(|(index, _)| *index); let dict = vm.ctx.new_dict(); - for (i, entry) in zelf.unpickler.memo.read().iter().enumerate() { - if let Some(obj) = entry { - let key: PyObjectRef = vm.ctx.new_int(i).into(); - dict.set_item(&*key, obj.clone(), vm)?; - } + for (i, obj) in snapshot { + let key: PyObjectRef = vm.ctx.new_int(i).into(); + dict.set_item(&*key, obj, vm)?; } Ok(dict.into()) } @@ -440,6 +450,10 @@ mod _pickle { // Unpickler + // PUT indices need not be contiguous. A sparse table also keeps MEMOIZE's + // index equal to the number of entries rather than the largest index. + type UnpicklerMemo = HashMap; + #[derive(Debug)] pub(super) struct UnpicklerConfig { initialized: bool, @@ -468,7 +482,7 @@ mod _pickle { #[derive(Debug, PyPayload)] pub(super) struct PyUnpickler { read_state: PyMutex, - memo: PyRwLock>>, + memo: PyRwLock, config: PyRwLock, } @@ -495,7 +509,7 @@ mod _pickle { fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { Ok(Self { read_state: PyMutex::new(ReadState::default()), - memo: PyRwLock::new(Vec::new()), + memo: PyRwLock::new(UnpicklerMemo::new()), config: PyRwLock::new(UnpicklerConfig::default()), }) } @@ -548,7 +562,7 @@ mod _pickle { peek, ..ReadState::default() }; - zelf.memo.write().clear(); + let old_memo = core::mem::take(&mut *zelf.memo.write()); *zelf.config.write() = UnpicklerConfig { initialized: true, proto: 0, @@ -557,6 +571,8 @@ mod _pickle { errors, buffers, }; + drop(state); + drop(old_memo); Ok(()) } } @@ -605,21 +621,21 @@ mod _pickle { } }; let new_memo = if let Some(proxy) = value.downcast_ref::() { - proxy.unpickler.memo.read().clone() + let snapshot = copy_memo(&proxy.unpickler.memo.read()); + snapshot.map_err(|_| vm.no_memory_error())? } else if let Some(dict) = value.downcast_ref::() { - let mut memo: Vec> = Vec::new(); + let mut memo = UnpicklerMemo::new(); + memo.try_reserve(dict.__len__()) + .map_err(|_| vm.no_memory_error())?; for (key, val) in dict { let idx = key .downcast_ref::() .ok_or_else(|| vm.new_type_error("memo key must be integers"))?; - let idx = idx - .as_bigint() - .to_usize() - .ok_or_else(|| vm.new_value_error("memo key must be positive integers."))?; - if memo.len() <= idx { - memo.resize(idx + 1, None); + let idx = idx.try_to_primitive::(vm)?; + if idx < 0 { + return Err(vm.new_value_error("memo key must be positive integers.")); } - memo[idx] = Some(val); + memo_insert(&mut memo, idx as usize, &val).map_err(|_| vm.no_memory_error())?; } memo } else { @@ -628,7 +644,8 @@ mod _pickle { value.class().name() ))); }; - *zelf.memo.write() = new_memo; + let old_memo = core::mem::replace(&mut *zelf.memo.write(), new_memo); + drop(old_memo); Ok(()) } } @@ -1253,13 +1270,13 @@ mod _pickle { Some(BINPUT) => { let i = st.read_n(1, vm)?[0] as usize; let value = top!(); - memo_put(zelf, i, value); + memo_put(zelf, Some(i), value, vm)?; } Some(LONG_BINPUT) => { let arr: [u8; 4] = st.read_n(4, vm)?.try_into().unwrap(); let i = u32::from_le_bytes(arr) as usize; let value = top!(); - memo_put(zelf, i, value); + memo_put(zelf, Some(i), value, vm)?; } Some(PUT) => { let line = st.read_line(vm)?; @@ -1269,12 +1286,11 @@ mod _pickle { let body = &line[..line.len() - 1]; let idx = parse_memo_index(body, vm)?; let value = top!(); - memo_put(zelf, idx, value); + memo_put(zelf, Some(idx), value, vm)?; } Some(MEMOIZE) => { let value = top!(); - let idx = zelf.memo.read().len(); - memo_put(zelf, idx, value); + memo_put(zelf, None, value, vm)?; } Some(PERSID) => { let line = st.read_line(vm)?; @@ -1503,19 +1519,43 @@ mod _pickle { } fn memo_get(zelf: &Py, idx: usize, vm: &VirtualMachine) -> PyResult { - zelf.memo - .read() - .get(idx) - .and_then(|o| o.clone()) + let value = zelf.memo.read().get(&idx).cloned(); + value .ok_or_else(|| new_unpickling_error(vm, format!("Memo value not found at index {idx}"))) } - fn memo_put(zelf: &Py, idx: usize, value: PyObjectRef) { - let mut memo = zelf.memo.write(); - if memo.len() <= idx { - memo.resize(idx + 1, None); + fn copy_memo(memo: &UnpicklerMemo) -> Result { + let mut copy = UnpicklerMemo::new(); + copy.try_reserve(memo.len())?; + copy.extend(memo.iter().map(|(&idx, obj)| (idx, obj.clone()))); + Ok(copy) + } + + fn memo_insert( + memo: &mut UnpicklerMemo, + idx: usize, + value: &PyObjectRef, + ) -> Result, TryReserveError> { + if !memo.contains_key(&idx) { + memo.try_reserve(1)?; } - memo[idx] = Some(value); + Ok(memo.insert(idx, value.clone())) + } + + fn memo_put( + zelf: &Py, + idx: Option, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let old_value = { + let mut memo = zelf.memo.write(); + let idx = idx.unwrap_or_else(|| memo.len()); + memo_insert(&mut memo, idx, &value) + }; + // Releasing an entry can call a finalizer that accesses the memo. + drop(old_value.map_err(|_| vm.no_memory_error())?); + Ok(()) } fn instantiate( @@ -1699,7 +1739,7 @@ mod _pickle { buf: data, ..ReadState::default() }), - memo: PyRwLock::new(Vec::new()), + memo: PyRwLock::new(UnpicklerMemo::new()), config: PyRwLock::new(config), } .into_ref(&vm.ctx); @@ -1745,7 +1785,7 @@ mod _pickle { peek, ..ReadState::default() }), - memo: PyRwLock::new(Vec::new()), + memo: PyRwLock::new(UnpicklerMemo::new()), config: PyRwLock::new(config), } .into_ref(&vm.ctx); diff --git a/extra_tests/snippets/stdlib_pickle.py b/extra_tests/snippets/stdlib_pickle.py new file mode 100644 index 00000000000..23d137df43b --- /dev/null +++ b/extra_tests/snippets/stdlib_pickle.py @@ -0,0 +1,11 @@ +import _pickle +import io + +# BINPUT leaves holes; MEMOIZE uses the number of present entries. +unpickler = _pickle.Unpickler(io.BytesIO(b"\x80\x04]q\x07\x94h\x01\x86.")) +left, right = unpickler.load() +assert left is right +memo = unpickler.memo.copy() +assert list(memo) == [1, 7] +assert memo[1] is left +assert memo[7] is left From e5f332fda45b80888fac0cd565078dc78154a589 Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:43:04 +0300 Subject: [PATCH 006/229] Release interpreter channel payloads outside internal locks (#8964) Assisted-by: Codex:gpt-6 --- crates/vm/src/stdlib/_interpchannels.rs | 78 ++++++++++++------- .../snippets/stdlib_subinterpreters.py | 47 +++++++++++ 2 files changed, 98 insertions(+), 27 deletions(-) diff --git a/crates/vm/src/stdlib/_interpchannels.rs b/crates/vm/src/stdlib/_interpchannels.rs index d9fde089fd9..6878ac425f2 100644 --- a/crates/vm/src/stdlib/_interpchannels.rs +++ b/crates/vm/src/stdlib/_interpchannels.rs @@ -366,18 +366,23 @@ pub(crate) mod _interpchannels { /// `channel_destroy`: forget the channel entirely. fn channel_destroy(cid: i64) -> ChanResult, bool)>> { - let mut table = channels().lock(); - let entry = table.refs.remove(&cid).ok_or(ChanErr::NotFound)?; - Ok(match entry.chan { - Some(chan) => drain_queue(&mut chan.state.lock()), - None => Vec::new(), - }) + let entry = channels() + .lock() + .refs + .remove(&cid) + .ok_or(ChanErr::NotFound)?; + let items = match entry.chan { + Some(chan) => core::mem::take(&mut chan.state.lock().queue), + None => VecDeque::new(), + }; + Ok(retire_items(items)) } - fn drain_queue(state: &mut ChannelState) -> Vec<(Arc, bool)> { - state - .queue - .drain(..) + // Shared buffers can run owner callbacks when released. Call only after + // unlocking both the directory and channel state. + fn retire_items(items: VecDeque) -> Vec<(Arc, bool)> { + items + .into_iter() .filter_map(|item| item.waiting.map(|w| (w, false))) .collect() } @@ -394,10 +399,12 @@ pub(crate) mod _interpchannels { return Vec::new(); } let entry = table.refs.remove(&cid).expect("just looked it up"); - match entry.chan { - Some(chan) => drain_queue(&mut chan.state.lock()), - None => Vec::new(), - } + drop(table); + let items = match entry.chan { + Some(chan) => core::mem::take(&mut chan.state.lock().queue), + None => VecDeque::new(), + }; + retire_items(items) } /// `_channels_add_id_object`. @@ -456,12 +463,13 @@ pub(crate) mod _interpchannels { state.closing = true; return Ok(Vec::new()); } - let waiters = drain_queue(&mut state); + let items = core::mem::take(&mut state.queue); state.open = false; state.ends.release_all(); drop(state); entry.chan = None; - Ok(waiters) + drop(table); + Ok(retire_items(items)) } /// `channel_release`: close one or both ends for the current interpreter. @@ -486,6 +494,7 @@ pub(crate) mod _interpchannels { .filter_map(|r| r.chan.clone()) .collect(); let mut waiters = Vec::new(); + let mut dropped = Vec::new(); for chan in chans { let mut state = chan.state.lock(); let mut i = 0; @@ -500,17 +509,19 @@ pub(crate) mod _interpchannels { if let Some(w) = item.waiting { waiters.push((w, false)); } + dropped.push(item.data); continue; } // UNBOUND_ERROR / UNBOUND_REPLACE keep the slot but throw the // data away; the item is now "unbound". - item.data = None; + dropped.push(item.data.take()); i += 1; } state.ends.clear_interpreter(interpid); state.open = state.ends.is_open(); } release_waiters_detached(waiters); + drop(dropped); } /// `resolve_unboundop`. @@ -894,12 +905,14 @@ pub(crate) mod _interpchannels { let cid = parse_cid(parsed[0].as_deref().unwrap(), vm)?.0; let send = flag(parsed[1].as_deref(), vm)?; let chan = channels_lookup(cid).map_err(|e| e.into_py(cid, vm))?; + let interpreters = crate::vm::runtime::list_interpreters(); let state = chan.state.lock(); if send && state.closing { + drop(state); return Err(ChanErr::Closed.into_py(cid, vm)); } let mut out = Vec::new(); - for info in crate::vm::runtime::list_interpreters() { + for info in interpreters { if state.ends.find(info.id, send).is_some_and(|i| { if send { state.ends.send[i].1 @@ -907,10 +920,18 @@ pub(crate) mod _interpchannels { state.ends.recv[i].1 } }) { - out.push(vm.ctx.new_int(info.id).into()); + out.push(info.id); } } - Ok(vm.ctx.new_list(out).into()) + drop(state); + Ok(vm + .ctx + .new_list( + out.into_iter() + .map(|id| vm.ctx.new_int(id).into()) + .collect(), + ) + .into()) } /// `channel_send` up to its `closing` check, which precedes converting the @@ -982,15 +1003,18 @@ pub(crate) mod _interpchannels { if timed_out { // The send is failing now, so make sure the item won't be received. let mut state = chan.state.lock(); - if let Some(pos) = state.queue.iter().position(|it| { - it.waiting - .as_ref() - .is_some_and(|w| Arc::ptr_eq(w, &waiting)) - }) { - state.queue.remove(pos); - } + let removed = state + .queue + .iter() + .position(|it| { + it.waiting + .as_ref() + .is_some_and(|w| Arc::ptr_eq(w, &waiting)) + }) + .and_then(|pos| state.queue.remove(pos)); drop(state); finish_closing(cid); + drop(removed); if !waiting.state.lock().received { return Err(vm .new_os_subtype_error( diff --git a/extra_tests/snippets/stdlib_subinterpreters.py b/extra_tests/snippets/stdlib_subinterpreters.py index 3ca0be27411..e2ad9d76be9 100644 --- a/extra_tests/snippets/stdlib_subinterpreters.py +++ b/extra_tests/snippets/stdlib_subinterpreters.py @@ -117,3 +117,50 @@ def receive(): encoding="utf-8", ) subprocess.run([sys.executable, str(script)], check=True, timeout=30) + + +# Releasing queued buffers can run a finalizer which reenters the channel API. +subprocess.run( + [ + sys.executable, + "-c", + """ +import _interpchannels as channels +import _interpreters +import sys + +released = [] + +class Exporter(bytearray): + def __del__(self): + channels.list_all() + released.append(True) + +for operation in ("destroy", "close", "drop", "timeout"): + # CPython 3.14.6 also deadlocks when close/drop releases a reentrant exporter. + if sys.implementation.name == "cpython" and operation in ("close", "drop"): + continue + channel = channels.create(3) + released.clear() + if operation == "timeout": + try: + channels.send_buffer(channel, Exporter(b"data"), timeout=0) + except TimeoutError: + pass + else: + raise AssertionError("send must time out without a receiver") + else: + channels.send_buffer(channel, Exporter(b"data"), blocking=False) + assert not released + if operation == "destroy": + channels.destroy(channel) + elif operation == "close": + channels.close(channel, force=True) + else: + del channel + assert released == [True], operation +""", + ], + check=True, + timeout=30, +) From f46ba48f10b08df003718030c2ab2589da42ee48 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 4 Oct 2026 23:23:22 -0700 Subject: [PATCH 007/229] Preserve callable addresses in ctypes function pointer casts (#8965) Read the stored executable address when casting a function pointer and accept function pointer types as cast destinations. The existing ownership dictionary retains the original callback through a cast roundtrip. Extracted independently on the unchanged Python 3.14 target, without typed callback argument or broader py_object transport changes. One unique callback-address/ownership regression extends the existing ctypes snippet; canonical coverage was checked for duplicates. Assisted-by: Codex:model-version-unavailable --- crates/vm/src/stdlib/_ctypes/function.rs | 8 +++++++- extra_tests/snippets/stdlib_ctypes.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/_ctypes/function.rs b/crates/vm/src/stdlib/_ctypes/function.rs index e67e3871d14..6066310d534 100644 --- a/crates/vm/src/stdlib/_ctypes/function.rs +++ b/crates/vm/src/stdlib/_ctypes/function.rs @@ -509,7 +509,10 @@ fn cast_check_pointertype(ctype: &PyObject, vm: &VirtualMachine) -> bool { return true; } - // PyCFuncPtrTypeObject_Check - TODO + // PyCFuncPtrTypeObject_Check + if ctype.class().fast_issubclass(PyCFuncPtrType::static_type()) { + return true; + } // simple pointer types via StgInfo.proto (c_void_p, c_char_p, etc.) if let Ok(type_attr) = ctype.get_attr("_type_", vm) @@ -562,6 +565,9 @@ pub(super) fn cast_impl( // Simple type (c_void_p, c_char_p, etc.) → value from buffer let buffer = simple.0.buffer.read(); rustpython_host_env::ctypes::read_pointer_from_buffer(&buffer) + } else if let Some(funcptr) = obj.downcast_ref::() { + // A function pointer stores the executable address in its buffer. + funcptr.get_func_ptr() } else if let Some(cdata) = obj.downcast_ref::() { // Array, Structure, Union → buffer address (b_ptr) cdata.buffer.read().as_ptr() as usize diff --git a/extra_tests/snippets/stdlib_ctypes.py b/extra_tests/snippets/stdlib_ctypes.py index 827fc598826..cd3d0206124 100644 --- a/extra_tests/snippets/stdlib_ctypes.py +++ b/extra_tests/snippets/stdlib_ctypes.py @@ -1,4 +1,5 @@ import ctypes +import gc import itertools import os as _os import sys as _sys @@ -458,4 +459,14 @@ def get_win_folder_via_ctypes(csidl_name: str) -> str: else: raise AssertionError("an unallocatable array was created") +# A function-pointer cast retains the callback and its executable address. +Unary = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) +callback = Unary(lambda value: value + 1) +address = ctypes.cast(callback, ctypes.c_void_p) +roundtrip = ctypes.cast(address, Unary) +assert roundtrip._objects[id(callback)] is callback +del callback, address +gc.collect() +assert roundtrip(41) == 42 + print("done") From f1ebf16ce0ce1b6d35269087a9fa8d84f51d2bce Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 5 Oct 2026 00:20:37 -0700 Subject: [PATCH 008/229] Implement the sentinel builtin (PEP 661) (#8945) * Implement the sentinel builtin from PEP 661 Add unique sentinel values with copy/pickle identity, caller module capture, union support, and cycle collection. Compare constructor behavior and six upstream tests with CPython v3.15.0rc2; retain the existing Python target and bundled libraries. Assisted-by: Codex:model-version-unavailable * Remove duplicate sentinel snippet tests Rely on the canonical Lib/test/test_builtin.py sentinel cases when the standard library is updated, as requested in PR #8945 review. Remove the custom snippet without changing the runtime implementation or existing standard-library tests. Assisted-by: Codex:model-version-unavailable --- crates/vm/src/builtins/mod.rs | 2 + crates/vm/src/builtins/sentinel.rs | 118 +++++++++++++++++++++++++++++ crates/vm/src/builtins/union.rs | 1 + crates/vm/src/stdlib/builtins.rs | 1 + crates/vm/src/types/slot.rs | 1 + crates/vm/src/types/zoo.rs | 7 +- 6 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 crates/vm/src/builtins/sentinel.rs diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs index c7404fcd169..a938b9b3dd1 100644 --- a/crates/vm/src/builtins/mod.rs +++ b/crates/vm/src/builtins/mod.rs @@ -75,6 +75,8 @@ pub(crate) mod type_; pub use type_::{PyType, PyTypeRef}; pub(crate) mod range; pub use range::PyRange; +pub(crate) mod sentinel; +pub use sentinel::PySentinel; pub(crate) mod set; pub use set::{PyFrozenSet, PySet}; pub(crate) mod singletons; diff --git a/crates/vm/src/builtins/sentinel.rs b/crates/vm/src/builtins/sentinel.rs new file mode 100644 index 00000000000..eefc33a58b0 --- /dev/null +++ b/crates/vm/src/builtins/sentinel.rs @@ -0,0 +1,118 @@ +use super::{PyStrRef, PyType, union_}; +use crate::{ + Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + object::{PyAtomicRef, Traverse, TraverseFn}, + protocol::PyNumberMethods, + types::{AsNumber, Constructor, Representable}, +}; + +/// Create a unique sentinel object with the given name. +#[pyclass(module = false, name = "sentinel", traverse = "manual")] +#[derive(Debug)] +pub struct PySentinel { + #[pymember(name = "__name__", type = "object_ex")] + name: PyStrRef, + #[pymember(name = "__module__", type = "object_ex", writable)] + module: PyAtomicRef>, + repr: Option, +} + +// SAFETY: Visit each owned Python reference once, without cloning it. +unsafe impl Traverse for PySentinel { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.name.traverse(tracer_fn); + self.module.traverse(tracer_fn); + self.repr.traverse(tracer_fn); + } + + fn clear(&mut self, out: &mut Vec) { + // String subclasses may own references back to this sentinel. + let name = core::mem::replace(&mut self.name, Context::genesis().empty_str.to_owned()); + out.push(name.into()); + if let Some(module) = self.module.store(None) { + out.push(module); + } + if let Some(repr) = self.repr.take() { + out.push(repr.into()); + } + } +} + +impl PyPayload for PySentinel { + #[inline] + fn class(ctx: &Context) -> &'static Py { + ctx.types.sentinel_type + } +} + +#[derive(FromArgs)] +pub struct SentinelArgs { + #[pyarg(positional, error_msg = "sentinel() argument 1 must be str")] + name: PyStrRef, + #[pyarg( + named, + optional, + error_msg = "sentinel() argument 'repr' must be str or None" + )] + repr: Option, +} + +impl Constructor for PySentinel { + type Args = SentinelArgs; + + fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let module = if let Some(frame) = vm.current_frame() + && let Some(function) = frame.iframe().func_obj() + { + // Use the module captured by the caller function, including the + // synthetic functions used for module, eval, and exec code. + function.get_attr(identifier!(vm, __module__), vm)? + } else { + vm.ctx.none() + }; + Ok(Self { + name: args.name, + module: PyAtomicRef::from(Some(module)), + repr: args.repr, + }) + } +} + +#[pyclass(with(Constructor, Representable, AsNumber), flags(HAVE_GC))] +impl PySentinel { + #[pymethod] + fn __copy__(zelf: PyRef) -> PyRef { + zelf + } + + #[pymethod] + fn __deepcopy__(zelf: PyRef, _memo: PyObjectRef) -> PyRef { + zelf + } + + #[pymethod] + fn __reduce__(zelf: &Py) -> PyStrRef { + zelf.name.clone() + } +} + +impl Representable for PySentinel { + fn repr(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok(zelf.repr.as_ref().unwrap_or(&zelf.name).clone()) + } +} + +impl AsNumber for PySentinel { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| union_::or_op(a.to_owned(), b.to_owned(), vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +pub(crate) fn init(context: &'static Context) { + PySentinel::extend_class(context, context.types.sentinel_type); +} diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs index 83ea3bb54b3..42d21d144ac 100644 --- a/crates/vm/src/builtins/union.rs +++ b/crates/vm/src/builtins/union.rs @@ -202,6 +202,7 @@ impl Py { fn is_unionable(obj: &PyObject, vm: &VirtualMachine) -> bool { let cls = obj.class(); cls.is(vm.ctx.types.none_type) + || cls.is(vm.ctx.types.sentinel_type) || obj.downcastable::() || cls.fast_issubclass(vm.ctx.types.generic_alias_type) || cls.is(vm.ctx.types.union_type) diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index fbced8423a7..d899f1ffca4 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -1479,6 +1479,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { "object" => ctx.types.object_type.to_owned(), "property" => ctx.types.property_type.to_owned(), "range" => ctx.types.range_type.to_owned(), + "sentinel" => ctx.types.sentinel_type.to_owned(), "set" => ctx.types.set_type.to_owned(), "slice" => ctx.types.slice_type.to_owned(), "staticmethod" => ctx.types.staticmethod_type.to_owned(), diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 03eb703f55b..50d3a7a730b 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -253,6 +253,7 @@ bitflagset::bitflagset! { const IMMUTABLETYPE = 8; const HEAPTYPE = 9; const BASETYPE = 10; + const HAVE_GC = 14; const METHOD_DESCRIPTOR = 17; const IS_ABSTRACT = 20; // Built-in types that match the subject itself in pattern matching diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 0d3c5d0fb92..9672e4cf3d2 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -4,8 +4,8 @@ use crate::{ asyncgenerator, bool_, builtin_func, bytearray, bytes, capsule, classmethod, code, complex, coroutine, descriptor, dict, enumerate, filter, float, frame, frame_locals_proxy, function, generator, genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, - module, namespace, object, property, pystr, range, set, singletons, slice, staticmethod, - super_, template, traceback, tuple, + module, namespace, object, property, pystr, range, sentinel, set, singletons, slice, + staticmethod, super_, template, traceback, tuple, type_::{self, PyType}, union_, weakproxy, weakref, zip, }, @@ -64,6 +64,7 @@ pub struct TypeZoo { pub memoryviewiterator_type: &'static Py, pub tuple_type: &'static Py, pub tuple_iterator_type: &'static Py, + pub sentinel_type: &'static Py, pub set_type: &'static Py, pub set_iterator_type: &'static Py, pub staticmethod_type: &'static Py, @@ -145,6 +146,7 @@ impl TypeZoo { memoryview_type: memory::PyMemoryView::init_builtin_type(), property_type: property::PyProperty::init_builtin_type(), range_type: range::PyRange::init_builtin_type(), + sentinel_type: sentinel::PySentinel::init_builtin_type(), set_type: set::PySet::init_builtin_type(), slice_type: slice::PySlice::init_builtin_type(), staticmethod_type: staticmethod::PyStaticMethod::init_builtin_type(), @@ -265,6 +267,7 @@ impl TypeZoo { weakref::init(context); weakproxy::init(context); singletons::init(context); + sentinel::init(context); module::init(context); namespace::init(context); mappingproxy::init(context); From 04d9990616b836da8ead2c8043d8cdbe524f71f6 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 5 Oct 2026 01:29:00 -0700 Subject: [PATCH 009/229] Add the eager-runtime hook required by Python 3.15 importlib (#8946) * Add the eager-runtime importlib lazy-attribute hook Expose _imp._set_lazy_attributes(modobj, name, /), used by CPython 3.15 importlib after eager loading. Match its empty pending-lazy-module registry behavior, including name type and string-subclass hashing checks. This does not implement lazy imports or change the target version or bundled libraries. Assisted-by: Codex:model-version-unavailable * Consolidate lazy import hook tests into stdlib_imp Move the distinct direct-hook argument and hashing contracts into the existing _imp snippet and remove the separate file. Preserve every pre-existing assertion and the CPython version guard; canonical 3.15 lazy-import tests already cover feature-level registry and reification behavior. Assisted-by: Codex:model-version-unavailable --- crates/vm/src/stdlib/_imp.rs | 15 +++++++++- extra_tests/snippets/stdlib_imp.py | 44 +++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index a2bb2a0abc6..b2b2783115b 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -193,7 +193,7 @@ fn find_frozen(name: &str, vm: &VirtualMachine) -> Result PyResult<()> { + // Even an empty set checks the hash of a str subclass. + name.as_object().hash(vm)?; + Ok(()) + } + #[pyfunction] fn is_builtin(name: PyUtf8StrRef, vm: &VirtualMachine) -> bool { vm.state.module_defs.contains_key(name.as_str()) diff --git a/extra_tests/snippets/stdlib_imp.py b/extra_tests/snippets/stdlib_imp.py index 9fd5f8a36fa..8153790c327 100644 --- a/extra_tests/snippets/stdlib_imp.py +++ b/extra_tests/snippets/stdlib_imp.py @@ -1,7 +1,7 @@ import _imp import time as import_time -from testutils import assert_raises +from testutils import assert_raises, skip_if_unsupported assert _imp.is_builtin("time") == True assert _imp.is_builtin("os") == False @@ -42,3 +42,45 @@ def __init__(self, name): assert ispkg is False assert origname == "__hello__" assert _imp.get_frozen_object("__hello__", data).co_name == "" + + +def test_lazy_attributes_eager_registry(): + # CPython accepts any first argument and a string name. In an eager-only + # interpreter there are no pending lazy module names to remove. + for modobj in (None, object(), {}, 42): + assert _imp._set_lazy_attributes(modobj, "package.module") is None + assert _imp._set_lazy_attributes(modobj, "\ud800") is None + + for name in (None, 1, b"package.module", [], {}): + with assert_raises(TypeError): + _imp._set_lazy_attributes(None, name) + + with assert_raises(TypeError): + _imp._set_lazy_attributes(modobj=None, name="package.module") + + class ModuleName(str): + hashes = 0 + + def __hash__(self): + self.hashes += 1 + return 42 + + name = ModuleName("package.module") + assert _imp._set_lazy_attributes(None, name) is None + assert name.hashes == 1 + + class UnhashableName(str): + __hash__ = None + + with assert_raises(TypeError): + _imp._set_lazy_attributes(None, UnhashableName("package.module")) + + class HashErrorName(str): + def __hash__(self): + raise RuntimeError("name hash failed") + + with assert_raises(RuntimeError): + _imp._set_lazy_attributes(None, HashErrorName("package.module")) + + +skip_if_unsupported(3, 15, test_lazy_attributes_eager_registry) From 8e581e6cd541b6489b68fe5aa41564176bc7f371 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:52:52 +0900 Subject: [PATCH 010/229] Truncate sequence index TypeError names to 200 characters (#8967) * Match CPython's str subscript TypeError message `str.__getitem__` with a non-index key raised the shared sequence wording ("str indices must be integers or slices or classes that override __index__ operator, not 'x'") instead of CPython's str-specific "string indices must be integers, not 'x'" (bpo-44110 / gh-88276, Python 3.11). Add `SequenceIndex::try_from_borrowed_object_with_err`, which takes a caller-supplied error closure, and have the existing `try_from_borrowed_object` delegate to it with the unchanged shared message. `str._getitem` now supplies the CPython-accurate message, which flows through both `__getitem__` and the mapping subscript slot. Other sequence types keep the shared wording. Removes the expectedFailure marker on `string_tests.StringLikeTest.test_subscript`, fixing it for both test_str and test_userstring. Co-Authored-By: Claude Opus 4.8 * Fold str's subscript TypeError into the shared index helper The previous commit added `SequenceIndex::try_from_borrowed_object_with_err` (a closure-taking function) so `str` could raise CPython's "string indices must be integers, not ..." message. Per review, a whole helper API plus a thin wrapper was overkill for a single caller. Instead, inline that one error case into `try_from_borrowed_object`: when `type_name` is "str", raise the str-specific message; every other type keeps the shared sequence wording. `str._getitem` goes back to the plain `match` over `SequenceIndex` with no duplicated index parsing. A bad slice key (e.g. `'a'[::'x']`) still surfaces the slice-specific error, since only the non-index/non-slice branch is customized. Co-Authored-By: Claude Opus 4.8 * Truncate index TypeError type names to 200 characters Route str's non-index TypeError through a function pointer message helper. Sequence index errors use the same 200-character type-name cap. Assisted-by: Grok:4.6 * Encode sequence index TypeError names as type params `&'static str` cannot be a const generic. SequenceIndexName holds the name as an associated constant, matching ArgName. Both str and sequence errors go through one fn(&str) -> String and a cold TypeError helper. Assisted-by: Grok:4.6 --------- Co-authored-by: Jaedeok Kim Co-authored-by: Claude Opus 4.8 --- crates/stdlib/src/array.rs | 8 +-- crates/stdlib/src/elementtree.rs | 4 +- crates/stdlib/src/mmap.rs | 6 +-- crates/vm/src/builtins/bytearray.rs | 8 +-- crates/vm/src/builtins/bytes.rs | 4 +- crates/vm/src/builtins/list.rs | 8 +-- crates/vm/src/builtins/tuple.rs | 4 +- crates/vm/src/sliceable.rs | 76 +++++++++++++++++++++++----- crates/vm/src/types/structseq.rs | 6 +-- extra_tests/snippets/builtin_list.py | 19 +++++++ extra_tests/snippets/builtin_str.py | 22 ++++++++ 11 files changed, 127 insertions(+), 38 deletions(-) diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index cb6e40d0d54..fd3e1bba429 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -33,7 +33,7 @@ pub mod array { }, sequence::{SequenceExt, SequenceMutExt}, sliceable::{ - SaturatedSlice, SequenceIndex, SequenceIndexOp, SliceableSequenceMutOp, + NameArray, SaturatedSlice, SequenceIndex, SequenceIndexOp, SliceableSequenceMutOp, SliceableSequenceOp, }, stdlib::_warnings, @@ -1205,7 +1205,7 @@ pub mod array { } fn getitem_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => self.read().getitem_by_index(i, vm), SequenceIndex::Slice(slice) => self.read().getitem_by_slice(slice, vm), } @@ -1221,7 +1221,7 @@ pub mod array { value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => { let typecode = zelf.read().typecode(); let item = ArrayContentType::item_from_object(typecode, value, vm)?; @@ -1266,7 +1266,7 @@ pub mod array { } fn delitem_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => self.try_resizable(vm)?.delitem_by_index(i, vm), SequenceIndex::Slice(slice) => self.try_resizable(vm)?.delitem_by_slice(slice, vm), } diff --git a/crates/stdlib/src/elementtree.rs b/crates/stdlib/src/elementtree.rs index 57c793d5494..1e49c9ec1bb 100644 --- a/crates/stdlib/src/elementtree.rs +++ b/crates/stdlib/src/elementtree.rs @@ -29,7 +29,7 @@ pub(crate) mod _elementtree { builtins::{PyDict, PyDictRef, PyList, PyModule, PyStr, PyType, PyTypeRef}, function::{FuncArgs, PySetterValue}, protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods}, - sliceable::{SequenceIndex, SliceableSequenceOp}, + sliceable::{NameElement, SequenceIndex, SliceableSequenceOp}, types::{ AsMapping, AsNumber, AsSequence, Constructor, DefaultConstructor, Initializer, IterNext, Iterable, Representable, SelfIter, @@ -1028,7 +1028,7 @@ pub(crate) mod _elementtree { } fn sequence_index(vm: &VirtualMachine, needle: &PyObject) -> PyResult { - SequenceIndex::try_from_borrowed_object(vm, needle, "element") + SequenceIndex::try_from_borrowed_object::(vm, needle) .map_err(|_| vm.new_type_error("element indices must be integers")) } diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index 2a3ee0bc460..317176c452b 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -18,7 +18,7 @@ mod mmap { protocol::{ BufferDescriptor, BufferMethods, PyBuffer, PyMappingMethods, PySequenceMethods, }, - sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp}, + sliceable::{NameMmap, SaturatedSlice, SequenceIndex, SequenceIndexOp}, types::{AsBuffer, AsMapping, AsSequence, Constructor, Representable}, }; use core::ops::{Deref, DerefMut}; @@ -1369,7 +1369,7 @@ mod mmap { } fn getitem_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "mmap")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => self.getitem_by_index(i, vm), SequenceIndex::Slice(slice) => self.getitem_by_slice(&slice, vm), } @@ -1381,7 +1381,7 @@ mod mmap { value: &PyObject, vm: &VirtualMachine, ) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "mmap")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => Self::setitem_by_index(zelf, i, value, vm), SequenceIndex::Slice(slice) => Self::setitem_by_slice(zelf, &slice, value, vm), } diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index 19e149d9df0..9e166ca24ca 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -28,7 +28,7 @@ use crate::{ BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods, }, - sliceable::{SequenceIndex, SliceableSequenceMutOp, SliceableSequenceOp}, + sliceable::{NameBytearray, SequenceIndex, SliceableSequenceMutOp, SliceableSequenceOp}, types::{ AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, DefaultConstructor, Initializer, IterNext, Iterable, PyComparisonOp, Representable, @@ -107,7 +107,7 @@ impl PyByteArray { value: &PyObject, vm: &VirtualMachine, ) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => zelf._setitem_by_index(i, value, vm), SequenceIndex::Slice(slice) => { let items = if zelf.is(&value) { @@ -126,7 +126,7 @@ impl PyByteArray { } fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => self .borrow_buf() .getitem_by_index(vm, i) @@ -139,7 +139,7 @@ impl PyByteArray { } pub fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "bytearray")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => self.try_resizable(vm)?.elements.delitem_by_index(vm, i), SequenceIndex::Slice(slice) => { // TODO: delete 0 elements don't need resizable diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index 415cf4df4df..f2d06706b04 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -22,7 +22,7 @@ use crate::{ BufferDescriptor, BufferFlags, BufferMethods, PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods, }, - sliceable::{SequenceIndex, SliceableSequenceOp}, + sliceable::{NameByte, SequenceIndex, SliceableSequenceOp}, types::{ AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, @@ -165,7 +165,7 @@ impl PyBytes { } fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "byte")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => self .getitem_by_index(vm, i) .map(|x| vm.ctx.new_int(x).into()), diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index d6a2c1669a7..dbd40b69c76 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -17,7 +17,7 @@ use crate::{ protocol::{PyIterReturn, PyMappingMethods, PySequenceMethods}, recursion::ReprGuard, sequence::{MutObjectSequenceOp, OptionalRangeArgs, SequenceExt, SequenceMutExt}, - sliceable::{SaturatedSlice, SequenceIndex, SliceableSequenceOp}, + sliceable::{NameList, SaturatedSlice, SequenceIndex, SliceableSequenceOp}, sorting::timsort, types::{ AsMapping, AsSequence, Comparable, Constructor, Initializer, IterNext, Iterable, @@ -264,7 +264,7 @@ impl PyList { } fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => { let vec = self.borrow_vec(); let pos = vec @@ -363,7 +363,7 @@ impl PyList { } fn _setitem(&self, needle: &PyObject, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(index) => self.assign_item(index, Some(value), vm), SequenceIndex::Slice(slice) => { let items = extract_cloned(&value, self, vm)?; @@ -396,7 +396,7 @@ impl PyList { } fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => self.assign_item(i, None, vm), SequenceIndex::Slice(slice) => self.assign_slice(slice, None, vm), } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index a1c06e49106..c8647057330 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -15,7 +15,7 @@ use crate::{ protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, recursion::ReprGuard, sequence::{OptionalRangeArgs, SequenceExt}, - sliceable::{SequenceIndex, SliceableSequenceOp}, + sliceable::{NameTuple, SequenceIndex, SliceableSequenceOp}, types::{ AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, @@ -470,7 +470,7 @@ impl PyTuple { } fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "tuple")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => { let index = self .elements diff --git a/crates/vm/src/sliceable.rs b/crates/vm/src/sliceable.rs index e859c7a3a37..d5a897f4308 100644 --- a/crates/vm/src/sliceable.rs +++ b/crates/vm/src/sliceable.rs @@ -257,6 +257,34 @@ pub enum SequenceIndex { Slice(SaturatedSlice), } +/// Compile-time type name used in sequence index TypeErrors. +/// +/// `&'static str` is not a legal const-generic type, so the name is an associated +/// constant on a marker type, the same pattern as [`crate::function::ArgName`]. +pub trait SequenceIndexName { + const NAME: &'static str; +} + +macro_rules! sequence_index_name { + ($($ty:ident = $name:literal),* $(,)?) => {$( + #[derive(Clone, Copy, Debug)] + pub struct $ty; + impl SequenceIndexName for $ty { + const NAME: &'static str = $name; + } + )*}; +} + +sequence_index_name! { + NameList = "list", + NameTuple = "tuple", + NameByte = "byte", + NameBytearray = "bytearray", + NameArray = "array", + NameMmap = "mmap", + NameElement = "element", +} + impl SequenceIndex { /// The index or slice `obj` stands for, or `None` when it is neither. fn try_from_object_opt(vm: &VirtualMachine, obj: &PyObject) -> Option> { @@ -280,27 +308,47 @@ impl SequenceIndex { } } - pub fn try_from_borrowed_object( + fn try_from_object( vm: &VirtualMachine, obj: &PyObject, - type_name: &str, + err_fmt: fn(&str) -> String, ) -> PyResult { - Self::try_from_object_opt(vm, obj).unwrap_or_else(|| { - Err(vm.new_type_error(format!( - "{type_name} indices must be integers or slices, not {}", - obj.class().slot_name() - ))) - }) + Self::try_from_object_opt(vm, obj) + .unwrap_or_else(|| Self::index_type_error(vm, obj, err_fmt)) + } + + #[cold] + #[inline(never)] + fn index_type_error( + vm: &VirtualMachine, + obj: &PyObject, + err_fmt: fn(&str) -> String, + ) -> PyResult { + let class_name = obj.class().slot_name(); + Err(vm.new_type_error(err_fmt(&class_name))) + } + + fn sequence_index_type_error(class_name: &str) -> String { + format!( + "{} indices must be integers or slices, not {class_name:.200}", + N::NAME + ) + } + + fn str_index_type_error(class_name: &str) -> String { + format!("string indices must be integers, not '{class_name:.200}'") + } + + pub fn try_from_borrowed_object( + vm: &VirtualMachine, + obj: &PyObject, + ) -> PyResult { + Self::try_from_object(vm, obj, Self::sequence_index_type_error::) } /// `unicode_subscript`, which turns down what it cannot use in its own words. pub fn try_from_str_subscript(vm: &VirtualMachine, obj: &PyObject) -> PyResult { - Self::try_from_object_opt(vm, obj).unwrap_or_else(|| { - Err(vm.new_type_error(format!( - "string indices must be integers, not '{}'", - obj.class().slot_name() - ))) - }) + Self::try_from_object(vm, obj, Self::str_index_type_error) } } diff --git a/crates/vm/src/types/structseq.rs b/crates/vm/src/types/structseq.rs index 09822495075..dee57fb92ca 100644 --- a/crates/vm/src/types/structseq.rs +++ b/crates/vm/src/types/structseq.rs @@ -11,7 +11,7 @@ use crate::{ }, iter::PyExactSizeIterator, protocol::{PyMappingMethods, PySequenceMethods}, - sliceable::{SequenceIndex, SliceableSequenceOp}, + sliceable::{NameTuple, SequenceIndex, SliceableSequenceOp}, types::PyComparisonOp, vm::Context, }; @@ -217,7 +217,7 @@ static STRUCT_SEQUENCE_AS_MAPPING: LazyLock = let tuple = mapping.obj.downcast_ref::().unwrap(); let visible_elements = &tuple.as_slice()[..n_seq]; - match SequenceIndex::try_from_borrowed_object(vm, needle, "tuple")? { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { SequenceIndex::Int(i) => visible_elements.getitem_by_index(vm, i), SequenceIndex::Slice(slice) => visible_elements .getitem_by_slice(vm, slice) @@ -376,7 +376,7 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { let n_seq = get_visible_len(zelf.as_ref(), vm)?; let visible_elements = &zelf.as_slice()[..n_seq]; - match SequenceIndex::try_from_borrowed_object(vm, &needle, "tuple")? { + match SequenceIndex::try_from_borrowed_object::(vm, &needle)? { SequenceIndex::Int(i) => visible_elements.getitem_by_index(vm, i), SequenceIndex::Slice(slice) => visible_elements .getitem_by_slice(vm, slice) diff --git a/extra_tests/snippets/builtin_list.py b/extra_tests/snippets/builtin_list.py index 189f794774b..59f77e5a9ba 100644 --- a/extra_tests/snippets/builtin_list.py +++ b/extra_tests/snippets/builtin_list.py @@ -1388,3 +1388,22 @@ def failing_source(): test_slice_replacement_lifetime() + + +def test_subscript_typeerror_message(): + def message(index): + try: + [0, 1, 2][index] + except TypeError as err: + return str(err) + raise AssertionError("TypeError was not raised") + + assert message("x") == "list indices must be integers or slices, not str" + + Long = type("A" * 250, (), {}) + assert message(Long()) == "list indices must be integers or slices, not {}".format( + "A" * 200 + ) + + +test_subscript_typeerror_message() diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index d114e2d5727..a8da6ab96bb 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -1160,3 +1160,25 @@ def __radd__(self, other): test_concat_error_message() + + +def test_subscript_typeerror_message(): + def message(index): + try: + "abc"[index] + except TypeError as err: + return str(err) + raise AssertionError("TypeError was not raised") + + assert message("def") == "string indices must be integers, not 'str'" + assert message(object()) == "string indices must be integers, not 'object'" + assert message(1.5) == "string indices must be integers, not 'float'" + assert message(None) == "string indices must be integers, not 'NoneType'" + + Long = type("A" * 250, (), {}) + assert message(Long()) == "string indices must be integers, not '{}'".format( + "A" * 200 + ) + + +test_subscript_typeerror_message() From d3ff1eb5ea657cd385a0eca90f8bcd84b68e015c Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 5 Oct 2026 04:48:57 -0700 Subject: [PATCH 011/229] Keep module cache entries present during import reordering (#8948) * Reorder the module cache atomically during imports Add a private exact-dict move primitive and importlib dispatcher so shutdown-order updates never temporarily remove a module. Preserve current values and custom-mapping fallback behavior; validate reentrant equality probe witnesses, invalidate layout caches, and compact moved entries. Cover all four existing bootstrap reorder sites, including legacy-loader cleanup. Add deterministic trace-window, loader replacement/removal, reentrancy, cache, iterator and compaction regressions. The private native primitive uses true-move semantics; it does not promise arbitrary pop/set callback equivalence. Assisted-by: Codex:model-version-unavailable * Keep atomic import regression compatible with modern bootstrap Run legacy-loader cases when the bootstrap exposes that helper, preserving all four 3.14 paths and the two remaining 3.15 paths. Always retain modern load and exec regressions. Assisted-by: Codex:model-version-unavailable * Keep the copied bootstrap unchanged in the native relocation PR Restore the exact CPython 3.14 bootstrap and retain only the private exact-dict relocation primitive and its direct native regressions. Remove the PR-added callback integration tests while preserving native helper assertions. This prepares a primitive only; imports do not call it and the import race remains unresolved. Assisted-by: Codex:model-version-unavailable --- crates/vm/src/builtins/dict.rs | 10 + crates/vm/src/dict_inner.rs | 236 ++++++++++++++++++ crates/vm/src/stdlib/_imp.rs | 14 +- extra_tests/snippets/import_atomic_reorder.py | 189 ++++++++++++++ 4 files changed, 447 insertions(+), 2 deletions(-) create mode 100644 extra_tests/snippets/import_atomic_reorder.py diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index d709521ea6d..2cc9034b1cf 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -1085,6 +1085,16 @@ impl Py { } impl PyExact { + pub(crate) fn move_to_end( + &self, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.entries + .move_to_end(vm, &*key)? + .ok_or_else(|| vm.new_key_error(key)) + } + /// Look up `key` in `self`, falling back to `other`. /// Both dicts must be exact `dict` types (enforced by `PyExact`). pub(crate) fn get_chain_exact( diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 4a901bfbf0e..ed6a2300528 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -1392,6 +1392,134 @@ impl Dict { Ok(removed) } + /// Move an existing entry to the end and return its current value without + /// making the key temporarily absent. Hashing and equality run unlocked. + pub(crate) fn move_to_end( + &self, + vm: &VirtualMachine, + key: &K, + ) -> PyResult> { + struct ProbeWitness { + index_index: IndexIndex, + entry_index: IndexEntry, + key: Option<(HashValue, PyObjectRef)>, + } + + let hash_value = key.key_hash(vm)?; + let mut idxs = None; + let mut prefix = Vec::::new(); + let mut compared: Option<(EntryIndex, IndexIndex, PyObjectRef, bool)> = None; + 'lookup: loop { + // Keep the compared key alive until after the guard is released, + // including when equality cleared the dictionary and recycled its + // entry and bucket for a different key. + let comparison = compared.take(); + let mut inner = self.write(); + let mask = (inner.indices.len() - 1) as i64; + let probes = idxs.get_or_insert_with(|| GenIndexes::new(hash_value, mask)); + if probes.mask != mask { + drop(inner); + prefix.clear(); + idxs = None; + continue; + } + let matched = if let Some((entry_index, index_index, candidate, equal)) = &comparison { + let valid = inner.indices.get(*index_index).and_then(|i| i.index()) + == Some(*entry_index) + && inner + .get_entry_checked(*entry_index, *index_index) + .is_some_and(|entry| entry.hash == hash_value && entry.key.is(candidate)); + // A false comparison can survive while another previously + // probed bucket changes (including after nested compaction). + // Revalidate the whole prefix before continuing past it. + let prefix_valid = *equal + || prefix.iter().all(|witness| { + inner.indices.get(witness.index_index) == Some(&witness.entry_index) + && witness.key.as_ref().is_none_or(|(hash, key)| { + inner + .get_entry_checked( + witness.entry_index.index().unwrap(), + witness.index_index, + ) + .is_some_and(|entry| entry.hash == *hash && entry.key.is(key)) + }) + }); + if !valid || !prefix_valid { + drop(inner); + prefix.clear(); + idxs = None; + continue; + } + equal.then_some(*entry_index) + } else { + None + }; + let entry_index = match matched { + Some(index) => index, + None => loop { + let index_index = probes.next(); + let index_entry = inner.indices[index_index]; + match index_entry { + IndexEntry::FREE => return Ok(None), + IndexEntry::DUMMY => { + prefix.push(ProbeWitness { + index_index, + entry_index: index_entry, + key: None, + }); + continue; + } + _ => {} + } + let entry_index = index_entry.index().unwrap(); + let entry = inner.entries[entry_index].as_ref().unwrap(); + if key.key_is(&entry.key) { + break entry_index; + } + prefix.push(ProbeWitness { + index_index, + entry_index: index_entry, + key: Some((entry.hash, entry.key.clone())), + }); + if entry.hash == hash_value { + let candidate = entry.key.clone(); + drop(inner); + drop(comparison); + let equal = key.key_eq(vm, &candidate)?; + compared = Some((entry_index, index_index, candidate, equal)); + continue 'lookup; + } + }, + }; + let value = inner.entries[entry_index].as_ref().unwrap().value.clone(); + if inner.entries[entry_index + 1..].iter().any(Option::is_some) { + // Reserve before taking the entry. No reader can observe an + // absent entry, and no hashing, equality or finalizer runs in + // this critical section. + inner.entries.reserve(1); + self.invalidate_keys_version(); + let entry = inner.entries[entry_index].take().unwrap(); + let new_index = inner.entries.len(); + inner.indices[entry.index] = unsafe { + // SAFETY: new_index is a valid nonnegative entry index. + IndexEntry::from_index_unchecked(new_index) + }; + inner.entries.push(Some(entry)); + let holes = inner.entries.len() - inner.used; + if holes >= 2 && holes >= inner.used { + // Moves do not increase `filled`, so ordinary insertion's + // resize threshold cannot bound their accumulated holes. + // Two holes also ensure this operation changes DictSize, + // allowing existing iterators to detect the relocation. + let indices_size = inner.indices.len(); + inner.resize(indices_size); + } + } + drop(inner); + return Ok(Some(value)); + } + } + pub(crate) fn pop_back(&self) -> Option<(PyObjectRef, T)> { let inner = &mut *self.write(); let entry = loop { @@ -1838,6 +1966,114 @@ mod tests { use super::*; use crate::{Interpreter, common::ascii}; + #[test] + fn move_to_end_preserves_values_and_invalidates_layout() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let dict = Dict::default(); + let keys = [ + vm.ctx.intern_str("first"), + vm.ctx.intern_str("middle"), + vm.ctx.intern_str("last"), + ]; + for (value, key) in keys.iter().enumerate() { + dict.insert(vm, *key, value).unwrap(); + } + let hint = dict.hint_for_key(vm, keys[1]).unwrap().unwrap() as usize; + let version = dict.assign_keys_version(); + let size = dict.size(); + assert_ne!(version, 0); + assert_eq!(dict.move_to_end(vm, keys[1]).unwrap(), Some(1)); + assert_eq!(dict.values(), vec![0, 2, 1]); + assert_eq!(dict.keys_version(), 0); + assert_eq!(dict.get_index_if_keys_version(version, hint), None); + assert_eq!(dict.get_hint(vm, keys[1], hint).unwrap(), None); + assert!(dict.next_entry_checked(0, &size, |_, v| *v).is_err()); + assert!( + dict.prev_entry_checked(size.entries_size - 1, &size, |_, v| *v) + .is_err() + ); + + let version = dict.assign_keys_version(); + let size = dict.size(); + assert_eq!(dict.move_to_end(vm, keys[1]).unwrap(), Some(1)); + assert_eq!(dict.keys_version(), version); + assert_eq!(dict.size(), size); + assert_eq!(dict.move_to_end(vm, "absent").unwrap(), None); + assert_eq!(dict.keys_version(), version); + assert_eq!(dict.size(), size); + assert_eq!(dict.values(), vec![0, 2, 1]); + }); + } + + #[test] + fn move_to_end_compacts_and_keeps_indices_consistent() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + for count in [2, 3, 32] { + let dict = Dict::default(); + for key in 0..count { + dict.insert(vm, &key, key).unwrap(); + } + for step in 0..1024 { + let key = step % count; + let size = dict.size(); + let hint = dict.hint_for_key(vm, &key).unwrap().unwrap() as usize; + let version = dict.assign_keys_version(); + assert_eq!(dict.move_to_end(vm, &key).unwrap(), Some(key)); + assert_eq!(dict.get_index_if_keys_version(version, hint), None); + assert!(dict.next_entry_checked(0, &size, |_, v| *v).is_err()); + assert!( + dict.prev_entry_checked(size.entries_size - 1, &size, |_, v| *v) + .is_err() + ); + let inner = dict.read(); + assert_eq!(inner.used, count); + assert_eq!(inner.filled, count); + assert!(inner.entries.len() < 2 * count); + assert!(inner.entries.capacity() <= 4 * count); + let values: Vec<_> = inner.entries.iter().flatten().map(|e| e.value).collect(); + assert_eq!( + values, + (1..=count).map(|i| (key + i) % count).collect::>() + ); + for (index, entry) in inner.entries.iter().enumerate() { + if let Some(entry) = entry { + assert_eq!(inner.indices[entry.index].index(), Some(index)); + assert_eq!(entry.hash, entry.value.key_hash(vm).unwrap()); + } + } + for (bucket, index) in inner.indices.iter().enumerate() { + if let Some(index) = index.index() { + assert_eq!(inner.entries[index].as_ref().unwrap().index, bucket); + } + } + } + } + }); + } + + #[test] + fn move_to_end_restores_only_matching_shared_shapes() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let first = vm.ctx.intern_str("move_first"); + let last = vm.ctx.intern_str("move_last"); + let dict = Dict::default(); + let peer = Dict::default(); + for table in [&dict, &peer] { + table.insert(vm, first, 1).unwrap(); + table.insert(vm, last, 2).unwrap(); + } + let version = dict.assign_keys_version(); + assert_eq!(peer.assign_keys_version(), version); + dict.move_to_end(vm, first).unwrap(); + assert_ne!(dict.assign_keys_version(), version); + dict.move_to_end(vm, last).unwrap(); + // The second move compacts back to the original hole-free layout. + assert_eq!(dict.assign_keys_version(), version); + assert_eq!(dict.get_index_if_keys_version(version, 0), Some(1)); + assert_eq!(dict.get_index_if_keys_version(version, 1), Some(2)); + }); + } + #[test] fn clone_compacts_deleted_entries() { Interpreter::without_stdlib(Default::default()).enter(|vm| { diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index b2b2783115b..07503ea91d8 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -193,13 +193,23 @@ fn find_frozen(name: &str, vm: &VirtualMachine) -> Result, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + modules.move_to_end(key, vm) + } + #[pyattr] fn check_hash_based_pycs(vm: &VirtualMachine) -> PyStrRef { vm.ctx diff --git a/extra_tests/snippets/import_atomic_reorder.py b/extra_tests/snippets/import_atomic_reorder.py new file mode 100644 index 00000000000..b7c8a9d356b --- /dev/null +++ b/extra_tests/snippets/import_atomic_reorder.py @@ -0,0 +1,189 @@ +"""Private exact-dict relocation and callback-reentry regression coverage.""" + +import _imp +import sys + + +def raises(kind, call, expected=None): + try: + call() + except kind as exc: + if expected is not None: + assert exc is expected + return exc + raise AssertionError(f"{kind.__name__} was not raised") + + +def check_native_helper(): + move = _imp._dict_move_to_end + first, middle, last = object(), object(), object() + mapping = {"first": first, "middle": middle, "last": last} + assert move(mapping, "middle") is middle + assert list(mapping) == ["first", "last", "middle"] + assert len(mapping) == 3 + assert move(mapping, "middle") is middle + assert list(mapping) == ["first", "last", "middle"] + single = {"only": None} + assert move(single, "only") is None + assert single == {"only": None} + missing = object() + exc = raises(KeyError, lambda: move(mapping, missing)) + assert exc.args[0] is missing + assert list(mapping) == ["first", "last", "middle"] + + class Subclass(dict): + pass + + raises(TypeError, lambda: move(Subclass(mapping), "middle")) + raises(TypeError, lambda: move([], "middle")) + stored = "".join(["equal", " key"]) + equal = "".join(["equal ", "key"]) + assert stored == equal and stored is not equal + mapping = {stored: middle, "after": last} + assert move(mapping, equal) is middle + assert list(mapping)[-1] is stored + + failure = ValueError("hash failure") + + class Hashed(str): + count = 0 + fail = False + + def __hash__(self): + self.count += 1 + if self.fail: + raise failure + return super().__hash__() + + lookup = Hashed(stored) + assert move(mapping, lookup) is middle + assert lookup.count == 1 + lookup.fail = True + raises(ValueError, lambda: move(mapping, lookup), failure) + assert lookup.count == 2 + + class Collision: + def __init__(self, label, action=None): + self.label = label + self.action = action + + def __hash__(self): + return 42 + + def __eq__(self, other): + action, self.action = self.action, None + if action is not None: + action() + return isinstance(other, Collision) and self.label == other.label + + lookup = Collision("target") + stored = Collision("target") + impostor = Collision("different") + mapping = {stored: first, "after": last} + + def replace_key(): + mapping.clear() + mapping[impostor] = middle + + stored.action = replace_key + raises(KeyError, lambda: move(mapping, lookup)) + assert list(mapping) == [impostor] + assert mapping[impostor] is middle + + stored = Collision("target") + mapping = {stored: first, "after": last} + stored.action = lambda: mapping.__setitem__(stored, middle) + assert move(mapping, lookup) is middle + assert list(mapping)[-1] is stored + + stored = Collision("target") + mapping = {stored: first, "after": last} + stored.action = lambda: mapping.clear() + raises(KeyError, lambda: move(mapping, lookup)) + assert mapping == {} + + stored = Collision("target") + mapping = {stored: first, "after": last} + + def resize(): + mapping.clear() + for i in range(100): + mapping[i] = i + mapping[stored] = middle + mapping["after"] = last + + stored.action = resize + assert move(mapping, lookup) is middle + assert list(mapping)[-1] is stored + assert len(mapping) == 102 + + # A surviving false candidate alone does not certify the probe prefix. + # Nested relocation compacts T into P's earlier bucket, while A keeps its + # key identity, bucket and physical entry index. T never becomes absent. + prefix_key = Collision("prefix") + candidate = Collision("candidate") + target = Collision("target") + mapping = {prefix_key: first, candidate: last, target: middle} + + def compact_prefix(): + del mapping[prefix_key] + assert move(mapping, candidate) is last + assert list(mapping) == [target, candidate] + + candidate.action = compact_prefix + assert move(mapping, lookup) is middle + assert list(mapping) == [candidate, target] + + # Inserting in a previously visited DUMMY bucket also requires a restart. + mapping = {prefix_key: first, candidate: last} + del mapping[prefix_key] + candidate.action = lambda: mapping.__setitem__(lookup, middle) + assert move(mapping, lookup) is middle + assert list(mapping) == [candidate, lookup] + + failure = RuntimeError("equality failure") + + def fail(): + raise failure + + stored = Collision("target", fail) + mapping = {stored: first, "after": last} + before = list(mapping) + raises(RuntimeError, lambda: move(mapping, lookup), failure) + assert list(mapping) == before + + class HashMutation: + def __hash__(self): + mapping.clear() + mapping["kept"] = last + return 42 + + raises(KeyError, lambda: move(mapping, HashMutation())) + assert mapping == {"kept": last} + + # Releasing the removed candidate must not run its finalizer under the + # dictionary lock. The only owners are the dict and the lookup's witness. + finalized = [] + + class Finalized(Collision): + def __del__(self): + mapping["finalized"] = True + finalized.append(True) + + mapping = {Finalized("target", lambda: mapping.clear()): first} + raises(KeyError, lambda: move(mapping, lookup)) + assert finalized == [True] + assert mapping == {"finalized": True} + + mapping = {"a": 1, "b": 2} + for i in range(100): + forward, backward = iter(mapping), reversed(mapping) + key = "a" if i % 2 == 0 else "b" + move(mapping, key) + raises(RuntimeError, lambda: next(forward)) + raises(RuntimeError, lambda: next(backward)) + assert list(mapping)[-1] == key + + +if sys.implementation.name == "rustpython": + check_native_helper() From 8cf8917f858988e093ab3d5678d157f25610207e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:59:36 +0900 Subject: [PATCH 012/229] build(deps): bump https://github.com/rbubley/mirrors-prettier (#8969) Bumps [https://github.com/rbubley/mirrors-prettier](https://github.com/rbubley/mirrors-prettier) from v3.9.8 to 3.9.9. - [Commits](https://github.com/rbubley/mirrors-prettier/compare/v3.9.8...v3.9.9) --- updated-dependencies: - dependency-name: https://github.com/rbubley/mirrors-prettier dependency-version: 3.9.9 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aeef06ed6a6..0a286052e77 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,7 @@ repos: priority: 0 - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.8 + rev: v3.9.9 hooks: - id: prettier files: '^wasm/.*$' From 524f1a2f4ca21929f8249864710db6cf2236dc0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:01:07 +0900 Subject: [PATCH 013/229] build(deps): bump github/gh-aw/actions/setup from 0.89.17 to 0.89.21 (#8975) Bumps [github/gh-aw/actions/setup](https://github.com/github/gh-aw) from 0.89.17 to 0.89.21. - [Release notes](https://github.com/github/gh-aw/releases) - [Changelog](https://github.com/github/gh-aw/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/gh-aw/compare/00457477720387bcc7d7baaf841dedb22ad06617...c35393777e5604a63721d09512263b1383301d4f) --- updated-dependencies: - dependency-name: github/gh-aw/actions/setup dependency-version: 0.89.21 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/upgrade-pylib.lock.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml index 23c1aeec47c..c08d19058e3 100644 --- a/.github/workflows/upgrade-pylib.lock.yml +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -58,7 +58,7 @@ jobs: comment_repo: "" steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17 + uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 with: destination: /opt/gh-aw/actions - name: Check workflow file timestamps @@ -99,7 +99,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17 + uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 with: destination: /opt/gh-aw/actions - name: Checkout repository @@ -806,7 +806,7 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17 + uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 with: destination: /opt/gh-aw/actions - name: Download agent output artifact @@ -927,7 +927,7 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17 + uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 with: destination: /opt/gh-aw/actions - name: Download agent artifacts @@ -1039,7 +1039,7 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17 + uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 with: destination: /opt/gh-aw/actions - name: Download agent output artifact From 52581b16f1d59759d5e800bfc2336442b6ffd495 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:12:58 +0900 Subject: [PATCH 014/229] build(deps): bump https://github.com/astral-sh/ruff-pre-commit (#8970) Bumps [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit) from v0.16.8 to 0.16.9. - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.16.8...v0.16.9) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.16.9 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a286052e77..a65592fac6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: priority: 0 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.8 + rev: v0.16.9 hooks: - id: ruff-format priority: 0 From 31267623587193a61ca3ae00ce69ff1ec21fd648 Mon Sep 17 00:00:00 2001 From: Hanjeong Lee Date: Tue, 6 Oct 2026 00:13:55 +0900 Subject: [PATCH 015/229] Give os.chmod a valid `__text_signature__` and keyword arguments (#8968) * Order chmod's arguments the way CPython declares them posix.chmod took dir_fd between path and mode. Since #8807 renders each argument's parameter kind, that put `/` after `*`, which inspect cannot parse, and path and mode stayed positional-only, so os.chmod(path=..., mode=...) raised TypeError. A ChmodArgs struct follows CPython's order, as nt.chmod already does. path and mode are positional-or-keyword. follow_symlinks shows CPython's text through py_default, like LinkArgs. Assisted-by: Claude Code:claude-opus-5 * Reject a text signature that puts `/` after `*` or `**` The writer counts positional-only parameters up front and emits `/` when they run out, which assumes they come first in the Rust argument list. Nothing enforced that, and posix.chmod violated it. The signature is composed in a const context, so the assert fails the build and names the fix. Assisted-by: Claude Code:claude-opus-5 --- crates/vm/src/function/signature.rs | 14 ++++- crates/vm/src/stdlib/posix.rs | 62 ++++++++++++++++------- extra_tests/snippets/builtin_signature.py | 9 ++++ 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/crates/vm/src/function/signature.rs b/crates/vm/src/function/signature.rs index 3dfae3ae9df..b43595ee358 100644 --- a/crates/vm/src/function/signature.rs +++ b/crates/vm/src/function/signature.rs @@ -242,6 +242,7 @@ struct St { emitted: bool, po_left: usize, var_pos_seen: bool, + var_kw_seen: bool, star_emitted: bool, } @@ -255,6 +256,7 @@ const fn write_signature_prefix(buf: &mut [u8], name: &str, args: &[SigArg]) -> emitted: false, po_left: count_po_args(args), var_pos_seen: false, + var_kw_seen: false, star_emitted: false, }, args, @@ -575,6 +577,12 @@ const fn write_one(buf: &mut [u8], mut st: St, param: Param, fallback: &str) -> st = emit_named(buf, st, "", name, param.default); st.po_left -= 1; if st.po_left == 0 { + // Python rejects `/` after `*` or `**`. The Rust arguments put a + // keyword or variadic one before a positional one; reorder them. + assert!( + !st.star_emitted && !st.var_pos_seen && !st.var_kw_seen, + "positional-only parameter after `*` or `**`: reorder the Rust arguments" + ); st = emit_text(buf, st, "/"); } st @@ -591,7 +599,10 @@ const fn write_one(buf: &mut [u8], mut st: St, param: Param, fallback: &str) -> st.var_pos_seen = true; emit_named(buf, st, "*", name, param.default) } - ParamKind::VarKeyword => emit_named(buf, st, "**", name, param.default), + ParamKind::VarKeyword => { + st.var_kw_seen = true; + emit_named(buf, st, "**", name, param.default) + } } } @@ -631,6 +642,7 @@ mod tests { emitted: false, po_left: 0, var_pos_seen: false, + var_kw_seen: false, star_emitted: true, }, Param { diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 1f123812fc2..a5fe3f4e88a 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1186,7 +1186,7 @@ pub mod module { path: OsPath, dir_fd: DirFd<'_, 0>, mode: u32, - follow_symlinks: FollowSymlinks, + follow_symlinks: bool, vm: &VirtualMachine, ) -> PyResult<()> { let [] = dir_fd.0; @@ -1195,7 +1195,7 @@ pub mod module { not(target_os = "redox"), not(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd")) ))] - if !follow_symlinks.0 { + if !follow_symlinks { let err_path = path.clone(); let c_path = path.into_cstring(vm)?; return rustpython_host_env::posix::fchmodat( @@ -1219,7 +1219,7 @@ pub mod module { let err_path = path.clone(); let body = move || { use std::os::unix::fs::PermissionsExt; - let meta = fs_metadata(&path, follow_symlinks.0)?; + let meta = fs_metadata(&path, follow_symlinks)?; let mut permissions = meta.permissions(); permissions.set_mode(mode); fs::set_permissions(&path, permissions) @@ -1233,18 +1233,32 @@ pub mod module { } #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn chmod( - path: OsPathOrFd<'_>, - dir_fd: DirFd<'_, 0>, + #[derive(FromArgs)] + struct ChmodArgs<'fd> { + #[pyarg(any)] + path: OsPathOrFd<'fd>, + #[pyarg(any)] mode: u32, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { + #[pyarg(flatten)] + dir_fd: DirFd<'fd, 0>, + // CPython writes the platform expression; on posix it is always true. + #[pyarg(named, default = true, py_default = "(os.name != 'nt')")] + follow_symlinks: bool, + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn chmod(args: ChmodArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + let ChmodArgs { + path, + mode, + dir_fd, + follow_symlinks, + } = args; match path { OsPathOrFd::Path(path) => { #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd",))] - if !follow_symlinks.0 && dir_fd == Default::default() { + if !follow_symlinks && dir_fd == Default::default() { return lchmod(LchmodArgs { path, mode }, vm); } _chmod(path, dir_fd, mode, follow_symlinks, vm) @@ -1254,14 +1268,28 @@ pub mod module { } #[cfg(target_os = "redox")] - #[pyfunction] - fn chmod( + #[derive(FromArgs)] + struct ChmodArgs { + #[pyarg(any)] path: OsPath, - dir_fd: DirFd<0>, + #[pyarg(any)] mode: u32, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { + #[pyarg(flatten)] + dir_fd: DirFd<'static, 0>, + // CPython writes the platform expression; on posix it is always true. + #[pyarg(named, default = true, py_default = "(os.name != 'nt')")] + follow_symlinks: bool, + } + + #[cfg(target_os = "redox")] + #[pyfunction] + fn chmod(args: ChmodArgs, vm: &VirtualMachine) -> PyResult<()> { + let ChmodArgs { + path, + mode, + dir_fd, + follow_symlinks, + } = args; _chmod(path, dir_fd, mode, follow_symlinks, vm) } diff --git a/extra_tests/snippets/builtin_signature.py b/extra_tests/snippets/builtin_signature.py index be0d53e9e3c..68998566a7e 100644 --- a/extra_tests/snippets/builtin_signature.py +++ b/extra_tests/snippets/builtin_signature.py @@ -64,3 +64,12 @@ assert sum.__text_signature__ == "($module, iterable, /, start=0)" assert str(inspect.signature(round)) == "(number, ndigits=None)" assert str(inspect.signature(sum)) == "(iterable, /, start=0)" + +# A keyword-only argument between two positional ones used to put `/` after +# `*`, which inspect cannot parse. Arguments are ordered like CPython's, so +# path and mode take keywords. Windows renders follow_symlinks differently. +if hasattr(os, "chmod"): + chmod_signature = os.chmod.__text_signature__ + assert chmod_signature.startswith("($module, /, path, mode, *, dir_fd=None"), ( + chmod_signature + ) From 5b5c0c7df70c5807e3a0bfd7990e2a5e92f3873f Mon Sep 17 00:00:00 2001 From: Jorge Polanco <55784702+Jorge-Polanco-Roque@users.noreply.github.com> Date: Mon, 5 Oct 2026 09:24:42 -0600 Subject: [PATCH 016/229] Give dict a CPython-style message for unhashable keys (#8610) * Give dict a CPython-style message for unhashable keys Dict operations (subscription, membership, get/pop/setdefault, ...) now raise "cannot use 'X' as a dict key (unhashable type: 'X')", matching CPython 3.14 and mirroring the existing PySetInner wrapping. The key is only materialized on the error path, so hashable keys pay no extra cost. Enables Lib/test/test_dict.py::DictTest::test_unhashable_key. Assisted-by: Claude Opus 4.8 (Anthropic) * dict: only rewrite hashing failures, not key-comparison errors wrap_unhashable_error previously rewrote any TypeError from a dict operation as an unhashable-key error, but those operations also compare keys on a hash collision, so a TypeError from a colliding key's __eq__ was mislabeled. Disambiguate on the error path by re-hashing the key, so the successful path still hashes exactly once (keeps the do-not-rehash / atomic invariants) while comparison errors propagate unchanged. Adds a colliding-key regression test. Assisted-by: Claude Opus 4.8 (Anthropic) * dict: hash unhashable keys once up front and cover every insertion path Addresses the review feedback (thanks @luantaraschi): - The message now reaches update(), the constructor and |= as well, not just __setitem__: merge_object_with_override and merge_from_seq2 now hash the key up front and thread it into contains_known_hash/insert_known_hash. - Hash the key once instead of re-hashing on the error path. This threads the hash through the *_known_hash operations (add dict_inner get_known_hash), so a __hash__ that fails only on its first call is still reported instead of escaping unwrapped. - Use the fully-qualified type name (matching CPython's %T). - Only an exact TypeError is rewritten; a __hash__ raising a TypeError subclass now propagates unchanged. pop() and setdefault() have no *_known_hash entry point on the inner map, so they hash once up front for the message and then do their own single lookup. Assisted-by: Claude Opus 4.8 * dict: thread the known hash through setdefault and pop as well Follow-up on the review: setdefault() and pop() previously hashed the key a second time (inside the inner map) after hash_or_unhashable already hashed it, so a stateful __hash__ failing on the second call would skip the dict-specific message. Add setdefault_known_hash / pop_known_hash to the inner map and thread the precomputed hash, so every operation now hashes exactly once. Adds a regression test asserting setdefault/pop call __hash__ only once. Assisted-by: Claude Opus 4.8 * test: hash the key on a non-empty dict so CPython 3.14 exercises the pop path `{}.pop(CountingHash())` on an empty dict does not hash the key under CPython 3.14 (the lookup short-circuits), so `CountingHash.calls == 1` was 0 and the snippet failed its CPython parity run. Use a non-empty dict so the lookup must hash the key exactly once, on both CPython and RustPython. Also applied `ruff format` (two blank lines before top-level defs). Assisted-by: Claude Opus 4.8 (Anthropic) * dict: drop unused internal pop wrapper Per review, the vm-taking Dict::pop wrapper was dead code (#[allow(dead_code)], kept only for API symmetry) with no callers. Remove it and fold its doc into pop_known_hash. Co-Authored-By: Claude Opus 4.8 (1M context) * dict: rename pop_known_hash to pop Per review, with the plain pop gone the _known_hash suffix no longer earns its keep, so drop it. Matches the remaining known-hash helpers that still pair with a base method. Co-Authored-By: Claude Opus 4.8 (1M context) * dict: fold setdefault wrapper into setdefault, drop unused setdefault_entry Per review, apply the same pattern as pop: the dead setdefault wrapper only hashed and delegated, so remove it and rename setdefault_known_hash to setdefault (callers already thread the hash). Also drop setdefault_entry, which this PR added but nothing uses. Co-Authored-By: Claude Opus 4.8 (1M context) * dict: use set_cause and Result from fully_qualified_name After rebasing onto main, fully_qualified_name returns PyResult and set___cause__ is a private Python setter. Assisted-by: Grok:4.6 * dict_inner: take a hash for every key operation Drop the hashing wrappers and *_known_hash aliases. Callers hash the key and pass it in. Assisted-by: Grok:4.6 * snippets: add blank line before BadEq class ruff format requires two blank lines before a top-level class. Assisted-by: Grok:4.6 --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Jeong YunWon --- Lib/test/test_dict.py | 1 - crates/vm/src/builtins/dict.rs | 131 +++++++++---- crates/vm/src/builtins/set.rs | 49 ++--- crates/vm/src/dict_inner.rs | 275 ++++++++++++++------------- crates/vm/src/stdlib/_collections.rs | 4 +- crates/vm/src/stdlib/_weakref.rs | 3 +- extra_tests/snippets/builtin_dict.py | 109 +++++++++++ 7 files changed, 379 insertions(+), 193 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index e11dd17f1d9..b0a999c54ef 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1614,7 +1614,6 @@ class Shenanigans: self.assertEqual(holds_reference.ref['data'], 42) self.assertEqual(holds_reference.attr, "whatever") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unhashable_key(self): d = {'a': 1} key = [1, 2, 3] diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 2cc9034b1cf..a0de012fc86 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -156,11 +156,12 @@ impl PyDict { Ok(keys_method) => { let keys = PyIter::try_from_object(vm, keys_method.call((), vm)?)?; while let PyIterReturn::Return(key) = keys.next(vm)? { - if !override_existing && dict.contains(vm, &*key)? { + let hash = Self::hash_or_unhashable(&*key, vm)?; + if !override_existing && dict.contains(vm, &*key, hash)? { continue; } let val = other.get_item(&*key, vm)?; - dict.insert(vm, &*key, val)?; + dict.insert(vm, &*key, hash, val)?; } true } @@ -264,10 +265,11 @@ impl PyDict { for (index, element) in iter.iter::(vm)?.enumerate() { let (key, value) = Self::update_sequence_pair(element?, index, vm)?; - if !override_existing && dict.contains(vm, &*key)? { + let hash = Self::hash_or_unhashable(&*key, vm)?; + if !override_existing && dict.contains(vm, &*key, hash)? { continue; } - dict.insert(vm, &*key, value)?; + dict.insert(vm, &*key, hash, value)?; } Ok(()) } @@ -284,10 +286,10 @@ impl PyDict { while let Some((next, key, value, hash)) = dict_other.entries.next_entry_with_hash(position) { position = next; - if !override_existing && dict.contains_known_hash(vm, &*key, hash)? { + if !override_existing && dict.contains(vm, &*key, hash)? { continue; } - dict.insert_known_hash(vm, &*key, hash, value)?; + dict.insert(vm, &*key, hash, value)?; } if dict_other.entries.has_changed_size(dict_size) { return Err(vm.new_runtime_error("dict mutated during update")); @@ -299,6 +301,33 @@ impl PyDict { self.entries.len() == 0 } + /// Hash `key`, turning a hashing failure into the dict-specific + /// "cannot use 'X' as a dict key (...)" wording. + /// + /// The returned hash is passed into the inner map, so the key is hashed + /// exactly once. A `TypeError` from key comparison (e.g. a colliding key's + /// `__eq__`) is raised by the operation itself and must propagate unchanged. + /// Hashing up front also means a `__hash__` that fails only intermittently + /// is still reported. + fn hash_or_unhashable(key: &K, vm: &VirtualMachine) -> PyResult { + match key.key_hash(vm) { + Ok(hash) => Ok(hash), + // Exact `TypeError` only: a `__hash__` raising a *subclass* of + // `TypeError` (e.g. a user `MyTypeError`) must propagate unchanged, + // matching CPython's exact-type check. + Err(cause) if cause.class().is(vm.ctx.exceptions.type_error) => { + let message = cause.as_object().str(vm)?; + let key = key.to_pyobject(vm); + let name = key.class().fully_qualified_name(vm)?; + let err = + vm.new_type_error(format!("cannot use '{name}' as a dict key ({message})")); + err.set_cause(Some(cause)); + Err(err) + } + Err(other) => Err(other), + } + } + /// Set item variant which can be called with multiple /// key types, such as str to name a notable one. pub fn inner_setitem( @@ -307,7 +336,8 @@ impl PyDict { value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - self.entries.insert(vm, key, value) + let hash = Self::hash_or_unhashable(key, vm)?; + self.entries.insert(vm, key, hash, value) } pub(crate) fn inner_delitem( @@ -315,7 +345,12 @@ impl PyDict { key: &K, vm: &VirtualMachine, ) -> PyResult<()> { - self.entries.delete(vm, key) + let hash = Self::hash_or_unhashable(key, vm)?; + if self.entries.delete_if_exists(vm, key, hash)? { + Ok(()) + } else { + Err(vm.new_key_error(key.to_pyobject(vm))) + } } pub fn get_or_insert( @@ -324,21 +359,24 @@ impl PyDict { key: &PyObject, default: impl FnOnce() -> PyObjectRef, ) -> PyResult { - self.entries.setdefault(vm, key, default) + let hash = Self::hash_or_unhashable(key, vm)?; + self.entries.setdefault(vm, key, hash, default) } pub fn from_attributes(attrs: PyAttributes, vm: &VirtualMachine) -> PyResult { let entries = DictContentType::default(); for (key, value) in attrs { - entries.insert(vm, key, value)?; + let hash = key.key_hash(vm)?; + entries.insert(vm, key, hash, value)?; } Ok(Self { entries }) } pub fn contains_key(&self, key: &K, vm: &VirtualMachine) -> bool { - self.entries.contains(vm, key).unwrap() + let hash = key.key_hash(vm).unwrap(); + self.entries.contains(vm, key, hash).unwrap() } pub fn size(&self) -> dict_inner::DictSize { @@ -354,7 +392,8 @@ impl PyDict { key: &K, vm: &VirtualMachine, ) -> PyResult> { - self.entries.get(vm, key) + let hash = Self::hash_or_unhashable(key, vm)?; + self.entries.get(vm, key, hash) } /// Keys of `obj` with their stored hashes, or `None` if it must be iterated @@ -395,7 +434,8 @@ impl PyDict { } fn __contains__(&self, key: &PyObject, vm: &VirtualMachine) -> PyResult { - self.entries.contains(vm, key) + let hash = Self::hash_or_unhashable(key, vm)?; + self.entries.contains(vm, key, hash) } fn __delitem__(&self, key: &PyObject, vm: &VirtualMachine) -> PyResult<()> { @@ -412,7 +452,8 @@ impl PyDict { default: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - self.entries.setdefault(vm, &*key, || default) + let hash = Self::hash_or_unhashable(&*key, vm)?; + self.entries.setdefault(vm, &*key, hash, || default) } fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -448,7 +489,8 @@ impl PyDict { self.merge_object(dict_obj, vm)?; } for (key, value) in kwargs { - self.entries.insert(vm, &key, value)?; + let hash = Self::hash_or_unhashable(&key, vm)?; + self.entries.insert(vm, &key, hash, value)?; } Ok(()) } @@ -538,9 +580,7 @@ impl Py { pydict.entries.reserve_for_empty(keys.len()); } for (key, hash) in keys { - pydict - .entries - .insert_known_hash(vm, &*key, hash, value.clone())?; + pydict.entries.insert(vm, &*key, hash, value.clone())?; } } else { for key in iterable.iter(vm)? { @@ -571,10 +611,9 @@ impl Py { #[pymethod] fn get(&self, args: DictGetArgs, vm: &VirtualMachine) -> PyResult { - Ok(self - .entries - .get(vm, &*args.key)? - .unwrap_or_else(|| args.default.unwrap_or_else(|| vm.ctx.none()))) + let hash = PyDict::hash_or_unhashable(&*args.key, vm)?; + let found = self.entries.get(vm, &*args.key, hash)?; + Ok(found.unwrap_or_else(|| args.default.unwrap_or_else(|| vm.ctx.none()))) } #[pymethod(name = "setdefault")] @@ -606,7 +645,8 @@ impl Py { default: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - match self.entries.pop(vm, &*key)? { + let hash = PyDict::hash_or_unhashable(&*key, vm)?; + match self.entries.pop(vm, &*key, hash)? { Some(value) => Ok(value), None => default.ok_or_else(|| vm.new_key_error(key)), } @@ -699,9 +739,12 @@ impl AsMapping for PyDict { impl AsSequence for PyDict { fn as_sequence() -> &'static PySequenceMethods { static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { - contains: atomic_func!(|seq, target, vm| PyDict::sequence_downcast(seq) - .entries - .contains(vm, target)), + contains: atomic_func!(|seq, target, vm| { + let hash = PyDict::hash_or_unhashable(target, vm)?; + PyDict::sequence_downcast(seq) + .entries + .contains(vm, target, hash) + }), ..PySequenceMethods::NOT_IMPLEMENTED }); &AS_SEQUENCE @@ -804,7 +847,9 @@ impl Py { key: &K, vm: &VirtualMachine, ) -> PyResult { - if let Some(value) = self.entries.get(vm, key)? { + let hash = PyDict::hash_or_unhashable(key, vm)?; + let found = self.entries.get(vm, key, hash)?; + if let Some(value) = found { Ok(value) } else if let Some(value) = self.missing_opt(key, vm)? { Ok(value) @@ -854,7 +899,7 @@ impl Py { hash: crate::common::hash::PyHash, vm: &VirtualMachine, ) -> PyResult> { - self.entries.get_known_hash(vm, key, hash) + self.entries.get(vm, key, hash) } pub(crate) fn set_item_known_hash( @@ -864,7 +909,7 @@ impl Py { value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - self.entries.insert_known_hash(vm, key, hash, value) + self.entries.insert(vm, key, hash, value) } pub(crate) fn del_item_known_hash( @@ -873,7 +918,7 @@ impl Py { hash: crate::common::hash::PyHash, vm: &VirtualMachine, ) -> PyResult<()> { - if self.entries.delete_if_exists_known_hash(vm, key, hash)? { + if self.entries.delete_if_exists(vm, key, hash)? { Ok(()) } else { Err(vm.new_key_error(key.to_owned())) @@ -886,7 +931,7 @@ impl Py { hash: crate::common::hash::PyHash, vm: &VirtualMachine, ) -> PyResult { - self.entries.contains_known_hash(vm, key, hash) + self.entries.contains(vm, key, hash) } pub fn get_item_opt( @@ -895,7 +940,8 @@ impl Py { vm: &VirtualMachine, ) -> PyResult> { if self.exact_dict(vm) { - self.entries.get(vm, key) + let hash = PyDict::hash_or_unhashable(key, vm)?; + self.entries.get(vm, key, hash) // Match CPython's exact-dict fast path: __missing__ only participates // for dict subclasses through the generic mapping lookup path below. } else { @@ -916,7 +962,8 @@ impl Py { vm: &VirtualMachine, ) -> PyResult> { if self.exact_dict(vm) { - self.entries.hint_for_key(vm, key) + let hash = PyDict::hash_or_unhashable(key, vm)?; + self.entries.hint_for_key(vm, key, hash) } else { Ok(None) } @@ -947,7 +994,8 @@ impl Py { if let Some(value) = self.entries.get_hint(vm, key, usize::from(hint))? { return Ok(Some((value, None))); } - self.entries.get_with_hint(vm, key) + let hash = PyDict::hash_or_unhashable(key, vm)?; + self.entries.get_with_hint(vm, key, hash) } else { Ok(self.get_item_opt(key, vm)?.map(|value| (value, None))) } @@ -965,8 +1013,9 @@ impl Py { vm: &VirtualMachine, ) -> PyResult> { if self.exact_dict(vm) { + let hash = PyDict::hash_or_unhashable(key, vm)?; self.entries - .insert_with_hint(vm, key, usize::from(hint), value) + .insert_with_hint(vm, key, hash, usize::from(hint), value) } else { self.as_object().set_item(key, value, vm)?; Ok(None) @@ -1055,7 +1104,8 @@ impl Py { vm: &VirtualMachine, ) -> PyResult> { if self.exact_dict(vm) { - self.entries.remove_if_exists(vm, key) + let hash = PyDict::hash_or_unhashable(key, vm)?; + self.entries.remove_if_exists(vm, key, hash) } else { let value = self.as_object().get_item(key, vm)?; self.as_object().del_item(key, vm)?; @@ -1090,8 +1140,9 @@ impl PyExact { key: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { + let hash = PyDict::hash_or_unhashable(&*key, vm)?; self.entries - .move_to_end(vm, &*key)? + .move_to_end(vm, &*key, hash)? .ok_or_else(|| vm.new_key_error(key)) } @@ -1105,7 +1156,8 @@ impl PyExact { ) -> PyResult> { debug_assert!(self.class().is(vm.ctx.types.dict_type)); debug_assert!(other.class().is(vm.ctx.types.dict_type)); - self.entries.get_chain(&other.entries, vm, key) + let hash = PyDict::hash_or_unhashable(key, vm)?; + self.entries.get_chain(&other.entries, vm, key, hash) } } @@ -1677,10 +1729,11 @@ impl AsSequence for PyDictKeys { static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyDictKeys::sequence_downcast(seq).__len__())), contains: atomic_func!(|seq, target, vm| { + let hash = PyDict::hash_or_unhashable(target, vm)?; PyDictKeys::sequence_downcast(seq) .dict .entries - .contains(vm, target) + .contains(vm, target, hash) }), ..PySequenceMethods::NOT_IMPLEMENTED }); diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index e3e8ade64c4..992f14ac2f4 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -275,8 +275,10 @@ impl PySetInner { } fn contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - let result = self - .retry_op_with_frozenset(needle, vm, |needle, vm| self.content.contains(vm, needle)); + let result = self.retry_op_with_frozenset(needle, vm, |needle, vm| { + let hash = needle.hash(vm)?; + self.content.contains(vm, needle, hash) + }); Self::wrap_unhashable_error(result, needle, vm) } @@ -287,7 +289,7 @@ impl PySetInner { hash: PyHash, vm: &VirtualMachine, ) -> PyResult { - self.content.contains_known_hash(vm, needle, hash) + self.content.contains(vm, needle, hash) } fn compare(&self, other: &Self, op: PyComparisonOp, vm: &VirtualMachine) -> PyResult { @@ -375,7 +377,7 @@ impl PySetInner { && self.len() >> 2 <= dict._as_dict_inner().len() { return self.difference_by( - |key, hash| dict._as_dict_inner().contains_known_hash(vm, key, hash), + |key, hash| dict._as_dict_inner().contains(vm, key, hash), vm, ); } @@ -412,9 +414,7 @@ impl PySetInner { if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { // the source is already duplicate-free for (item, hash) in elements { - new_inner - .content - .delete_or_insert_known_hash(vm, &item, hash, ())?; + new_inner.content.delete_or_insert(vm, &item, hash, ())?; } return Ok(new_inner); } @@ -423,9 +423,7 @@ impl PySetInner { let other_set = Self::from_iter(other.iter(vm)?, vm)?; for (item, hash) in other_set.content.keys_with_hashes() { - new_inner - .content - .delete_or_insert_known_hash(vm, &item, hash, ())?; + new_inner.content.delete_or_insert(vm, &item, hash, ())?; } Ok(new_inner) @@ -491,25 +489,32 @@ impl PySetInner { } fn add(&self, item: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let result = self.content.insert(vm, item, ()); + let result = (|| { + let hash = item.hash(vm)?; + self.content.insert(vm, item, hash, ()) + })(); Self::wrap_unhashable_error(result, item, vm) } /// [`Self::add`] with a known hash. fn add_known_hash(&self, item: &PyObject, hash: PyHash, vm: &VirtualMachine) -> PyResult<()> { - let result = self.content.insert_known_hash(vm, item, hash, ()); + let result = self.content.insert(vm, item, hash, ()); Self::wrap_unhashable_error(result, item, vm) } fn remove(&self, item: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let result = - self.retry_op_with_frozenset(item, vm, |item, vm| self.content.delete(vm, item)); + let result = self.retry_op_with_frozenset(item, vm, |item, vm| { + let hash = item.hash(vm)?; + self.content.delete(vm, item, hash) + }); Self::wrap_unhashable_error(result, item, vm) } fn discard(&self, item: &PyObject, vm: &VirtualMachine) -> PyResult { - let result = self - .retry_op_with_frozenset(item, vm, |item, vm| self.content.delete_if_exists(vm, item)); + let result = self.retry_op_with_frozenset(item, vm, |item, vm| { + let hash = item.hash(vm)?; + self.content.delete_if_exists(vm, item, hash) + }); Self::wrap_unhashable_error(result, item, vm) } @@ -598,12 +603,14 @@ impl PySetInner { }; if let Some(elements) = elements { for (item, hash) in elements { - self.content.delete_if_exists_known_hash(vm, &*item, hash)?; + self.content.delete_if_exists(vm, &*item, hash)?; } continue; } for item in iterable.iter(vm)? { - self.content.delete_if_exists(vm, &*item?)?; + let item = item?; + let hash = item.hash(vm)?; + self.content.delete_if_exists(vm, &*item, hash)?; } } Ok(()) @@ -618,16 +625,14 @@ impl PySetInner { if let Some(elements) = Self::cached_hashes(iterable.as_object(), vm) { // the source is already duplicate-free for (item, hash) in elements { - self.content - .delete_or_insert_known_hash(vm, &item, hash, ())?; + self.content.delete_or_insert(vm, &item, hash, ())?; } continue; } // We want to remove duplicates in iterable let iterable_set = Self::from_iter(iterable.iter(vm)?, vm)?; for (item, hash) in iterable_set.content.keys_with_hashes() { - self.content - .delete_or_insert_known_hash(vm, &item, hash, ())?; + self.content.delete_or_insert(vm, &item, hash, ())?; } } Ok(()) diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index ed6a2300528..513b2ca7089 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -578,22 +578,13 @@ impl Dict { } } - /// Store a key - pub(crate) fn insert(&self, vm: &VirtualMachine, key: &K, value: T) -> PyResult<()> - where - K: DictKey + ?Sized, - { - let hash = key.key_hash(vm)?; - self.insert_known_hash(vm, key, hash, value) - } - - /// Store a key whose hash the caller already knows. + /// Store a key. /// /// `hash` must be computed for `key` or read from [`Self::keys_with_hashes`] /// or [`Self::next_entry_with_hash`] on a container holding this same key. /// A wrong hash lands the entry in a bucket no lookup probes, silently /// losing the key. - pub(crate) fn insert_known_hash( + pub(crate) fn insert( &self, vm: &VirtualMachine, key: &K, @@ -603,16 +594,16 @@ impl Dict { where K: DictKey + ?Sized, { - self.insert_known_hash_indexed(vm, key, hash, value)?; + self.insert_indexed(vm, key, hash, value)?; Ok(()) } - /// [`Self::insert_known_hash`], also reporting the entry index it stored to. + /// [`Self::insert`], also reporting the entry index it stored to. /// /// The index doubles as a `hint` for [`Self::get_hint`] / /// [`Self::insert_with_hint`], so a caller that wants one gets it from the /// store itself instead of probing the dict a second time. - fn insert_known_hash_indexed( + fn insert_indexed( &self, vm: &VirtualMachine, key: &K, @@ -669,43 +660,23 @@ impl Dict { Ok(stored_index) } + /// [`Self::insert`] for membership. Same hash contract. pub(crate) fn contains( &self, vm: &VirtualMachine, key: &K, - ) -> PyResult { - let key_hash = key.key_hash(vm)?; - self.contains_known_hash(vm, key, key_hash) - } - - /// [`Self::contains`] with a known hash. Same contract as - /// [`Self::insert_known_hash`]. - pub(crate) fn contains_known_hash( - &self, - vm: &VirtualMachine, - key: &K, hash: HashValue, ) -> PyResult { let (entry, _) = self.lookup(vm, key, hash, None)?; Ok(entry.index().is_some()) } - /// Retrieve a key + /// Retrieve a key. Same hash contract as [`Self::insert`]. #[cfg_attr(feature = "flame-it", flame("Dict"))] pub(crate) fn get( &self, vm: &VirtualMachine, key: &K, - ) -> PyResult> { - let hash = key.key_hash(vm)?; - self._get_inner(vm, key, hash) - } - - /// `_PyDict_GetItem_KnownHash`: lookup using a caller-supplied hash. - pub(crate) fn get_known_hash( - &self, - vm: &VirtualMachine, - key: &K, hash: HashValue, ) -> PyResult> { self._get_inner(vm, key, hash) @@ -719,8 +690,8 @@ impl Dict { &self, vm: &VirtualMachine, key: &K, + hash: HashValue, ) -> PyResult> { - let hash = key.key_hash(vm)?; let (entry, _) = self.lookup(vm, key, hash, None)?; let Some(index) = entry.index() else { return Ok(None); @@ -736,8 +707,8 @@ impl Dict { &self, vm: &VirtualMachine, key: &K, + hash: HashValue, ) -> PyResult)>> { - let hash = key.key_hash(vm)?; let ret = loop { let (entry, index_index) = self.lookup(vm, key, hash, None)?; if let Some(index) = entry.index() { @@ -763,6 +734,7 @@ impl Dict { &self, vm: &VirtualMachine, key: &K, + hash: HashValue, hint: usize, value: T, ) -> PyResult> { @@ -779,8 +751,7 @@ impl Dict { _ => value, } }; - let hash = key.key_hash(vm)?; - let stored = self.insert_known_hash_indexed(vm, key, hash, value)?; + let stored = self.insert_indexed(vm, key, hash, value)?; Ok(u16::try_from(stored).ok()) } @@ -860,8 +831,8 @@ impl Dict { other: &Self, vm: &VirtualMachine, key: &K, + hash: HashValue, ) -> PyResult> { - let hash = key.key_hash(vm)?; if let Some(x) = self._get_inner(vm, key, hash)? { Ok(Some(x)) } else { @@ -892,28 +863,19 @@ impl Dict { }; } - /// Delete a key - pub(crate) fn delete(&self, vm: &VirtualMachine, key: &K) -> PyResult<()> + /// Delete a key. Same hash contract as [`Self::insert`]. + pub(crate) fn delete(&self, vm: &VirtualMachine, key: &K, hash: HashValue) -> PyResult<()> where K: DictKey + ?Sized, { - if self.remove_if_exists(vm, key)?.is_some() { + if self.remove_if_exists(vm, key, hash)?.is_some() { Ok(()) } else { Err(vm.new_key_error(key.to_pyobject(vm))) } } - pub(crate) fn delete_if_exists(&self, vm: &VirtualMachine, key: &K) -> PyResult - where - K: DictKey + ?Sized, - { - self.remove_if_exists(vm, key).map(|opt| opt.is_some()) - } - - /// [`Self::delete_if_exists`] with a known hash. Same contract as - /// [`Self::insert_known_hash`]. - pub(crate) fn delete_if_exists_known_hash( + pub(crate) fn delete_if_exists( &self, vm: &VirtualMachine, key: &K, @@ -922,44 +884,39 @@ impl Dict { where K: DictKey + ?Sized, { - self.remove_if_known_hash(vm, key, hash, |_| Ok(true)) + self.remove_if_exists(vm, key, hash) .map(|opt| opt.is_some()) } - pub(crate) fn delete_if(&self, vm: &VirtualMachine, key: &K, pred: F) -> PyResult + pub(crate) fn delete_if( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + pred: F, + ) -> PyResult where K: DictKey + ?Sized, F: Fn(&T) -> PyResult, { - self.remove_if(vm, key, pred).map(|opt| opt.is_some()) - } - - pub(crate) fn remove_if_exists(&self, vm: &VirtualMachine, key: &K) -> PyResult> - where - K: DictKey + ?Sized, - { - self.remove_if(vm, key, |_| Ok(true)) + self.remove_if(vm, key, hash, pred).map(|opt| opt.is_some()) } - /// pred should be VERY CAREFUL about what it does as it is called while - /// the dict's internal mutex is held - pub(crate) fn remove_if( + pub(crate) fn remove_if_exists( &self, vm: &VirtualMachine, key: &K, - pred: F, + hash: HashValue, ) -> PyResult> where K: DictKey + ?Sized, - F: Fn(&T) -> PyResult, { - let hash = key.key_hash(vm)?; - self.remove_if_known_hash(vm, key, hash, pred) + self.remove_if(vm, key, hash, |_| Ok(true)) } - /// [`Self::remove_if`] with a known hash. Same contract as - /// [`Self::insert_known_hash`]. - fn remove_if_known_hash( + /// pred should be VERY CAREFUL about what it does as it is called while + /// the dict's internal mutex is held. Same hash contract as [`Self::insert`]. + pub(crate) fn remove_if( &self, vm: &VirtualMachine, key: &K, @@ -980,9 +937,9 @@ impl Dict { Ok(removed.map(|entry| entry.value)) } - /// Delete an existing key or insert a missing key using its known hash. - /// Same contract as [`Self::insert_known_hash`]. - pub(crate) fn delete_or_insert_known_hash( + /// Delete an existing key or insert a missing key. Same hash contract as + /// [`Self::insert`]. + pub(crate) fn delete_or_insert( &self, vm: &VirtualMachine, key: &PyObject, @@ -1010,12 +967,19 @@ impl Dict { Ok(()) } - pub(crate) fn setdefault(&self, vm: &VirtualMachine, key: &K, default: F) -> PyResult + /// Get the value for `key`, inserting `default()` if it is absent. + /// Same hash contract as [`Self::insert`]. + pub(crate) fn setdefault( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + default: F, + ) -> PyResult where K: DictKey + ?Sized, F: FnOnce() -> T, { - let hash = key.key_hash(vm)?; let mut default = Some(default); loop { let (index_entry, index_index) = self.lookup(vm, key, hash, None)?; @@ -1050,13 +1014,13 @@ impl Dict { &self, vm: &VirtualMachine, key: &K, + hash: HashValue, default: F, ) -> PyResult<(PyObjectRef, T)> where K: DictKey + ?Sized, F: FnOnce() -> T, { - let hash = key.key_hash(vm)?; let mut default = Some(default); loop { let (index_entry, index_index) = self.lookup(vm, key, hash, None)?; @@ -1193,7 +1157,7 @@ impl Dict { } /// All keys paired with the hash stored in their entry, for feeding - /// [`Self::insert_known_hash`] without re-calling `__hash__`. + /// [`Self::insert`] without re-calling `__hash__`. pub(crate) fn keys_with_hashes(&self) -> Vec<(PyObjectRef, HashValue)> { self.read() .entries @@ -1375,13 +1339,13 @@ impl Dict { Ok(ControlFlow::Break(removed)) } - /// Retrieve and delete a key + /// Retrieve and delete a key. Same hash contract as [`Self::insert`]. pub(crate) fn pop( &self, vm: &VirtualMachine, key: &K, + hash_value: HashValue, ) -> PyResult> { - let hash_value = key.key_hash(vm)?; let removed = loop { let lookup = self.lookup(vm, key, hash_value, None)?; match self.pop_inner(lookup) { @@ -1393,11 +1357,12 @@ impl Dict { } /// Move an existing entry to the end and return its current value without - /// making the key temporarily absent. Hashing and equality run unlocked. + /// making the key temporarily absent. Equality runs unlocked. pub(crate) fn move_to_end( &self, vm: &VirtualMachine, key: &K, + hash_value: HashValue, ) -> PyResult> { struct ProbeWitness { index_index: IndexIndex, @@ -1405,7 +1370,6 @@ impl Dict { key: Option<(HashValue, PyObjectRef)>, } - let hash_value = key.key_hash(vm)?; let mut idxs = None; let mut prefix = Vec::::new(); let mut compared: Option<(EntryIndex, IndexIndex, PyObjectRef, bool)> = None; @@ -1966,6 +1930,10 @@ mod tests { use super::*; use crate::{Interpreter, common::ascii}; + fn hash_key(vm: &VirtualMachine, key: &K) -> HashValue { + key.key_hash(vm).unwrap() + } + #[test] fn move_to_end_preserves_values_and_invalidates_layout() { Interpreter::without_stdlib(Default::default()).enter(|vm| { @@ -1976,13 +1944,20 @@ mod tests { vm.ctx.intern_str("last"), ]; for (value, key) in keys.iter().enumerate() { - dict.insert(vm, *key, value).unwrap(); + dict.insert(vm, *key, hash_key(vm, *key), value).unwrap(); } - let hint = dict.hint_for_key(vm, keys[1]).unwrap().unwrap() as usize; + let hint = dict + .hint_for_key(vm, keys[1], hash_key(vm, keys[1])) + .unwrap() + .unwrap() as usize; let version = dict.assign_keys_version(); let size = dict.size(); assert_ne!(version, 0); - assert_eq!(dict.move_to_end(vm, keys[1]).unwrap(), Some(1)); + assert_eq!( + dict.move_to_end(vm, keys[1], hash_key(vm, keys[1])) + .unwrap(), + Some(1) + ); assert_eq!(dict.values(), vec![0, 2, 1]); assert_eq!(dict.keys_version(), 0); assert_eq!(dict.get_index_if_keys_version(version, hint), None); @@ -1995,10 +1970,18 @@ mod tests { let version = dict.assign_keys_version(); let size = dict.size(); - assert_eq!(dict.move_to_end(vm, keys[1]).unwrap(), Some(1)); + assert_eq!( + dict.move_to_end(vm, keys[1], hash_key(vm, keys[1])) + .unwrap(), + Some(1) + ); assert_eq!(dict.keys_version(), version); assert_eq!(dict.size(), size); - assert_eq!(dict.move_to_end(vm, "absent").unwrap(), None); + assert_eq!( + dict.move_to_end(vm, "absent", hash_key(vm, "absent")) + .unwrap(), + None + ); assert_eq!(dict.keys_version(), version); assert_eq!(dict.size(), size); assert_eq!(dict.values(), vec![0, 2, 1]); @@ -2011,14 +1994,20 @@ mod tests { for count in [2, 3, 32] { let dict = Dict::default(); for key in 0..count { - dict.insert(vm, &key, key).unwrap(); + dict.insert(vm, &key, hash_key(vm, &key), key).unwrap(); } for step in 0..1024 { let key = step % count; let size = dict.size(); - let hint = dict.hint_for_key(vm, &key).unwrap().unwrap() as usize; + let hint = dict + .hint_for_key(vm, &key, hash_key(vm, &key)) + .unwrap() + .unwrap() as usize; let version = dict.assign_keys_version(); - assert_eq!(dict.move_to_end(vm, &key).unwrap(), Some(key)); + assert_eq!( + dict.move_to_end(vm, &key, hash_key(vm, &key)).unwrap(), + Some(key) + ); assert_eq!(dict.get_index_if_keys_version(version, hint), None); assert!(dict.next_entry_checked(0, &size, |_, v| *v).is_err()); assert!( @@ -2059,14 +2048,14 @@ mod tests { let dict = Dict::default(); let peer = Dict::default(); for table in [&dict, &peer] { - table.insert(vm, first, 1).unwrap(); - table.insert(vm, last, 2).unwrap(); + table.insert(vm, first, hash_key(vm, first), 1).unwrap(); + table.insert(vm, last, hash_key(vm, last), 2).unwrap(); } let version = dict.assign_keys_version(); assert_eq!(peer.assign_keys_version(), version); - dict.move_to_end(vm, first).unwrap(); + dict.move_to_end(vm, first, hash_key(vm, first)).unwrap(); assert_ne!(dict.assign_keys_version(), version); - dict.move_to_end(vm, last).unwrap(); + dict.move_to_end(vm, last, hash_key(vm, last)).unwrap(); // The second move compacts back to the original hole-free layout. assert_eq!(dict.assign_keys_version(), version); assert_eq!(dict.get_index_if_keys_version(version, 0), Some(1)); @@ -2079,10 +2068,10 @@ mod tests { Interpreter::without_stdlib(Default::default()).enter(|vm| { let dict = Dict::default(); for key in 0..128usize { - dict.insert(vm, &key, key).unwrap(); + dict.insert(vm, &key, hash_key(vm, &key), key).unwrap(); } for key in 0..120usize { - dict.delete(vm, &key).unwrap(); + dict.delete(vm, &key, hash_key(vm, &key)).unwrap(); } let version = dict.assign_keys_version(); let copied = dict.clone(); @@ -2099,10 +2088,12 @@ mod tests { } assert_eq!(copied.values(), (120..128).collect::>()); for key in 120..128usize { - assert_eq!(copied.get(vm, &key).unwrap(), Some(key)); + assert_eq!(copied.get(vm, &key, hash_key(vm, &key)).unwrap(), Some(key)); } - copied.insert(vm, &0usize, 0).unwrap(); - assert!(!dict.contains(vm, &0usize).unwrap()); + copied + .insert(vm, &0usize, hash_key(vm, &0usize), 0) + .unwrap(); + assert!(!dict.contains(vm, &0usize, hash_key(vm, &0usize)).unwrap()); }); } @@ -2111,7 +2102,7 @@ mod tests { Interpreter::without_stdlib(Default::default()).enter(|vm| { let dict = Dict::default(); for key in 0..128usize { - dict.insert(vm, &key, ()).unwrap(); + dict.insert(vm, &key, hash_key(vm, &key), ()).unwrap(); } for _ in 0..124 { dict.pop_back().unwrap(); @@ -2120,8 +2111,8 @@ mod tests { assert_eq!(copied.len(), 4); assert!(copied.read().indices.len() < dict.read().indices.len()); for key in 0..4usize { - assert!(copied.contains(vm, &key).unwrap()); - dict.delete(vm, &key).unwrap(); + assert!(copied.contains(vm, &key, hash_key(vm, &key)).unwrap()); + dict.delete(vm, &key, hash_key(vm, &key)).unwrap(); } let empty = dict.clone(); assert_eq!(dict.len(), 0); @@ -2141,10 +2132,14 @@ mod tests { let padding = vm.ctx.intern_str("padding"); let first: Dict = Dict::default(); let second: Dict = Dict::default(); - first.insert(vm, name, 10).unwrap(); - first.insert(vm, padding, 20).unwrap(); - second.insert(vm, padding, 30).unwrap(); - second.insert(vm, name, 40).unwrap(); + first.insert(vm, name, hash_key(vm, name), 10).unwrap(); + first + .insert(vm, padding, hash_key(vm, padding), 20) + .unwrap(); + second + .insert(vm, padding, hash_key(vm, padding), 30) + .unwrap(); + second.insert(vm, name, hash_key(vm, name), 40).unwrap(); let (first_version, first_index) = first.module_attr_cache(name, vm).unwrap(); let (second_version, second_index) = second.module_attr_cache(name, vm).unwrap(); assert_ne!(first_version, second_version); @@ -2169,21 +2164,35 @@ mod tests { ); let foreign_key: PyObjectRef = vm.ctx.new_int(7).into(); - first.insert(vm, &*foreign_key, 50).unwrap(); + first + .insert(vm, &*foreign_key, hash_key(vm, &*foreign_key), 50) + .unwrap(); assert!(first.module_attr_cache(name, vm).is_none()); assert_eq!( first.get_cached_module_attr(name, first_version as usize, first_index.into(), vm), None ); - first.delete(vm, &*foreign_key).unwrap(); first - .insert(vm, vm.ctx.intern_str("__getattr__"), 60) + .delete(vm, &*foreign_key, hash_key(vm, &*foreign_key)) + .unwrap(); + first + .insert( + vm, + vm.ctx.intern_str("__getattr__"), + hash_key(vm, vm.ctx.intern_str("__getattr__")), + 60, + ) .unwrap(); assert!(first.module_attr_cache(name, vm).is_none()); let noninterned: Dict = Dict::default(); noninterned - .insert(vm, &*vm.ctx.new_str("target"), 70) + .insert( + vm, + &*vm.ctx.new_str("target"), + hash_key(vm, &*vm.ctx.new_str("target")), + 70, + ) .unwrap(); let (version, index) = noninterned.module_attr_cache(name, vm).unwrap(); assert_eq!( @@ -2201,27 +2210,31 @@ mod tests { let key1 = vm.new_pyobj(true); let value1 = vm.new_pyobj(ascii!("abc")); - dict.insert(vm, &*key1, value1).unwrap(); + dict.insert(vm, &*key1, hash_key(vm, &*key1), value1) + .unwrap(); assert_eq!(1, dict.len()); let key2 = vm.new_pyobj(ascii!("x")); let value2 = vm.new_pyobj(ascii!("def")); - dict.insert(vm, &*key2, value2.clone()).unwrap(); + dict.insert(vm, &*key2, hash_key(vm, &*key2), value2.clone()) + .unwrap(); assert_eq!(2, dict.len()); - dict.insert(vm, &*key1, value2.clone()).unwrap(); + dict.insert(vm, &*key1, hash_key(vm, &*key1), value2.clone()) + .unwrap(); assert_eq!(2, dict.len()); - dict.delete(vm, &*key1).unwrap(); + dict.delete(vm, &*key1, hash_key(vm, &*key1)).unwrap(); assert_eq!(1, dict.len()); - dict.insert(vm, &*key1, value2.clone()).unwrap(); + dict.insert(vm, &*key1, hash_key(vm, &*key1), value2.clone()) + .unwrap(); assert_eq!(2, dict.len()); - assert!(dict.contains(vm, &*key1).unwrap()); - assert!(dict.contains(vm, "x").unwrap()); + assert!(dict.contains(vm, &*key1, hash_key(vm, &*key1)).unwrap()); + assert!(dict.contains(vm, "x", hash_key(vm, "x")).unwrap()); - let val = dict.get(vm, "x").unwrap().unwrap(); + let val = dict.get(vm, "x", hash_key(vm, "x")).unwrap().unwrap(); vm.bool_eq(&val, &value2) .expect("retrieved value must be equal to inserted value."); }) @@ -2244,12 +2257,16 @@ mod tests { (inner.indices.len(), inner.entries.capacity()) }; for key in 0..count { - dict.insert(vm, &key, vm.ctx.none()).unwrap(); + dict.insert(vm, &key, hash_key(vm, &key), vm.ctx.none()) + .unwrap(); let inner = dict.read(); assert_eq!((inner.indices.len(), inner.entries.capacity()), capacity); } assert_eq!(dict.len(), count); - assert!(dict.contains(vm, &(count - 1)).unwrap()); + assert!( + dict.contains(vm, &(count - 1), hash_key(vm, &(count - 1))) + .unwrap() + ); // Existing and tombstoned layouts are left intact. dict.reserve_for_empty(count * 4); @@ -2258,7 +2275,7 @@ mod tests { assert_eq!((inner.indices.len(), inner.entries.capacity()), capacity); } for key in 0..count { - dict.delete(vm, &key).unwrap(); + dict.delete(vm, &key, hash_key(vm, &key)).unwrap(); } let old_size = dict.read().size(); dict.reserve_for_empty(count * 4); @@ -2297,19 +2314,21 @@ mod tests { fn replace_contents_invalidates_cached_indices() { Interpreter::without_stdlib(Default::default()).enter(|vm| { let dict = Dict::default(); - dict.insert(vm, &1usize, ()).unwrap(); + dict.insert(vm, &1usize, hash_key(vm, &1usize), ()).unwrap(); let old_version = dict.assign_keys_version(); assert_ne!(old_version, 0); let replacement = Dict::default(); - replacement.insert(vm, &2usize, ()).unwrap(); + replacement + .insert(vm, &2usize, hash_key(vm, &2usize), ()) + .unwrap(); assert_ne!(replacement.assign_keys_version(), 0); dict.replace_contents(replacement); assert_eq!(dict.keys_version(), 0); assert_eq!(dict.get_index_if_keys_version(old_version, 0), None); - assert!(!dict.contains(vm, &1usize).unwrap()); - assert!(dict.contains(vm, &2usize).unwrap()); + assert!(!dict.contains(vm, &1usize, hash_key(vm, &1usize)).unwrap()); + assert!(dict.contains(vm, &2usize, hash_key(vm, &2usize)).unwrap()); }); } } diff --git a/crates/vm/src/stdlib/_collections.rs b/crates/vm/src/stdlib/_collections.rs index 1e562f0815b..9307f7c67fc 100644 --- a/crates/vm/src/stdlib/_collections.rs +++ b/crates/vm/src/stdlib/_collections.rs @@ -1193,11 +1193,11 @@ mod _collections { for key in iter { let key = key?; let hash = key.hash(vm)?; - let count = match entries.get_known_hash(vm, &*key, hash)? { + let count = match entries.get(vm, &*key, hash)? { Some(old) => vm._add(&old, one.as_object())?, None => one.clone().into(), }; - entries.insert_known_hash(vm, &*key, hash, count)?; + entries.insert(vm, &*key, hash, count)?; } return Ok(()); } diff --git a/crates/vm/src/stdlib/_weakref.rs b/crates/vm/src/stdlib/_weakref.rs index 767f793fdf9..624a1d511b2 100644 --- a/crates/vm/src/stdlib/_weakref.rs +++ b/crates/vm/src/stdlib/_weakref.rs @@ -52,8 +52,9 @@ mod _weakref { #[pyfunction] fn _remove_dead_weakref(dct: PyDictRef, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let hash = key.hash(vm)?; dct._as_dict_inner() - .delete_if(vm, &*key, |wr| { + .delete_if(vm, &*key, hash, |wr| { let wr = wr .downcast_ref::() .ok_or_else(|| vm.new_type_error("not a weakref"))?; diff --git a/extra_tests/snippets/builtin_dict.py b/extra_tests/snippets/builtin_dict.py index 5dca75b9b9d..fb84bf48a90 100644 --- a/extra_tests/snippets/builtin_dict.py +++ b/extra_tests/snippets/builtin_dict.py @@ -745,3 +745,112 @@ def __getitem__(self, key): hash(od.keys()) with assert_raises(TypeError): hash(od.items()) + + +# A TypeError raised while *comparing* keys (e.g. from a colliding key's +# __eq__) must propagate unchanged, not be rewritten as an unhashable-key +# error. Only genuine hashing failures get the dict-specific "unhashable" +# wording. +class BadEq: + def __hash__(self): + return 42 # fixed hash forces a collision + + def __eq__(self, other): + raise TypeError("nope") + + +bad = {} +bad[BadEq()] = 1 +with assert_raises(TypeError) as cm: + bad[BadEq()] # hashes fine (42), then compares against the colliding key +assert "nope" in str(cm.exception), str(cm.exception) +assert "dict key" not in str(cm.exception), ( + f"comparison error mislabeled as unhashable: {cm.exception}" +) + +# A genuinely unhashable key still reports the dict-specific message. +with assert_raises(TypeError) as cm: + {}[[]] +assert "as a dict key" in str(cm.exception), str(cm.exception) + +# The message reaches every insertion path, not just __setitem__: the +# constructor, update() and |= all go through the same wrapping. +for make in ( + lambda: dict([([], 1)]), + lambda: {}.update([([], 1)]), + lambda: {}.__ior__([([], 1)]), +): + with assert_raises(TypeError) as cm: + make() + assert "as a dict key" in str(cm.exception), str(cm.exception) + + +# The key is hashed once up front, so a __hash__ that fails only on its first +# call is still reported (a re-hash on the error path would let it escape). +class FlakyHash: + _calls = 0 + + def __hash__(self): + FlakyHash._calls += 1 + if FlakyHash._calls == 1: + raise TypeError("first call fails") + return 0 + + +with assert_raises(TypeError) as cm: + {}[FlakyHash()] +assert "as a dict key" in str(cm.exception), str(cm.exception) + + +# The type name is the fully qualified one, like CPython's %T. +def _make_nested(): + class Nested: + __hash__ = None + + return Nested + + +with assert_raises(TypeError) as cm: + {}[_make_nested()()] +assert "_make_nested..Nested" in str(cm.exception), str(cm.exception) + + +# A __hash__ raising a *subclass* of TypeError is left unchanged (CPython +# checks the exact type), so `except MySubclass` still catches it. +class MyTypeError(TypeError): + pass + + +class SubclassHash: + def __hash__(self): + raise MyTypeError("custom") + + +with assert_raises(MyTypeError) as cm: + {}[SubclassHash()] +assert "as a dict key" not in str(cm.exception), str(cm.exception) + + +# Every operation hashes the key exactly once (the hash is threaded into the +# inner map), so a __hash__ that would fail on a second call is never called +# twice. setdefault() and pop() went through their own lookup before. +class CountingHash: + calls = 0 + + def __hash__(self): + CountingHash.calls += 1 + if CountingHash.calls >= 2: + raise TypeError("must not hash twice") + return 7 + + +CountingHash.calls = 0 +{}.setdefault(CountingHash(), 1) +assert CountingHash.calls == 1, CountingHash.calls + +CountingHash.calls = 0 +with assert_raises(KeyError): + # A non-empty dict so the lookup must hash the key (CPython skips hashing + # entirely when popping from an empty dict). + {1: 1}.pop(CountingHash()) +assert CountingHash.calls == 1, CountingHash.calls From 55f7abb2f4c12f460743977eed431831cf831a3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:10:10 +0000 Subject: [PATCH 017/229] build(deps): bump siphasher from 1.0.3 to 1.0.4 (#8974) Bumps [siphasher](https://github.com/jedisct1/rust-siphash) from 1.0.3 to 1.0.4. - [Commits](https://github.com/jedisct1/rust-siphash/compare/1.0.3...1.0.4) --- updated-dependencies: - dependency-name: siphasher dependency-version: 1.0.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f3a54435c0..15e59964bb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4475,9 +4475,9 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "siphasher" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" [[package]] name = "slab" From 54e119d3c18aece4fee094419386cb34a99e07f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:14:56 +0000 Subject: [PATCH 018/229] build(deps): bump glob from 0.3.3 to 0.3.4 (#8972) Bumps [glob](https://github.com/rust-lang/glob) from 0.3.3 to 0.3.4. - [Release notes](https://github.com/rust-lang/glob/releases) - [Changelog](https://github.com/rust-lang/glob/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/glob/compare/v0.3.3...v0.3.4) --- updated-dependencies: - dependency-name: glob dependency-version: 0.3.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15e59964bb1..6e5d55f915d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1726,9 +1726,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "graviola" From 3d1c174d17b6e21a64034f2aa7e6a8bb5cb16a4a Mon Sep 17 00:00:00 2001 From: Seonghun An <53287605+shAn-kor@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:39:46 +0900 Subject: [PATCH 019/229] Match invalid format specifier error messages (#8477) * Match invalid format specifier error details Assisted-by: Codex:gpt-5.6-sol * format: convert FormatSpecError in one helper InvalidFormatSpecifier needs the spec and the object type. Drop IntoPyException so that path cannot produce a generic message. Assisted-by: Grok:4.6 --------- Co-authored-by: Jeong YunWon --- Lib/test/test_format.py | 3 - crates/vm/src/builtins/bool.rs | 11 +-- crates/vm/src/builtins/complex.rs | 19 +++-- crates/vm/src/builtins/float.rs | 19 +++-- crates/vm/src/builtins/int.rs | 20 +++-- crates/vm/src/builtins/str.rs | 14 ++-- crates/vm/src/format.rs | 126 +++++++++++++++--------------- 7 files changed, 105 insertions(+), 107 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index aa28108312e..1f626d87fa6 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -529,7 +529,6 @@ def test_with_an_underscore_and_a_comma_in_format_specifier(self): with self.assertRaisesRegex(ValueError, error_msg): '{:._,f}'.format(1.1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_better_error_message_format(self): # https://bugs.python.org/issue20524 for value in [12j, 12, 12.0, "12"]: @@ -551,7 +550,6 @@ def test_better_error_message_format(self): with self.assertRaisesRegex(ValueError, err): eval("f'xx{value:{bad_format_spec}}yy'") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_in_error_message(self): str_err = re.escape( "Invalid format specifier '%ЫйЯЧ' for object of type 'str'") @@ -615,7 +613,6 @@ def test_negative_zero(self): self.assertEqual(f"{-0.:x>z6.1f}", "xxx0.0") self.assertEqual(f"{-0.:🖤>z6.1f}", "🖤🖤🖤0.0") # multi-byte fill char - @unittest.expectedFailure # TODO: RUSTPYTHON def test_specifier_z_error(self): error_msg = re.compile("Invalid format specifier '.*z.*'") with self.assertRaisesRegex(ValueError, error_msg): diff --git a/crates/vm/src/builtins/bool.rs b/crates/vm/src/builtins/bool.rs index a8c2c019582..26086605a9e 100644 --- a/crates/vm/src/builtins/bool.rs +++ b/crates/vm/src/builtins/bool.rs @@ -1,9 +1,8 @@ use super::{PyInt, PyStrRef, PyType, PyTypeRef, PyUtf8StrRef}; -use crate::common::format::FormatSpec; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, class::{PyClassDef, PyClassImpl}, - convert::{IntoPyException, ToPyObject, ToPyResult}, + convert::{ToPyObject, ToPyResult}, function::{FuncArgs, OptionalArg}, protocol::PyNumberMethods, types::{AsNumber, Constructor, Representable}, @@ -124,9 +123,11 @@ impl Py { vm: &VirtualMachine, ) -> PyResult { let new_bool = zelf.try_to_bool(vm)?; - FormatSpec::parse(format_spec.as_str()) - .and_then(|format_spec| format_spec.format_bool(new_bool)) - .map_err(|err| err.into_pyexception(vm)) + crate::format::parse_format_spec(zelf.as_object(), format_spec.as_str(), vm)? + .format_bool(new_bool) + .map_err(|err| { + crate::format::format_spec_error(err, zelf.as_object(), format_spec.as_str(), vm) + }) } } diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index f471c43c580..eea71f475a2 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -3,8 +3,8 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyUtf8StrRef, class::{PyClassDef, PyClassImpl}, - common::{format::FormatSpec, wtf8::Wtf8Buf}, - convert::{IntoPyException, ToPyObject, ToPyResult}, + common::wtf8::Wtf8Buf, + convert::{ToPyObject, ToPyResult}, function::{FuncArgs, OptionalArg, PyComparisonValue}, protocol::PyNumberMethods, stdlib::_warnings, @@ -571,17 +571,16 @@ impl Py { if format_spec.is_empty() { return Ok(zelf.as_object().str(vm)?.as_wtf8().to_owned()); } - let format_spec = - FormatSpec::parse(format_spec.as_str()).map_err(|err| err.into_pyexception(vm))?; - let result = if format_spec.has_locale_format() { + let spec = crate::format::parse_format_spec(zelf.as_object(), format_spec.as_str(), vm)?; + let result = if spec.has_locale_format() { let locale = crate::format::get_locale_info(); - format_spec.format_complex_locale(&zelf.as_complex(), &locale) + spec.format_complex_locale(&zelf.as_complex(), &locale) } else { - format_spec.format_complex(&zelf.as_complex()) + spec.format_complex(&zelf.as_complex()) }; - result - .map(Wtf8Buf::from_string) - .map_err(|err| err.into_pyexception(vm)) + result.map(Wtf8Buf::from_string).map_err(|err| { + crate::format::format_spec_error(err, zelf.as_object(), format_spec.as_str(), vm) + }) } #[pyclassmethod] diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 78d3bef7794..3d21596f658 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -6,8 +6,8 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, class::{PyClassDef, PyClassImpl}, - common::{float_ops, format::FormatSpec, hash, wtf8::Wtf8Buf}, - convert::{IntoPyException, ToPyObject, ToPyResult}, + common::{float_ops, hash, wtf8::Wtf8Buf}, + convert::{ToPyObject, ToPyResult}, function::{ArgBytesLike, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, protocol::PyNumberMethods, types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, @@ -291,17 +291,16 @@ impl Py { if format_spec.is_empty() { return Ok(zelf.as_object().str(vm)?.as_wtf8().to_owned()); } - let format_spec = - FormatSpec::parse(format_spec.as_str()).map_err(|err| err.into_pyexception(vm))?; - let result = if format_spec.has_locale_format() { + let spec = crate::format::parse_format_spec(zelf.as_object(), format_spec.as_str(), vm)?; + let result = if spec.has_locale_format() { let locale = crate::format::get_locale_info(); - format_spec.format_float_locale(zelf.to_f64(), &locale) + spec.format_float_locale(zelf.to_f64(), &locale) } else { - format_spec.format_float(zelf.to_f64()) + spec.format_float(zelf.to_f64()) }; - result - .map(Wtf8Buf::from_string) - .map_err(|err| err.into_pyexception(vm)) + result.map(Wtf8Buf::from_string).map_err(|err| { + crate::format::format_spec_error(err, zelf.as_object(), format_spec.as_str(), vm) + }) } #[pystaticmethod] diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index d6799474582..76522e14b62 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -6,12 +6,11 @@ use crate::{ byte::bytes_from_object, class::{PyClassDef, PyClassImpl}, common::{ - format::FormatSpec, hash, int::{bigint_to_finite_float, bytes_to_int, true_div}, wtf8::Wtf8Buf, }, - convert::{IntoPyException, ToPyObject, ToPyResult}, + convert::{ToPyObject, ToPyResult}, function::{ ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue, PySsize, @@ -601,20 +600,19 @@ impl Py { if format_spec.is_empty() && !zelf.class().is(vm.ctx.types.int_type) { return Ok(zelf.as_object().str(vm)?.as_wtf8().to_owned()); } - let format_spec = - FormatSpec::parse(format_spec.as_str()).map_err(|err| err.into_pyexception(vm))?; - if format_spec.is_decimal_int_format() { + let spec = crate::format::parse_format_spec(zelf.as_object(), format_spec.as_str(), vm)?; + if spec.is_decimal_int_format() { check_int_to_str_digits(zelf.as_bigint(), vm)?; } - let result = if format_spec.has_locale_format() { + let result = if spec.has_locale_format() { let locale = crate::format::get_locale_info(); - format_spec.format_int_locale(zelf.as_bigint(), &locale) + spec.format_int_locale(zelf.as_bigint(), &locale) } else { - format_spec.format_int(zelf.as_bigint()) + spec.format_int(zelf.as_bigint()) }; - result - .map(Wtf8Buf::from_string) - .map_err(|err| err.into_pyexception(vm)) + result.map(Wtf8Buf::from_string).map_err(|err| { + crate::format::format_spec_error(err, zelf.as_object(), format_spec.as_str(), vm) + }) } #[pymethod] diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index c66b87cfb5c..fa4e905cdd9 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -15,7 +15,7 @@ use crate::{ lock::LazyLock, str::{PyKindStr, StrData, StrKind}, }, - convert::{IntoPyException, ToPyException, ToPyObject, ToPyResult}, + convert::{ToPyException, ToPyObject, ToPyResult}, format::{format, format_map}, function::{ArgIterable, FuncArgs, OptionalArg, PyComparisonValue, PySsize}, intern::PyInterned, @@ -41,7 +41,7 @@ use num_traits::ToPrimitive; use rustpython_common::{ ascii, atomic::{self, PyAtomic, Radium}, - format::{FormatSpec, FormatString, FromTemplate}, + format::{FormatString, FromTemplate}, hash, lock::PyMutex, str::DeduceStrKind, @@ -1167,11 +1167,11 @@ impl Py { }; } let zelf = zelf.try_into_utf8(vm)?; - let s = FormatSpec::parse(format_spec.as_str()) - .and_then(|format_spec| { - format_spec.format_string(&CharLenStr(zelf.as_str(), zelf.char_len())) - }) - .map_err(|err| err.into_pyexception(vm))?; + let s = crate::format::parse_format_spec(zelf.as_object(), format_spec.as_str(), vm)? + .format_string(&CharLenStr(zelf.as_str(), zelf.char_len())) + .map_err(|err| { + crate::format::format_spec_error(err, zelf.as_object(), format_spec.as_str(), vm) + })?; Ok(vm.ctx.new_str(s)) } diff --git a/crates/vm/src/format.rs b/crates/vm/src/format.rs index d95def76813..3a6155f26e6 100644 --- a/crates/vm/src/format.rs +++ b/crates/vm/src/format.rs @@ -1,9 +1,6 @@ use crate::{ - PyObject, PyResult, VirtualMachine, - builtins::PyBaseExceptionRef, - convert::{IntoPyException, ToPyException}, - function::FuncArgs, - stdlib::builtins, + PyObject, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, convert::ToPyException, + function::FuncArgs, stdlib::builtins, }; use crate::common::format::*; @@ -41,63 +38,70 @@ pub(crate) fn get_locale_info() -> LocaleInfo { } } -impl IntoPyException for FormatSpecError { - fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - match self { - Self::DecimalDigitsTooMany => { - vm.new_value_error("Too many decimal digits in format string") - } - Self::PrecisionTooBig => vm.new_value_error("Precision too big"), - Self::PrecisionMissing => vm.new_value_error("Format specifier missing precision"), - Self::InvalidFormatSpecifier => vm.new_value_error("Invalid format specifier"), - Self::UnspecifiedFormat(c1, c2) => { - let msg = format!("Cannot specify '{c1}' with '{c2}'."); - vm.new_value_error(msg) - } - Self::ExclusiveFormat(c1, c2) => { - let msg = format!("Cannot specify both '{c1}' and '{c2}'."); - vm.new_value_error(msg) - } - Self::UnknownFormatCode(c, s) => { - let msg = format!("Unknown format code '{c}' for object of type '{s}'"); - vm.new_value_error(msg) - } - Self::PrecisionNotAllowed => { - vm.new_value_error("Precision not allowed in integer format specifier") - } - Self::NotAllowed(s) => { - let msg = format!("{s} not allowed with integer format specifier 'c'"); - vm.new_value_error(msg) - } - Self::UnableToConvert => vm.new_value_error("Unable to convert int to float"), - Self::CodeNotInRange => vm.new_overflow_error("%c arg not in range(0x110000)"), - Self::IntTooLargeForCLong => { - vm.new_overflow_error("Python int too large to convert to C long") - } - Self::ZeroPadding => { - vm.new_value_error("Zero padding is not allowed in complex format specifier") - } - Self::AlignmentFlag => { - vm.new_value_error("'=' alignment flag is not allowed in complex format specifier") - } - Self::NegativeZeroCoercionNotAllowed(type_name) => { - let msg = format!( - "Negative zero coercion (z) not allowed in {type_name} format specifier" - ); - vm.new_value_error(msg) - } - Self::StringAlignmentFlag => { - vm.new_value_error("'=' alignment not allowed in string format specifier") - } - Self::StringSpecNotAllowed(s) => { - let msg = format!("{s} not allowed in string format specifier"); - vm.new_value_error(msg) - } - Self::NotImplemented(c, s) => { - let msg = format!("Format code '{c}' for object of type '{s}' not implemented yet"); - vm.new_value_error(msg) - } +pub(crate) fn parse_format_spec( + object: &PyObject, + spec: &str, + vm: &VirtualMachine, +) -> PyResult { + FormatSpec::parse(spec).map_err(|err| format_spec_error(err, object, spec, vm)) +} + +pub(crate) fn format_spec_error( + err: FormatSpecError, + object: &PyObject, + spec: &str, + vm: &VirtualMachine, +) -> PyBaseExceptionRef { + match err { + FormatSpecError::InvalidFormatSpecifier => vm.new_value_error(format!( + "Invalid format specifier '{spec}' for object of type '{}'", + object.class().name() + )), + FormatSpecError::DecimalDigitsTooMany => { + vm.new_value_error("Too many decimal digits in format string") + } + FormatSpecError::PrecisionTooBig => vm.new_value_error("Precision too big"), + FormatSpecError::PrecisionMissing => { + vm.new_value_error("Format specifier missing precision") + } + FormatSpecError::UnspecifiedFormat(c1, c2) => { + vm.new_value_error(format!("Cannot specify '{c1}' with '{c2}'.")) + } + FormatSpecError::ExclusiveFormat(c1, c2) => { + vm.new_value_error(format!("Cannot specify both '{c1}' and '{c2}'.")) + } + FormatSpecError::UnknownFormatCode(c, s) => vm.new_value_error(format!( + "Unknown format code '{c}' for object of type '{s}'" + )), + FormatSpecError::PrecisionNotAllowed => { + vm.new_value_error("Precision not allowed in integer format specifier") + } + FormatSpecError::NotAllowed(s) => { + vm.new_value_error(format!("{s} not allowed with integer format specifier 'c'")) + } + FormatSpecError::UnableToConvert => vm.new_value_error("Unable to convert int to float"), + FormatSpecError::CodeNotInRange => vm.new_overflow_error("%c arg not in range(0x110000)"), + FormatSpecError::IntTooLargeForCLong => { + vm.new_overflow_error("Python int too large to convert to C long") + } + FormatSpecError::ZeroPadding => { + vm.new_value_error("Zero padding is not allowed in complex format specifier") + } + FormatSpecError::AlignmentFlag => { + vm.new_value_error("'=' alignment flag is not allowed in complex format specifier") + } + FormatSpecError::NegativeZeroCoercionNotAllowed(type_name) => vm.new_value_error(format!( + "Negative zero coercion (z) not allowed in {type_name} format specifier" + )), + FormatSpecError::StringAlignmentFlag => { + vm.new_value_error("'=' alignment not allowed in string format specifier") + } + FormatSpecError::StringSpecNotAllowed(s) => { + vm.new_value_error(format!("{s} not allowed in string format specifier")) } + FormatSpecError::NotImplemented(c, s) => vm.new_value_error(format!( + "Format code '{c}' for object of type '{s}' not implemented yet" + )), } } From 052549a55074c7644afc386bceb8312fecf77b92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:42:49 +0000 Subject: [PATCH 020/229] build(deps): bump bstr from 1.13.0 to 1.13.1 (#8973) Bumps [bstr](https://github.com/BurntSushi/bstr) from 1.13.0 to 1.13.1. - [Commits](https://github.com/BurntSushi/bstr/compare/1.13.0...1.13.1) --- updated-dependencies: - dependency-name: bstr dependency-version: 1.13.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e5d55f915d..8a7e9248a50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -445,9 +445,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", "regex-automata", From 3e0e4013e18e194e9ec5b826f41d80925cc8bd2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:05:59 +0000 Subject: [PATCH 021/229] build(deps): bump reviewdog/action-actionlint from 1.76.0 to 1.77.0 (#8977) Bumps [reviewdog/action-actionlint](https://github.com/reviewdog/action-actionlint) from 1.76.0 to 1.77.0. - [Release notes](https://github.com/reviewdog/action-actionlint/releases) - [Commits](https://github.com/reviewdog/action-actionlint/compare/320fcdd9c860767cf17fab3b20e22e739d5d02b8...2085657ab2c7f48c58edcc767fba576f63bea76b) --- updated-dependencies: - dependency-name: reviewdog/action-actionlint dependency-version: 1.77.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cd0b1349c09..7acd8710c53 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -754,7 +754,7 @@ jobs: components: rustfmt - name: actionlint - uses: reviewdog/action-actionlint@320fcdd9c860767cf17fab3b20e22e739d5d02b8 # v1.76.0 + uses: reviewdog/action-actionlint@2085657ab2c7f48c58edcc767fba576f63bea76b # v1.77.0 - name: zizmor uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 From 64dafe0d5638c176b3e9299bda701d6096a066f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:29:11 +0900 Subject: [PATCH 022/229] build(deps): bump thin-vec from 0.2.19 to 0.2.20 (#8980) Bumps [thin-vec](https://github.com/mozilla/thin-vec) from 0.2.19 to 0.2.20. - [Changelog](https://github.com/mozilla/thin-vec/blob/main/RELEASES.md) - [Commits](https://github.com/mozilla/thin-vec/compare/v0.2.19...v0.2.20) --- updated-dependencies: - dependency-name: thin-vec dependency-version: 0.2.20 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a7e9248a50..00cf83184be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4702,9 +4702,9 @@ checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" [[package]] name = "thin-vec" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79def32ffcd477db1ff26f76dab9e3a91f0bd42a85ca96577089b24623056f9d" +checksum = "4568d7e143ec86d2021c338bae2afa88699e84b8e0af523626654fe6f03a1748" [[package]] name = "thiserror" From 3b881b6a08eecf1d0d89e27794887fa30205e0b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:29:29 +0900 Subject: [PATCH 023/229] build(deps): bump which from 8.0.5 to 8.0.6 (#8978) Bumps [which](https://github.com/harryfei/which-rs) from 8.0.5 to 8.0.6. - [Release notes](https://github.com/harryfei/which-rs/releases) - [Changelog](https://github.com/harryfei/which-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/harryfei/which-rs/compare/8.0.5...8.0.6) --- updated-dependencies: - dependency-name: which dependency-version: 8.0.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00cf83184be..8898a28e412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5131,9 +5131,9 @@ dependencies = [ [[package]] name = "which" -version = "8.0.5" +version = "8.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" dependencies = [ "libc", ] From f14f8bce3525d4dc3e82b498efb508a07d6022d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:29:46 +0900 Subject: [PATCH 024/229] build(deps): bump taiki-e/install-action from 2.87.17 to 2.87.21 (#8976) * build(deps): bump taiki-e/install-action from 2.87.17 to 2.87.21 Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.87.17 to 2.87.21. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/94c31af3204a9f15ab40b35ad084410b905bbc73...4cef1412cce204788f482e778a0b9187f9626a29) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.87.21 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Ignore spurious warning from shellcheck We do not put the `EXTRA_TEST_ARGS` and `SKIPS` variables in quotes by design. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: fanninpm <27117322+fanninpm@users.noreply.github.com> --- .github/workflows/ci.yaml | 3 ++- .github/workflows/cron-ci.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7acd8710c53..61d097997a3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -559,6 +559,7 @@ jobs: - name: Run CPython tests run: | + # shellcheck disable=SC2086 target/release/rustpython -u -m test --slow-ci -j "$CORES" $EXTRA_TEST_ARGS $SKIPS timeout-minutes: ${{ matrix.timeout }} shell: bash @@ -725,7 +726,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@94c31af3204a9f15ab40b35ad084410b905bbc73 # v2.87.17 + - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 with: tool: cargo-shear fallback: none diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 16c74bdc43a..f5e7bddc29e 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -33,7 +33,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@94c31af3204a9f15ab40b35ad084410b905bbc73 # v2.87.17 + - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 with: tool: cargo-llvm-cov From f2f8f95a808b9d96faf4456e6b40330f20f8b94b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:30:17 +0900 Subject: [PATCH 025/229] build(deps): bump bitflags from 2.13.1 to 2.13.2 (#8979) Bumps [bitflags](https://github.com/bitflags/bitflags) from 2.13.1 to 2.13.2. - [Release notes](https://github.com/bitflags/bitflags/releases) - [Changelog](https://github.com/bitflags/bitflags/blob/main/CHANGELOG.md) - [Commits](https://github.com/bitflags/bitflags/compare/2.13.1...2.13.2) --- updated-dependencies: - dependency-name: bitflags dependency-version: 2.13.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8898a28e412..c640c8afb86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -338,7 +338,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.10.5", @@ -358,7 +358,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.10.5", @@ -380,9 +380,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitflagset" @@ -2527,7 +2527,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -2540,7 +2540,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -2706,7 +2706,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "foreign-types", "libc", @@ -3392,7 +3392,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -3577,7 +3577,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", @@ -3722,7 +3722,7 @@ dependencies = [ name = "rustpython-capi" version = "0.6.1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "itertools 0.15.0", "libc", "malachite-bigint", @@ -3737,7 +3737,7 @@ dependencies = [ name = "rustpython-codegen" version = "0.6.1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "indexmap", "itertools 0.15.0", "log", @@ -3762,7 +3762,7 @@ version = "0.6.1" dependencies = [ "ascii", "base64", - "bitflags 2.13.1", + "bitflags 2.13.2", "blake2b_simd", "blake2s_simd", "bzip2", @@ -3812,7 +3812,7 @@ dependencies = [ name = "rustpython-compiler-core" version = "0.6.1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bitflagset", "itertools 0.15.0", "lz4_flex", @@ -3857,7 +3857,7 @@ dependencies = [ name = "rustpython-host_env" version = "0.6.1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bitflagset", "cc", "der 0.8.2", @@ -3941,7 +3941,7 @@ checksum = "bc19eeb4cfa2c35428143693eadf85ee78816f22a829298593581a56700c0a21" dependencies = [ "aho-corasick", "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "char_str", "compact_str", "get-size2", @@ -3961,7 +3961,7 @@ version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "480eec9ac4dda77b6724f0c9a54a81d3c57868d6a707c3d7c5133fd83c933f66" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bstr", "drop_bomb", "get-size2", @@ -4014,7 +4014,7 @@ dependencies = [ name = "rustpython-sre_engine" version = "0.6.1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "codspeed-criterion-compat", "num_enum", "optional", @@ -4099,7 +4099,7 @@ name = "rustpython-vm" version = "0.6.1" dependencies = [ "ascii", - "bitflags 2.13.1", + "bitflags 2.13.2", "bitflagset", "bstr", "constant_time_eq 0.6.0", @@ -4200,7 +4200,7 @@ version = "18.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53f6a737db68eb1a8ccff86b584b2fc13eca6a7bb6f78ebc7c529547e3ab9684" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "clipboard-win", "home", @@ -4296,7 +4296,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation", "core-foundation-sys", "libc", @@ -4642,7 +4642,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "501336eb7ba9e417300a6a0fa985721065467aa83a6dcf0422a8e43e4c0328fa" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation", "system-configuration-sys", ] From bb1c28852cd0a4b8db4847cbaacb376b17cd7003 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:43:52 +0900 Subject: [PATCH 026/229] build(deps-dev): bump proxy-addr from 2.0.7 to 2.0.8 in /wasm/demo (#8984) Bumps [proxy-addr](https://github.com/jshttp/proxy-addr) from 2.0.7 to 2.0.8. - [Release notes](https://github.com/jshttp/proxy-addr/releases) - [Changelog](https://github.com/jshttp/proxy-addr/blob/master/HISTORY.md) - [Commits](https://github.com/jshttp/proxy-addr/compare/v2.0.7...v2.0.8) --- updated-dependencies: - dependency-name: proxy-addr dependency-version: 2.0.8 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 3860bd7bd44..e0a498d2011 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -4604,9 +4604,9 @@ "license": "MIT" }, "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", "dev": true, "license": "MIT", "dependencies": { @@ -4615,6 +4615,10 @@ }, "engines": { "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/proxy-addr/node_modules/ipaddr.js": { From 1fbb58bf01092cea2b3179e59d0e9536b4aeb566 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:44:01 +0900 Subject: [PATCH 027/229] build(deps): bump rustls from 0.23.43 to 0.23.45 (#8985) Bumps [rustls](https://github.com/rustls/rustls) from 0.23.43 to 0.23.45. - [Release notes](https://github.com/rustls/rustls/releases) - [Changelog](https://github.com/rustls/rustls/blob/main/CHANGELOG.md) - [Commits](https://github.com/rustls/rustls/compare/v/0.23.43...v/0.23.45) --- updated-dependencies: - dependency-name: rustls dependency-version: 0.23.45 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c640c8afb86..1c8c03190d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,9 +277,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-fips-sys" -version = "0.13.15" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c0e6249c249b8916c98ebae7bc06216c8dcab3002f32872b4abe642d17063b1" +checksum = "03367707e92796b190a4207d4d39b0a4271d574503d2969c2b0cfbf5c87658ee" dependencies = [ "bindgen 0.72.1", "cc", @@ -292,9 +292,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-fips-sys", "aws-lc-sys", @@ -303,9 +303,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -3586,14 +3586,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "once_cell", "rustls-pki-types", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.15", "subtle", "zeroize", ] @@ -3682,9 +3682,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -3881,7 +3881,7 @@ dependencies = [ "rustls", "rustls-native-certs", "rustls-pemfile", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.15", "rustpython-wtf8", "rustyline", "schannel", From abdf2553350aebdc9b07f2876ab0f1938a6edd64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:44:25 +0900 Subject: [PATCH 028/229] build(deps-dev): bump source-map-js from 1.2.1 to 1.2.2 in /wasm/demo (#8987) Bumps [source-map-js](https://github.com/7rulnik/source-map-js) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/7rulnik/source-map-js/releases) - [Changelog](https://github.com/7rulnik/source-map-js/blob/main/CHANGELOG.md) - [Commits](https://github.com/7rulnik/source-map-js/compare/v1.2.1...v1.2.2) --- updated-dependencies: - dependency-name: source-map-js dependency-version: 1.2.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index e0a498d2011..83a8d1874da 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -5375,9 +5375,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { From 03c57206921257706c1ab127be6218d28d6be22b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:44:35 +0900 Subject: [PATCH 029/229] build(deps-dev): bump shell-quote from 1.10.0 to 1.12.0 in /wasm/demo (#8988) Bumps [shell-quote](https://github.com/ljharb/shell-quote) from 1.10.0 to 1.12.0. - [Changelog](https://github.com/ljharb/shell-quote/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/shell-quote/compare/v1.10.0...v1.12.0) --- updated-dependencies: - dependency-name: shell-quote dependency-version: 1.12.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 83a8d1874da..5080f8a69ab 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -5257,9 +5257,9 @@ } }, "node_modules/shell-quote": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.10.0.tgz", - "integrity": "sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==", + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.12.0.tgz", + "integrity": "sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q==", "dev": true, "license": "MIT", "engines": { From 34a7d5ceb92b854f23f0fd9607a9f7ad0c690102 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:44:56 +0900 Subject: [PATCH 030/229] build(deps-dev): bump postcss-selector-parser in /wasm/demo (#8989) Bumps [postcss-selector-parser](https://github.com/postcss/postcss-selector-parser) from 7.1.5 to 7.1.6. - [Release notes](https://github.com/postcss/postcss-selector-parser/releases) - [Changelog](https://github.com/postcss/postcss-selector-parser/blob/main/CHANGELOG.md) - [Commits](https://github.com/postcss/postcss-selector-parser/compare/7.1.5...7.1.6) --- updated-dependencies: - dependency-name: postcss-selector-parser dependency-version: 7.1.6 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 5080f8a69ab..6f85ed43ff2 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -4565,9 +4565,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "7.1.5", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.5.tgz", - "integrity": "sha512-KvvtD7SrlBP7dlgkBghEE3r84CABm5SmV2aNcG4oCA+qDnJ/tvKonFVvwWAyyWUEwxuNawdfEAZKP9zM3oZ2Uw==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.6.tgz", + "integrity": "sha512-7qASPzhKF2l2KLboRZux8CCTRMdGiV08vWmyKzPz22qZ7ZjQBOeY7rNzNoCLSUiftJ7HUq0GERHmxw/t0dCdMw==", "dev": true, "license": "MIT", "dependencies": { From 4b6ad58469a0b44c2244f77d471b549b59f8932b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:45:06 +0900 Subject: [PATCH 031/229] build(deps): bump rustix from 1.1.4 to 1.1.5 (#8990) Bumps [rustix](https://github.com/bytecodealliance/rustix) from 1.1.4 to 1.1.5. - [Release notes](https://github.com/bytecodealliance/rustix/releases) - [Changelog](https://github.com/bytecodealliance/rustix/blob/main/CHANGES.md) - [Commits](https://github.com/bytecodealliance/rustix/compare/v1.1.4...v1.1.5) --- updated-dependencies: - dependency-name: rustix dependency-version: 1.1.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c8c03190d2..aa1ec9cf24a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1469,7 +1469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2048,7 +2048,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3573,15 +3573,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4682,7 +4682,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5176,7 +5176,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From a40334179123949fbeef1cdc7557539c06cd187b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:54:27 +0900 Subject: [PATCH 032/229] Take SyntaxError messages and ranges from the parser (#8983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Match CPython's SyntaxError range for unparenthesized `except` types CPython's `invalid_except_stmt` rule raises the error only once the whole clause has matched, and reports a range that starts at the first exception type and ends at the `:` closing the clause, so it covers the `as NAME` part as well: invalid_except_stmt: | 'except' a=expression ',' expressions 'as' NAME ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") } The parser reports the exception types alone, so look up that `:` in the source and widen the range to match. Only `as NAME` may follow the exception types, which is why the first `:` after them is the one closing the clause. try: pass except A, B, C as e: pass CPython 3.14: ('x.py', 3, 8, 'except A, B, C as e:\n', 3, 20) before: ('x.py', 3, 8, 'except A, B, C as e:\n', 3, 15) Fixes #8496 Co-Authored-By: Claude Opus 5 * Clarify that the `except` error range stops before the closing colon `invalid_except_stmt_end` returns the exclusive end of the range, so the column it yields is the one the `:` sits on even though the `:` is not part of the range. The previous "ending at the `:`" wording read as if the colon were included, which invites an off-by-one "fix". Comments only; no behavior change. Co-Authored-By: Claude Opus 5 (1M context) * Fix the `except ... as` error column for non-ASCII names and line joins `invalid_except_stmt_end` recovers the end of CPython's range by scanning the source, which has to redo by hand what CPython gets from its tokenizer: - The column was the byte distance from the line start, but `end_offset` is a character column. `except Ä, B as e:` reported 18 instead of 17. CPython converts explicitly, in `_PyPegen_byte_offset_to_character_offset`. - The colon scan already spans explicit line joins, but the `as` check ran on the raw slice, so a `\` before `as` left the range unextended. `except A, B \` + `as exc:` reported (3, 12) instead of (4, 7). Add regression cases to `syntax_invalid.py` covering both, plus the whitespace, `except*` and line-join placements that already worked. All values are verified against CPython 3.14. Co-Authored-By: Claude Opus 5 (1M context) * Update the Ruff fork to 0.16.10 Point the rustpython-ruff_* dependencies at the 0.16.10-rustpython tag of the RustPython Ruff fork and port to its AST changes: `ExprCompare` stores all operands in `operands`, call arguments use `ThinVec`, dict comprehension generators and f-string elements are boxed slices, `FStringValue` iterates `FStringPartRef`, and `TokenKind::Name` is now `TokenKind::Identifier`. Assisted-by: Grok:grok-4.6 Assisted-by: Claude Code:claude-opus-5-5 * Drop the source scan for the `except ... as` error range The parser now reports the unparenthesized exception types error with a range ending at the `:` that closes the clause, so `invalid_except_stmt_end` and the message match that widened the range are removed. Assisted-by: Claude Code:claude-opus-5-5 * Take parse error messages from the parser as they are The Ruff fork now emits the final message texts, so `vm_new.rs` no longer rewrites parse error messages by matching their text and no longer lowercases the first character of every message. `analyze_compile_error` keeps only the caret narrowing for starred expressions and the line number in the unterminated string message. The compiler matches the new lowercase texts for indented block and mixed bytes literal errors. The fork is referenced by revision until it is published. Assisted-by: Claude Code:claude-opus-5-5 * Take starred and bytes literal error ranges from the parser The Ruff fork now reports misplaced starred expressions at the `*` and non-ASCII bytes literal errors over the whole literal. `vm_new.rs` drops `narrow_caret` together with `SyntaxErrorInfo`, which only carried the message after that, and the compiler drops `bytes_literal_span`. Assisted-by: Claude Code:claude-opus-5-5 * Take indentation error kinds and positions from the parser The Ruff fork now reports `TabError` and `TooDeepIndentation` as their own lexer errors and places indentation and line continuation errors itself. `vm_new.rs` picks TabError or IndentationError from the error kind instead of scanning the source for mixed indentation, drops the TabError message override, and sets the end offset of tokenizer errors to 0 or -1 by kind. The compiler drops the line-end relocation of unindent errors and the line continuation rewrite. `_tokenize` raises TabError and depth errors for the token they replace, also for sources containing tabs. Remove expectedFailure from test_tokenize.test_max_indent and four test_tabnanny tests. Assisted-by: Claude Code:claude-opus-5-5 * Take unterminated string messages from the parser The Ruff fork's unterminated string errors now include the detected line and are reported at the start of the string. `vm_new.rs` drops its message override for `UnclosedStringError`, and the shell continues a line on an unterminated triple-quoted string by the error's `triple_quoted` flag instead of reading the quotes from the source. Assisted-by: Claude Code:claude-opus-5-5 * Take bracket errors and tokenizer error priority from the parser Bump the ruff fork, which now reports unmatched, mismatched, unclosed and too deeply nested brackets and decides whether a tokenizer error replaces the parser's first error. Remove the bracket, unterminated string and nesting depth source scanners and `pre_parse_source_error`. A tokenizer error from the parser now wins over source scanner diagnostics found after it. An unclosed bracket is incomplete input only when the parser reports it as such, and other tokenizer errors end where they start. Remove the expected failure markers from test_unicode_identifiers test_invalid and the `import ä £` doctest in test_syntax. Assisted-by: Claude Code:claude-opus-5-5 * Take number literal, string prefix and non-printable errors from the parser Bump the ruff fork, which now reports malformed number literals, incompatible string prefixes and non-printable characters itself. Remove the number literal, string prefix and non-printable character source scanners and the override classes that ranked them. Leading zero and string prefix errors keep their range instead of ending where they start. Assisted-by: Claude Code:claude-opus-5-5 * Take the missing indented block error from the parser Bump the ruff fork, which now reports `ExpectedIndentedBlock` with the clause and the line of its header keyword. Remove the message rewrite that derived the clause and line from the source. The VM chooses IndentationError or incomplete input by the error variant instead of its message. Assisted-by: Claude Code:claude-opus-5-5 * Take binding target errors from the parser Bump the ruff fork, which now names the invalid expression in assignment, augmented assignment, delete, `for`, `with`, import, pattern and `except` target errors. Remove the source scanners for those targets and the message rewrites of `InvalidAssignmentTarget` and `InvalidNamedAssignmentTarget`. Remove the expected failure markers from test_syntax doctests that now pass. Assisted-by: Claude Code:claude-opus-5-5 * Take parameter, argument, star, comma and dictionary errors from the parser Bump the ruff fork to 36919b62fa. Remove the source scanners for type parameters, parameter lists, star annotations, call arguments, def type parameters, parenthesized groups, missing commas and dictionary items. Set end_offset 0 for ExpectedColonAfterDictionaryKey. Remove 14 EXPECTED_FAILURE markers in test_syntax and test_unpack_ex. Assisted-by: Claude Code:claude-opus-5-5 * Take expression errors from the parser Bump the ruff fork to 4f922d2124. Remove the source scanners for comprehensions, collection and condition assignments, expressions between strings, named expression targets, missing `in` after for-loop variables and statements in `if` expressions. Remove the now passing expected failure markers in test_syntax and test_named_expressions. Assisted-by: Claude Code:claude-opus-5-5 * Take statement errors from the parser Bump the ruff fork to c7f6594ed6. Remove the source scanners for standalone except, `import ... from`, `**_` mapping rests, `elif` after `else` and mixed except handlers, and the post-parse scanners for call arguments, match targets, yield after a comma and parenthesized star imports. Assisted-by: Claude Code:claude-opus-5-5 * Take string and f-string errors from the parser Bump the ruff fork to 6b46d0dfc8. Remove the source scanners for malformed `\N` escapes, f-string and t-string replacement fields, mixed t-string literals and `print`/`exec` statements, and the override plumbing they used. Drop the `f'{x'; '` case from the compiler test, whose expected message differs from CPython 3.14. Assisted-by: Claude Code:claude-opus-5-5 * Restore the bracket depth check and fix incomplete unclosed strings Restore `pre_parse_source_error`, which rejects bracket nesting past 200 levels before parsing. The parser reports the same error but still builds the full tree, which overflows the native stack when dropped. Remove the VM's unclosed string scanner, which ignored the mode and treated `'abc` in exec mode as incomplete input. The compiler's check now skips escaped newlines. Assisted-by: Claude Code:claude-opus-5-5 * Report multiple statements before later errors in single mode In single mode, when the first statement is a simple statement, report code after its line as multiple statements before any error found later, at the newline token or the comment before it. This replaces the multiple statements check on the parsed body and also applies to `ast.parse(mode='single')`. An unclosed string that follows an earlier syntax error is no longer incomplete input. The bracket depth error range is now empty. Assisted-by: Claude Code:claude-opus-5-5 * Update the Ruff fork for parser speed and starred annotations Bump the ruff fork to 8b59b85344, which skips the tokenizer error pass when parsing finds no errors, stops allocating for every identifier's string prefix check, and reports `def f(*args: *b = ...)` as invalid syntax. Remove expected failure markers from test_code_module test_indentation_error and test_sysexcepthook_indentation_error, and from test_dictcomps test_illegal_assignment. Assisted-by: Claude Code:claude-opus-5-5 * Use the published Ruff fork 0.16.10 crates Switch the rustpython-ruff_* dependencies from the git revision to the 0.16.10 crates.io release, published from the same commit (8b59b85344). Assisted-by: Claude Code:claude-opus-5-5 --------- Co-authored-by: zzarbttoo Co-authored-by: Claude Opus 5 --- Cargo.lock | 36 +- Cargo.toml | 8 +- Lib/test/test_code_module.py | 2 - Lib/test/test_dictcomps.py | 1 - Lib/test/test_fstring.py | 1 - Lib/test/test_named_expressions.py | 1 - Lib/test/test_syntax.py | 42 +- Lib/test/test_tabnanny.py | 4 - Lib/test/test_tokenize.py | 1 - Lib/test/test_unicode_identifiers.py | 1 - Lib/test/test_unpack_ex.py | 4 +- crates/codegen/src/compile.rs | 70 +- crates/codegen/src/symboltable.rs | 8 +- crates/codegen/src/unparse.rs | 17 +- crates/compiler/src/lib.rs | 6757 ++--------------------- crates/stdlib/src/_tokenize.rs | 49 +- crates/vm/src/stdlib/_ast.rs | 6 + crates/vm/src/stdlib/_ast/argument.rs | 40 +- crates/vm/src/stdlib/_ast/expression.rs | 24 +- crates/vm/src/stdlib/_ast/string.rs | 51 +- crates/vm/src/stdlib/_ast/validate.rs | 9 +- crates/vm/src/vm/compile.rs | 6 +- crates/vm/src/vm/vm_new.rs | 590 +- extra_tests/snippets/syntax_invalid.py | 43 + src/shell.rs | 21 +- 25 files changed, 753 insertions(+), 7039 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa1ec9cf24a..4b15f9af776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,9 +572,9 @@ dependencies = [ [[package]] name = "char_str" -version = "0.0.2" +version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576ba56f6ca18ebb069d0d07260407171712aff4dfe15ee3f8982dc16a455bcf" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" dependencies = [ "castaway", "get-size2", @@ -1612,9 +1612,9 @@ dependencies = [ [[package]] name = "get-size-derive2" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c736d226c32e496b8377813b52269e11ad3a48d8373b68862d0364f04fd1229d" +checksum = "7efe46d2adb529015748742f1a38594039ce357dbb21e6d178de342cfce468d4" dependencies = [ "attribute-derive", "quote", @@ -1623,9 +1623,9 @@ dependencies = [ [[package]] name = "get-size2" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b411f34418305908ab15a82ff78958c2a9aee9a272b2a2e663836b20a4e4b9d3" +checksum = "d4d338de392104417fe6fdbe588c52383b9236956d8afe5654f0077176edc632" dependencies = [ "compact_str", "get-size-derive2", @@ -3935,9 +3935,9 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_ast" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc19eeb4cfa2c35428143693eadf85ee78816f22a829298593581a56700c0a21" +checksum = "5b7ffeb85f9a68c461d538839c79548584768df08f2d68cb1ee693be697e0355" dependencies = [ "aho-corasick", "arrayvec", @@ -3957,14 +3957,16 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_parser" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "480eec9ac4dda77b6724f0c9a54a81d3c57868d6a707c3d7c5133fd83c933f66" +checksum = "7c9729e35cac02839c7a67aaccfea3eba32032f7aeacca2e82ad79e8d7b7fd95" dependencies = [ "bitflags 2.13.2", "bstr", "drop_bomb", "get-size2", + "hashbrown 0.17.1", + "icu_properties", "memchr", "rustc-hash", "rustpython-ruff_python_ast", @@ -3980,22 +3982,24 @@ dependencies = [ [[package]] name = "rustpython-ruff_python_trivia" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d69a5d9bb610a76bee05d23adce0de55f0fa06f251c0e1c7f7a0c625a366a58" +checksum = "764f43ea060f888164fbd360b0e1a36f2a35f5c03def85407bab7039fc62a645" dependencies = [ "itertools 0.15.0", + "memchr", "rustc-hash", "rustpython-ruff_source_file", "rustpython-ruff_text_size", "unicode-ident", + "unicode-normalization", ] [[package]] name = "rustpython-ruff_source_file" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3aa9c91c8a4e86bf0fa8a67aec284fa6c0c533cdefb900b08584be15748c606" +checksum = "cfe26430c3f2eadb40524b6df7ac9a3970dd2b58f921b01ed75df0121e332fad" dependencies = [ "memchr", "rustpython-ruff_text_size", @@ -4003,9 +4007,9 @@ dependencies = [ [[package]] name = "rustpython-ruff_text_size" -version = "0.16.5" +version = "0.16.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fedbd1af4f91cd828185123d95119662db9c4a0b21c02b695783d74360d4f46" +checksum = "5e5377a04f2855030600b93a05754242950ea9b070507ef9aef67965a52eeb09" dependencies = [ "get-size2", ] diff --git a/Cargo.toml b/Cargo.toml index 74565c4b09e..111ba98687e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -188,10 +188,10 @@ rustpython-wtf8 = { path = "crates/wtf8", version = "0.6.1" } rustpython-doc = { path = "crates/doc", version = "0.6.1" } # Use the RustPython Ruff fork for RustPython public `_ast` metadata. -ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.16.5" } -ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.16.5" } -ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.16.5" } -ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.16.5" } +ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.16.10" } +ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.16.10" } +ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.16.10" } +ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.16.10" } der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] } phf = { version = "0.14.0", default-features = false, features = ["macros"]} diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index fb519878cd8..3642b47c2c1 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -111,7 +111,6 @@ def test_syntax_error(self): self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - @unittest.expectedFailure # TODO: RUSTPYTHON; - 'IndentationError: unexpected indentation'] def test_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] self.console.interact() @@ -187,7 +186,6 @@ def test_sysexcepthook_syntax_error(self): ' ^\n', 'SyntaxError: invalid syntax\n']) - @unittest.expectedFailure # TODO: RUSTPYTHON; + 'IndentationError: unexpected indent\n'] def test_sysexcepthook_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] hook = mock.Mock() diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index fc7ebb0f5a6..26b56dac503 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -75,7 +75,6 @@ def test_local_visibility(self): self.assertEqual(actual, expected) self.assertEqual(v, "Local variable") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal_assignment(self): with self.assertRaisesRegex(SyntaxError, "cannot assign"): compile("{x: y for y, x in ((1, 2), (3, 4))} = 5", "", diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index b68e356e04d..baa638e07fa 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -865,7 +865,6 @@ def __format__(self, spec): x = X() self.assertEqual(f'{x} {x}', '1 2') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_missing_expression(self): self.assertAllRaise(SyntaxError, "f-string: valid expression required before '}'", diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index a859e051de2..cf44080670d 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -98,7 +98,6 @@ def test_named_expression_invalid_16(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_expression_invalid_17(self): code = "[i := 0, j := 1 for i, j in [(1, 2), (3, 4)]]" diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index d3cb80bbb70..51a4a8155f5 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -326,7 +326,7 @@ # Incorrectly closed strings ->>> "The interesting object "The important object" is very important" # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> "The interesting object "The important object" is very important" Traceback (most recent call last): SyntaxError: invalid syntax. Is this intended to be part of the string? @@ -423,7 +423,7 @@ Traceback (most recent call last): SyntaxError: invalid syntax ->>> def foo(/,a,b=,c): # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> def foo(/,a,b=,c): ... pass Traceback (most recent call last): SyntaxError: at least one argument must precede / @@ -1045,7 +1045,7 @@ a complex 'if' (one with 'elif') would fail to notice an invalid suite, leading to spurious errors. - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... x() = 1 ... elif 1: ... pass @@ -1053,7 +1053,7 @@ ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... pass ... elif 1: ... x() = 1 @@ -1061,7 +1061,7 @@ ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... x() = 1 ... elif 1: ... pass @@ -1071,7 +1071,7 @@ ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... pass ... elif 1: ... x() = 1 @@ -1081,7 +1081,7 @@ ... SyntaxError: cannot assign to function call here. Maybe you meant '==' instead of '='? - >>> if 1: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> if 1: ... pass ... elif 1: ... pass @@ -1185,7 +1185,7 @@ Traceback (most recent call last): SyntaxError: expected ':' - >>> with (blech as something) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> with (blech as something) ... pass Traceback (most recent call last): SyntaxError: expected ':' @@ -1195,12 +1195,12 @@ Traceback (most recent call last): SyntaxError: expected ':' - >>> with (blech, block as something) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> with (blech, block as something) ... pass Traceback (most recent call last): SyntaxError: expected ':' - >>> with (blech, block as something, bluch) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> with (blech, block as something, bluch) ... pass Traceback (most recent call last): SyntaxError: expected ':' @@ -1464,7 +1464,7 @@ Traceback (most recent call last): SyntaxError: invalid syntax - >>> dict(x=34, (x for x in range 10), 1); x $ y # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> dict(x=34, (x for x in range 10), 1); x $ y Traceback (most recent call last): SyntaxError: invalid syntax @@ -2077,7 +2077,7 @@ Traceback (most recent call last): SyntaxError: cannot use literal as import target ->>> from a import (b as c.d) # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import (b as c.d) Traceback (most recent call last): SyntaxError: cannot use attribute as import target @@ -2085,18 +2085,18 @@ Traceback (most recent call last): SyntaxError: cannot use literal as import target ->>> from a import ( # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import ( ... b as f()) Traceback (most recent call last): SyntaxError: cannot use function call as import target ->>> from a import ( # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import ( ... b as [], ... ) Traceback (most recent call last): SyntaxError: cannot use list as import target ->>> from a import ( # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE +>>> from a import ( ... b, ... c as () ... ) @@ -2233,7 +2233,7 @@ Traceback (most recent call last): SyntaxError: cannot assign to __debug__ - >>> import ä £ # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> import ä £ Traceback (most recent call last): SyntaxError: invalid character '£' (U+00A3) @@ -2245,7 +2245,7 @@ Traceback (most recent call last): SyntaxError: cannot use '_' as a target - >>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> match ...: ... case 42 as 1+2+4: ... ... Traceback (most recent call last): @@ -2263,13 +2263,13 @@ Traceback (most recent call last): SyntaxError: cannot use tuple as pattern target - >>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> match ...: ... case 42 as (a + 1): ... ... Traceback (most recent call last): SyntaxError: cannot use expression as pattern target - >>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> match ...: ... case (32 as x) | (42 as a()): ... ... Traceback (most recent call last): @@ -2375,7 +2375,7 @@ A[*:] and A[:*] - >>> A[*:] # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> A[*:] Traceback (most recent call last): ... SyntaxError: Invalid star expression @@ -2386,7 +2386,7 @@ A[*] - >>> A[*] # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> A[*] Traceback (most recent call last): ... SyntaxError: Invalid star expression diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index d7a77eb26e4..11b3f8ea016 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -217,7 +217,6 @@ def test_when_tokenize_tokenerror(self): with self.assertRaises(SystemExit): self.verify_tabnanny_check(file_path, err=err) - @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error_verbose(self): """A python source code file eligible for raising `tabnanny.NannyNag`. @@ -231,7 +230,6 @@ def test_when_nannynag_error_verbose(self): tabnanny.verbose = 1 self.verify_tabnanny_check(file_path, out=out) - @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error(self): """A python source code file eligible for raising `tabnanny.NannyNag`.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: @@ -341,7 +339,6 @@ def test_quiet_flag(self): stdout = f"{file_path}\n" self.validate_cmd("-q", file_path, stdout=stdout) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_verbose_mode(self): """Should display more error information if verbose mode is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: @@ -350,7 +347,6 @@ def test_verbose_mode(self): ).strip() self.validate_cmd("-v", path, stdout=stdout, partial=True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_double_verbose_mode(self): """Should display detailed error information if double verbose is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 0e81c6f6db2..6649de68285 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3193,7 +3193,6 @@ def get_tokens(string): with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: IndentationError not raised by @support.skip_wasi_stack_overflow() def test_max_indent(self): MAXINDENT = 100 diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 27749a0805c..3680072d643 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -17,7 +17,6 @@ def test_non_bmp_normalized(self): 𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1 self.assertIn("Unicode", dir()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid(self): try: from test.tokenizedata import badsyntax_3131 # noqa: F401 diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index a5a20025930..6f49e62e8dc 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -361,7 +361,7 @@ ... SyntaxError: cannot use starred expression here - >>> (((*x))),y = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + >>> (((*x))),y = 1, 2 # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: cannot use starred expression here @@ -376,7 +376,7 @@ ... SyntaxError: cannot use starred expression here - >>> ((*x),y) = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + >>> ((*x),y) = 1, 2 # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: cannot use starred expression here diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c2b6446ee79..3aea9089de0 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -8604,14 +8604,15 @@ impl<'warnings> Compiler<'warnings> { self.use_cpython_label_block(end); Ok(()) } - ast::Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - .. - }) if ops.len() > 1 => { + ast::Expr::Compare(compare) if compare.ops.len() > 1 => { self.set_source_range(expression.range()); - self.compile_jump_if_compare(left, ops, comparators, condition, target_block) + self.compile_jump_if_compare( + compare.first_operand(), + &compare.ops, + compare.comparators(), + condition, + target_block, + ) } _ => { // Fall back case which always will work! @@ -8940,13 +8941,8 @@ impl<'warnings> Compiler<'warnings> { self.emit_load_attr(idx); } } - ast::Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - .. - }) => { - self.compile_compare(left, ops, comparators)?; + ast::Expr::Compare(compare) => { + self.compile_compare(compare.first_operand(), &compare.ops, compare.comparators())?; } ast::Expr::Constant(ast::ExprConstant { value, .. }) => { self.emit_load_const(ast_constant_value_to_constant_data(value.clone())); @@ -12521,7 +12517,11 @@ impl<'warnings> Compiler<'warnings> { ast::Expr::Await(_) => self.found = true, ast::Expr::ListComp(ast::ExprListComp { generators, .. }) | ast::Expr::SetComp(ast::ExprSetComp { generators, .. }) - | ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) + if generators.iter().any(|generator| generator.is_async) => + { + self.found = true + } + ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) if generators.iter().any(|generator| generator.is_async) => { self.found = true @@ -12556,16 +12556,16 @@ impl<'warnings> Compiler<'warnings> { fn compile_expr_fstring(&mut self, fstring: &ast::ExprFString) -> CompileResult<()> { let fstring_range = fstring.range; - let fstring = fstring.value.as_slice(); - if self.count_fstring_parts(fstring) > STACK_USE_GUIDELINE { - return self.compile_fstring_parts_joined(fstring, fstring_range); + let value = &fstring.value; + if self.count_fstring_parts(value) > STACK_USE_GUIDELINE { + return self.compile_fstring_parts_joined(value, fstring_range); } let mut element_count = 0; let mut pending_literal = None; let mut pending_literal_range = None; let mut pending_literal_no_location = false; - for part in fstring { + for part in value { self.compile_fstring_part_into( part, &mut pending_literal, @@ -12629,7 +12629,7 @@ impl<'warnings> Compiler<'warnings> { fn compile_fstring_parts_joined( &mut self, - fstring: &[ast::FStringPart], + fstring: &ast::FStringValue, fstring_range: TextRange, ) -> CompileResult<()> { self.set_source_range(fstring_range); @@ -12666,7 +12666,7 @@ impl<'warnings> Compiler<'warnings> { fn compile_fstring_part_into( &mut self, - part: &ast::FStringPart, + part: ast::FStringPartRef<'_>, pending_literal: &mut Option, pending_literal_range: &mut Option, pending_literal_no_location: &mut bool, @@ -12674,7 +12674,7 @@ impl<'warnings> Compiler<'warnings> { join_append_range: Option, ) -> CompileResult<()> { match part { - ast::FStringPart::Literal(string) => { + ast::FStringPartRef::Literal(string) => { let value = string_literal_part_value(&self.source_file, string); if pending_literal.is_none() { *pending_literal_range = Some(string.range); @@ -12687,7 +12687,7 @@ impl<'warnings> Compiler<'warnings> { } Ok(()) } - ast::FStringPart::FString(fstring) => self.compile_fstring_elements_into( + ast::FStringPartRef::FString(fstring) => self.compile_fstring_elements_into( fstring.flags, &fstring.elements, pending_literal, @@ -12801,7 +12801,7 @@ impl<'warnings> Compiler<'warnings> { } } - fn count_fstring_parts(&self, fstring: &[ast::FStringPart]) -> u32 { + fn count_fstring_parts(&self, fstring: &ast::FStringValue) -> u32 { let mut element_count = 0; let mut pending_literal = None; for part in fstring { @@ -12813,12 +12813,12 @@ impl<'warnings> Compiler<'warnings> { fn count_fstring_part_into( &self, - part: &ast::FStringPart, + part: ast::FStringPartRef<'_>, pending_literal: &mut Option, element_count: &mut u32, ) { match part { - ast::FStringPart::Literal(string) => { + ast::FStringPartRef::Literal(string) => { let value = string_literal_part_value(&self.source_file, string); if let Some(pending) = pending_literal.as_mut() { pending.push_wtf8(value.as_ref()); @@ -12826,7 +12826,7 @@ impl<'warnings> Compiler<'warnings> { *pending_literal = Some(value); } } - ast::FStringPart::FString(fstring) => self.count_fstring_elements_into( + ast::FStringPartRef::FString(fstring) => self.count_fstring_elements_into( fstring.flags, &fstring.elements, pending_literal, @@ -13972,7 +13972,7 @@ mod tests { arguments: ast::Arguments { node_index: ast::AtomicNodeIndex::NONE, range: TextRange::default(), - args: Box::default(), + args: Default::default(), keywords: Default::default(), runtime_args: None, runtime_bases: None, @@ -14120,7 +14120,7 @@ mod tests { arguments: ast::Arguments { node_index: ast::AtomicNodeIndex::NONE, range: TextRange::default(), - args: Box::default(), + args: Default::default(), keywords: Default::default(), runtime_args: None, runtime_bases: None, @@ -14196,12 +14196,14 @@ mod tests { let message = first_ast_constant_warning(ast::Expr::Compare(ast::ExprCompare { node_index: ast::AtomicNodeIndex::NONE, range: TextRange::default(), - left: Box::new(left), ops: Box::new([ast::CmpOp::Is]), - comparators: Box::new([ast::Expr::NoneLiteral(ast::ExprNoneLiteral { - node_index: ast::AtomicNodeIndex::NONE, - range: TextRange::default(), - })]), + operands: Box::new([ + left, + ast::Expr::NoneLiteral(ast::ExprNoneLiteral { + node_index: ast::AtomicNodeIndex::NONE, + range: TextRange::default(), + }), + ]), runtime_comparators: None, })); assert!( diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index bff326087e6..56512f4bb7d 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -2207,11 +2207,9 @@ impl SymbolTableBuilder { Expr::BoolOp(ExprBoolOp { values, .. }) => { self.scan_expressions(values, context)?; } - Expr::Compare(ExprCompare { - left, comparators, .. - }) => { - self.scan_expression(left, context)?; - self.scan_expressions(comparators, context)?; + Expr::Compare(compare) => { + self.scan_expression(compare.first_operand(), context)?; + self.scan_expressions(compare.comparators(), context)?; } Expr::Subscript(ExprSubscript { value, slice, .. }) => { self.scan_expression(value, ExpressionContext::Load)?; diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 86744a6010b..3ada6644cbb 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -380,18 +380,11 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { UnparseExpr::new(value, self.source) )?; } - ast::Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - node_index: _, - range: _range, - .. - }) => { + ast::Expr::Compare(compare) => { group_if!(precedence::CMP, { let new_lvl = precedence::CMP + 1; - self.unparse_expr(left, new_lvl)?; - for (op, cmp) in ops.iter().zip(comparators) { + self.unparse_expr(compare.first_operand(), new_lvl)?; + for (_, op, cmp) in compare.iter() { self.p(" ")?; self.p(op.as_str())?; self.p(" ")?; @@ -700,8 +693,8 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p("f")?; let body = fmt::from_fn(|f| { value.iter().try_for_each(|part| match part { - ast::FStringPart::Literal(lit) => f.write_str(lit), - ast::FStringPart::FString(ast::FString { elements, .. }) => { + ast::FStringPartRef::Literal(lit) => f.write_str(lit), + ast::FStringPartRef::FString(ast::FString { elements, .. }) => { Unparser::new(f, self.source).unparse_fstring_body(elements) } }) diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 534173d2c11..71c2dfad9a1 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -63,7 +63,21 @@ impl CompileError { let raw_location = error.location; let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) { Some(diagnostic) => diagnostic, - None => default_parse_diagnostic(error, source_file), + None => { + let is_unclosed_bracket = matches!( + error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::UnclosedBracket { + incomplete: true, + .. + }) + ); + let is_unclosed_string = + unclosed_string_is_continuable(&error, source_file.source_text(), mode); + let mut diagnostic = default_parse_diagnostic(error, source_file); + diagnostic.is_unclosed_bracket = is_unclosed_bracket; + diagnostic.is_unclosed_string = is_unclosed_string; + diagnostic + } }; Self::Parse(ParseError { @@ -89,8 +103,8 @@ impl CompileError { location, end_location, source_path: source_file.name().to_owned(), - is_unclosed_bracket: diagnostic.is_unclosed_bracket, - is_unclosed_string: diagnostic.is_unclosed_string, + is_unclosed_bracket: false, + is_unclosed_string: false, }) } @@ -182,22 +196,6 @@ impl NormalizedParseDiagnostic { } } - fn other(source_file: &SourceFile, diagnostic: CpythonDiagnostic) -> Self { - let (location, end_location) = source_locations( - source_file, - diagnostic.range.start(), - diagnostic.range.end(), - ); - let mut diagnostic_out = Self::new( - parser::ParseErrorType::OtherError(diagnostic.message), - location, - end_location, - ); - diagnostic_out.is_unclosed_string = diagnostic.is_unclosed_string; - diagnostic_out.is_unclosed_bracket = diagnostic.is_unclosed_bracket; - diagnostic_out - } - const fn with_unclosed_bracket(mut self, is_unclosed_bracket: bool) -> Self { self.is_unclosed_bracket = is_unclosed_bracket; self @@ -212,8 +210,6 @@ impl NormalizedParseDiagnostic { struct CpythonDiagnostic { message: String, range: ruff_text_size::TextRange, - is_unclosed_string: bool, - is_unclosed_bracket: bool, } impl CpythonDiagnostic { @@ -228,66 +224,8 @@ impl CpythonDiagnostic { TextSize::new(start as u32), TextSize::new(end as u32), ), - is_unclosed_string: false, - is_unclosed_bracket: false, } } - - const fn with_unclosed_string(mut self) -> Self { - self.is_unclosed_string = true; - self - } - - const fn with_unclosed_bracket(mut self) -> Self { - self.is_unclosed_bracket = true; - self - } -} - -/// Lexer-class failures outrank print hints. Decode and f-string diagnostics -/// compete with lexer failures by offset, and an unterminated quote is only -/// a fallback when no decode/f-string diagnostic exists. Print is considered -/// against the final winner: a later 0x still beats print, but an f-string -/// that replaced that 0x must not hide an earlier print. A decode diagnostic -/// also suppresses an EOF unclosed opener. -#[derive(Clone, Copy, PartialEq, Eq)] -enum OverrideClass { - Lexer, - Decode, - Print, -} - -struct RankedOverride { - diagnostic: CpythonDiagnostic, - unclosed_bracket: bool, - class: OverrideClass, -} - -fn consider_override( - best: &mut Option, - diagnostic: CpythonDiagnostic, - class: OverrideClass, -) { - let unclosed_bracket = diagnostic.is_unclosed_bracket; - consider_ranked(best, diagnostic, unclosed_bracket, class); -} - -fn consider_ranked( - best: &mut Option, - diagnostic: CpythonDiagnostic, - unclosed_bracket: bool, - class: OverrideClass, -) { - if best - .as_ref() - .is_none_or(|current| diagnostic.range.start() < current.diagnostic.range.start()) - { - *best = Some(RankedOverride { - diagnostic, - unclosed_bracket, - class, - }); - } } fn cpython_parse_diagnostic_override( @@ -297,109 +235,42 @@ fn cpython_parse_diagnostic_override( ) -> Option { let source_text = source_file.source_text(); - macro_rules! source_error { - ($expr:expr) => { - if let Some(error) = $expr { - return Some(NormalizedParseDiagnostic::other(source_file, error)); - } - }; - } - - let mut earliest: Option = None; - if let Some(diagnostic) = invalid_number_literal_error(source_text) { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - if let Some(diagnostic) = incompatible_string_prefix_error(source_text) { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - if let Some(diagnostic) = non_printable_character_error(source_text) { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - let bracket = bracket_syntax_error(source_text); - if let Some(bracket) = bracket.as_ref() { - // Unclosed openers are reported at the opener and only become errors - // at EOF. A later token-time diagnostic (invalid number, prefix, …) - // must keep winning. Mismatched closers stay in the lexer-class - // positional ranking. - if !bracket.unclosed { - consider_ranked( - &mut earliest, - bracket.diagnostic.clone(), - false, - OverrideClass::Lexer, - ); - } - } - let mut saw_decode = false; - if let Some(diagnostic) = malformed_unicode_n_escape_error(source_text) { - saw_decode = true; - consider_override(&mut earliest, diagnostic, OverrideClass::Decode); - } - if let Some(diagnostic) = invalid_interpolated_string_error(source_text) { - saw_decode = true; - consider_override(&mut earliest, diagnostic, OverrideClass::Decode); - } - if let Some(diagnostic) = mixed_tstring_literal_error(error, source_text) { - saw_decode = true; - consider_override(&mut earliest, diagnostic, OverrideClass::Decode); - } - // A later ordinary unterminated quote is only a fallback. Format-spec - // newlines and empty fields are decode diagnostics and must keep winning. - let line_continuation = matches!( + let tokenizer_error = matches!( &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) + parser::ParseErrorType::Lexical(lexical) if lexical.is_tokenizer_error() ); - if !saw_decode - && !line_continuation - && let Some(diagnostic) = unterminated_string_error(source_text, mode) - { - consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); - } - // Indentation errors outrank a print/exec missing-parentheses rewrite. - let indent_error = matches!( - &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) - | parser::ParseErrorType::UnexpectedIndentation - ) || expected_indented_block_error(error, source_text).is_some(); - if !indent_error - && earliest - .as_ref() - .is_none_or(|current| current.class != OverrideClass::Lexer) - && let Some(diagnostic) = invalid_legacy_statement_error(source_text) - { - consider_override(&mut earliest, diagnostic, OverrideClass::Print); - } - if !saw_decode - && earliest - .as_ref() - .is_none_or(|current| current.class == OverrideClass::Print) - && let Some(bracket) = bracket.filter(|bracket| bracket.unclosed) - { - consider_ranked( - &mut earliest, - bracket.diagnostic, - true, - OverrideClass::Lexer, - ); - } - if let Some(override_diag) = earliest { - return Some( - NormalizedParseDiagnostic::other(source_file, override_diag.diagnostic) - .with_unclosed_bracket(override_diag.unclosed_bracket), - ); - } - - if matches!( + let line_continuation = matches!( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) - ) { + ); + if line_continuation { // exec input gets an implicit trailing newline, so a final `\` is a // continuation that then hits EOF (`E_EOF`). single/eval see `\` at // EOF as `E_LINECONT` instead. - let terminal_backslash = source_text.len().checked_sub(1); - if matches!(mode, Mode::Exec) - && terminal_backslash == Some(error.location.start().to_usize()) - { + if matches!(mode, Mode::Exec) && error.location.start().to_usize() == source_text.len() { + let source_with_newline = format!("{source_text}\n"); + let parsed = parser::parse_unchecked( + &source_with_newline, + parser::ParseOptions::from(parser::Mode::Module), + ); + if let Some(unclosed) = parsed.errors().first().filter(|first| { + matches!( + first.error, + parser::ParseErrorType::Lexical( + parser::LexicalErrorType::UnclosedBracket { .. } + ) + ) + }) { + let mut diagnostic = default_parse_diagnostic(unclosed.clone(), source_file); + diagnostic.is_unclosed_bracket = matches!( + unclosed.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::UnclosedBracket { + incomplete: true, + .. + }) + ); + return Some(diagnostic); + } let loc = source_line_end_location(source_file, error.location.start()); return Some(NormalizedParseDiagnostic::new( parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()), @@ -407,126 +278,22 @@ fn cpython_parse_diagnostic_override( loc, )); } - let loc = source_location(source_file, error.location.start() + TextSize::from(1)); - return Some(NormalizedParseDiagnostic::new( - parser::ParseErrorType::OtherError( - "unexpected character after line continuation character".to_owned(), - ), - loc, - loc, - )); + return None; } - source_error!(unterminated_string_error(source_text, mode)); - source_error!(expected_indented_block_error(error, source_text)); - if matches!( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::Eof) ) { return Some(eof_parse_diagnostic(error, source_file)); } - - source_error!(invalid_type_param_error(source_text)); - source_error!(invalid_comprehension_error(source_text)); - source_error!(invalid_parameter_star_annotation_error(source_text)); - source_error!(invalid_parameter_list_error(source_text)); - source_error!(invalid_call_argument_error(source_text)); - - if is_missing_comma_between_literals(error) { - let (loc, end_loc) = adjusted_error_locations(source_file, error.location); - let msg = "invalid syntax. Perhaps you forgot a comma?".into(); - return Some(NormalizedParseDiagnostic::new( - parser::ParseErrorType::OtherError(msg), - loc, - end_loc, - )); - } - - source_error!(invalid_dict_error(source_text)); - if !matches!(mode, Mode::Eval) { - source_error!(invalid_collection_assignment_error(source_text)); - } - source_error!(invalid_group_error(source_text)); - source_error!(invalid_def_type_params_error(source_text)); - source_error!(invalid_expression_error(source_text)); - source_error!(invalid_named_expression_error(source_text)); - // Assignment and other statements are not expressions. Eval keeps the - // generic parse error rather than a statement-target rewrite. - if !matches!(mode, Mode::Eval) { - source_error!(invalid_plain_assignment_error(source_text)); - source_error!(expression_assignment_error(source_text)); - source_error!(invalid_annotation_target_error(source_text)); - source_error!(invalid_assignment_target_error(source_text)); - source_error!(invalid_condition_assignment_error( - source_text, - error.location.start().to_usize() - )); - source_error!(invalid_augassign_target_error(source_text)); - source_error!(invalid_for_target_error(source_text)); - source_error!(invalid_with_target_error(source_text)); - source_error!(invalid_delete_target_error(source_text)); - source_error!(invalid_standalone_except_error(source_text)); - source_error!(invalid_import_statement_error(source_text)); - source_error!(invalid_import_target_error(source_text)); - source_error!(invalid_except_as_target_error(source_text)); - source_error!(invalid_match_mapping_rest_wildcard_error(source_text)); - source_error!(invalid_match_as_target_error(source_text)); - source_error!(invalid_for_if_clause_error(source_text)); - source_error!(invalid_if_expression_statement_error(source_text)); - source_error!(invalid_else_elif_error(source_text)); - source_error!(mixed_except_handlers_error(source_text)); - } - - if matches!( - &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::InvalidByteLiteral) - ) && let Some((start, end)) = - bytes_literal_span(source_text, error.location.start().to_usize()) + if tokenizer_error + || matches!( + &error.error, + parser::ParseErrorType::ExpectedIndentedBlock { .. } + ) { - let (loc, end_loc) = source_locations( - source_file, - TextSize::new(start as u32), - TextSize::new(end as u32), - ); - return Some(NormalizedParseDiagnostic::new( - error.error.clone(), - loc, - end_loc, - )); - } - - if matches!( - &error.error, - parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) - ) { - let end_loc = source_line_end_location(source_file, error.location.start()); - return Some(NormalizedParseDiagnostic::new( - error.error.clone(), - end_loc, - end_loc, - )); - } - - if matches!( - &error.error, - parser::ParseErrorType::InvalidAssignmentTarget - ) { - return Some(invalid_assignment_target_diagnostic(error, source_file)); - } - - if matches!( - &error.error, - parser::ParseErrorType::InvalidNamedAssignmentTarget - ) { - let (loc, end_loc) = adjusted_error_locations(source_file, error.location); - let target = source_file.source_text().slice(error.location); - let msg = format!("cannot use assignment expressions with {target}"); - return Some(NormalizedParseDiagnostic::new( - parser::ParseErrorType::OtherError(msg), - loc, - end_loc, - )); + return None; } // CPython's PEG parser collapses a bare "expected an expression" failure @@ -569,41 +336,52 @@ fn eof_parse_diagnostic( } } -fn invalid_assignment_target_diagnostic( - error: &parser::ParseError, - source_file: &SourceFile, -) -> NormalizedParseDiagnostic { - let (loc, end_loc) = adjusted_error_locations(source_file, error.location); - let expr_str = source_file.source_text().slice(error.location); - - let msg = parser::parse_expression(expr_str).map_or_else( - |_| match expr_str { - "yield" => "assignment to yield expression not possible".into(), - _ => format!("cannot assign to {expr_str}"), - }, - |parsed| match *parsed.syntax().body { - ast::Expr::Call(_) => "cannot assign to function call".into(), - ast::Expr::BinOp(_) => "cannot assign to expression".into(), - ast::Expr::If(_) => "cannot assign to conditional expression".into(), - ast::Expr::Generator(_) => "cannot assign to generator expression".into(), - ast::Expr::Yield(_) | ast::Expr::YieldFrom(_) => { - "cannot assign to yield expression here. Maybe you meant '==' instead of '='?" - .into() - } - ast::Expr::FString(_) => "invalid syntax".into(), - ast::Expr::StringLiteral(_) - | ast::Expr::BytesLiteral(_) - | ast::Expr::NumberLiteral(_) => { - "cannot assign to literal here. Maybe you meant '==' instead of '='?".into() - } - ast::Expr::EllipsisLiteral(_) => { - "cannot assign to ellipsis here. Maybe you meant '==' instead of '='?".into() - } - _ => format!("cannot assign to {expr_str}"), - }, - ); +/// Returns whether more input could still terminate the string that `error` reports as +/// unterminated. +/// +/// A string that reached the end of the source can continue, except a single-quoted f/t-string, +/// a single-quoted string in exec input whose implicit final newline ends it, and a string after +/// an assignment in eval input. +fn unclosed_string_is_continuable(error: &parser::ParseError, source: &str, mode: Mode) -> bool { + use parser::{InterpolatedStringErrorType, LexicalErrorType}; + + let parser::ParseErrorType::Lexical(lexical) = &error.error else { + return false; + }; + let start = error.location.start().to_usize(); + let at_eof = match lexical { + LexicalErrorType::UnclosedStringError { + triple_quoted: true, + .. + } + | LexicalErrorType::FStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, + ) + | LexicalErrorType::TStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, + ) => true, + LexicalErrorType::UnclosedStringError { .. } => { + // Exec mode appends a newline, which only a final `\` can escape. + let rest = &source[start..]; + !has_unescaped_newline(rest) && (!matches!(mode, Mode::Exec) || rest.ends_with('\\')) + } + _ => false, + }; + at_eof && !(matches!(mode, Mode::Eval) && eval_has_assignment_before(source.as_bytes(), start)) +} - NormalizedParseDiagnostic::new(parser::ParseErrorType::OtherError(msg), loc, end_loc) +fn has_unescaped_newline(text: &str) -> bool { + let mut chars = text.chars(); + while let Some(c) = chars.next() { + match c { + '\\' => { + chars.next(); + } + '\n' | '\r' => return true, + _ => {} + } + } + false } fn default_parse_diagnostic( @@ -646,14 +424,6 @@ fn source_line_end_location(source_file: &SourceFile, offset: TextSize) -> Sourc } } -fn is_missing_comma_between_literals(error: &parser::ParseError) -> bool { - matches!( - &error.error, - parser::ParseErrorType::ExpectedToken { expected, found } - if matches!((expected, found), (TokenKind::Comma, TokenKind::Int)) - ) -} - fn is_ascii_identifier_char(byte: u8) -> bool { byte == b'_' || byte.is_ascii_alphanumeric() } @@ -675,5895 +445,188 @@ fn identifier_continue_before(bytes: &[u8], index: usize) -> bool { .is_some_and(|ch| ch == '_' || ch.is_alphanumeric()) } -fn numeric_keyword_suffix(rest: &[u8]) -> bool { - rest.starts_with(b"and") - || rest.starts_with(b"else") - || rest.starts_with(b"for") - || rest.starts_with(b"if") - || rest.starts_with(b"in") - || rest.starts_with(b"is") - || rest.starts_with(b"or") - || rest.starts_with(b"not") -} - -fn consume_decimal_digits(bytes: &[u8], mut index: usize) -> usize { - while index < bytes.len() { - match bytes[index] { - b'0'..=b'9' => index += 1, - b'_' if bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit()) => - { - index += 2; - } - _ => break, - } +fn quoted_string_is_closed(bytes: &[u8], start: usize) -> bool { + let quote = bytes[start]; + let triple = bytes.get(start + 1) == Some("e) && bytes.get(start + 2) == Some("e); + let end = skip_quoted_string(bytes, start); + if triple { + end >= start + 6 + && bytes[end - 3] == quote + && bytes[end - 2] == quote + && bytes[end - 1] == quote + } else { + end > start + 1 && bytes[end - 1] == quote } - index } -fn consume_radix_digits(bytes: &[u8], mut index: usize, is_digit: impl Fn(u8) -> bool) -> usize { +fn skip_quoted_string(bytes: &[u8], mut index: usize) -> usize { + let quote = bytes[index]; + let triple = bytes.get(index + 1) == Some("e) && bytes.get(index + 2) == Some("e); + let quote_len = if triple { 3 } else { 1 }; + index += quote_len; while index < bytes.len() { - if is_digit(bytes[index]) { - index += 1; - } else if bytes.get(index) == Some(&b'_') - && bytes.get(index + 1).is_some_and(|&byte| is_digit(byte)) + if bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + } else if triple + && bytes.get(index) == Some("e) + && bytes.get(index + 1) == Some("e) + && bytes.get(index + 2) == Some("e) { - index += 2; + return index + 3; + } else if !triple && bytes[index] == quote { + return index + 1; } else { - break; + index += 1; } } index } -fn invalid_radix_literal_error( - bytes: &[u8], - start: usize, - kind: &'static str, - is_digit: impl Fn(u8) -> bool, -) -> Option<(String, usize)> { - let mut index = start + 2; - let mut has_digit = false; - loop { - let Some(&byte) = bytes.get(index) else { - return if has_digit { - None - } else { - Some((format!("invalid {kind} literal"), start + 1)) - }; - }; - if byte == b'_' { - let Some(&next) = bytes.get(index + 1) else { - return Some((format!("invalid {kind} literal"), index)); - }; - if is_digit(next) { - has_digit = true; - index += 2; - continue; +fn starts_identifier(bytes: &[u8], index: usize, word: &[u8]) -> bool { + bytes.get(index..index + word.len()) == Some(word) + && index + .checked_sub(1) + .and_then(|before| bytes.get(before)) + .is_none_or(|byte| !is_ascii_identifier_char(*byte)) + && bytes + .get(index + word.len()) + .is_none_or(|byte| !is_ascii_identifier_char(*byte)) +} + +fn eval_has_assignment_before(bytes: &[u8], end: usize) -> bool { + let mut index = 0; + let mut level = 0usize; + let mut in_lambda_params = false; + while index < end { + match bytes[index] { + b'#' => { + while index < end && bytes[index] != b'\n' { + index += 1; + } } - if next.is_ascii_digit() && matches!(kind, "binary" | "octal") { - return Some(( - format!("invalid digit '{}' in {kind} literal", next as char), - index + 1, - )); + b'\'' | b'"' => index = skip_quoted_string(bytes, index), + b'(' | b'[' | b'{' => { + level += 1; + index += 1; } - return Some((format!("invalid {kind} literal"), index)); - } - if is_digit(byte) { - has_digit = true; - index += 1; - continue; - } - if byte.is_ascii_digit() && matches!(kind, "binary" | "octal") { - return Some(( - format!("invalid digit '{}' in {kind} literal", byte as char), - index, - )); - } - if has_digit { - return None; + b')' | b']' | b'}' => { + level = level.saturating_sub(1); + index += 1; + } + b':' if level == 0 => { + in_lambda_params = false; + index += 1; + } + b'=' if level == 0 + && !in_lambda_params + && bytes.get(index + 1) != Some(&b'=') + && !matches!( + bytes.get(index.saturating_sub(1)), + Some(b':' | b'!' | b'<' | b'>') + ) => + { + return true; + } + _ if level == 0 && starts_identifier(bytes, index, b"lambda") => { + in_lambda_params = true; + index += b"lambda".len(); + } + _ => index += 1, } - return Some((format!("invalid {kind} literal"), start + 1)); } + false } -fn decimal_tail_error(bytes: &[u8], mut index: usize) -> Option { - loop { - while bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - index += 1; - } - if bytes.get(index) != Some(&b'_') { - return None; - } - let underscore = index; - index += 1; - if !bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - return Some(underscore); - } - } -} - -fn decimal_tail_end(bytes: &[u8], mut index: usize) -> usize { - loop { - while bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - index += 1; - } - if bytes.get(index) == Some(&b'_') - && bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit()) - { - index += 2; +fn interpolated_string_prefix(bytes: &[u8], quote: usize) -> Option<&'static str> { + let prev = quote.checked_sub(1).and_then(|index| bytes.get(index))?; + let lower_prev = prev.to_ascii_lowercase(); + let (prefix_start, marker) = if matches!(lower_prev, b'f' | b't') { + if quote >= 2 && bytes[quote - 2].eq_ignore_ascii_case(&b'r') { + (quote - 2, lower_prev) } else { - return index; + (quote - 1, lower_prev) } - } -} - -fn invalid_decimal_literal_error(bytes: &[u8], start: usize) -> Option<(String, usize)> { - if bytes.get(start) == Some(&b'.') { + } else if lower_prev == b'r' + && quote >= 2 + && matches!(bytes[quote - 2].to_ascii_lowercase(), b'f' | b't') + { + (quote - 2, bytes[quote - 2].to_ascii_lowercase()) + } else { return None; - } - let message = "invalid decimal literal".to_owned(); - if let Some(offset) = decimal_tail_error(bytes, start) { - return Some((message, offset)); - } - - let mut index = decimal_tail_end(bytes, start); - if bytes.get(index) == Some(&b'.') { - if bytes.get(index + 1) == Some(&b'_') { - return Some((message, index)); - } - if let Some(offset) = decimal_tail_error(bytes, index + 1) { - return Some((message, offset)); - } - index = decimal_tail_end(bytes, index + 1); - } - if matches!(bytes.get(index), Some(b'e' | b'E')) { - let exponent = index; - index += 1; - let sign = if matches!(bytes.get(index), Some(b'+' | b'-')) { - let sign = index; - index += 1; - Some(sign) - } else { - None - }; - if !bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - // Without a sign the exponent letter is put back, so the position - // is the digit before it rather than the letter. - return Some((message, sign.unwrap_or_else(|| exponent.saturating_sub(1)))); - } - if let Some(offset) = decimal_tail_error(bytes, index) { - return Some((message, offset)); - } - } - None -} + }; -fn leading_zero_decimal_literal_error(bytes: &[u8], start: usize) -> Option { - if bytes.get(start) != Some(&b'0') { + if prefix_start > 0 && identifier_continue_before(bytes, prefix_start) { return None; } - let mut index = start; - loop { - match bytes.get(index) { - Some(b'0') => index += 1, - Some(b'_') - if bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit()) => - { - index += 1; - } - _ => break, - } - } - if bytes.get(index).is_some_and(|byte| byte.is_ascii_digit()) { - let after_digits = decimal_tail_end(bytes, index); - if !matches!( - bytes.get(after_digits), - Some(b'.' | b'e' | b'E' | b'j' | b'J') - ) { - return Some(CpythonDiagnostic::new( - "leading zeros in decimal integer literals are not permitted; use an 0o prefix for octal integers" - .to_owned(), - start, - index, - )); - } - } - None -} - -fn invalid_numeric_literal_error(bytes: &[u8], start: usize) -> Option { - if bytes.get(start) == Some(&b'0') { - let radix = match bytes.get(start + 1) { - Some(b'x' | b'X') => invalid_radix_literal_error(bytes, start, "hexadecimal", |byte| { - byte.is_ascii_hexdigit() - }), - Some(b'o' | b'O') => invalid_radix_literal_error(bytes, start, "octal", |byte| { - matches!(byte, b'0'..=b'7') - }), - Some(b'b' | b'B') => invalid_radix_literal_error(bytes, start, "binary", |byte| { - matches!(byte, b'0' | b'1') - }), - _ => None, - }; - if let Some(radix) = radix { - return Some(point_span(radix)); - } - if let Some(err) = leading_zero_decimal_literal_error(bytes, start) { - return Some(err); - } - } - invalid_decimal_literal_error(bytes, start).map(point_span) -} -fn consume_exponent(bytes: &[u8], index: usize) -> usize { - if !matches!(bytes.get(index), Some(b'e' | b'E')) { - return index; - } - let mut cursor = index + 1; - if matches!(bytes.get(cursor), Some(b'+' | b'-')) { - cursor += 1; - } - if bytes.get(cursor).is_some_and(|byte| byte.is_ascii_digit()) { - consume_decimal_digits(bytes, cursor) + Some(if marker == b'f' { + "f-string" } else { - index - } + "t-string" + }) } -fn number_literal_end(bytes: &[u8], start: usize) -> Option<(&'static str, usize)> { - if bytes.get(start) == Some(&b'.') { - if !bytes - .get(start + 1) - .is_some_and(|byte| byte.is_ascii_digit()) - { - return None; - } - let mut index = consume_decimal_digits(bytes, start + 1); - index = consume_exponent(bytes, index); - if matches!(bytes.get(index), Some(b'j' | b'J')) { - return Some(("imaginary", index + 1)); - } - return Some(("decimal", index)); - } - - if !bytes.get(start).is_some_and(|byte| byte.is_ascii_digit()) { - return None; - } - - if bytes.get(start) == Some(&b'0') { - match bytes.get(start + 1) { - Some(b'x' | b'X') => { - let end = consume_radix_digits(bytes, start + 2, |byte| byte.is_ascii_hexdigit()); - return Some(("hexadecimal", end)); - } - Some(b'o' | b'O') => { - let end = - consume_radix_digits(bytes, start + 2, |byte| matches!(byte, b'0'..=b'7')); - return Some(("octal", end)); - } - Some(b'b' | b'B') => { - let end = - consume_radix_digits(bytes, start + 2, |byte| matches!(byte, b'0' | b'1')); - return Some(("binary", end)); - } - _ => {} - } - } - - let mut index = consume_decimal_digits(bytes, start); - if bytes.get(index) == Some(&b'.') { - index = consume_decimal_digits(bytes, index + 1); - } - index = consume_exponent(bytes, index); - if matches!(bytes.get(index), Some(b'j' | b'J')) { - return Some(("imaginary", index + 1)); - } - Some(("decimal", index)) -} +const MAXFSTRINGLEVEL: usize = 150; -fn quoted_string_is_closed(bytes: &[u8], start: usize) -> bool { - let quote = bytes[start]; - let triple = bytes.get(start + 1) == Some("e) && bytes.get(start + 2) == Some("e); - let end = skip_quoted_string(bytes, start); - if triple { - end >= start + 6 - && bytes[end - 3] == quote - && bytes[end - 2] == quote - && bytes[end - 1] == quote +fn skip_string_token(bytes: &[u8], quote_index: usize) -> usize { + if interpolated_string_prefix(bytes, quote_index).is_some() { + interpolated_string_end(bytes, quote_index).unwrap_or(bytes.len()) } else { - end > start + 1 && bytes[end - 1] == quote - } -} - -fn bytes_literal_span(source: &str, error_at: usize) -> Option<(usize, usize)> { - let bytes = source.as_bytes(); - if error_at > bytes.len() || bytes.is_empty() { - return None; - } - let mut quote_idx = error_at.min(bytes.len().saturating_sub(1)); - loop { - if matches!(bytes[quote_idx], b'\'' | b'"') { - break; - } - if quote_idx == 0 { - return None; - } - quote_idx -= 1; - } - let mut start = quote_idx; - while start > 0 && matches!(bytes[start - 1], b'b' | b'B' | b'r' | b'R') { - start -= 1; - } - if !matches!(bytes.get(start), Some(b'b' | b'B' | b'r' | b'R')) { - return None; + skip_quoted_string(bytes, quote_index) } - let end = skip_quoted_string(bytes, quote_idx); - Some((start, end)) } -fn skip_quoted_string(bytes: &[u8], mut index: usize) -> usize { - let quote = bytes[index]; - let triple = bytes.get(index + 1) == Some("e) && bytes.get(index + 2) == Some("e); +fn interpolated_string_end(bytes: &[u8], quote_index: usize) -> Option { + let quote = bytes[quote_index]; + let triple = + bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e); let quote_len = if triple { 3 } else { 1 }; - index += quote_len; - while index < bytes.len() { - if bytes[index] == b'\\' { - index = (index + 2).min(bytes.len()); - } else if triple - && bytes.get(index) == Some("e) - && bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - return index + 3; - } else if !triple && bytes[index] == quote { - return index + 1; - } else { - index += 1; - } - } - index -} - -// An error the tokenizer reports at a single position spans nothing. -fn point_span((message, offset): (String, usize)) -> CpythonDiagnostic { - CpythonDiagnostic::new(message, offset, offset) -} - -fn invalid_number_literal_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; + let mut index = quote_index + quote_len; + let mut brace_depth = 0usize; while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - byte if byte >= 0x80 || byte == b'_' || byte.is_ascii_alphabetic() => { - index += 1; - while index < bytes.len() - && (bytes[index] >= 0x80 || is_ascii_identifier_char(bytes[index])) - { - index += 1; - } - } - b'.' | b'0'..=b'9' => { - if let Some(err) = invalid_numeric_literal_error(bytes, index) { - return Some(err); - } - let Some((kind, end)) = number_literal_end(bytes, index) else { - index += 1; - continue; - }; - if end > index { - if source[end..].starts_with('⁄') { - return Some(CpythonDiagnostic::new( - "invalid character '⁄' (U+2044)".to_owned(), - end, - end, - )); - } - if bytes - .get(end) - .is_some_and(|byte| *byte < 128 && is_ascii_identifier_char(*byte)) - && !numeric_keyword_suffix(&bytes[end..]) - { - let offset = end.saturating_sub(1); - return Some(CpythonDiagnostic::new( - format!("invalid {kind} literal"), - offset, - offset, - )); - } - } - index = end.max(index + 1); - } - _ => index += 1, - } - } - None -} - -fn cpython_indented_block_clause(message: &str) -> Option<&'static str> { - let clause = message.strip_prefix("Expected an indented block after ")?; - Some(match clause { - "`if` statement" => "'if' statement", - "`elif` clause" => "'elif' statement", - "`else` clause" => "'else' statement", - "`for` statement" => "'for' statement", - "`with` statement" => "'with' statement", - "`while` statement" => "'while' statement", - "`try` statement" => "'try' statement", - "`except` clause" => "'except' statement", - "`finally` clause" => "'finally' statement", - "`match` statement" => "'match' statement", - "`case` block" => "'case' statement", - "`class` definition" => "class definition", - "function definition" => "function definition", - _ => return None, - }) -} - -fn previous_non_empty_line_number(source: &str, offset: usize) -> Option { - let bytes = source.as_bytes(); - let mut index = offset.min(bytes.len()); - while index > 0 { - let line_end = index; - while index > 0 && bytes[index - 1] != b'\n' { - index -= 1; - } - let line_start = index; - let content_start = skip_horizontal_whitespace(bytes, line_start); - let mut content_end = line_end; - while content_end > content_start - && matches!( - bytes.get(content_end - 1), - Some(b' ' | b'\t' | b'\r' | b'\x0c') - ) - { - content_end -= 1; - } - if content_start < content_end { - return Some( - source[..line_start] - .bytes() - .filter(|byte| *byte == b'\n') - .count() - + 1, - ); - } - index = line_start.saturating_sub(1); - } - None -} - -fn expected_indented_block_error( - error: &parser::ParseError, - source: &str, -) -> Option { - let parser::ParseErrorType::OtherError(message) = &error.error else { - return None; - }; - let mut clause = cpython_indented_block_clause(message)?; - let start = error.location.start().to_usize(); - let end = error.location.end().to_usize(); - let line = previous_non_empty_line_number(source, start)?; - if clause == "'except' statement" - && let Some(previous_line) = previous_non_empty_line(source, start) - && matches!( - previous_line.trim_start(), - line if line.starts_with("except*") || line.starts_with("except *") - ) - { - clause = "'except*' statement"; - } - Some(CpythonDiagnostic::new( - format!("expected an indented block after {clause} on line {line}"), - start, - end, - )) -} - -fn previous_non_empty_line(source: &str, offset: usize) -> Option<&str> { - let bytes = source.as_bytes(); - let mut index = offset.min(bytes.len()); - while index > 0 { - let line_end = index; - while index > 0 && bytes[index - 1] != b'\n' { - index -= 1; - } - let line_start = index; - let mut content_start = line_start; - while content_start < line_end - && matches!(bytes[content_start], b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - { - content_start += 1; - } - let mut content_end = line_end; - while content_end > content_start - && matches!(bytes[content_end - 1], b' ' | b'\t' | b'\r' | b'\x0c') - { - content_end -= 1; - } - if content_start < content_end { - return source.get(line_start..line_end); - } - index = line_start.saturating_sub(1); - } - None -} - -fn starts_identifier(bytes: &[u8], index: usize, word: &[u8]) -> bool { - bytes.get(index..index + word.len()) == Some(word) - && index - .checked_sub(1) - .and_then(|before| bytes.get(before)) - .is_none_or(|byte| !is_ascii_identifier_char(*byte)) - && bytes - .get(index + word.len()) - .is_none_or(|byte| !is_ascii_identifier_char(*byte)) -} - -fn is_plain_assignment_operator(bytes: &[u8], index: usize) -> bool { - bytes.get(index) == Some(&b'=') - && bytes.get(index + 1) != Some(&b'=') - && !matches!( - index.checked_sub(1).and_then(|before| bytes.get(before)), - Some(b'=' | b'!' | b'<' | b'>' | b':') - ) -} - -fn is_simple_keyword_name(bytes: &[u8], mut start: usize, mut end: usize) -> bool { - while matches!( - bytes.get(start), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) { - start += 1; - } - while end > start - && matches!( - bytes.get(end - 1), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) - { - end -= 1; - } - let Some(&first) = bytes.get(start) else { - return false; - }; - if !(first == b'_' || first.is_ascii_alphabetic() || first >= 0x80) { - return false; - } - let mut index = start + 1; - while index < end { - if bytes[index] < 0x80 && !is_ascii_identifier_char(bytes[index]) { - return false; - } - index += 1; - } - true -} - -fn is_function_parameter_list(bytes: &[u8], paren: usize) -> bool { - let mut cursor = paren; - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - if cursor > 0 && bytes.get(cursor - 1) == Some(&b']') { - let mut bracket = cursor; - let mut level = 0usize; - while bracket > 0 { - bracket -= 1; - match bytes[bracket] { - b']' => level += 1, - b'[' => { - level = level.saturating_sub(1); - if level == 0 { - cursor = bracket; - break; - } - } - _ => {} + if brace_depth == 0 { + if bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + continue; } - } - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - } - while cursor > 0 - && bytes - .get(cursor - 1) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - cursor -= 1; - } - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - cursor >= 3 - && starts_identifier(bytes, cursor - 3, b"def") - && cursor - .checked_sub(4) - .and_then(|before| bytes.get(before)) - .is_none_or(|byte| !is_ascii_identifier_char(*byte)) -} - -#[derive(Clone, Copy)] -enum ParameterListKind { - Function, - Lambda, -} - -fn matching_delimiter(bytes: &[u8], open: usize, close: u8) -> Option { - let mut index = open; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - byte if byte == close => { - level = level.saturating_sub(1); - if level == 0 { - return Some(index); - } - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn find_lambda_parameter_end(bytes: &[u8], mut index: usize) -> Option { - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn top_level_byte(bytes: &[u8], mut index: usize, end: usize, needle: u8) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - byte if level == 0 && byte == needle => return Some(index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn identifier_end(bytes: &[u8], mut index: usize, end: usize) -> usize { - if !bytes - .get(index) - .is_some_and(|byte| *byte >= 0x80 || *byte == b'_' || byte.is_ascii_alphabetic()) - { - return index; - } - index += 1; - while index < end - && bytes - .get(index) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - index += 1; - } - index -} - -fn expression_slice_is_tuple(source: &str, start: usize, end: usize) -> bool { - let bytes = source.as_bytes(); - let (start, end) = trim_target_range(bytes, start, end); - if start >= end { - return false; - } - let Ok(parsed) = parser::parse(&source[start..end], parser::Mode::Expression.into()) else { - return false; - }; - matches!(parsed.into_syntax(), ast::Mod::Expression(expression) if matches!(*expression.body, ast::Expr::Tuple(_))) -} - -fn type_param_list_open(bytes: &[u8], open: usize) -> bool { - let mut cursor = open; - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - while cursor > 0 - && bytes - .get(cursor - 1) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - cursor -= 1; - } - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - (cursor >= 3 && starts_identifier(bytes, cursor - 3, b"def")) - || (cursor >= 5 && starts_identifier(bytes, cursor - 5, b"class")) - || (cursor >= 4 && starts_identifier(bytes, cursor - 4, b"type")) -} - -fn invalid_type_param_item_error( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (start, end) = trim_target_range(bytes, start, end); - if start >= end || bytes.get(start) != Some(&b'*') { - return None; - } - let is_param_spec = bytes.get(start + 1) == Some(&b'*'); - let name_start = start + if is_param_spec { 2 } else { 1 }; - let name_end = identifier_end(bytes, name_start, end); - if name_start == name_end { - return None; - } - let colon = next_non_horizontal_whitespace(bytes, name_end); - if colon >= end || bytes.get(colon) != Some(&b':') { - return None; - } - let has_constraints = expression_slice_is_tuple(source, colon + 1, end); - let message = match (is_param_spec, has_constraints) { - (false, false) => "cannot use bound with TypeVarTuple", - (false, true) => "cannot use constraints with TypeVarTuple", - (true, false) => "cannot use bound with ParamSpec", - (true, true) => "cannot use constraints with ParamSpec", - }; - Some(CpythonDiagnostic::new(message.to_owned(), colon, colon + 1)) -} - -fn invalid_type_param_list_error( - source: &str, - open: usize, - close: usize, -) -> Option { - let bytes = source.as_bytes(); - let mut item_start = open + 1; - let mut index = item_start; - let mut level = 0usize; - while index <= close { - if index == close || (level == 0 && bytes.get(index) == Some(&b',')) { - if let Some(error) = invalid_type_param_item_error(source, item_start, index) { - return Some(error); - } - item_start = index + 1; - index += 1; - continue; - } - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_type_param_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'[' if type_param_list_open(bytes, index) => { - let Some(close) = matching_delimiter(bytes, index, b']') else { - index += 1; - continue; - }; - if let Some(error) = invalid_type_param_list_error(source, index, close) { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_comprehension_in_slice( - bytes: &[u8], - open: usize, - close: usize, -) -> Option { - let for_index = find_keyword_at_level(bytes, open + 1, close, b"for")?; - let item_start = next_non_horizontal_whitespace(bytes, open + 1); - if item_start >= for_index { - return None; - } - if bytes.get(item_start..item_start + 2) == Some(b"**") && bytes.get(open) == Some(&b'{') { - return Some(CpythonDiagnostic::new( - "dict unpacking cannot be used in dict comprehension".to_owned(), - item_start, - item_start + 2, - )); - } - if bytes.get(item_start..item_start + 2) == Some(b"**") && bytes.get(open) == Some(&b'(') { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - for_index, - for_index + 3, - )); - } - if bytes.get(item_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "iterable unpacking cannot be used in comprehension".to_owned(), - item_start, - item_start + 1, - )); - } - if !matches!(bytes.get(open), Some(b'[' | b'{')) { - return None; - } - if top_level_colon(bytes, open + 1, for_index).is_none() - && let Some(comma) = top_level_byte(bytes, open + 1, for_index, b',') - { - let (start, _) = trim_target_range(bytes, open + 1, comma); - return Some(CpythonDiagnostic::new( - "did you forget parentheses around the comprehension target?".to_owned(), - start, - comma + 1, - )); - } - None -} - -fn invalid_comprehension_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - let close_byte = match bytes[index] { - b'(' => b')', - b'[' => b']', - _ => b'}', - }; - let Some(close) = matching_delimiter(bytes, index, close_byte) else { - index += 1; - continue; - }; - if let Some(error) = invalid_comprehension_in_slice(bytes, index, close) { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_group_in_slice(bytes: &[u8], open: usize, close: usize) -> Option { - let (item_start, item_end) = trim_target_range(bytes, open + 1, close); - if item_start >= item_end - || top_level_byte(bytes, item_start, item_end, b',').is_some() - || top_level_colon(bytes, item_start, item_end).is_some() - || find_keyword_at_level(bytes, item_start, item_end, b"for").is_some() - { - return None; - } - if bytes.get(item_start..item_start + 2) == Some(b"**") { - return Some(CpythonDiagnostic::new( - "cannot use double starred expression here".to_owned(), - item_start, - item_start + 2, - )); - } - if bytes.get(item_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot use starred expression here".to_owned(), - item_start, - item_start + 1, - )); - } - None -} - -fn invalid_group_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' => { - let Some(close) = matching_delimiter(bytes, index, b')') else { - index += 1; - continue; - }; - if let Some(error) = invalid_group_in_slice(bytes, index, close) { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_parameter_star_annotation_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' => { - let Some(close) = matching_delimiter(bytes, index, b')') else { - index += 1; - continue; - }; - let mut param_start = index + 1; - while param_start < close { - let param_end = - find_byte_at_level(bytes, param_start, close, b',').unwrap_or(close); - if let Some(colon) = top_level_colon(bytes, param_start, param_end) { - let value_start = next_non_horizontal_whitespace(bytes, colon + 1); - if bytes.get(value_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - value_start, - value_start + 1, - )); - } - } - param_start = param_end.saturating_add(1); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_def_type_params_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"def") => { - let name_start = skip_horizontal_whitespace(bytes, index + 3); - let name_end = identifier_end(bytes, name_start, bytes.len()); - let bracket = skip_horizontal_whitespace(bytes, name_end); - if bytes.get(bracket) == Some(&b'[') { - let Some(close) = matching_delimiter(bytes, bracket, b']') else { - index = bracket + 1; - continue; - }; - let after_close = skip_horizontal_whitespace(bytes, close + 1); - if bytes.get(after_close) == Some(&b'(') - && type_param_list_is_malformed(bytes, bracket + 1, close) - { - return Some(CpythonDiagnostic::new( - "expected '('".to_owned(), - bracket, - bracket + 1, - )); - } - } - index = name_end.max(index + 3); - } - _ => index += 1, - } - } - None -} - -fn type_param_list_is_malformed(bytes: &[u8], start: usize, end: usize) -> bool { - let mut index = start; - let mut expect_item = true; - while index < end { - index = skip_horizontal_whitespace(bytes, index); - if index >= end { - break; - } - if bytes[index] == b',' { - if expect_item { - return true; - } - expect_item = true; - index += 1; - continue; - } - if !expect_item { - return true; - } - if bytes.get(index..index + 2) == Some(b"**") { - index += 2; - } else if bytes.get(index) == Some(&b'*') { - index += 1; - } - let item_start = skip_horizontal_whitespace(bytes, index); - let item_end = identifier_end(bytes, item_start, end); - if item_end == item_start { - return true; - } - index = item_end; - if bytes.get(skip_horizontal_whitespace(bytes, index)) == Some(&b':') { - index = skip_horizontal_whitespace(bytes, index) + 1; - while index < end && bytes[index] != b',' { - index = match bytes[index] { - b'\'' | b'"' => skip_quoted_string(bytes, index), - _ => index + 1, - }; - } - } - expect_item = false; - } - false -} - -fn invalid_parameter_list_slice_error( - source: &str, - start: usize, - end: usize, - kind: ParameterListKind, -) -> Option { - let bytes = source.as_bytes(); - let mut index = start; - let mut level = 0usize; - let mut default_seen = false; - let mut keyword_only = false; - let mut slash_seen = false; - let mut var_keyword_seen = false; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if level == 0 - && var_keyword_seen - && bytes.get(index).is_some_and(|byte| { - *byte >= 0x80 || *byte == b'_' || byte.is_ascii_alphabetic() - }) => - { - let name_end = identifier_end(bytes, index, end); - return Some(CpythonDiagnostic::new( - "arguments cannot follow var-keyword argument".to_owned(), - index, - name_end, - )); - } - _ if level == 0 - && !keyword_only - && bytes.get(index).is_some_and(|byte| { - *byte >= 0x80 || *byte == b'_' || byte.is_ascii_alphabetic() - }) => - { - let param_end = find_byte_at_level(bytes, index, end, b',') - .or_else(|| top_level_byte(bytes, index, end, b')')) - .or_else(|| { - matches!(kind, ParameterListKind::Lambda) - .then(|| top_level_byte(bytes, index, end, b':')) - .flatten() - }) - .unwrap_or(end); - let name_end = identifier_end(bytes, index, param_end); - if let Some(eq) = top_level_byte(bytes, index, param_end, b'=') { - let value_start = next_non_horizontal_whitespace(bytes, eq + 1); - if value_start >= param_end - || matches!(bytes.get(value_start), Some(b',' | b')' | b':')) - { - let message = if matches!(kind, ParameterListKind::Lambda) - && matches!(bytes.get(value_start), Some(b':')) - { - "invalid syntax" - } else { - "expected default value expression" - }; - return Some(CpythonDiagnostic::new(message.to_owned(), eq, eq + 1)); - } - default_seen = true; - } else if default_seen { - return Some(CpythonDiagnostic::new( - "parameter without a default follows parameter with a default".to_owned(), - index, - name_end, - )); - } - index = param_end; - } - b'(' if level == 0 => { - let close = matching_delimiter(bytes, index, b')') - .filter(|close| *close <= end) - .unwrap_or(index + 1); - let message = match kind { - ParameterListKind::Function => "Function parameters cannot be parenthesized", - ParameterListKind::Lambda => { - "Lambda expression parameters cannot be parenthesized" - } - }; - return Some(CpythonDiagnostic::new(message.to_owned(), index, close + 1)); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'/' if level == 0 => { - if var_keyword_seen { - return Some(CpythonDiagnostic::new( - "arguments cannot follow var-keyword argument".to_owned(), - index, - index + 1, - )); - } - if slash_seen { - return Some(CpythonDiagnostic::new( - "/ may appear only once".to_owned(), - index, - index + 1, - )); - } - slash_seen = true; - let next = next_non_horizontal_whitespace(bytes, index + 1); - if bytes.get(next) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "expected comma between / and *".to_owned(), - next, - next + 1, - )); - } - index += 1; - } - b'*' if level == 0 => { - if var_keyword_seen { - return Some(CpythonDiagnostic::new( - "arguments cannot follow var-keyword argument".to_owned(), - index, - index + 1, - )); - } - keyword_only = true; - let stars = usize::from(bytes.get(index + 1) == Some(&b'*')) + 1; - let name_start = next_non_horizontal_whitespace(bytes, index + stars); - for keyword in [b"True".as_slice(), b"False".as_slice(), b"None".as_slice()] { - if starts_identifier(bytes, name_start, keyword) { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - name_start, - name_start + keyword.len(), - )); - } - } - let param_end = find_byte_at_level(bytes, name_start, end, b',') - .or_else(|| top_level_byte(bytes, name_start, end, b')')) - .or_else(|| { - matches!(kind, ParameterListKind::Lambda) - .then(|| top_level_byte(bytes, name_start, end, b':')) - .flatten() - }) - .unwrap_or(end); - if stars == 1 && matches!(bytes.get(name_start), Some(b')' | b',' | b':')) { - return Some(CpythonDiagnostic::new( - "named arguments must follow bare *".to_owned(), - index, - index + 1, - )); - } - if stars == 1 && top_level_byte(bytes, name_start, param_end, b'=').is_some() { - return Some(CpythonDiagnostic::new( - "var-positional argument cannot have default value".to_owned(), - index, - index + 1, - )); - } - if stars == 2 && top_level_byte(bytes, name_start, param_end, b'=').is_some() { - return Some(CpythonDiagnostic::new( - "var-keyword argument cannot have default value".to_owned(), - index, - index + 2, - )); - } - if stars == 2 { - var_keyword_seen = true; - index = param_end; - continue; - } - index += stars; - } - b'=' if level == 0 => { - let value_start = next_non_horizontal_whitespace(bytes, index + 1); - if value_start >= end || matches!(bytes.get(value_start), Some(b',' | b')' | b':')) - { - if matches!(kind, ParameterListKind::Lambda) - && matches!(bytes.get(value_start), Some(b':')) - { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - index, - index + 1, - )); - } - return Some(CpythonDiagnostic::new( - "expected default value expression".to_owned(), - index, - index + 1, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_parameter_list_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"def") => { - let Some(paren) = top_level_byte(bytes, index + 3, bytes.len(), b'(') else { - index += 3; - continue; - }; - let Some(close) = matching_delimiter(bytes, paren, b')') else { - index = paren + 1; - continue; - }; - if let Some(error) = invalid_parameter_list_slice_error( - source, - paren + 1, - close, - ParameterListKind::Function, - ) { - return Some(error); - } - index = close + 1; - } - _ if starts_identifier(bytes, index, b"lambda") => { - let params_start = index + 6; - let Some(params_end) = find_lambda_parameter_end(bytes, params_start) else { - index = params_start; - continue; - }; - if let Some(error) = invalid_parameter_list_slice_error( - source, - params_start, - params_end, - ParameterListKind::Lambda, - ) { - return Some(error); - } - index = params_end + 1; - } - _ => index += 1, - } - } - None -} - -#[derive(Clone, Copy)] -struct CallArgFrame { - level: usize, - arg_start: Option, - in_call: bool, -} - -fn next_non_horizontal_whitespace(bytes: &[u8], mut index: usize) -> usize { - while matches!(bytes.get(index), Some(b' ' | b'\t' | b'\x0c')) { - index += 1; - } - index -} - -fn invalid_call_argument_assignment_error( - source: &str, - arg_start: usize, - equal: usize, -) -> Option { - let bytes = source.as_bytes(); - let start = bytes[arg_start..equal] - .iter() - .rposition(|byte| *byte == b'\n') - .map_or(arg_start, |newline| arg_start + newline + 1); - let (target_start, target_end) = trim_target_range(bytes, start, equal); - if target_start >= target_end { - return None; - } - let value_start = next_non_horizontal_whitespace(bytes, equal + 1); - if matches!(bytes.get(value_start), None | Some(b',' | b')')) { - return Some(CpythonDiagnostic::new( - "expected argument value expression".to_owned(), - target_start, - equal + 1, - )); - } - if bytes.get(target_start..target_start + 2) == Some(b"**") { - return Some(CpythonDiagnostic::new( - "cannot assign to keyword argument unpacking".to_owned(), - target_start, - value_start, - )); - } - if bytes.get(target_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot assign to iterable argument unpacking".to_owned(), - target_start, - value_start, - )); - } - for keyword in [b"True".as_slice(), b"False".as_slice(), b"None".as_slice()] { - if bytes.get(target_start..target_end) == Some(keyword) { - let keyword = ::core::str::from_utf8(keyword).ok()?; - return Some(CpythonDiagnostic::new( - format!("cannot assign to {keyword}"), - target_start, - target_end, - )); - } - } - if is_simple_keyword_name(bytes, target_start, target_end) { - // NAME '=' expression for_if_clauses - if unparenthesized_comprehension(bytes, equal + 1) { - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - target_start, - equal + 1, - )); - } - return None; - } - Some(CpythonDiagnostic::new( - "expression cannot contain assignment, perhaps you meant \"==\"?".to_owned(), - target_start, - equal, - )) -} - -fn unparenthesized_comprehension(bytes: &[u8], mut index: usize) -> bool { - index = next_non_horizontal_whitespace(bytes, index); - if index >= bytes.len() { - return false; - } - let start = index; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - if level == 0 { - return false; - } - level -= 1; - index += 1; - } - b',' | b':' if level == 0 => return false, - _ if level == 0 && index > start && starts_identifier(bytes, index, b"for") => { - return true; - } - _ => index += 1, - } - } - false -} - -fn invalid_call_star_expression_error( - bytes: &[u8], - arg_start: usize, - index: usize, -) -> Option { - let start = next_non_horizontal_whitespace(bytes, arg_start); - if start != index || bytes.get(index) != Some(&b'*') { - return None; - } - let after_star = next_non_horizontal_whitespace(bytes, index + 1); - if matches!(bytes.get(after_star), None | Some(b',' | b')' | b':')) { - return Some(CpythonDiagnostic::new( - "Invalid star expression".to_owned(), - index, - (index + 1).min(bytes.len()), - )); - } - None -} - -fn invalid_call_argument_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - let mut level = 0usize; - let mut frames: Vec = Vec::new(); - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"lambda") => { - let params_start = index + 6; - if let Some(params_end) = find_lambda_parameter_end(bytes, params_start) { - index = params_end + 1; - } else { - index = params_start; - } - } - b'(' => { - level += 1; - let in_call = opening_paren_is_call(bytes, index) - || frames.last().is_some_and(|frame| frame.in_call); - frames.push(CallArgFrame { - level, - arg_start: (in_call && !is_function_parameter_list(bytes, index)) - .then_some(index + 1), - in_call, - }); - index += 1; - } - b')' => { - if matches!(frames.last(), Some(frame) if frame.level == level) { - frames.pop(); - } - level = level.saturating_sub(1); - index += 1; - } - b'[' | b'{' => { - level += 1; - index += 1; - } - b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' => { - if let Some(frame) = frames.last_mut() - && frame.level == level - && frame.arg_start.is_some() - { - frame.arg_start = Some(index + 1); - } - index += 1; - } - b'*' => { - if let Some(CallArgFrame { - level: frame_level, - arg_start: Some(arg_start), - in_call: true, - }) = frames.last().copied() - && frame_level == level - && let Some(error) = invalid_call_star_expression_error(bytes, arg_start, index) - { - return Some(error); - } - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) => { - if let Some(CallArgFrame { - level: frame_level, - arg_start: Some(arg_start), - in_call: true, - }) = frames.last().copied() - && frame_level == level - && let Some(error) = - invalid_call_argument_assignment_error(source, arg_start, index) - { - return Some(error); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn top_level_colon(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn expression_slice_is_valid(source: &str, start: usize, end: usize) -> bool { - let bytes = source.as_bytes(); - let (start, end) = trim_target_range(bytes, start, end); - start < end - && parser::parse(&source[start..end], parser::Mode::Expression.into()) - .is_ok_and(|parsed| matches!(parsed.into_syntax(), ast::Mod::Expression(_))) -} - -fn invalid_dict_entry_error( - source: &str, - item_start: usize, - item_end: usize, - colon: Option, - saw_dict_item: bool, -) -> Option { - let bytes = source.as_bytes(); - let (item_start, item_end) = trim_target_range(bytes, item_start, item_end); - if item_start >= item_end { - return None; - } - if let Some(colon) = colon { - let value_start = next_non_horizontal_whitespace(bytes, colon + 1); - if value_start >= item_end { - return Some(CpythonDiagnostic::new( - "expression expected after dictionary key and ':'".to_owned(), - colon, - colon + 1, - )); - } - if bytes.get(value_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot use a starred expression in a dictionary value".to_owned(), - value_start, - value_start + 1, - )); - } - if !expression_slice_is_valid(source, value_start, item_end) { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - value_start, - value_start, - )); - } - } else if saw_dict_item { - return Some(CpythonDiagnostic::new( - "':' expected after dictionary key".to_owned(), - item_end.saturating_sub(1), - item_end, - )); - } - None -} - -fn invalid_dict_literal_error( - source: &str, - open: usize, - close: usize, -) -> Option { - let bytes = source.as_bytes(); - let mut item_start = open + 1; - let mut index = item_start; - let mut level = 0usize; - let mut saw_dict_item = false; - let mut item_colon = None; - while index <= close { - if index == close || (level == 0 && bytes.get(index) == Some(&b',')) { - if let Some(error) = - invalid_dict_entry_error(source, item_start, index, item_colon, saw_dict_item) - { - return Some(error); - } - saw_dict_item |= item_colon.is_some(); - item_start = index + 1; - item_colon = None; - index += 1; - continue; - } - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b':' if level == 0 && item_colon.is_none() => { - item_colon = Some(index); - saw_dict_item = true; - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_dict_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'{' => { - let Some(close) = matching_delimiter(bytes, index, b'}') else { - index += 1; - continue; - }; - if top_level_colon(bytes, index + 1, close).is_some() - && let Some(error) = invalid_dict_literal_error(source, index, close) - { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn collection_open_is_call(bytes: &[u8], open: usize) -> bool { - if bytes.get(open) != Some(&b'(') { - return false; - } - let mut cursor = open; - while cursor > 0 && matches!(bytes.get(cursor - 1), Some(b' ' | b'\t' | b'\x0c')) { - cursor -= 1; - } - matches!( - cursor.checked_sub(1).and_then(|before| bytes.get(before)), - Some(b')' | b']' | b'_' | b'a'..=b'z' | b'A'..=b'Z' | 0x80..=0xff) - ) -} - -fn invalid_collection_assignment_in_slice( - source: &str, - bytes: &[u8], - start: usize, - end: usize, -) -> Option { - let mut item_start = start; - let mut index = start; - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' if level == 0 => { - item_start = index + 1; - index += 1; - } - b'=' if level == 0 && is_plain_assignment_operator(bytes, index) => { - if top_level_colon(bytes, item_start, index).is_none() { - let start = next_non_horizontal_whitespace(bytes, item_start); - let target_end = trim_end_horizontal_whitespace(bytes, start, index); - if start < target_end - && let Some((expr_name, expr_start, expr_end, _)) = - expression_name_and_range(&source[start..target_end]) - { - if matches!(expr_name, "list" | "tuple") { - return None; - } - if matches!(expr_name, "expression" | "attribute" | "subscript") { - return Some(CpythonDiagnostic::new( - format!( - "cannot assign to {expr_name} here. Maybe you meant '==' instead of '='?" - ), - start + expr_start, - start + expr_end, - )); - } - } - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - start, - index + 1, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_collection_assignment_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - let close_byte = match bytes[index] { - b'(' => b')', - b'[' => b']', - _ => b'}', - }; - let Some(close) = matching_delimiter(bytes, index, close_byte) else { - index += 1; - continue; - }; - if !collection_open_is_call(bytes, index) - && let Some(error) = - invalid_collection_assignment_in_slice(source, bytes, index + 1, close) - { - return Some(error); - } - index = close + 1; - } - _ => index += 1, - } - } - None -} - -fn expression_assignment_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - let mut paren_arg_starts: Vec<(Option, bool)> = Vec::new(); - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - _ if starts_identifier(bytes, index, b"lambda") => { - let params_start = index + 6; - if let Some(params_end) = find_lambda_parameter_end(bytes, params_start) { - index = params_end + 1; - } else { - index = params_start; - } - } - b'(' => { - let in_call_context = opening_paren_is_call(bytes, index) - || paren_arg_starts.last().is_some_and(|(_, in_call)| *in_call); - paren_arg_starts.push(( - (!is_function_parameter_list(bytes, index)).then_some(index + 1), - in_call_context, - )); - index += 1; - } - b')' => { - paren_arg_starts.pop(); - index += 1; - } - b',' => { - if let Some((start, _)) = paren_arg_starts.last_mut() - && start.is_some() - { - *start = Some(index + 1); - } - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) => { - if let Some((Some(start), true)) = paren_arg_starts.last().copied() - && !is_simple_keyword_name(bytes, start, index) - { - let mut expr_start = start; - while matches!(bytes.get(expr_start), Some(b' ' | b'\t' | b'\x0c')) { - expr_start += 1; - } - return Some(CpythonDiagnostic::new( - "expression cannot contain assignment, perhaps you meant \"==\"?" - .to_owned(), - expr_start, - index, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_named_expression_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index + 1 < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b':' if bytes.get(index + 1) == Some(&b'=') => { - let target_start = named_expression_target_start(bytes, index); - let target_end = trim_end_horizontal_whitespace(bytes, target_start, index); - if target_start < target_end - && let Some((expr_name, start, end, is_name)) = - expression_name_and_range(&source[target_start..target_end]) - && !is_name - { - return Some(CpythonDiagnostic::new( - format!("cannot use assignment expressions with {expr_name}"), - target_start + start, - target_start + end, - )); - } - index += 2; - } - _ => index += 1, - } - } - None -} - -#[derive(Clone, Copy)] -struct AssignmentContext { - start: usize, - call: bool, -} - -fn invalid_plain_assignment_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut stack: Vec = Vec::new(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - stack.push(AssignmentContext { - start: index + 1, - call: bytes[index] == b'(' && opening_paren_is_call(bytes, index), - }); - index += 1; - } - b')' | b']' | b'}' => { - stack.pop(); - index += 1; - } - b',' => { - if let Some(context) = stack.last_mut() - && !context.call - { - context.start = index + 1; - } - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) => { - if let Some(context) = stack.last().copied() - && !context.call - { - let target_start = skip_horizontal_whitespace(bytes, context.start); - let target_end = trim_end_horizontal_whitespace(bytes, target_start, index); - if target_start < target_end - && let Some((expr_name, start, end, _)) = - expression_name_and_range(&source[target_start..target_end]) - && matches!(expr_name, "expression" | "attribute" | "subscript") - { - return Some(CpythonDiagnostic::new( - format!( - "cannot assign to {expr_name} here. Maybe you meant '==' instead of '='?" - ), - target_start + start, - target_start + end, - )); - } - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn opening_paren_is_call(bytes: &[u8], paren: usize) -> bool { - let mut cursor = paren; - while cursor > 0 && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\x0c') { - cursor -= 1; - } - cursor > 0 - && (bytes[cursor - 1] >= 0x80 - || is_ascii_identifier_char(bytes[cursor - 1]) - || matches!(bytes[cursor - 1], b')' | b']')) -} - -fn named_expression_target_start(bytes: &[u8], walrus: usize) -> usize { - let mut index = walrus; - let mut level = 0usize; - while index > 0 { - index -= 1; - match bytes[index] { - b')' | b']' | b'}' => level += 1, - b'(' | b'[' | b'{' if level > 0 => level -= 1, - b'(' | b'[' | b'{' if level == 0 => return index + 1, - b',' | b'\n' | b';' if level == 0 => return index + 1, - _ => {} - } - } - 0 -} - -fn trim_end_horizontal_whitespace(bytes: &[u8], start: usize, mut end: usize) -> usize { - while end > start && matches!(bytes[end - 1], b' ' | b'\t' | b'\x0c') { - end -= 1; - } - end -} - -fn annotation_target_error_for_slice( - source: &str, - start: usize, - colon: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, colon); - if target_start >= target_end { - return None; - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - match expression.body.as_ref() { - ast::Expr::Name(_) | ast::Expr::Attribute(_) | ast::Expr::Subscript(_) => None, - ast::Expr::List(_) => Some(CpythonDiagnostic::new( - "only single target (not list) can be annotated".to_owned(), - target_start, - target_end, - )), - ast::Expr::Tuple(_) => Some(CpythonDiagnostic::new( - "only single target (not tuple) can be annotated".to_owned(), - target_start, - target_end, - )), - _ => Some(CpythonDiagnostic::new( - "illegal target for annotation".to_owned(), - target_start, - target_end, - )), - } -} - -fn invalid_annotation_line_start(bytes: &[u8], line_start: usize) -> bool { - let column = skip_horizontal_whitespace(bytes, line_start); - for keyword in [ - b"async".as_slice(), - b"case", - b"class", - b"def", - b"elif", - b"else", - b"except", - b"finally", - b"for", - b"if", - b"match", - b"try", - b"while", - b"with", - ] { - if starts_identifier(bytes, column, keyword) { - return false; - } - } - true -} - -fn invalid_annotation_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - if invalid_annotation_line_start(bytes, line_start) - && let Some(colon) = find_byte_at_level(bytes, line_start, line_end, b':') - && bytes.get(colon + 1) != Some(&b'=') - && colon.checked_sub(1).and_then(|before| bytes.get(before)) != Some(&b':') - && let Some(error) = annotation_target_error_for_slice(source, line_start, colon) - { - return Some(error); - } - line_start = line_end; - } - None -} - -fn statement_target_end(bytes: &[u8], mut index: usize) -> usize { - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => return index, - b'\n' | b';' if level == 0 => return index, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - index -} - -fn invalid_assignment_target(expression: &ast::Expr) -> Option<&ast::Expr> { - match expression { - ast::Expr::List(ast::ExprList { elts, .. }) - | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { - elts.iter().find_map(invalid_assignment_target) - } - ast::Expr::Starred(ast::ExprStarred { value, .. }) => invalid_assignment_target(value), - ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => None, - _ => Some(expression), - } -} - -fn invalid_for_target(expression: &ast::Expr) -> Option<&ast::Expr> { - match expression { - ast::Expr::List(ast::ExprList { elts, .. }) - | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => elts.iter().find_map(invalid_for_target), - ast::Expr::Starred(ast::ExprStarred { value, .. }) => invalid_for_target(value), - ast::Expr::Compare(ast::ExprCompare { left, ops, .. }) => { - if matches!(ops.first(), Some(ast::CmpOp::In)) { - invalid_for_target(left) - } else { - None - } - } - ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => None, - _ => Some(expression), - } -} - -fn invalid_delete_target(expression: &ast::Expr) -> Option<&ast::Expr> { - match expression { - ast::Expr::List(ast::ExprList { elts, .. }) - | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { - elts.iter().find_map(invalid_delete_target) - } - ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => None, - ast::Expr::Starred(_) => Some(expression), - ast::Expr::Compare(_) => Some(expression), - _ => Some(expression), - } -} - -fn delete_target_expr_name(expression: &ast::Expr) -> &'static str { - match expression { - ast::Expr::Attribute(_) => "attribute", - ast::Expr::Subscript(_) => "subscript", - ast::Expr::Starred(_) => "starred", - ast::Expr::Name(_) => "name", - ast::Expr::List(_) => "list", - ast::Expr::Tuple(_) => "tuple", - ast::Expr::Lambda(_) => "lambda", - ast::Expr::Call(_) => "function call", - ast::Expr::BoolOp(_) | ast::Expr::BinOp(_) | ast::Expr::UnaryOp(_) => "expression", - ast::Expr::Generator(_) => "generator expression", - ast::Expr::Yield(_) | ast::Expr::YieldFrom(_) => "yield expression", - ast::Expr::Await(_) => "await expression", - ast::Expr::ListComp(_) => "list comprehension", - ast::Expr::SetComp(_) => "set comprehension", - ast::Expr::DictComp(_) => "dict comprehension", - ast::Expr::Dict(_) => "dict literal", - ast::Expr::Set(_) => "set display", - ast::Expr::FString(_) => "f-string expression", - ast::Expr::TString(_) => "t-string expression", - ast::Expr::NumberLiteral(_) | ast::Expr::StringLiteral(_) | ast::Expr::BytesLiteral(_) => { - "literal" - } - ast::Expr::Constant(expr) => match &expr.value { - ast::ConstantValue::None => "None", - ast::ConstantValue::Boolean(true) => "True", - ast::ConstantValue::Boolean(false) => "False", - ast::ConstantValue::Ellipsis => "ellipsis", - ast::ConstantValue::Tuple(_) => "tuple", - ast::ConstantValue::Frozenset(_) => "literal", - ast::ConstantValue::Str(_) - | ast::ConstantValue::Bytes(_) - | ast::ConstantValue::Integer(_) - | ast::ConstantValue::Float(_) - | ast::ConstantValue::Complex { .. } => "literal", - }, - ast::Expr::BooleanLiteral(boolean) => { - if boolean.value { - "True" - } else { - "False" - } - } - ast::Expr::NoneLiteral(_) => "None", - ast::Expr::EllipsisLiteral(_) => "ellipsis", - ast::Expr::Compare(_) => "comparison", - ast::Expr::If(_) => "conditional expression", - ast::Expr::Named(_) => "named expression", - ast::Expr::Slice(_) | ast::Expr::IpyEscapeCommand(_) => "expression", - } -} - -fn parenthesized_single_starred_delete_target(bytes: &[u8], start: usize, end: usize) -> bool { - let mut cursor = start; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if bytes.get(cursor) != Some(&b'(') { - return false; - } - cursor += 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if bytes.get(cursor) != Some(&b'*') { - return false; - } - let mut level = 1usize; - cursor += 1; - while cursor < end { - match bytes[cursor] { - b'\'' | b'"' => { - cursor = skip_quoted_string(bytes, cursor); - } - b'(' | b'[' | b'{' => { - level += 1; - cursor += 1; - } - b')' => { - level = level.saturating_sub(1); - if level == 0 { - cursor += 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - return cursor == end; - } - cursor += 1; - } - b',' if level == 1 => return false, - b']' | b'}' => { - level = level.saturating_sub(1); - cursor += 1; - } - _ => cursor += 1, - } - } - false -} - -fn assignment_target_expr_range(source: &str, start: usize, end: usize) -> Option<(usize, usize)> { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end { - return None; - } - if parser::parse( - &source[target_start..target_end], - parser::Mode::Expression.into(), - ) - .is_ok() - { - return Some((target_start, target_end)); - } - // `def f(): (yield bar)` — skip the suite header so the remaining - // text is the assignment target expression. Other colons (`x: int += 1`) - // are not suite headers for this diagnostic. Grouping parentheses may - // wrap the yield expression. - let colon = top_level_colon(bytes, target_start, target_end)?; - let after = skip_horizontal_whitespace(bytes, colon + 1); - let yield_at = skip_opening_parentheses(bytes, after, target_end); - if yield_at >= target_end || !starts_identifier(bytes, yield_at, b"yield") { - return None; - } - Some((after, target_end)) -} - -fn skip_opening_parentheses(bytes: &[u8], mut index: usize, end: usize) -> usize { - loop { - index = skip_horizontal_whitespace(bytes, index); - if index >= end || bytes[index] != b'(' { - return index; - } - index += 1; - } -} - -fn trim_target_range(bytes: &[u8], mut start: usize, mut end: usize) -> (usize, usize) { - while start < end - && matches!( - bytes.get(start), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) - { - start += 1; - } - while end > start - && matches!( - bytes.get(end - 1), - Some(b' ' | b'\t' | b'\n' | b'\r' | b'\x0c') - ) - { - end -= 1; - } - (start, end) -} - -fn invalid_assignment_message(name: &'static str, top_level_bitwise: bool) -> String { - if top_level_bitwise { - format!("cannot assign to {name} here. Maybe you meant '==' instead of '='?") - } else { - format!("cannot assign to {name}") - } -} - -fn assignment_target_error_for_slice( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = assignment_target_expr_range(source, start, end)?; - if starts_identifier(bytes, target_start, b"yield") { - return Some(CpythonDiagnostic::new( - "assignment to yield expression not possible".to_owned(), - target_start, - target_start + 5, - )); - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let invalid_target = invalid_assignment_target(&expression.body)?; - let invalid_start = target_start + invalid_target.range().start().to_usize(); - let invalid_end = target_start + invalid_target.range().end().to_usize(); - let name = delete_target_expr_name(invalid_target); - let top_level = invalid_target.range() == expression.body.range(); - let bitwise_like = matches!( - invalid_target, - ast::Expr::Call(_) - | ast::Expr::BoolOp(_) - | ast::Expr::BinOp(_) - | ast::Expr::UnaryOp(_) - | ast::Expr::NumberLiteral(_) - | ast::Expr::StringLiteral(_) - | ast::Expr::BytesLiteral(_) - | ast::Expr::EllipsisLiteral(_) - | ast::Expr::Yield(_) - | ast::Expr::YieldFrom(_) - | ast::Expr::Set(_) - | ast::Expr::Dict(_) - | ast::Expr::FString(_) - | ast::Expr::TString(_) - ); - Some(CpythonDiagnostic::new( - invalid_assignment_message(name, top_level && bitwise_like), - invalid_start, - invalid_end, - )) -} - -fn star_target_error_for_slice( - source: &str, - start: usize, - end: usize, -) -> Option { - invalid_target_error_for_slice(source, start, end, invalid_assignment_target) -} - -fn for_target_error_for_slice(source: &str, start: usize, end: usize) -> Option { - invalid_target_error_for_slice(source, start, end, invalid_for_target) -} - -fn invalid_target_error_for_slice( - source: &str, - start: usize, - end: usize, - invalid_target: for<'a> fn(&'a ast::Expr) -> Option<&'a ast::Expr>, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end { - return None; - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let invalid_target = invalid_target(&expression.body)?; - let name = delete_target_expr_name(invalid_target); - let invalid_start = target_start + invalid_target.range().start().to_usize(); - let invalid_end = target_start + invalid_target.range().end().to_usize(); - Some(CpythonDiagnostic::new( - format!("cannot assign to {name}"), - invalid_start, - invalid_end, - )) -} - -fn first_compare_operator_at_level(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'<' | b'>' if level == 0 => return Some(index), - b'=' if level == 0 && bytes.get(index + 1) == Some(&b'=') => return Some(index), - b'!' if level == 0 && bytes.get(index + 1) == Some(&b'=') => return Some(index), - _ if level == 0 && starts_identifier(bytes, index, b"is") => return Some(index), - _ if level == 0 && starts_identifier(bytes, index, b"not") => return Some(index), - _ => index += 1, - } - } - None -} - -fn non_in_compare_for_target_error( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end { - return None; - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let ast::Expr::Compare(ast::ExprCompare { ops, .. }) = expression.body.as_ref() else { - return None; - }; - if matches!(ops.first(), Some(ast::CmpOp::In)) { - return None; - } - let operator = first_compare_operator_at_level(bytes, target_start, target_end)?; - Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - operator, - (operator + 1).min(target_end), - )) -} - -fn top_level_plain_assignment_offsets(bytes: &[u8]) -> Vec { - let mut offsets = Vec::new(); - let mut index = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'=' if level == 0 && is_plain_assignment_operator(bytes, index) => { - offsets.push(index); - index += 1; - } - _ => index += 1, - } - } - offsets -} - -fn invalid_condition_assignment_error( - source: &str, - parse_error_offset: usize, -) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - let mut line_start = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\n' => { - line_start = index + 1; - index += 1; - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 - && index == skip_horizontal_whitespace(bytes, line_start) - && (starts_identifier(bytes, index, b"if") - || starts_identifier(bytes, index, b"elif") - || starts_identifier(bytes, index, b"while")) => - { - let keyword_len = if starts_identifier(bytes, index, b"while") { - 5 - } else if starts_identifier(bytes, index, b"elif") { - 4 - } else { - 2 - }; - let cond_start = skip_horizontal_whitespace(bytes, index + keyword_len); - let Some(colon) = top_level_colon(bytes, cond_start, bytes.len()) else { - index += keyword_len; - continue; - }; - if parse_error_offset < index || parse_error_offset > colon { - index += keyword_len; - continue; - } - let Some(equal) = condition_plain_assignment(bytes, cond_start, colon) else { - index += keyword_len; - continue; - }; - let (target_start, target_end) = trim_target_range(bytes, cond_start, equal); - if target_start >= target_end { - index += keyword_len; - continue; - } - if is_simple_keyword_name(bytes, target_start, target_end) { - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - target_start, - equal + 1, - )); - } - if let Some((expr_name, start, end, _)) = - expression_name_and_range(&source[target_start..target_end]) - { - return Some(CpythonDiagnostic::new( - format!( - "cannot assign to {expr_name} here. Maybe you meant '==' instead of '='?" - ), - target_start + start, - target_start + end, - )); - } - return Some(CpythonDiagnostic::new( - "invalid syntax. Maybe you meant '==' or ':=' instead of '='?".to_owned(), - target_start, - equal + 1, - )); - } - _ => index += 1, - } - } - None -} - -fn condition_plain_assignment(bytes: &[u8], start: usize, end: usize) -> Option { - let mut index = start; - let mut nest = Vec::new(); - while index < end { - match bytes[index] { - b'#' => { - while index < end && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' => { - nest.push(if is_call_open(bytes, start, index) { - b'c' - } else { - b'g' - }); - index += 1; - } - b'[' => { - nest.push(b'['); - index += 1; - } - b'{' => { - nest.push(b'{'); - index += 1; - } - b')' | b']' | b'}' => { - nest.pop(); - index += 1; - } - b':' if nest.last() == Some(&b'l') => { - nest.pop(); - index += 1; - } - b'=' if is_plain_assignment_operator(bytes, index) - && !nest.contains(&b'c') - && !nest.contains(&b'l') => - { - return Some(index); - } - _ if starts_identifier(bytes, index, b"lambda") => { - nest.push(b'l'); - index += 6; - } - _ => index += 1, - } - } - None -} - -fn is_call_open(bytes: &[u8], start: usize, open: usize) -> bool { - let mut index = open; - while index > start { - index -= 1; - match bytes[index] { - b' ' | b'\t' | b'\n' | b'\r' | b'\x0c' => {} - b')' | b']' => return true, - byte if byte >= 0x80 || is_ascii_identifier_char(byte) => { - let mut ident_start = index; - while ident_start > start - && bytes - .get(ident_start - 1) - .is_some_and(|b| *b >= 0x80 || is_ascii_identifier_char(*b)) - { - ident_start -= 1; - } - return !is_condition_keyword(&bytes[ident_start..=index]); - } - _ => return false, - } - } - false -} - -fn is_condition_keyword(word: &[u8]) -> bool { - matches!( - word, - b"not" | b"and" | b"or" | b"in" | b"is" | b"if" | b"elif" | b"while" | b"await" | b"lambda" - ) -} - -fn invalid_assignment_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let offsets = top_level_plain_assignment_offsets(bytes); - if offsets.is_empty() { - return None; - } - let mut start = 0usize; - for offset in offsets { - if let Some(error) = assignment_target_error_for_slice(source, start, offset) { - return Some(error); - } - start = offset + 1; - } - None -} - -fn top_level_augassign_offset(bytes: &[u8]) -> Option<(usize, usize)> { - let mut index = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b'+' | b'-' | b'*' | b'@' | b'/' | b'%' | b'&' | b'|' | b'^' - if level == 0 && bytes.get(index + 1) == Some(&b'=') => - { - return Some((index, 2)); - } - b'<' | b'>' - if level == 0 - && bytes.get(index + 1) == Some(&bytes[index]) - && bytes.get(index + 2) == Some(&b'=') => - { - return Some((index, 3)); - } - b'*' if level == 0 - && bytes.get(index + 1) == Some(&b'*') - && bytes.get(index + 2) == Some(&b'=') => - { - return Some((index, 3)); - } - b'/' if level == 0 - && bytes.get(index + 1) == Some(&b'/') - && bytes.get(index + 2) == Some(&b'=') => - { - return Some((index, 3)); - } - _ => index += 1, - } - } - None -} - -fn invalid_augassign_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let (operator, _) = top_level_augassign_offset(bytes)?; - let (target_start, target_end) = assignment_target_expr_range(source, 0, operator)?; - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - return None; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let name = delete_target_expr_name(&expression.body); - Some(CpythonDiagnostic::new( - format!("'{name}' is an illegal expression for augmented assignment"), - target_start, - target_end, - )) -} - -fn find_for_target_delimiter(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, b"in") => return Some(index), - b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn invalid_for_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - _ if starts_identifier(bytes, index, b"for") => { - let target_start = skip_horizontal_whitespace(bytes, index + 3); - let line_end = source[index..] - .find('\n') - .map_or(bytes.len(), |newline| index + newline); - if let Some(target_end) = find_for_target_delimiter(bytes, target_start, line_end) { - if let Some(error) = - for_target_error_for_slice(source, target_start, target_end) - { - return Some(error); - } - if let Some(error) = - non_in_compare_for_target_error(source, target_start, target_end) - { - return Some(error); - } - } - index = target_start.max(index + 3); - } - _ => index += 1, - } - } - None -} - -fn find_with_target_delimiter(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' if level == 0 => return Some(index), - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' | b':' if level == 0 => return Some(index), - _ => index += 1, - } - } - None -} - -fn invalid_with_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let mut column = skip_horizontal_whitespace(bytes, line_start); - if starts_identifier(bytes, column, b"async") { - column = skip_horizontal_whitespace(bytes, column + 5); - } - if !starts_identifier(bytes, column, b"with") { - line_start = line_end; - continue; - } - let mut index = column + 4; - while let Some(as_index) = find_keyword_at_level(bytes, index, line_end, b"as") { - let target_start = skip_horizontal_whitespace(bytes, as_index + 2); - if let Some(target_end) = find_with_target_delimiter(bytes, target_start, line_end) { - if let Some(error) = star_target_error_for_slice(source, target_start, target_end) { - return Some(error); - } - index = target_end.saturating_add(1); - } else { - break; - } - } - line_start = line_end; - } - None -} - -fn find_missing_in_if_keyword(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level == 0 => return None, - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, b"in") => return None, - _ if level == 0 && starts_identifier(bytes, index, b"if") => return Some(index), - _ => index += 1, - } - } - None -} - -fn invalid_for_if_clause_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0usize; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level > 0 && starts_identifier(bytes, index, b"for") => { - let target_start = skip_horizontal_whitespace(bytes, index + 3); - let line_end = source[index..] - .find('\n') - .map_or(bytes.len(), |newline| index + newline); - if let Some(if_index) = find_missing_in_if_keyword(bytes, target_start, line_end) { - return Some(CpythonDiagnostic::new( - "'in' expected after for-loop variables".to_owned(), - if_index, - (if_index + 2).min(line_end), - )); - } - index = target_start.max(index + 3); - } - _ => index += 1, - } - } - None -} - -fn invalid_delete_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'd' if starts_identifier(bytes, index, b"del") => { - let mut target_start = index + 3; - if !matches!(bytes.get(target_start), Some(b' ' | b'\t' | b'\x0c')) { - index += 3; - continue; - } - while matches!(bytes.get(target_start), Some(b' ' | b'\t' | b'\x0c')) { - target_start += 1; - } - let mut target_end = statement_target_end(bytes, target_start); - while target_end > target_start - && matches!(bytes.get(target_end - 1), Some(b' ' | b'\t' | b'\x0c')) - { - target_end -= 1; - } - if target_start >= target_end { - index = target_end.max(index + 3); - continue; - } - if parenthesized_single_starred_delete_target(bytes, target_start, target_end) { - return Some(CpythonDiagnostic::new( - "cannot use starred expression here".to_owned(), - target_start, - target_end, - )); - } - if bytes.get(target_start) == Some(&b'*') { - return Some(CpythonDiagnostic::new( - "cannot delete starred".to_owned(), - target_start, - (target_start + 1).min(target_end), - )); - } - let target_text = &source[target_start..target_end]; - let Ok(parsed) = parser::parse(target_text, parser::Mode::Expression.into()) else { - index = target_end; - continue; - }; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - index = target_end; - continue; - }; - let Some(invalid_target) = invalid_delete_target(&expression.body) else { - index = target_end; - continue; - }; - let start = target_start + invalid_target.range().start().to_usize(); - let end = target_start + invalid_target.range().end().to_usize(); - if matches!(invalid_target, ast::Expr::FString(_)) { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - start, - end, - )); - } - let name = delete_target_expr_name(invalid_target); - return Some(CpythonDiagnostic::new( - format!("cannot delete {name}"), - start, - end, - )); - } - _ => index += 1, - } - } - None -} - -fn skip_horizontal_whitespace(bytes: &[u8], mut index: usize) -> usize { - while matches!(bytes.get(index), Some(b' ' | b'\t' | b'\x0c')) { - index += 1; - } - index -} - -fn find_keyword_at_level( - bytes: &[u8], - mut index: usize, - end: usize, - keyword: &[u8], -) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, keyword) => return Some(index), - _ => index += 1, - } - } - None -} - -fn find_byte_at_level(bytes: &[u8], mut index: usize, end: usize, needle: u8) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'#' if level == 0 => return None, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - byte if level == 0 && byte == needle => return Some(index), - _ => index += 1, - } - } - None -} - -fn expression_name_and_range(source: &str) -> Option<(&'static str, usize, usize, bool)> { - let parsed = parser::parse(source, parser::Mode::Expression.into()).ok()?; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let is_name = matches!(expression.body.as_ref(), ast::Expr::Name(_)); - Some(( - delete_target_expr_name(&expression.body), - expression.body.range().start().to_usize(), - expression.body.range().end().to_usize(), - is_name, - )) -} - -fn invalid_standalone_except_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut seen_try = false; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let column = skip_horizontal_whitespace(bytes, line_start); - if column >= line_end { - line_start = line_end; - continue; - } - if starts_identifier(bytes, column, b"try") { - seen_try = true; - } else if (bytes.get(column..column + 7) == Some(b"except*") - || starts_identifier(bytes, column, b"except")) - && !seen_try - { - let end = if bytes.get(column..column + 7) == Some(b"except*") { - column + 7 - } else { - column + 6 - }; - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - column, - end, - )); - } - line_start = line_end; - } - None -} - -fn invalid_import_statement_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let column = skip_horizontal_whitespace(bytes, line_start); - if column < line_end - && starts_identifier(bytes, column, b"import") - && find_keyword_at_level(bytes, column + 6, line_end, b"from").is_some() - { - return Some(CpythonDiagnostic::new( - "Did you mean to use 'from ... import ...' instead?".to_owned(), - column, - column + 6, - )); - } - line_start = line_end; - } - None -} - -fn import_as_target_end(bytes: &[u8], mut index: usize) -> usize { - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' if level == 0 => return index, - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' if level == 0 => return index, - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b',' | b';' | b'\n' if level == 0 => return index, - _ => index += 1, - } - } - index -} - -fn valid_import_alias_name(bytes: &[u8], mut start: usize, end: usize) -> bool { - start = skip_horizontal_whitespace(bytes, start); - let Some(&first) = bytes.get(start) else { - return false; - }; - if !(first == b'_' || first.is_ascii_alphabetic() || first >= 0x80) { - return false; - } - let mut index = start + 1; - while index < end { - match bytes[index] { - b' ' | b'\t' | b'\x0c' => break, - byte if byte >= 0x80 || is_ascii_identifier_char(byte) => index += 1, - _ => return false, - } - } - let index = skip_horizontal_whitespace(bytes, index); - matches!( - bytes.get(index), - None | Some(b',' | b')' | b';' | b'\n' | b'\r') - ) -} - -fn import_target_error_for_slice( - source: &str, - start: usize, - end: usize, -) -> Option { - let bytes = source.as_bytes(); - let (target_start, target_end) = trim_target_range(bytes, start, end); - if target_start >= target_end || valid_import_alias_name(bytes, target_start, target_end) { - return None; - } - let parsed = parser::parse( - &source[target_start..target_end], - parser::Mode::Expression.into(), - ) - .ok()?; - let ast::Mod::Expression(expression) = parsed.into_syntax() else { - return None; - }; - let name = delete_target_expr_name(&expression.body); - let start = target_start + expression.body.range().start().to_usize(); - let end = target_start + expression.body.range().end().to_usize(); - Some(CpythonDiagnostic::new( - format!("cannot use {name} as import target"), - start, - end, - )) -} - -fn statement_starts_import(bytes: &[u8], line_start: usize, line_end: usize) -> bool { - let column = skip_horizontal_whitespace(bytes, line_start); - if starts_identifier(bytes, column, b"import") { - return true; - } - starts_identifier(bytes, column, b"from") - && find_keyword_at_level(bytes, column + 4, line_end, b"import").is_some() -} - -fn invalid_import_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut in_parenthesized_from_import = false; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let starts_import = statement_starts_import(bytes, line_start, line_end); - if starts_import && bytes[line_start..line_end].contains(&b'(') { - in_parenthesized_from_import = true; - } - if starts_import || in_parenthesized_from_import { - let mut index = line_start; - while index < line_end { - if starts_identifier(bytes, index, b"as") { - let target_start = skip_horizontal_whitespace(bytes, index + 2); - let target_end = import_as_target_end(bytes, target_start); - if let Some(error) = - import_target_error_for_slice(source, target_start, target_end) - { - return Some(error); - } - index = target_end.max(index + 2); - } else { - index += 1; - } - } - } - if in_parenthesized_from_import && bytes[line_start..line_end].contains(&b')') { - in_parenthesized_from_import = false; - } - line_start = line_end; - } - None -} - -fn invalid_except_as_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut seen_try = false; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let mut column = skip_horizontal_whitespace(bytes, line_start); - if column >= line_end { - line_start = line_end; - continue; - } - if starts_identifier(bytes, column, b"try") { - seen_try = true; - line_start = line_end; - continue; - } - let (keyword_len, starred) = if bytes.get(column..column + 7) == Some(b"except*") { - (7, true) - } else if starts_identifier(bytes, column, b"except") { - (6, false) - } else { - line_start = line_end; - continue; - }; - if !seen_try { - line_start = line_end; - continue; - } - column += keyword_len; - let Some(as_index) = find_keyword_at_level(bytes, column, line_end, b"as") else { - line_start = line_end; - continue; - }; - let target_start = skip_horizontal_whitespace(bytes, as_index + 2); - let Some(delimiter) = find_byte_at_level(bytes, target_start, line_end, b':') - .into_iter() - .chain(find_byte_at_level(bytes, target_start, line_end, b',')) - .min() - else { - line_start = line_end; - continue; - }; - let mut target_end = delimiter; - while target_end > target_start - && matches!(bytes.get(target_end - 1), Some(b' ' | b'\t' | b'\x0c')) - { - target_end -= 1; - } - let Some((expr_name, start, end, is_name)) = - expression_name_and_range(&source[target_start..target_end]) - else { - line_start = line_end; - continue; - }; - if !is_name { - let statement = if starred { "except*" } else { "except" }; - return Some(CpythonDiagnostic::new( - format!("cannot use {statement} statement with {expr_name}"), - target_start + start, - target_start + end, - )); - } - line_start = line_end; - } - None -} - -fn invalid_match_as_target_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let quoted_ranges = quoted_string_ranges(bytes); - let mut quoted_range = 0usize; - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let mut column = skip_horizontal_whitespace(bytes, line_start); - if column >= line_end - || offset_in_ranges("ed_ranges, &mut quoted_range, column) - || !starts_identifier(bytes, column, b"case") - { - line_start = line_end; - continue; - } - column += 4; - let Some(as_index) = find_keyword_at_level(bytes, column, line_end, b"as") else { - line_start = line_end; - continue; - }; - let target_start = skip_horizontal_whitespace(bytes, as_index + 2); - let Some(delimiter) = find_byte_at_level(bytes, target_start, line_end, b':') - .into_iter() - .chain(find_byte_at_level(bytes, target_start, line_end, b',')) - .min() - else { - line_start = line_end; - continue; - }; - let mut target_end = delimiter; - while target_end > target_start - && matches!(bytes.get(target_end - 1), Some(b' ' | b'\t' | b'\x0c')) - { - target_end -= 1; - } - if source[target_start..target_end].trim() == "_" { - return Some(CpythonDiagnostic::new( - "cannot use '_' as a target".to_owned(), - target_start, - target_end, - )); - } - let Some((expr_name, start, end, is_name)) = - expression_name_and_range(&source[target_start..target_end]) - else { - line_start = line_end; - continue; - }; - if !is_name { - if matches!(expr_name, "expression" | "subscript") { - line_start = line_end; - continue; - } - return Some(CpythonDiagnostic::new( - format!("cannot use {expr_name} as pattern target"), - target_start + start, - target_start + end, - )); - } - line_start = line_end; - } - None -} - -fn quoted_string_ranges(bytes: &[u8]) -> Vec<(usize, usize)> { - let mut ranges = Vec::new(); - let mut index = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - let end = skip_quoted_string(bytes, index); - ranges.push((index, end)); - index = end; - } - _ => index += 1, - } - } - ranges -} - -fn offset_in_ranges(ranges: &[(usize, usize)], range_index: &mut usize, offset: usize) -> bool { - while ranges - .get(*range_index) - .is_some_and(|(_, end)| *end <= offset) - { - *range_index += 1; - } - ranges - .get(*range_index) - .is_some_and(|(start, end)| *start <= offset && offset < *end) -} - -fn invalid_match_mapping_rest_wildcard_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let next_line_end = |line_start: usize| { - line_start - + bytes[line_start..] - .iter() - .position(|byte| *byte == b'\n') - .unwrap_or(bytes.len() - line_start) - }; - let mut index = 0usize; - let mut line_start = 0usize; - let mut line_end = next_line_end(line_start); - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'\n' => { - index += 1; - line_start = index; - line_end = next_line_end(line_start); - } - _ => { - let column = skip_horizontal_whitespace(bytes, line_start); - if index != column - || column >= line_end - || !starts_identifier(bytes, column, b"case") - { - index += 1; - continue; - } - let mut cursor = column + 4; - while cursor < line_end { - match bytes[cursor] { - b'#' => break, - b'\'' | b'"' => cursor = skip_quoted_string(bytes, cursor), - b'{' => { - let rest = next_non_horizontal_whitespace(bytes, cursor + 1); - if bytes.get(rest..rest + 2) == Some(b"**") { - let name_start = next_non_horizontal_whitespace(bytes, rest + 2); - let name_end = identifier_end(bytes, name_start, line_end); - if source.get(name_start..name_end) == Some("_") { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - name_start, - name_end, - )); - } - } - cursor += 1; - } - _ => cursor += 1, - } - } - index = line_end; - } - } - } - None -} - -fn invalid_if_expression_statement_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - if let Some(if_index) = find_keyword_at_level(bytes, line_start, line_end, b"if") - && let Some((start, end)) = statement_before_if_expression(bytes, line_start, if_index) - && find_keyword_at_level(bytes, if_index + 2, line_end, b"else").is_some() - { - return Some(CpythonDiagnostic::new( - "expected expression before 'if', but statement is given".to_owned(), - start, - end, - )); - } - if let Some(else_index) = find_keyword_at_level(bytes, line_start, line_end, b"else") - && find_keyword_at_level(bytes, line_start, else_index, b"if").is_some() - && let Some((start, end)) = - statement_after_else_expression(bytes, else_index + 4, line_end) - { - return Some(CpythonDiagnostic::new( - "expected expression after 'else', but statement is given".to_owned(), - start, - end, - )); - } - line_start = line_end; - } - None -} - -fn statement_before_if_expression( - bytes: &[u8], - line_start: usize, - if_index: usize, -) -> Option<(usize, usize)> { - let mut start = if_index; - while start > line_start && matches!(bytes.get(start - 1), Some(b' ' | b'\t' | b'\x0c')) { - start -= 1; - } - while start > line_start - && !matches!( - bytes.get(start - 1), - Some(b'=' | b':' | b',' | b'(' | b'[' | b'{') - ) - { - start -= 1; - } - start = skip_horizontal_whitespace(bytes, start); - for keyword in [b"pass".as_slice(), b"break", b"continue"] { - if starts_identifier(bytes, start, keyword) { - return Some((start, start + keyword.len())); - } - } - None -} - -fn statement_after_else_expression( - bytes: &[u8], - else_end: usize, - line_end: usize, -) -> Option<(usize, usize)> { - let start = skip_horizontal_whitespace(bytes, else_end); - for keyword in [ - b"pass".as_slice(), - b"return", - b"raise", - b"del", - b"yield", - b"assert", - b"break", - b"continue", - b"import", - b"from", - ] { - if starts_identifier(bytes, start, keyword) { - let end = statement_target_end(bytes, start).min(line_end); - return Some((start, end.max(start + keyword.len()))); - } - } - None -} - -fn invalid_else_elif_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut line_start = 0usize; - let mut else_indents: Vec = Vec::new(); - for line in source.split_inclusive('\n') { - let line_end = line_start + line.len(); - let column = skip_horizontal_whitespace(bytes, line_start); - let line_column = column.saturating_sub(line_start); - if column >= line_end { - line_start = line_end; - continue; - } - while else_indents - .last() - .is_some_and(|indent| line_column < *indent) - { - else_indents.pop(); - } - if starts_identifier(bytes, column, b"else") - && find_byte_at_level(bytes, column + 4, line_end, b':').is_some() - { - else_indents.push(line_column); - } else if starts_identifier(bytes, column, b"elif") && else_indents.contains(&line_column) { - return Some(CpythonDiagnostic::new( - "'elif' block follows an 'else' block".to_owned(), - column, - column + 4, - )); - } - line_start = line_end; - } - None -} - -fn mixed_except_handlers_error(source: &str) -> Option { - let message = "cannot have both 'except' and 'except*' on the same 'try'".to_owned(); - let mut seen_except = false; - let mut seen_except_star = false; - let mut line_start = 0usize; - for line in source.split_inclusive('\n') { - let bytes = line.as_bytes(); - let mut column = 0usize; - while matches!(bytes.get(column), Some(b' ' | b'\t' | b'\x0c')) { - column += 1; - } - let token_start = line_start + column; - if bytes.get(column..column + 7) == Some(b"except*") { - if seen_except { - return Some(CpythonDiagnostic::new( - message, - token_start, - token_start + 7, - )); - } - seen_except_star = true; - } else if starts_identifier(bytes, column, b"except") { - if seen_except_star { - return Some(CpythonDiagnostic::new( - message, - token_start, - token_start + 6, - )); - } - seen_except = true; - } - line_start += line.len(); - } - None -} - -fn non_printable_character_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - byte if byte.is_ascii_control() && !matches!(byte, b'\t' | b'\n' | b'\r' | b'\x0c') => { - return Some(CpythonDiagnostic::new( - format!("invalid non-printable character U+{byte:04X}"), - index, - index + 1, - )); - } - byte if byte >= 0x80 => { - let ch = source[index..].chars().next()?; - if ch.is_control() { - return Some(CpythonDiagnostic::new( - format!("invalid non-printable character U+{:04X}", ch as u32), - index, - index + ch.len_utf8(), - )); - } - index += ch.len_utf8(); - } - _ => index += 1, - } - } - None -} - -fn unterminated_string_error(source: &str, mode: Mode) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - let mut line = 1usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\n' => { - line += 1; - index += 1; - } - quote @ (b'\'' | b'"') => { - let start = index; - let start_line = line; - let quote_size = if bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - 3 - } else { - 1 - }; - index += quote_size; - let mut has_escaped_quote = false; - let mut ended_with_escape = false; - let mut closed = false; - while index < bytes.len() { - let c = bytes[index]; - if c == b'\n' { - if quote_size == 1 { - // A single-quoted literal cannot span a line, so this is the same - // "the literal never ended" case as running out of source; CPython - // spells the two as one condition in lexer.c too. - break; - } - line += 1; - index += 1; - } else if c == quote { - if quote_size == 3 { - if bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - index += 3; - closed = true; - break; - } - index += 1; - } else { - index += 1; - closed = true; - break; - } - } else if c == b'\\' { - if bytes.get(index + 1) == Some("e) { - has_escaped_quote = true; - } - ended_with_escape = index + 1 >= bytes.len(); - index = (index + 2).min(bytes.len()); - } else { - index += 1; - } - } - if !closed { - if let Some(error) = - unclosed_replacement_field_error(bytes, start, start + quote_size, index) - { - return Some(error); - } - let detected_line = if quote_size == 3 { line } else { start_line }; - let interpolated = interpolated_string_prefix(bytes, start); - let diagnostic = CpythonDiagnostic::new( - unterminated_string_message( - detected_line, - quote_size == 3, - has_escaped_quote, - interpolated, - ), - start, - start, - ); - // E_EOLS is only for a plain single-quoted string at EOF. - // Single-quoted f/t-strings never set it. exec input appends a - // newline, so a single-quoted literal becomes the newline case - // unless a final `\` consumes that newline. - let exec_implicit_newline = - matches!(mode, Mode::Exec) && quote_size == 1 && !ended_with_escape; - let eval_assignment = - matches!(mode, Mode::Eval) && eval_has_assignment_before(bytes, start); - let continuable = index >= bytes.len() - && !(interpolated.is_some() && quote_size == 1) - && !exec_implicit_newline - && !eval_assignment; - return Some(if continuable { - diagnostic.with_unclosed_string() - } else { - diagnostic - }); - } - } - _ => index += 1, - } - } - None -} - -fn eval_has_assignment_before(bytes: &[u8], end: usize) -> bool { - let mut index = 0; - let mut level = 0usize; - let mut in_lambda_params = false; - while index < end { - match bytes[index] { - b'#' => { - while index < end && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - b':' if level == 0 => { - in_lambda_params = false; - index += 1; - } - b'=' if level == 0 - && !in_lambda_params - && bytes.get(index + 1) != Some(&b'=') - && !matches!( - bytes.get(index.saturating_sub(1)), - Some(b':' | b'!' | b'<' | b'>') - ) => - { - return true; - } - _ if level == 0 && starts_identifier(bytes, index, b"lambda") => { - in_lambda_params = true; - index += b"lambda".len(); - } - _ => index += 1, - } - } - false -} - -fn invalid_interpolated_string_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - quote @ (b'\'' | b'"') => { - let Some(prefix) = interpolated_string_prefix(bytes, index) else { - index = skip_quoted_string(bytes, index); - continue; - }; - if let Some(error) = - single_quoted_format_spec_newline_error(bytes, index, quote, prefix) - { - return Some(error); - } - let Some((content_start, content_end)) = - quoted_string_content_range(bytes, index, quote) - else { - index = skip_quoted_string(bytes, index); - continue; - }; - if let Some(error) = - invalid_replacement_field_error(bytes, content_start, content_end, prefix) - { - return Some(error); - } - index = skip_quoted_string(bytes, index); - } - _ => index += 1, - } - } - None -} - -/// Whether the literal opened at `quote` carries a `b` in its prefix. -fn is_bytes_literal(bytes: &[u8], quote: usize) -> bool { - let mut start = quote; - while quote - start < 2 - && start > 0 - && matches!( - bytes[start - 1].to_ascii_lowercase(), - b'r' | b'b' | b'u' | b'f' | b't' - ) - { - start -= 1; - } - if start > 0 && is_ascii_identifier_char(bytes[start - 1]) { - return false; - } - bytes[start..quote] - .iter() - .any(|byte| byte.eq_ignore_ascii_case(&b'b')) -} - -/// Which of CPython's two messages a mixed literal concatenation gets. Ruff reports only the -/// bytes/non-bytes mix, so `t"x" b"y"` arrives here as a bytes error where CPython names the -/// t-string, and this scanner owns the choice between them. -/// -/// CPython's `invalid_string_tstring_concat` is an `invalid_` rule, reached only on the error -/// pass, and the `strings` rule tries `(fstring|string)+` first. That alternative consumes the -/// concatenation's leading run of non-t-string literals, so a bytes/non-bytes mix among *those* -/// raises from `_PyPegen_concatenate_strings` on the first pass and the t-string rule never runs. -/// The bytes message therefore keeps precedence for `"a" b"b" t"c"` but not for `t"x" b"y"`. -fn mixed_tstring_literal_error( - error: &parser::ParseError, - source: &str, -) -> Option { - let parser::ParseErrorType::OtherError(message) = &error.error else { - return None; - }; - if !message.eq_ignore_ascii_case("bytes literal cannot be mixed with non-bytes literals") { - return None; - } - - let start = error.location.start().to_usize(); - let end = error.location.end().to_usize(); - let bytes = source.as_bytes(); - if end > bytes.len() { - return None; - } - - let mut index = start; - let (mut bytes_seen, mut nonbytes_seen) = (false, false); - while index < end { - match bytes[index] { - b'\'' | b'"' => { - if interpolated_string_prefix(bytes, index) == Some("t-string") { - let message = if bytes_seen && nonbytes_seen { - // Reported here rather than left to ruff's error so that the scanners - // further down the chain cannot claim the concatenation instead. - "cannot mix bytes and nonbytes literals" - } else { - "cannot mix t-string literals with string or bytes literals" - }; - return Some(CpythonDiagnostic::new(message.to_owned(), start, end)); - } - if is_bytes_literal(bytes, index) { - bytes_seen = true; - } else { - nonbytes_seen = true; - } - index = skip_quoted_string(bytes, index); - } - _ => index += 1, - } - } - None -} - -fn single_quoted_format_spec_newline_error( - bytes: &[u8], - quote_index: usize, - quote: u8, - prefix: &str, -) -> Option { - if bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e) { - return None; - } - - let (content_start, content_end) = quoted_string_content_range(bytes, quote_index, quote)?; - let mut index = content_start; - while index < content_end { - match bytes[index] { - b'{' if bytes.get(index + 1) == Some(&b'{') => index += 2, - b'}' if bytes.get(index + 1) == Some(&b'}') => index += 2, - b'{' => { - let expr_start = skip_ascii_whitespace(bytes, index + 1, content_end); - if let Some(separator) = replacement_field_separator(bytes, expr_start, content_end) - && bytes[separator] == b':' - { - let format_end = - replacement_field_closing_brace(bytes, separator + 1, content_end) - .unwrap_or(content_end); - if bytes[separator + 1..format_end].contains(&b'\n') { - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: newlines are not allowed in format specifiers for single quoted {prefix}s" - ), - quote_index, - quote_index + 1, - )); - } - } - index += 1; - } - _ => index += 1, - } - } - None -} - -fn interpolated_string_prefix(bytes: &[u8], quote: usize) -> Option<&'static str> { - let prev = quote.checked_sub(1).and_then(|index| bytes.get(index))?; - let lower_prev = prev.to_ascii_lowercase(); - let (prefix_start, marker) = if matches!(lower_prev, b'f' | b't') { - if quote >= 2 && bytes[quote - 2].eq_ignore_ascii_case(&b'r') { - (quote - 2, lower_prev) - } else { - (quote - 1, lower_prev) - } - } else if lower_prev == b'r' - && quote >= 2 - && matches!(bytes[quote - 2].to_ascii_lowercase(), b'f' | b't') - { - (quote - 2, bytes[quote - 2].to_ascii_lowercase()) - } else { - return None; - }; - - if prefix_start > 0 && identifier_continue_before(bytes, prefix_start) { - return None; - } - - Some(if marker == b'f' { - "f-string" - } else { - "t-string" - }) -} - -const MAXFSTRINGLEVEL: usize = 150; - -fn skip_string_token(bytes: &[u8], quote_index: usize) -> usize { - if interpolated_string_prefix(bytes, quote_index).is_some() { - interpolated_string_end(bytes, quote_index).unwrap_or(bytes.len()) - } else { - skip_quoted_string(bytes, quote_index) - } -} - -fn interpolated_string_end(bytes: &[u8], quote_index: usize) -> Option { - let quote = bytes[quote_index]; - let triple = - bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e); - let quote_len = if triple { 3 } else { 1 }; - let mut index = quote_index + quote_len; - let mut brace_depth = 0usize; - while index < bytes.len() { - if brace_depth == 0 { - if bytes[index] == b'\\' { - index = (index + 2).min(bytes.len()); - continue; - } - if triple - && bytes.get(index) == Some("e) - && bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - return Some(index + 3); - } - if !triple && bytes[index] == quote { - return Some(index + 1); - } - if bytes[index] == b'{' { - if bytes.get(index + 1) == Some(&b'{') { - index += 2; - } else { - brace_depth = 1; - index += 1; - } - } else if bytes[index] == b'}' && bytes.get(index + 1) == Some(&b'}') { - index += 2; - } else { - index += 1; - } - } else { - match bytes[index] { - quote_ch @ (b'\'' | b'"') => { - if quoted_string_is_closed(bytes, index) { - index = skip_string_token(bytes, index).max(index + 1); - } else if !triple && quote_ch == quote { - return Some(index + 1); - } else { - index = skip_string_token(bytes, index).max(index + 1); - } - } - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'{' => { - brace_depth += 1; - index += 1; - } - b'}' => { - brace_depth -= 1; - index += 1; - } - b'\\' => index = (index + 2).min(bytes.len()), - _ => index += 1, - } - } - } - None -} - -fn quote_len_at(bytes: &[u8], quote_index: usize) -> usize { - let quote = bytes[quote_index]; - if bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e) { - 3 - } else { - 1 - } -} - -fn interpolated_string_content_range(bytes: &[u8], quote_index: usize) -> Option<(usize, usize)> { - let end = interpolated_string_end(bytes, quote_index)?; - let quote_len = quote_len_at(bytes, quote_index); - Some((quote_index + quote_len, end - quote_len)) -} - -fn quoted_string_content_range( - bytes: &[u8], - quote_index: usize, - quote: u8, -) -> Option<(usize, usize)> { - let triple = - bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e); - let quote_len = if triple { 3 } else { 1 }; - let content_start = quote_index + quote_len; - let mut index = content_start; - while index < bytes.len() { - if bytes[index] == b'\\' { - index = (index + 2).min(bytes.len()); - } else if (triple - && bytes.get(index) == Some("e) - && bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e)) - || (!triple && bytes[index] == quote) - { - return Some((content_start, index)); - } else { - index += 1; - } - } - None -} - -fn invalid_replacement_field_error( - bytes: &[u8], - start: usize, - end: usize, - prefix: &str, -) -> Option { - let mut index = start; - while index < end { - match bytes[index] { - b'{' if bytes.get(index + 1) == Some(&b'{') => index += 2, - b'}' if bytes.get(index + 1) == Some(&b'}') => index += 2, - b'{' => { - if let Some(error) = replacement_field_error(bytes, index, end, prefix, 0) { - return Some(error); - } - // Resume in the literal text. Inside the field a `#` starts a comment and a `{` - // opens no new field, so walking on byte by byte would misread both. - index = replacement_field_end(bytes, index, end).unwrap_or(end); - } - _ => index += 1, - } - } - None -} - -/// Position just past a replacement field's closing brace. The expression part is code, where a -/// comment runs to the end of the line; past the separator only nested fields nest. -fn replacement_field_end(bytes: &[u8], open: usize, end: usize) -> Option { - let expr_start = skip_replacement_field_trivia(bytes, open + 1, end); - let separator = replacement_field_separator(bytes, expr_start, end)?; - if bytes[separator] == b'}' { - return Some(separator + 1); - } - replacement_field_closing_brace(bytes, separator + 1, end).map(|brace| brace + 1) -} - -/// The one-based line holding `offset`. Counted on demand: these scanners only run once the -/// source has failed to parse, and tracking a line through the spans they skip would have to -/// re-scan them anyway. -fn line_of_offset(bytes: &[u8], offset: usize) -> usize { - 1 + bytes[..offset] - .iter() - .filter(|byte| **byte == b'\n') - .count() -} - -/// Brackets opened in a replacement field's expression must close before the field does. CPython -/// words these exactly as it does for brackets anywhere else, except that a closer with nothing -/// open is attributed to the literal. `start..end` must cover only the expression: a format spec -/// is plain text, so a bracket there opens nothing. -fn replacement_field_bracket_error( - bytes: &[u8], - start: usize, - end: usize, - prefix: &str, -) -> Option { - // Each entry carries where its bracket opened, so a mismatch can name that line. - let mut stack: Vec<(u8, usize)> = Vec::new(); - let mut index = start; - while index < end { - let byte = bytes[index]; - match byte { - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - continue; - } - b'#' => { - index = skip_replacement_field_comment(bytes, index, end); - continue; - } - b'(' | b'[' | b'{' => stack.push((byte, index)), - b')' | b']' | b'}' => { - let expected = expected_opening_bracket(byte as char) as u8; - match stack.last() { - // The field's own closing brace: everything inside it balanced. - None if byte == b'}' => return None, - None => { - return Some(CpythonDiagnostic::new( - format!("{prefix}: unmatched '{}'", byte as char), - index, - index + 1, - )); - } - Some(&(opening, _)) if opening == expected => { - stack.pop(); - } - Some(&(opening, opened_at)) => { - // CPython names the opening line only when it is not the closing one, - // comparing `parenlinenostack[level]` against the current `lineno`. - let opening_line = line_of_offset(bytes, opened_at); - let suffix = if opening_line == line_of_offset(bytes, index) { - String::new() - } else { - format!(" on line {opening_line}") - }; - return Some(CpythonDiagnostic::new( - format!( - "closing parenthesis '{}' does not match opening parenthesis '{}'{suffix}", - byte as char, opening as char - ), - index, - index + 1, - )); - } - } - } - _ => {} - } - index += 1; - } - None -} - -/// A `#` in a replacement field's expression starts a comment. When no newline follows it before -/// the end of the literal, the comment swallows the closing brace and the field can never close. -/// `start..end` must cover only the expression: `#` is an ordinary character in a format spec, -/// where it selects the alternate form. -fn replacement_field_comment_error( - bytes: &[u8], - open: usize, - start: usize, - end: usize, - literal_end: usize, -) -> Option { - let mut index = start; - while index < end { - match bytes[index] { - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - continue; - } - b'#' => { - if !bytes[index..literal_end].contains(&b'\n') { - return Some( - CpythonDiagnostic::new("'{' was never closed".to_owned(), open, open + 1) - .with_unclosed_bracket(), - ); - } - index = skip_replacement_field_comment(bytes, index, end); - continue; - } - _ => index += 1, - } - } - None -} - -/// Advances past a comment, stopping on the newline that ends it. -fn skip_replacement_field_comment(bytes: &[u8], mut index: usize, end: usize) -> usize { - while index < end && bytes[index] != b'\n' { - index += 1; - } - index -} - -/// Whitespace and comments may both precede a replacement field's expression. -fn skip_replacement_field_trivia(bytes: &[u8], mut index: usize, end: usize) -> usize { - loop { - index = skip_ascii_whitespace(bytes, index, end); - if index < end && bytes[index] == b'#' { - index = skip_replacement_field_comment(bytes, index, end); - continue; - } - return index; - } -} - -/// Bounds the mutual recursion between a replacement field and the format spec it contains, -/// so that pathologically nested input cannot exhaust the stack. -const MAX_REPLACEMENT_FIELD_DEPTH: usize = 32; - -fn replacement_field_error( - bytes: &[u8], - open: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - let expr_start = skip_replacement_field_trivia(bytes, open + 1, end); - - // The expression runs up to the field's first top-level `=`, `!`, `:` or `}`. Everything past - // it is a conversion or a format spec, which are text rather than code, so the checks that - // treat the field as code have to stop here. - let separator = replacement_field_separator(bytes, expr_start, end); - let expr_end = separator.unwrap_or(end); - - if let Some(backslash) = replacement_field_line_continuation(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - "unexpected character after line continuation character".to_owned(), - backslash + 1, - (backslash + 2).min(end), - )); - } - if let Some(quote) = unterminated_string_in_replacement_field(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - unterminated_string_message(1, false, false, None), - quote, - quote + 1, - )); - } - // Brackets are reported ahead of everything else in the field, and a comment that swallows - // the closing brace ahead of the expression itself. The comment scan starts at the brace - // because a comment may also sit between it and the expression. - if let Some(error) = replacement_field_bracket_error(bytes, expr_start, expr_end, prefix) { - return Some(error); - } - if let Some(error) = replacement_field_comment_error(bytes, open, open + 1, expr_end, end) { - return Some(error); - } - - // Nothing follows `{` at all, so the field simply never closed. CPython asks for the - // brace here; an expression that merely fails to start is reported further down. - if expr_start >= end { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}'"), - open, - open + 1, - )); - } - - if let marker @ (b'=' | b'!' | b':' | b'}') = bytes[expr_start] - && is_replacement_field_marker(bytes, expr_start) - { - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: valid expression required before '{}'", - marker as char - ), - expr_start, - expr_start + 1, - )); - } - - // A `{` display is a valid expression, but only if something can follow it. CPython - // reports the inner brace when it cannot, as in `f'{3:{{>10}'`. - if bytes[expr_start] == b'{' { - let inner = skip_ascii_whitespace(bytes, expr_start + 1, end); - if inner < end - && bytes[inner] != b'}' - && invalid_replacement_expression_start(bytes, inner, end) - { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting a valid expression after '{{'"), - expr_start, - expr_start + 1, - )); - } - } - - if starts_identifier(bytes, expr_start, b"lambda") { - return Some(CpythonDiagnostic::new( - format!("{prefix}: lambda expressions are not allowed without parentheses"), - expr_start, - expr_start + b"lambda".len(), - )); - } - - if invalid_replacement_expression_start(bytes, expr_start, end) { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting a valid expression after '{{'"), - open, - open + 1, - )); - } - - // Adjacent atoms in a replacement field are a missing comma, not an - // f-string "expecting '}'" at the opening brace. Prefix `yield`/`await`/ - // `not` and continuation keywords (`and`, `for`, ...) are not a second atom. - if !starts_identifier(bytes, expr_start, b"yield") - && !starts_identifier(bytes, expr_start, b"await") - && !starts_identifier(bytes, expr_start, b"not") - && let Some(atom_end) = adjacent_atom_end(bytes, expr_start) - { - let next = skip_ascii_whitespace(bytes, atom_end, expr_end); - if next > atom_end - && expression_atom_start(bytes, next) - && !expression_continuation_keyword(bytes, next) - { - let second_end = adjacent_atom_end(bytes, next).unwrap_or(next + 1); - return Some(CpythonDiagnostic::new( - "invalid syntax. Perhaps you forgot a comma?".to_owned(), - expr_start, - second_end, - )); - } - } - - // The expression started fine but ran into a character that cannot continue it. CPython - // points at that character and lists the separators it wanted instead. - if let Some(stray) = replacement_expression_stray_character(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '=', or '!', or ':', or '}}'"), - stray, - stray + 1, - )); - } - - let Some(separator) = separator else { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}'"), - open, - open + 1, - )); - }; - - // Only now that the field is known to close. A stray character above is a finished token - // the parser rejects on lookahead, but a dangling operator makes it ask for one more, and - // producing that token runs into the literal's closing quote: lexer.c then answers from - // `INSIDE_FSTRING(tok)` with "%c-string: expecting '}'" before any `invalid_` rule is tried. - if let Some(operator) = dangling_operator(bytes, expr_start, expr_end) { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '=', or '!', or ':', or '}}'"), - operator, - operator + 1, - )); - } - - if let Some(lambda_at) = top_level_lambda(bytes, expr_start, expr_end) - && lambda_allowed_at_expression_position(bytes, expr_start, lambda_at) - { - return Some(CpythonDiagnostic::new( - format!("{prefix}: lambda expressions are not allowed without parentheses"), - lambda_at, - lambda_at + b"lambda".len(), - )); - } - - if bytes[separator] == b':' - && replacement_expression_has_parse_error(bytes, expr_start, separator) - { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - expr_start, - separator, - )); - } - - match bytes[separator] { - b'=' => invalid_debug_expression_error(bytes, separator, end, prefix, depth), - b'!' => invalid_conversion_error(bytes, separator, end, prefix, depth), - b':' => invalid_format_spec_error(bytes, separator, end, prefix, depth), - b'}' => None, - _ => unreachable!(), - } -} - -fn replacement_field_line_continuation( - bytes: &[u8], - mut index: usize, - end: usize, -) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b'\\' => return Some(index), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - index += 1; - } - b'=' | b'!' | b':' | b'}' - if level == 0 && is_replacement_field_marker(bytes, index) => - { - return None; - } - _ => index += 1, - } - } - None -} - -fn unterminated_string_in_replacement_field( - bytes: &[u8], - mut index: usize, - end: usize, -) -> Option { - while index < end { - match bytes[index] { - quote @ (b'\'' | b'"') => { - let string_end = skip_quoted_string(bytes, index); - if string_end >= end && !bytes[index + 1..end].contains("e) { - return Some(index); - } - index = string_end; - } - b'#' => index = skip_replacement_field_comment(bytes, index, end), - _ => index += 1, - } - } - None -} - -fn top_level_lambda(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - index += 1; - } - _ if level == 0 && starts_identifier(bytes, index, b"lambda") => { - return Some(index); - } - _ => index += 1, - } - } - None -} - -/// Unparenthesized `lambda` is allowed in the same places a `lambdef` can appear: -/// at the start of the field or after a top-level comma. -fn lambda_allowed_at_expression_position(bytes: &[u8], mut index: usize, lambda_at: usize) -> bool { - let mut after_comma = true; - let mut level = 0usize; - while index < lambda_at { - match bytes[index] { - b'\'' | b'"' => { - after_comma = false; - index = skip_quoted_string(bytes, index); - } - b'#' => index = skip_replacement_field_comment(bytes, index, lambda_at), - b'(' | b'[' | b'{' => { - level += 1; - after_comma = false; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - after_comma = false; - index += 1; - } - b',' if level == 0 => { - after_comma = true; - index += 1; - } - b' ' | b'\t' | b'\n' | b'\r' | b'\x0c' => index += 1, - _ => { - after_comma = false; - index += 1; - } - } - } - after_comma -} - -fn replacement_expression_has_parse_error(bytes: &[u8], start: usize, end: usize) -> bool { - let Ok(expression) = ::core::str::from_utf8(&bytes[start..end]) else { - return false; - }; - parser::parse_expression(expression).is_err() -} - -fn invalid_replacement_expression_start(bytes: &[u8], index: usize, end: usize) -> bool { - if index >= end { - return true; - } - - // A leading `.` is Ellipsis or a float without an integer part, both of which start a - // perfectly good expression. - if bytes[index] == b'.' - && (bytes[index..end].starts_with(b"...") - || bytes - .get(index + 1) - .is_some_and(|byte| byte.is_ascii_digit())) - { - return false; - } - - if matches!( - bytes[index], - b'.' | b',' | b'*' | b'/' | b'%' | b'&' | b'|' | b'^' | b'<' | b'>' | b'@' | b'=' | b'!' - ) || is_stray_in_replacement_expression(bytes[index]) - { - return true; - } - - if matches!(bytes[index], b'+' | b'-' | b'~') { - let operand = skip_ascii_whitespace(bytes, index + 1, end); - if starts_identifier(bytes, operand, b"lambda") { - return true; - } - return !bytes.get(operand).is_some_and(|byte| { - *byte >= 0x80 - || *byte == b'_' - || byte.is_ascii_alphabetic() - || byte.is_ascii_digit() - || matches!(*byte, b'\'' | b'"' | b'(' | b'[' | b'{') - // `-.5` is a signed float. - || (*byte == b'.' - && bytes - .get(operand + 1) - .is_some_and(|next| next.is_ascii_digit())) - }); - } - - [ - b"and".as_slice(), - b"as".as_slice(), - b"else".as_slice(), - b"for".as_slice(), - b"if".as_slice(), - b"in".as_slice(), - b"is".as_slice(), - b"or".as_slice(), - ] - .iter() - .any(|keyword| starts_identifier(bytes, index, keyword)) -} - -/// A top-level `=` or `!` marks a debug specifier or a conversion only when it is not part of a -/// longer operator. CPython's tokenizer consumes `==`, `!=`, `<=`, `+=` and the rest as single -/// tokens before it ever considers the debug marker, so those must not end the expression here. -fn is_replacement_field_marker(bytes: &[u8], index: usize) -> bool { - match bytes[index] { - b'=' => { - if bytes.get(index + 1) == Some(&b'=') { - return false; - } - !index - .checked_sub(1) - .and_then(|previous| bytes.get(previous).copied()) - .is_some_and(precedes_equals_in_one_operator) - } - b'!' => bytes.get(index + 1) != Some(&b'='), - _ => true, - } -} - -fn replacement_field_separator(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' if level > 0 => { - level -= 1; - index += 1; - } - b'=' | b'!' | b':' | b'}' - if level == 0 && is_replacement_field_marker(bytes, index) => - { - return Some(index); - } - _ => index += 1, - } - } - None -} - -/// Characters that can neither start nor continue a replacement field expression. CPython's -/// interpolated-string tokenizer stops at them and asks for a separator instead. -const fn is_stray_in_replacement_expression(byte: u8) -> bool { - matches!(byte, b';' | b'$' | b'?' | b'`') -} - -fn replacement_expression_stray_character( - bytes: &[u8], - mut index: usize, - end: usize, -) -> Option { - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'#' => index = skip_replacement_field_comment(bytes, index, end), - byte if is_stray_in_replacement_expression(byte) => return Some(index), - _ => index += 1, - } - } - None -} - -/// A byte that pairs with a following `=` to make one operator token, so that the `=` belongs -/// to it rather than opening a debug specifier: `==`, `!=`, `<=`, `>=`, the augmented -/// assignments, and the `:=` walrus. Against `is_dangling_operator_byte` below, this has `:` -/// and lacks `.` and `~`, because neither `.=` nor `~=` is a Python operator. -const fn precedes_equals_in_one_operator(byte: u8) -> bool { - matches!( - byte, - b'!' | b'%' - | b'&' - | b'*' - | b'+' - | b'-' - | b'/' - | b':' - | b'<' - | b'=' - | b'>' - | b'@' - | b'^' - | b'|' - ) -} - -/// Characters that make up an operator which cannot end an expression. Against -/// `precedes_equals_in_one_operator` above, this has `.` and `~` for `a.b.` and `~a`, and lacks -/// `:`, which separates a replacement field's parts rather than operating on anything. -const fn is_dangling_operator_byte(byte: u8) -> bool { - matches!( - byte, - b'!' | b'%' - | b'&' - | b'*' - | b'+' - | b'-' - | b'.' - | b'/' - | b'<' - | b'=' - | b'>' - | b'@' - | b'^' - | b'|' - | b'~' - ) -} - -/// The identifier that ends just before `word`, skipping the whitespace between them. -fn preceding_keyword(bytes: &[u8], start: usize, word: usize) -> Option<(usize, &[u8])> { - let mut cursor = word; - while cursor > start && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\r' | b'\n' | 0x0c) { - cursor -= 1; - } - let stop = cursor; - while cursor > start && is_ascii_identifier_char(bytes[cursor - 1]) { - cursor -= 1; - } - (cursor < stop).then(|| (cursor, &bytes[cursor..stop])) -} - -/// A replacement field expression cannot end on an operator that still wants an operand. -/// CPython's tokenizer then reaches the field brace with the expression unfinished and asks for -/// a separator, pointing at the operator that left it that way. Returns that operator's start. -fn dangling_operator(bytes: &[u8], start: usize, end: usize) -> Option { - // The region cannot be read backwards: the newline that ends a comment is whitespace, so - // trimming from `end` would step into the comment body. Walk forward instead and keep the - // position just past the last byte that is really part of the expression. - let mut index = start; - let mut tail = start; - while index < end { - match bytes[index] { - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index).min(end); - tail = index; - } - b'#' => index = skip_replacement_field_comment(bytes, index, end), - b' ' | b'\t' | b'\r' | b'\n' | 0x0c => index += 1, - _ => { - index += 1; - tail = index; - } - } - } - if tail == start { - return None; - } - - // A trailing keyword operator reads back as an identifier. - if is_ascii_identifier_char(bytes[tail - 1]) { - let mut word = tail; - while word > start && is_ascii_identifier_char(bytes[word - 1]) { - word -= 1; - } - if ![ - b"and".as_slice(), - b"or".as_slice(), - b"not".as_slice(), - b"in".as_slice(), - b"is".as_slice(), - b"if".as_slice(), - b"for".as_slice(), - ] - .iter() - .any(|keyword| &bytes[word..tail] == *keyword) - { - return None; - } - // `is not` and `not in` are single operators, and CPython points at their first word. - let previous = preceding_keyword(bytes, start, word); - return Some(match (previous, &bytes[word..tail]) { - (Some((at, b"is")), b"not") | (Some((at, b"not")), b"in") => at, - _ => word, - }); - } - - if !is_dangling_operator_byte(bytes[tail - 1]) { - return None; - } - let mut token = tail; - while token > start && is_dangling_operator_byte(bytes[token - 1]) { - token -= 1; - } - - // `1.` is a float: that dot finishes the literal rather than dangling. - if bytes[token..tail] == *b"." && token > start && bytes[token - 1].is_ascii_digit() { - return None; - } - // `...` is Ellipsis, so consume whole triples: a leftover dot is what dangles. - if bytes[token..tail].iter().all(|byte| *byte == b'.') { - let leftover = (tail - token) % 3; - return (leftover != 0).then(|| tail - leftover); - } - Some(token) -} - -fn invalid_debug_expression_error( - bytes: &[u8], - equals: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - let next = equals + 1; - if next >= end { - return None; - } - // A conversion or format spec may follow `=`, and is validated exactly as it would be - // when following the expression directly. - match bytes[next] { - b'!' => return invalid_conversion_error(bytes, next, end, prefix, depth), - b':' => return invalid_format_spec_error(bytes, next, end, prefix, depth), - b'}' => return None, - _ => {} - } - Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '!', or ':', or '}}'"), - next, - next.saturating_add(1).min(end), - )) -} - -fn invalid_conversion_error( - bytes: &[u8], - bang: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - let next = bang + 1; - if next >= end { - return Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}'"), - bang, - bang + 1, - )); - } - - if bytes[next].is_ascii_whitespace() { - let following = skip_ascii_whitespace(bytes, next, end); - let message = if bytes - .get(following) - .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') - { - "conversion type must come right after the exclamation mark" - } else { - "missing conversion character" - }; - return Some(CpythonDiagnostic::new( - format!("{prefix}: {message}"), - next, - next + 1, - )); - } - - if matches!(bytes[next], b':' | b'}') { - return Some(CpythonDiagnostic::new( - format!("{prefix}: missing conversion character"), - next, - next + 1, - )); - } - - // A non-ASCII character is still a conversion character as far as CPython is concerned, - // so it gets named in the message like any other invalid one. - if bytes[next] >= 0x80 - && let Some(character) = ::core::str::from_utf8(&bytes[next..end]) - .ok() - .and_then(|text| text.chars().next()) - { - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: invalid conversion character '{character}': expected 's', 'r', or 'a'" - ), - next, - next + character.len_utf8(), - )); - } - - if !bytes[next].is_ascii_alphabetic() && bytes[next] != b'_' { - return Some(CpythonDiagnostic::new( - format!("{prefix}: invalid conversion character"), - next, - next + 1, - )); - } - - let conversion_end = identifier_end(bytes, next, end); - let conversion = &bytes[next..conversion_end]; - if !matches!(conversion, b"s" | b"r" | b"a") { - let conversion = ::core::str::from_utf8(conversion).unwrap_or(""); - return Some(CpythonDiagnostic::new( - format!( - "{prefix}: invalid conversion character '{conversion}': expected 's', 'r', or 'a'" - ), - next, - conversion_end, - )); - } - - if conversion_end >= end { - return None; - } - - match bytes[conversion_end] { - b':' => return invalid_format_spec_error(bytes, conversion_end, end, prefix, depth), - b'}' => return None, - _ => {} - } - - Some(CpythonDiagnostic::new( - format!("{prefix}: expecting ':' or '}}'"), - conversion_end, - conversion_end + 1, - )) -} - -fn invalid_format_spec_error( - bytes: &[u8], - colon: usize, - end: usize, - prefix: &str, - depth: usize, -) -> Option { - // Unlike literal text, a format spec has no `{{` escape: every `{` opens a nested - // replacement field, so the outer scanner cannot validate these for us. - let mut index = colon + 1; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'{' => { - if depth < MAX_REPLACEMENT_FIELD_DEPTH - && let Some(error) = - replacement_field_error(bytes, index, end, prefix, depth + 1) - { - return Some(error); - } - // Resume after the nested field. Walking into it again from every enclosing - // spec would re-scan the same bytes once per level, which costs O(2^depth). - let Some(after) = replacement_field_end(bytes, index, end) else { - break; - }; - index = after; - } - b'}' => return None, - _ => index += 1, - } - } - Some(CpythonDiagnostic::new( - format!("{prefix}: expecting '}}', or format specs"), - colon, - colon + 1, - )) -} - -fn replacement_field_closing_brace(bytes: &[u8], mut index: usize, end: usize) -> Option { - let mut level = 0usize; - while index < end { - match bytes[index] { - b'\'' | b'"' => index = skip_quoted_string(bytes, index), - b'{' => { - level += 1; - index += 1; - } - b'}' if level > 0 => { - level -= 1; - index += 1; - } - b'}' => return Some(index), - _ => index += 1, - } - } - None -} - -fn skip_ascii_whitespace(bytes: &[u8], mut index: usize, end: usize) -> usize { - while index < end && matches!(bytes[index], b' ' | b'\t' | b'\r' | b'\n' | 0x0c) { - index += 1; - } - index -} - -fn string_literal_end_at(bytes: &[u8], index: usize) -> Option { - match bytes.get(index).copied()? { - b'\'' | b'"' => Some(skip_quoted_string(bytes, index)), - first if first.is_ascii_alphabetic() => { - if matches!(bytes.get(index + 1), Some(b'\'' | b'"')) { - return string_literal_prefix(bytes, index, index + 1) - .then(|| skip_quoted_string(bytes, index + 1)); - } - if matches!(bytes.get(index + 2), Some(b'\'' | b'"')) { - return string_literal_prefix(bytes, index, index + 2) - .then(|| skip_quoted_string(bytes, index + 2)); - } - None - } - _ => None, - } -} - -fn string_literal_prefix(bytes: &[u8], start: usize, quote: usize) -> bool { - let prefix = &bytes[start..quote]; - let valid = matches!( - prefix, - b"b" | b"B" - | b"r" - | b"R" - | b"u" - | b"U" - | b"f" - | b"F" - | b"t" - | b"T" - | b"br" - | b"bR" - | b"Br" - | b"BR" - | b"rb" - | b"rB" - | b"Rb" - | b"RB" - | b"fr" - | b"fR" - | b"Fr" - | b"FR" - | b"rf" - | b"rF" - | b"Rf" - | b"RF" - | b"tr" - | b"tR" - | b"Tr" - | b"TR" - | b"rt" - | b"rT" - | b"Rt" - | b"RT" - ); - valid && (start == 0 || !is_ascii_identifier_char(bytes[start - 1])) -} - -fn invalid_expression_error(source: &str) -> Option { - invalid_string_expression_error(source).or_else(|| missing_comma_expression_error(source)) -} - -fn invalid_string_expression_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - if let Some(first_string_end) = string_literal_end_at(bytes, index) { - let expr_start = skip_ascii_whitespace(bytes, first_string_end, bytes.len()); - if expression_atom_start(bytes, expr_start) - && let Some(expr_end) = adjacent_atom_end(bytes, expr_start) - { - let next = skip_ascii_whitespace(bytes, expr_end, bytes.len()); - if string_literal_end_at(bytes, next).is_some() { - return Some(CpythonDiagnostic::new( - "invalid syntax. Is this intended to be part of the string?".to_owned(), - expr_start, - expr_end, - )); - } - } - index = first_string_end; - } else { - index += 1; - } - } - None -} - -fn missing_comma_expression_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut stack: Vec = Vec::new(); - let mut index = 0; - while index < bytes.len() { - if bytes[index] == b'#' { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } else if let Some(string_end) = string_literal_end_at(bytes, index) { - index = string_end; - } else { - match bytes[index] { - b'(' | b'[' | b'{' => { - if bytes[index] == b'[' && opening_bracket_is_class_type_params(bytes, index) { - let Some(close) = matching_delimiter(bytes, index, b']') else { - index += 1; - continue; - }; - index = close + 1; - continue; - } - stack.push(bytes[index]); - index += 1; - } - b')' | b']' | b'}' => { - stack.pop(); - index += 1; - } - _ if !stack.is_empty() && expression_continuation_keyword(bytes, index) => { - index = identifier_end(bytes, index, bytes.len()); - } - byte if !stack.is_empty() && expression_atom_start_byte(byte) => { - // `yield x` / `await x` are prefix expressions, not two - // adjacent atoms missing a comma. - if starts_identifier(bytes, index, b"yield") { - index = identifier_end(bytes, index, bytes.len()); - let after_yield = skip_ascii_whitespace(bytes, index, bytes.len()); - if starts_identifier(bytes, after_yield, b"from") { - index = identifier_end(bytes, after_yield, bytes.len()); - } - continue; - } - if starts_identifier(bytes, index, b"await") { - index = identifier_end(bytes, index, bytes.len()); - continue; - } - let atom_end = adjacent_atom_end(bytes, index).unwrap_or(index + 1); - let next = skip_ascii_whitespace(bytes, atom_end, bytes.len()); - if next > atom_end - && expression_atom_start(bytes, next) - && !expression_continuation_keyword(bytes, next) - { - // The diagnostic covers both atoms, so it ends past - // the whole second one. `next + 1` ended past its - // first byte instead, which is only the same thing - // when that atom is one ASCII character. - let second_end = adjacent_atom_end(bytes, next).unwrap_or(next + 1); - return Some(CpythonDiagnostic::new( - "invalid syntax. Perhaps you forgot a comma?".to_owned(), - index, - second_end, - )); - } - index = atom_end; - } - _ => index += 1, - } - } - } - None -} - -fn opening_bracket_is_class_type_params(bytes: &[u8], bracket: usize) -> bool { - let mut cursor = bracket; - while cursor > 0 && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\x0c') { - cursor -= 1; - } - while cursor > 0 - && bytes - .get(cursor - 1) - .is_some_and(|byte| *byte >= 0x80 || is_ascii_identifier_char(*byte)) - { - cursor -= 1; - } - while cursor > 0 && matches!(bytes[cursor - 1], b' ' | b'\t' | b'\x0c') { - cursor -= 1; - } - cursor >= 5 && starts_identifier(bytes, cursor - 5, b"class") -} - -fn expression_continuation_keyword(bytes: &[u8], index: usize) -> bool { - [ - b"and".as_slice(), - b"else".as_slice(), - b"for".as_slice(), - b"if".as_slice(), - b"in".as_slice(), - b"is".as_slice(), - b"not".as_slice(), - b"or".as_slice(), - ] - .iter() - .any(|keyword| starts_identifier(bytes, index, keyword)) -} - -fn expression_atom_start(bytes: &[u8], index: usize) -> bool { - bytes - .get(index) - .is_some_and(|byte| expression_atom_start_byte(*byte)) - || string_literal_end_at(bytes, index).is_some() -} - -fn expression_atom_start_byte(byte: u8) -> bool { - byte >= 0x80 - || byte == b'_' - || byte.is_ascii_alphabetic() - || byte.is_ascii_digit() - || matches!(byte, b'\'' | b'"' | b'(' | b'[' | b'{') -} - -fn adjacent_atom_end(bytes: &[u8], index: usize) -> Option { - if let Some(string_end) = string_literal_end_at(bytes, index) { - return Some(string_end); - } - match bytes.get(index).copied()? { - byte if byte >= 0x80 || byte == b'_' || byte.is_ascii_alphabetic() => { - Some(identifier_end(bytes, index, bytes.len())) - } - byte if byte.is_ascii_digit() => { - let mut end = index + 1; - while end < bytes.len() && (bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_') { - end += 1; - } - Some(end) - } - b'(' | b'[' | b'{' => Some(index + 1), - _ => None, - } -} - -struct OpenDelimiter { - position: usize, - opener: u8, - in_format_spec: bool, -} - -/// An f-string or t-string that runs off the end with a replacement field still open reports -/// the brace rather than the missing quote, matching CPython — but only while the tokenizer is -/// still reading the field's *expression*. Past the field's own `:` it is emitting FSTRING_MIDDLE -/// again, so running out of input there is an unterminated literal like any other, and -/// `f'{a:>5` gets the quote message where `f'{a` gets the brace. -fn unclosed_replacement_field_error( - bytes: &[u8], - quote_start: usize, - content_start: usize, - content_end: usize, -) -> Option { - interpolated_string_prefix(bytes, quote_start)?; - - // Brackets and quotes are delimiters only once a field is open; in the literal's text they - // are ordinary characters. And past a field's own format spec the tokenizer emits literal - // text again, so a field still open at end of input is an unclosed brace only before that. - let mut open: Vec = Vec::new(); - let mut index = content_start; - while index < content_end { - let inside_expression = matches!( - open.last(), - Some(OpenDelimiter { - opener: b'{', - in_format_spec: false, - .. - }) - ); - match bytes[index] { - // `{{` and `}}` escape only in literal text, which is where a brace at depth zero is. - b'{' if open.is_empty() && bytes.get(index + 1) == Some(&b'{') => index += 2, - b'}' if open.is_empty() && bytes.get(index + 1) == Some(&b'}') => index += 2, - b'{' => { - open.push(OpenDelimiter { - position: index, - opener: b'{', - in_format_spec: false, - }); - index += 1; - } - b'}' => { - open.pop(); - index += 1; - } - byte @ (b'(' | b'[') if !open.is_empty() => { - open.push(OpenDelimiter { - position: index, - opener: byte, - in_format_spec: false, - }); - index += 1; - } - b')' | b']' if !open.is_empty() => { - open.pop(); - index += 1; - } - // Only the field's own `:` opens a format spec — one in a slice, a display or a - // lambda belongs to whatever bracket encloses it. - b':' if inside_expression => { - open.last_mut().expect("a brace is open").in_format_spec = true; - index += 1; - } - b'\'' | b'"' if !open.is_empty() => { - index = skip_quoted_string(bytes, index).min(content_end); - } - b'#' if inside_expression => { - index = skip_replacement_field_comment(bytes, index, content_end); - } - _ => index += 1, - } - } - - // CPython names the innermost unclosed delimiter, so a bracket opened inside the expression - // takes the message from the field that encloses it. - let &OpenDelimiter { - position, - opener, - in_format_spec, - } = open.last()?; - (!in_format_spec).then(|| { - CpythonDiagnostic::new( - format!("'{}' was never closed", opener as char), - position, - position + 1, - ) - .with_unclosed_bracket() - }) -} - -fn unterminated_string_message( - detected_line: usize, - triple: bool, - has_escaped_quote: bool, - prefix: Option<&str>, -) -> String { - // CPython names the literal kind, e.g. "unterminated f-string literal". - let kind = prefix.unwrap_or("string"); - if triple { - format!("unterminated triple-quoted {kind} literal (detected at line {detected_line})") - // The escaped-quote hint belongs to the plain-string branch of CPython's tokenizer alone. - // Its interpolated branch has only the two forms above and below, so pairing the hint with - // a prefix would invent a message CPython never emits. - } else if has_escaped_quote && prefix.is_none() { - format!( - "unterminated {kind} literal (detected at line {detected_line}); perhaps you escaped the end quote?" - ) - } else { - format!("unterminated {kind} literal (detected at line {detected_line})") - } -} - -fn expected_opening_bracket(closing: char) -> char { - match closing { - ')' => '(', - ']' => '[', - '}' => '{', - _ => unreachable!(), - } -} - -/// A bracket diagnostic, and whether it is an opener that was never closed. The caller needs -/// that apart from the message because ruff reports the unclosed case as an EOF error. -#[derive(Clone)] -struct BracketError { - diagnostic: CpythonDiagnostic, - unclosed: bool, -} - -fn bracket_syntax_error(source: &str) -> Option { - let mut stack: Vec<(char, usize, usize)> = Vec::new(); - let mut in_string = false; - let mut string_quote = '\0'; - let mut triple_quote = false; - let mut escape_next = false; - let mut is_raw_string = false; - let mut line = 1usize; - - let chars: Vec<(usize, char)> = source.char_indices().collect(); - let mut index = 0; - while index < chars.len() { - let (byte_offset, ch) = chars[index]; - - if ch == '\n' { - line += 1; - } - - if escape_next { - escape_next = false; - index += 1; - continue; - } - - if in_string { - if ch == '\\' && !is_raw_string { - escape_next = true; - } else if triple_quote { - if ch == string_quote - && index + 2 < chars.len() - && chars[index + 1].1 == string_quote - && chars[index + 2].1 == string_quote - { - in_string = false; - index += 3; - continue; - } - } else if ch == string_quote { - in_string = false; - } - index += 1; - continue; - } - - if ch == '#' { - while index < chars.len() && chars[index].1 != '\n' { - index += 1; - } - continue; - } - - if ch == '\\' { - match chars.get(index + 1).map(|(_, next)| *next) { - Some('\n' | '\r') => { - escape_next = true; - index += 1; - continue; - } - Some(_) => { - index += 2; - continue; - } - None => { - index += 1; - continue; - } - } - } - - if ch == '\'' || ch == '"' { - is_raw_string = false; - for look_back in 1..=2.min(index) { - let prev = chars[index - look_back].1; - if matches!(prev, 'r' | 'R') { - is_raw_string = true; - break; - } - if !matches!(prev, 'b' | 'B' | 'f' | 'F' | 'u' | 'U') { - break; - } - } - string_quote = ch; - if index + 2 < chars.len() && chars[index + 1].1 == ch && chars[index + 2].1 == ch { - triple_quote = true; - in_string = true; - index += 3; - continue; - } - triple_quote = false; - in_string = true; - index += 1; - continue; - } - - match ch { - '(' | '[' | '{' => stack.push((ch, byte_offset, line)), - ')' | ']' | '}' => { - let expected = expected_opening_bracket(ch); - let Some(&(opening, _, opening_line)) = stack.last() else { - return Some(BracketError { - diagnostic: CpythonDiagnostic::new( - format!("unmatched '{ch}'"), - byte_offset, - byte_offset, - ), - unclosed: false, - }); - }; - if opening == expected { - stack.pop(); - } else { - let suffix = if opening_line != line { - format!(" on line {opening_line}") - } else { - String::new() - }; - return Some(BracketError { - diagnostic: CpythonDiagnostic::new( - format!( - "closing parenthesis '{ch}' does not match opening parenthesis '{opening}'{suffix}" - ), - byte_offset, - byte_offset, - ), - unclosed: false, - }); - } - } - _ => {} - } - - index += 1; - } - - stack.last().map(|(opening, byte_offset, _)| BracketError { - diagnostic: CpythonDiagnostic::new( - format!("'{opening}' was never closed"), - *byte_offset, - *byte_offset, - ), - unclosed: true, - }) -} - -fn is_legacy_statement_expression_start(byte: u8) -> bool { - byte >= 0x80 - || byte == b'_' - || byte.is_ascii_alphabetic() - || byte.is_ascii_digit() - || matches!(byte, b'\'' | b'"' | b'{' | b'[') -} - -fn legacy_statement_container_has_invalid_attribute(bytes: &[u8], start: usize) -> bool { - let Some(&opening) = bytes.get(start) else { - return false; - }; - if !matches!(opening, b'{' | b'[') { - return false; - } - - let mut index = start; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\n' | b';' if level == 0 => return false, - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - if level == 0 { - return false; - } - } - b'.' => { - let mut cursor = index + 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if matches!(bytes.get(cursor), Some(b')' | b']' | b'}')) { - return true; - } - index += 1; - } - _ => index += 1, - } - } - false -} - -fn invalid_legacy_statement_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'p' | b'e' => { - let keyword = if starts_identifier(bytes, index, b"print") { - Some("print") - } else if starts_identifier(bytes, index, b"exec") { - Some("exec") - } else { - None - }; - let Some(keyword) = keyword else { - index += 1; - continue; - }; - let after_keyword = index + keyword.len(); - if !matches!(bytes.get(after_keyword), Some(b' ' | b'\t' | b'\x0c')) { - index = after_keyword; - continue; - } - let mut cursor = after_keyword; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if legacy_statement_container_has_invalid_attribute(bytes, cursor) { - index = after_keyword; - continue; - } - if bytes.get(cursor).is_some_and(|byte| { - *byte != b'(' && is_legacy_statement_expression_start(*byte) - }) { - return Some(CpythonDiagnostic::new( - format!( - "Missing parentheses in call to '{keyword}'. Did you mean {keyword}(...)?" - ), - index, - after_keyword, - )); - } - index = after_keyword; - } - _ => index += 1, - } - } - None -} - -/// Return the syntax error for a decimal integer literal exceeding the configured limit. -/// -/// The parser has already distinguished integer literals from strings, comments, floats, and -/// complex numbers. Inspecting its tokens keeps the limit consistent for every source parsing -/// entry point without reimplementing Python's lexer here. -#[must_use] -pub fn long_decimal_integer_literal_error( - source_file: &SourceFile, - tokens: &Tokens, - max_str_digits: usize, -) -> Option { - if max_str_digits == 0 { - return None; - } - tokens.iter().find_map(|token| { - if token.kind() != TokenKind::Int { - return None; - } - let literal = source_file.source_text().slice(token.range()); - if literal - .as_bytes() - .get(..2) - .is_some_and(|prefix| matches!(prefix, b"0x" | b"0X" | b"0o" | b"0O" | b"0b" | b"0B")) - { - return None; - } - let digits = literal.bytes().filter(u8::is_ascii_digit).count(); - (digits > max_str_digits).then(|| { - let start = token.range().start().to_usize(); - CompileError::from_source_error(source_file, CpythonDiagnostic::new(format!( - "Exceeds the limit ({max_str_digits} digits) for integer string conversion: value has {digits} digits; use sys.set_int_max_str_digits() to increase the limit - Consider hexadecimal for huge integer literals to avoid decimal conversion limits." - ), start, start)) - }) - }) -} - -fn invalid_parenthesized_import_star_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'f' if starts_identifier(bytes, index, b"from") => { - let mut cursor = index + 4; - while cursor < bytes.len() && !matches!(bytes[cursor], b'\n' | b';') { - if starts_identifier(bytes, cursor, b"import") { - cursor += 6; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\r')) { - cursor += 1; - } - if bytes.get(cursor) == Some(&b'(') { - cursor += 1; - while cursor < bytes.len() - && !matches!(bytes[cursor], b')' | b'\n' | b';') - { - if bytes[cursor] == b'*' { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - cursor, - cursor + 1, - )); - } - cursor += 1; - } - } - break; - } - cursor += 1; - } - index = cursor; - } - _ => index += 1, - } - } - None -} - -fn prefix_letters_before_quote(bytes: &[u8], quote_index: usize) -> Option<(usize, Vec)> { - let mut letters = Vec::new(); - let mut index = quote_index; - while index > 0 { - let byte = bytes[index - 1]; - if !matches!( - byte, - b'r' | b'R' | b'b' | b'B' | b'u' | b'U' | b'f' | b'F' | b't' | b'T' - ) { - break; - } - letters.push(byte.to_ascii_lowercase()); - index -= 1; - } - if letters.is_empty() { - return None; - } - if index > 0 && (bytes[index - 1] == b'_' || bytes[index - 1].is_ascii_alphabetic()) { - return None; - } - letters.reverse(); - Some((index, letters)) -} - -fn incompatible_string_prefix_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - if let Some((start, letters)) = prefix_letters_before_quote(bytes, index) - && let Some(message) = incompatible_prefix_message(&letters) - { - return Some(CpythonDiagnostic::new(message, start, start + 1)); - } - index = skip_string_token(bytes, index); - } - _ => index += 1, - } - } - None -} - -fn incompatible_prefix_message(letters: &[u8]) -> Option { - if letters.len() < 2 { - return None; - } - let mut seen_r = false; - let mut seen_b = false; - let mut seen_u = false; - let mut seen_f = false; - let mut seen_t = false; - for &letter in letters { - match letter { - b'r' if seen_r => return None, - b'b' if seen_b => return None, - b'u' if seen_u => return None, - b'f' if seen_f => return None, - b't' if seen_t => return None, - b'r' => seen_r = true, - b'b' => seen_b = true, - b'u' => seen_u = true, - b'f' => seen_f = true, - b't' => seen_t = true, - _ => {} - } - } - let pair = if seen_u && seen_b { - ("u", "b") - } else if seen_u && seen_r { - ("u", "r") - } else if seen_u && seen_f { - ("u", "f") - } else if seen_u && seen_t { - ("u", "t") - } else if seen_b && seen_f { - ("b", "f") - } else if seen_b && seen_t { - ("b", "t") - } else if seen_f && seen_t { - ("f", "t") - } else { - return None; - }; - Some(format!( - "'{}' and '{}' prefixes are incompatible", - pair.0, pair.1 - )) -} - -fn malformed_unicode_n_escape_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } + if triple + && bytes.get(index) == Some("e) + && bytes.get(index + 1) == Some("e) + && bytes.get(index + 2) == Some("e) + { + return Some(index + 3); } - b'\'' | b'"' => { - let quote_index = index; - let interpolated = interpolated_string_prefix(bytes, quote_index).is_some(); - let Some((content_start, content_end)) = - quoted_string_content_range(bytes, quote_index, bytes[quote_index]) - else { - index = skip_quoted_string(bytes, quote_index); - continue; - }; - if let Some(error) = malformed_unicode_n_in_content( - bytes, - content_start, - content_end, - quote_index, - interpolated, - ) { - return Some(error); - } - index = skip_string_token(bytes, quote_index); + if !triple && bytes[index] == quote { + return Some(index + 1); } - _ => index += 1, - } - } - None -} - -fn malformed_unicode_n_in_content( - bytes: &[u8], - start: usize, - end: usize, - quote_index: usize, - interpolated: bool, -) -> Option { - let mut index = start; - let mut brace_depth = 0usize; - while index + 1 < end { - if interpolated && brace_depth == 0 && bytes[index] == b'{' { - if bytes.get(index + 1) == Some(&b'{') { + if bytes[index] == b'{' { + if bytes.get(index + 1) == Some(&b'{') { + index += 2; + } else { + brace_depth = 1; + index += 1; + } + } else if bytes[index] == b'}' && bytes.get(index + 1) == Some(&b'}') { index += 2; - continue; + } else { + index += 1; } - brace_depth = 1; - index += 1; - continue; - } - if interpolated && brace_depth > 0 { + } else { match bytes[index] { - b'\'' | b'"' => index = skip_string_token(bytes, index).max(index + 1), + quote_ch @ (b'\'' | b'"') => { + if quoted_string_is_closed(bytes, index) { + index = skip_string_token(bytes, index).max(index + 1); + } else if !triple && quote_ch == quote { + return Some(index + 1); + } else { + index = skip_string_token(bytes, index).max(index + 1); + } + } + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } b'{' => { brace_depth += 1; index += 1; @@ -6572,54 +635,63 @@ fn malformed_unicode_n_in_content( brace_depth -= 1; index += 1; } + b'\\' => index = (index + 2).min(bytes.len()), _ => index += 1, } - continue; - } - if bytes[index] == b'\\' && bytes[index + 1] == b'N' { - let escape_start = index; - if bytes.get(index + 2) == Some(&b'{') { - let mut look = index + 3; - while look < end && bytes[look] != b'}' { - look += 1; - } - if look >= end { - return Some(unicode_n_diagnostic(escape_start, end, start, quote_index)); - } - index = look + 1; - continue; - } - return Some(unicode_n_diagnostic( - escape_start, - escape_start + 2, - start, - quote_index, - )); - } - if interpolated && bytes[index] == b'}' && bytes.get(index + 1) == Some(&b'}') { - index += 2; - continue; } - index += 1; } None } -fn unicode_n_diagnostic( - escape_start: usize, - escape_end: usize, - content_start: usize, - quote_index: usize, -) -> CpythonDiagnostic { - let start = escape_start - content_start; - let end = escape_end.saturating_sub(content_start).saturating_sub(1); - CpythonDiagnostic::new( - format!( - "(unicode error) 'unicodeescape' codec can't decode bytes in position {start}-{end}: malformed \\N character escape" - ), - quote_index, - quote_index + 1, - ) +fn quote_len_at(bytes: &[u8], quote_index: usize) -> usize { + let quote = bytes[quote_index]; + if bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e) { + 3 + } else { + 1 + } +} + +fn interpolated_string_content_range(bytes: &[u8], quote_index: usize) -> Option<(usize, usize)> { + let end = interpolated_string_end(bytes, quote_index)?; + let quote_len = quote_len_at(bytes, quote_index); + Some((quote_index + quote_len, end - quote_len)) +} + +/// Return the syntax error for a decimal integer literal exceeding the configured limit. +/// +/// The parser has already distinguished integer literals from strings, comments, floats, and +/// complex numbers. Inspecting its tokens keeps the limit consistent for every source parsing +/// entry point without reimplementing Python's lexer here. +#[must_use] +pub fn long_decimal_integer_literal_error( + source_file: &SourceFile, + tokens: &Tokens, + max_str_digits: usize, +) -> Option { + if max_str_digits == 0 { + return None; + } + tokens.iter().find_map(|token| { + if token.kind() != TokenKind::Int { + return None; + } + let literal = source_file.source_text().slice(token.range()); + if literal + .as_bytes() + .get(..2) + .is_some_and(|prefix| matches!(prefix, b"0x" | b"0X" | b"0o" | b"0O" | b"0b" | b"0B")) + { + return None; + } + let digits = literal.bytes().filter(u8::is_ascii_digit).count(); + (digits > max_str_digits).then(|| { + let start = token.range().start().to_usize(); + CompileError::from_source_error(source_file, CpythonDiagnostic::new(format!( + "Exceeds the limit ({max_str_digits} digits) for integer string conversion: value has {digits} digits; use sys.set_int_max_str_digits() to increase the limit - Consider hexadecimal for huge integer literals to avoid decimal conversion limits." + ), start, start)) + }) + }) } fn too_many_nested_interpolated_strings(source: &str) -> Option { @@ -6668,76 +740,6 @@ fn too_many_nested_interpolated_strings_in( None } -fn too_many_nested_parentheses_error(source: &str) -> Option { - const MAXLEVEL: usize = 200; - - let bytes = source.as_bytes(); - let mut index = 0; - let mut level = 0usize; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b'(' | b'[' | b'{' => { - if level >= MAXLEVEL { - return Some(CpythonDiagnostic::new( - "too many nested parentheses".to_owned(), - index, - index + 1, - )); - } - level += 1; - index += 1; - } - b')' | b']' | b'}' => { - level = level.saturating_sub(1); - index += 1; - } - _ => index += 1, - } - } - None -} - -fn invalid_unparenthesized_yield_after_comma_error(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => { - index = skip_quoted_string(bytes, index); - } - b',' => { - let mut cursor = index + 1; - while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\x0c')) { - cursor += 1; - } - if starts_identifier(bytes, cursor, b"yield") { - return Some(CpythonDiagnostic::new( - "invalid syntax".to_owned(), - cursor, - cursor + 5, - )); - } - index += 1; - } - _ => index += 1, - } - } - None -} - /// Horizontal and vertical space the tokenizer skips, not `str::trim()`. const fn is_ascii_tokenizer_whitespace(c: char) -> bool { matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c') @@ -6796,6 +798,43 @@ pub fn leading_byte_order_mark_error(source_file: &SourceFile) -> Option Option { + const MAXLEVEL: usize = 200; + + let bytes = source.as_bytes(); + let mut index = 0; + let mut level = 0usize; + while index < bytes.len() { + match bytes[index] { + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'\'' | b'"' => { + index = skip_quoted_string(bytes, index); + } + b'(' | b'[' | b'{' => { + if level >= MAXLEVEL { + return Some(CpythonDiagnostic::new( + "too many nested parentheses".to_owned(), + index, + index, + )); + } + level += 1; + index += 1; + } + b')' | b']' | b'}' => { + level = level.saturating_sub(1); + index += 1; + } + _ => index += 1, + } + } + None +} + /// Source-level errors raised before parsing, as CPython's tokenizer does. /// /// Checking after the parse would build the tree first, and one nested that @@ -6818,17 +857,7 @@ fn post_parse_source_error( if let Some(error) = too_many_nested_interpolated_strings(source_file.source_text()) { return Some(CompileError::from_source_error(source_file, error)); } - if let Some(error) = - long_decimal_integer_literal_error(source_file, tokens, opts.int_max_str_digits) - { - return Some(error); - } - invalid_call_argument_error(source_file.source_text()) - .or_else(|| invalid_match_mapping_rest_wildcard_error(source_file.source_text())) - .or_else(|| invalid_match_as_target_error(source_file.source_text())) - .or_else(|| invalid_unparenthesized_yield_after_comma_error(source_file.source_text())) - .or_else(|| invalid_parenthesized_import_star_error(source_file.source_text())) - .map(|error| CompileError::from_source_error(source_file, error)) + long_decimal_integer_literal_error(source_file, tokens, opts.int_max_str_digits) } fn is_compound_stmt(stmt: &ast::Stmt) -> bool { @@ -7006,7 +1035,7 @@ pub fn unsupported_grammar_error(ast: &ast::Mod, source_file: &SourceFile) -> Op match expr { ast::Expr::FString(fstring) => { for part in &fstring.value { - if let ast::FStringPart::FString(part) = part { + if let ast::FStringPartRef::FString(part) = part { self.check_format_specs("f-string", &part.elements, 0); } } @@ -7042,22 +1071,6 @@ fn single_mode_body_error(body: &[ast::Stmt], source_file: &SourceFile) -> Optio let first_start = source_code.source_location(first.range().start(), PositionEncoding::Utf8); let first_end = source_code.source_location(first.range().end(), PositionEncoding::Utf8); - if body.iter().skip(1).any(|stmt| { - source_code - .source_location(stmt.range().start(), PositionEncoding::Utf8) - .line - > first_start.line - }) { - return Some(CompileError::from_source_error( - source_file, - CpythonDiagnostic::new( - "multiple statements found while compiling a single statement".to_owned(), - first.range().end().to_usize(), - first.range().end().to_usize(), - ), - )); - } - if is_compound_stmt(first) && first_start.line == first_end.line && !ends_with_line_break(source_file.source_text()) @@ -7074,6 +1087,77 @@ fn single_mode_body_error(body: &[ast::Stmt], source_file: &SourceFile) -> Optio None } +/// Converts the first parse error. A tokenizer error that follows an earlier syntax error is +/// only found by the pass over the rest of the source, so it never asks for more input. +fn first_parse_error( + parsed: parser::Parsed, + source_file: &SourceFile, + mode: Mode, +) -> Result, CompileError> { + let Some((first, rest)) = parsed.errors().split_first() else { + return Ok(parsed); + }; + let after_syntax_error = rest.iter().any(|error| { + !matches!(error.error, ParseErrorType::Lexical(_)) + && error.location.start() < first.location.start() + }); + let mut error = CompileError::from_ruff_parse_error(first.clone(), source_file, mode); + if after_syntax_error && let CompileError::Parse(error) = &mut error { + error.is_unclosed_string = false; + } + Err(error) +} + +/// Single mode reads only the first statement when it is a simple statement, so any code after +/// its line is reported as multiple statements, ahead of errors the rest of the source has. +#[must_use] +pub fn single_mode_multiple_statements_error( + source_file: &SourceFile, + parsed: &parser::Parsed, +) -> Option { + let ast::Mod::Module(module) = parsed.syntax() else { + return None; + }; + if is_compound_stmt(module.body.first()?) { + return None; + } + let tokens = parsed.tokens(); + let newline_index = tokens + .iter() + .position(|token| token.kind() == TokenKind::Newline)?; + let newline = &tokens[newline_index]; + if parsed + .errors() + .iter() + .any(|error| error.location.start() < newline.end()) + { + return None; + } + let mut rest = &source_file.source_text()[newline.end().to_usize()..]; + loop { + rest = rest.trim_start_matches([' ', '\t', '\n', '\r', '\x0c']); + match rest.strip_prefix('#') { + Some(comment) => rest = comment.find('\n').map_or("", |end| &comment[end..]), + None if rest.is_empty() => return None, + None => break, + } + } + // The error is at the newline token, which starts at a trailing comment. + let position = match newline_index.checked_sub(1).map(|index| &tokens[index]) { + Some(comment) if comment.kind() == TokenKind::Comment => comment.start(), + _ => newline.start(), + } + .to_usize(); + Some(CompileError::from_source_error( + source_file, + CpythonDiagnostic::new( + "multiple statements found while compiling a single statement".to_owned(), + position, + position, + ), + )) +} + fn single_mode_source_error(ast: &ast::Mod, source_file: &SourceFile) -> Option { let ast::Mod::Module(module) = ast else { return None; @@ -7277,12 +1361,16 @@ fn _compile_with_syntax_warning_handler<'a>( .contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), ); pre_parse_source_error(&source_file)?; - let parsed = parser::parse(barry_source.source(), parser_options); - if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + let parsed = parser::parse_unchecked(barry_source.source(), parser_options); + if matches!(mode, Mode::Single) + && let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) + { return Err(error); } - let parsed = - parsed.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; + if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { + return Err(error); + } + let parsed = first_parse_error(parsed, &source_file, mode)?; if matches!(mode, Mode::Single) && let Some(error) = single_mode_blank_source_error(&source_file) { @@ -7580,12 +1668,16 @@ pub fn _compile_symtable( let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { pre_parse_source_error(&source_file)?; - let parsed = ruff_python_parser::parse(barry_source.source(), parser_options); - if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), parser_options); + if matches!(mode, Mode::Single) + && let Some(error) = single_mode_multiple_statements_error(&source_file, &parsed) + { return Err(error); } - let ast = - parsed.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; + if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { + return Err(error); + } + let ast = first_parse_error(parsed, &source_file, mode)?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -7608,12 +1700,11 @@ pub fn _compile_symtable( } Mode::Eval => { pre_parse_source_error(&source_file)?; - let parsed = ruff_python_parser::parse(barry_source.source(), parser_options); - if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), parser_options); + if let Some(error) = barry_source.diagnostic(parsed.errors().first(), &source_file) { return Err(error); } - let ast = - parsed.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; + let ast = first_parse_error(parsed, &source_file, mode)?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -7732,11 +1823,11 @@ mod tests { ), ( "t\"x\" b\"y\"", - "Cannot mix t-string literals with string or bytes literals", + "cannot mix t-string literals with string or bytes literals", ), ( "b\"x\" t\"y\"", - "Cannot mix t-string literals with string or bytes literals", + "cannot mix t-string literals with string or bytes literals", ), // The literals ahead of the first t-string already mix, so CPython's first pass // raises from `_PyPegen_concatenate_strings` before the t-string rule is reached. @@ -7819,7 +1910,6 @@ mod tests { "(print x; '\\N'", "Missing parentheses in call to 'print'. Did you mean print(...)?", ), - ("f'{x'; '", "f-string: expecting '}'"), ("f'{x'; 0x", "f-string: expecting '}'"), ( "print x; f'{x'; 0x", @@ -7835,7 +1925,10 @@ mod tests { "f-string: valid expression required before '}'", ), ("{\\'a\\'}", "unexpected character after line continuation"), - ("\"\\\n\"(1 for c in I,\\\n\\", "'(' was never closed"), + ( + "\"\\\n\"(1 for c in I,\\\n\\", + "unexpected character after line continuation character", + ), ( r"'\N'", "(unicode error) 'unicodeescape' codec can't decode bytes in position 0-1: malformed \\N character escape", diff --git a/crates/stdlib/src/_tokenize.rs b/crates/stdlib/src/_tokenize.rs index b439ddb6e0b..65d6755632a 100644 --- a/crates/stdlib/src/_tokenize.rs +++ b/crates/stdlib/src/_tokenize.rs @@ -210,22 +210,22 @@ mod _tokenize { let kind = token.kind(); let range = token.range(); - // Check for lexical indentation errors. - // Skip when source has tabs — ruff and CPython handle tab - // indentation differently (CPython uses tabsize=8), so ruff may - // report false IndentationErrors for valid mixed-tab code. - if !source.contains('\t') { - for err in errors.iter() { - if !matches!( - err.error, - ParseErrorType::Lexical(LexicalErrorType::IndentationError) - ) { - continue; - } - if err.location.start() <= range.start() && range.start() < err.location.end() { - return Err(raise_indentation_error(vm, err, source, line_index)); - } - } + // An indentation error replaces the token the lexer raised it for. + if kind == TokenKind::Unknown + && let Some(err) = errors.iter().find(|err| { + line_index.line_index(err.location.start()) + == line_index.line_index(range.start()) + && matches!( + err.error, + ParseErrorType::Lexical( + LexicalErrorType::IndentationError + | LexicalErrorType::TabError + | LexicalErrorType::TooDeepIndentation + ) + ) + }) + { + return Err(raise_indentation_error(vm, err, source, line_index)); } if kind == TokenKind::EndOfFile { @@ -480,7 +480,7 @@ mod _tokenize { exc } - /// Raise an IndentationError from a parse error. + /// Raise an IndentationError or TabError from a parse error. fn raise_indentation_error( vm: &VirtualMachine, err: &ParseError, @@ -491,10 +491,15 @@ mod _tokenize { let err_line_text = source.full_line_str(err.location.start()); let err_text = err_line_text.trim_end_matches('\n').trim_end_matches('\r'); let msg = format!("{}", err.error); - let exc = vm.new_exception_msg( - vm.ctx.exceptions.indentation_error.to_owned(), - msg.clone().into(), - ); + let exc_type = if matches!( + err.error, + ParseErrorType::Lexical(LexicalErrorType::TabError) + ) { + vm.ctx.exceptions.tab_error + } else { + vm.ctx.exceptions.indentation_error + }; + let exc = vm.new_exception_msg(exc_type.to_owned(), msg.clone().into()); let obj = exc.as_object(); let _ = obj.set_attr("lineno", vm.ctx.new_int(err_lc.line.get()), vm); let _ = obj.set_attr("offset", vm.ctx.new_int(err_text.len() as i64 + 1), vm); @@ -658,7 +663,7 @@ mod _tokenize { const fn token_kind_value(kind: TokenKind) -> u8 { match kind { TokenKind::EndOfFile => 0, - TokenKind::Name + TokenKind::Identifier | TokenKind::For | TokenKind::In | TokenKind::Pass diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index bba073d5b0c..241ff35eaef 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -1815,6 +1815,12 @@ pub(crate) fn parse( ); rustpython_compiler::pre_parse_source_error(&source_file)?; let parsed = parser::parse_unchecked(barry_source.source(), options); + if interactive + && let Some(error) = + rustpython_compiler::single_mode_multiple_statements_error(&source_file, &parsed) + { + return Err(error); + } let type_comment_source = type_comments.then(|| TypeCommentSource::new(source, parsed.tokens())); if let Some(lines) = &type_comment_source diff --git a/crates/vm/src/stdlib/_ast/argument.rs b/crates/vm/src/stdlib/_ast/argument.rs index 0ea9725a168..687c557b429 100644 --- a/crates/vm/src/stdlib/_ast/argument.rs +++ b/crates/vm/src/stdlib/_ast/argument.rs @@ -7,7 +7,7 @@ pub(super) struct PositionalArguments { } enum PositionalArgumentsKind { - Args(Box<[ast::Expr]>), + Args(thin_vec::ThinVec), RuntimeValues(Vec>), } @@ -24,7 +24,7 @@ impl PositionalArguments { Ok(Self::from_values(TextRange::default(), values)) } - fn from_args(range: TextRange, args: Box<[ast::Expr]>) -> Self { + fn from_args(range: TextRange, args: thin_vec::ThinVec) -> Self { Self { range, kind: PositionalArgumentsKind::Args(args), @@ -42,14 +42,7 @@ impl PositionalArguments { if values.iter().any(Option::is_none) { Self::from_runtime_values(range, values) } else { - Self::from_args( - range, - values - .into_iter() - .flatten() - .collect::>() - .into_boxed_slice(), - ) + Self::from_args(range, values.into_iter().flatten().collect()) } } @@ -57,13 +50,14 @@ impl PositionalArguments { self.range } - fn into_args_and_runtime_values(self) -> (Box<[ast::Expr]>, Option>>) { + fn into_args_and_runtime_values( + self, + ) -> (thin_vec::ThinVec, Option>>) { match self.kind { PositionalArgumentsKind::Args(args) => (args, None), - PositionalArgumentsKind::RuntimeValues(values) => ( - lower_runtime_expr_list(values.clone()).into_boxed_slice(), - Some(values), - ), + PositionalArgumentsKind::RuntimeValues(values) => { + (lower_runtime_expr_list(values.clone()).into(), Some(values)) + } } } } @@ -71,7 +65,7 @@ impl PositionalArguments { impl Node for PositionalArguments { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { match self.kind { - PositionalArgumentsKind::Args(args) => BoxedSlice(args).ast_to_object(vm, source_file), + PositionalArgumentsKind::Args(args) => args.ast_to_object(vm, source_file), PositionalArgumentsKind::RuntimeValues(values) => values.ast_to_object(vm, source_file), } } @@ -81,8 +75,8 @@ impl Node for PositionalArguments { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult { - let args: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; - Ok(Self::from_args(TextRange::default(), args.0)) + let args: thin_vec::ThinVec<_> = Node::ast_from_object(vm, source_file, object)?; + Ok(Self::from_args(TextRange::default(), args)) } } @@ -137,7 +131,7 @@ pub(super) fn merge_function_call_arguments( node_index: Default::default(), range, args, - keywords: key_args.keywords.into(), + keywords: key_args.keywords.into_vec().into(), runtime_args, runtime_bases: None, } @@ -173,7 +167,7 @@ pub(super) fn split_function_call_arguments( // debug_assert!(range.contains_range(keyword_arguments_range)); let keyword_arguments = KeywordArguments { range: keyword_arguments_range, - keywords: keywords.into(), + keywords: keywords.into_iter().collect::>().into_boxed_slice(), }; (positional_arguments, keyword_arguments) @@ -214,7 +208,7 @@ pub(super) fn split_class_def_args( // debug_assert!(range.contains_range(keyword_arguments_range)); let keyword_arguments = KeywordArguments { range: keyword_arguments_range, - keywords: keywords.into(), + keywords: keywords.into_iter().collect::>().into_boxed_slice(), }; (Some(positional_arguments), Some(keyword_arguments)) @@ -231,7 +225,7 @@ pub(super) fn merge_class_def_args( let (args, runtime_bases) = if let Some(positional_arguments) = positional_arguments { positional_arguments.into_args_and_runtime_values() } else { - (vec![].into_boxed_slice(), None) + (thin_vec::ThinVec::new(), None) }; let keywords = if let Some(keyword_arguments) = keyword_arguments { keyword_arguments.keywords @@ -243,7 +237,7 @@ pub(super) fn merge_class_def_args( node_index: Default::default(), range: Default::default(), // TODO args, - keywords: keywords.into(), + keywords: keywords.into_vec().into(), runtime_args: None, runtime_bases, })) diff --git a/crates/vm/src/stdlib/_ast/expression.rs b/crates/vm/src/stdlib/_ast/expression.rs index 6d5e21302fe..aae354e50eb 100644 --- a/crates/vm/src/stdlib/_ast/expression.rs +++ b/crates/vm/src/stdlib/_ast/expression.rs @@ -866,7 +866,8 @@ fn expr_dict_comp_from_object_with_range( "DictComp", )?), value: get_required_node_field(vm, source_file, object, "value", "DictComp")?, - generators: get_node_list_field(vm, source_file, object, "generators", "DictComp")?, + generators: get_node_list_field(vm, source_file, object, "generators", "DictComp")? + .into_boxed_slice(), range, }) } @@ -888,8 +889,12 @@ impl Node for ast::ExprDictComp { .unwrap(); dict.set_item("value", value.ast_to_object(vm, source_file), vm) .unwrap(); - dict.set_item("generators", generators.ast_to_object(vm, source_file), vm) - .unwrap(); + dict.set_item( + "generators", + BoxedSlice(generators).ast_to_object(vm, source_file), + vm, + ) + .unwrap(); node_add_location(&dict, range, vm, source_file); node.into() } @@ -1082,14 +1087,17 @@ fn expr_compare_from_object_with_range( object: &PyObject, range: TextRange, ) -> PyResult { + let left: ast::Expr = get_required_node_field(vm, source_file, object, "left", "Compare")?; let comparators: Vec> = get_node_list_field(vm, source_file, object, "comparators", "Compare")?; let (runtime_comparators, comparators) = runtime_expr_boxed_slice_from_values(comparators); + let mut operands = Vec::with_capacity(comparators.len() + 1); + operands.push(left); + operands.extend(comparators); Ok(ast::ExprCompare { node_index: Default::default(), - left: get_required_node_field(vm, source_file, object, "left", "Compare")?, ops: get_node_boxed_slice_field(vm, source_file, object, "ops", "Compare")?, - comparators, + operands: operands.into_boxed_slice(), range, runtime_comparators, }) @@ -1099,9 +1107,8 @@ impl Node for ast::ExprCompare { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { node_index: _, - left, ops, - comparators, + operands, range, runtime_comparators, } = self; @@ -1109,10 +1116,13 @@ impl Node for ast::ExprCompare { .into_ref_with_type(vm, pyast::NodeExprCompare::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); + let mut operands = operands.into_vec().into_iter(); + let left = operands.next().expect("A comparison has a left operand"); dict.set_item("left", left.ast_to_object(vm, source_file), vm) .unwrap(); dict.set_item("ops", BoxedSlice(ops).ast_to_object(vm, source_file), vm) .unwrap(); + let comparators: Box<[ast::Expr]> = operands.collect(); let comparators = runtime_comparators.map_or_else( || BoxedSlice(comparators).ast_to_object(vm, source_file), |values| values.ast_to_object(vm, source_file), diff --git a/crates/vm/src/stdlib/_ast/string.rs b/crates/vm/src/stdlib/_ast/string.rs index 553b235c46c..e3b1c94754e 100644 --- a/crates/vm/src/stdlib/_ast/string.rs +++ b/crates/vm/src/stdlib/_ast/string.rs @@ -320,6 +320,35 @@ fn ruff_format_spec_to_joined_str( } } +fn dummy_string_literal() -> ast::StringLiteral { + ast::StringLiteral { + node_index: Default::default(), + range: Default::default(), + value: "".into(), + flags: ast::StringLiteralFlags::empty(), + } +} + +fn dummy_fstring() -> ast::FString { + ast::FString { + node_index: Default::default(), + range: Default::default(), + elements: Default::default(), + flags: ast::FStringFlags::empty(), + } +} + +fn take_fstring_part(part: ast::FStringPartMut<'_>) -> ast::FStringPart { + match part { + ast::FStringPartMut::Literal(literal) => { + ast::FStringPart::Literal(core::mem::replace(literal, dummy_string_literal())) + } + ast::FStringPartMut::FString(fstring) => { + ast::FStringPart::FString(core::mem::replace(fstring, dummy_fstring())) + } + } +} + fn ruff_fstring_element_to_ruff_fstring_part( element: ast::InterpolatedStringElement, ) -> ast::FStringPart { @@ -362,16 +391,9 @@ fn format_spec_expr_to_ruff_format_spec( runtime_joined_str: _, runtime_values: _, } = fstring; - let default_part = ast::FStringPart::FString(ast::FString { - node_index: Default::default(), - range: Default::default(), - elements: Default::default(), - flags: ast::FStringFlags::empty(), - }); let mut elements = Vec::new(); - for i in 0..value.as_slice().len() { - let part = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_part.clone()); - match part { + for part in value { + match take_fstring_part(part) { ast::FStringPart::Literal(ast::StringLiteral { range, value, @@ -718,16 +740,9 @@ pub(super) fn fstring_to_object( .ast_to_object(vm, source_file); } - let default_part = ast::FStringPart::FString(ast::FString { - node_index: Default::default(), - range: Default::default(), - elements: Default::default(), - flags: ast::FStringFlags::empty(), - }); let mut values = Vec::new(); - for i in 0..value.as_slice().len() { - let part = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_part.clone()); - match part { + for part in &mut value { + match take_fstring_part(part) { ast::FStringPart::Literal(literal) => { values.push(JoinedStrPart::Constant(Constant::new_str( rustpython_codegen::string_literal_part_value(source_file, &literal), diff --git a/crates/vm/src/stdlib/_ast/validate.rs b/crates/vm/src/stdlib/_ast/validate.rs index cbdc15b53a4..2bd51611e12 100644 --- a/crates/vm/src/stdlib/_ast/validate.rs +++ b/crates/vm/src/stdlib/_ast/validate.rs @@ -616,10 +616,11 @@ fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) - validate_expr(vm, &await_expr.value, ast::ExprContext::Load) } ast::Expr::Compare(compare) => { - if compare.comparators.is_empty() { + let comparators = compare.operands.get(1..).unwrap_or(&[]); + if comparators.is_empty() { return Err(vm.new_value_error("Compare with no comparators")); } - if compare.comparators.len() != compare.ops.len() { + if comparators.len() != compare.ops.len() { return Err(vm.new_value_error( "Compare has a different number of comparators and operands", )); @@ -629,8 +630,8 @@ fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) - compare.runtime_comparators.as_ref(), ast::ExprContext::Load, )?; - validate_exprs(vm, &compare.comparators, ast::ExprContext::Load, false)?; - validate_expr(vm, &compare.left, ast::ExprContext::Load) + validate_exprs(vm, comparators, ast::ExprContext::Load, false)?; + validate_expr(vm, compare.first_operand(), ast::ExprContext::Load) } ast::Expr::Call(call) => { validate_expr(vm, &call.func, ast::ExprContext::Load)?; diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 626659f19e1..b5f7c77c772 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -1506,9 +1506,9 @@ mod escape_warnings { } // F-string literals — tokenizer + string_parser paths ast::Expr::FString(fstring_expr) => { - for part in fstring_expr.value.as_slice() { + for part in &fstring_expr.value { match part { - ast::FStringPart::Literal(string_lit) => { + ast::FStringPartRef::Literal(string_lit) => { // Plain string part in f-string concatenation if !matches!( string_lit.flags.prefix(), @@ -1517,7 +1517,7 @@ mod escape_warnings { self.check_quoted_literal(string_lit.range, false); } } - ast::FStringPart::FString(fstring) => { + ast::FStringPartRef::FString(fstring) => { if matches!( fstring.flags.prefix(), ast::str_prefix::FStringPrefix::Raw { .. } diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index c8f8d72efb2..1a1696b7106 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -1,7 +1,5 @@ #[cfg(feature = "parser")] -use ruff_python_ast::token::TokenKind; - -use ruff_python_parser::{InterpolatedStringErrorType, LexicalErrorType, ParseErrorType}; +use ruff_python_parser::{LexicalErrorType, ParseErrorType}; use rustpython_common::wtf8::Wtf8Buf; use rustpython_compiler_core::SourceLocation; @@ -9,7 +7,7 @@ use rustpython_compiler_core::SourceLocation; use core::ops::RangeInclusive; #[cfg(feature = "parser")] -use rustpython_compiler::{CompileError, ParseError, is_blank_python_source}; +use rustpython_compiler::is_blank_python_source; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, @@ -49,262 +47,6 @@ macro_rules! define_exception_fn { }; } -#[derive(Clone, Debug)] -struct SyntaxErrorInfo { - msg: String, - narrow_caret: bool, -} - -impl SyntaxErrorInfo { - #[must_use] - const fn new(msg: String, narrow_caret: bool) -> Self { - Self { msg, narrow_caret } - } - - #[cfg(feature = "parser")] - #[must_use] - const fn handle_expected_token(expected: TokenKind, found: TokenKind) -> &'static str { - match (expected, found) { - (TokenKind::Colon, TokenKind::Newline) => "expected ':'", - - (TokenKind::Lpar, _) => "expected '('", - - (TokenKind::Else, y) if !matches!(y, TokenKind::Colon) => { - "expected 'else' after 'if' expression" - } - - _ => "invalid syntax", - } - } - - #[cfg(feature = "parser")] - fn analyze_compile_error(&mut self, compile_error: &CompileError) { - let CompileError::Parse(ParseError { - error, location, .. - }) = compile_error - else { - return; - }; - - let msg = match error { - ParseErrorType::FStringError(InterpolatedStringErrorType::UnterminatedString) - | ParseErrorType::Lexical(LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedString, - )) => "unterminated f-string literal".into(), - - ParseErrorType::TStringError(InterpolatedStringErrorType::UnterminatedString) - | ParseErrorType::Lexical(LexicalErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedString, - )) => "unterminated t-string literal".into(), - - ParseErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - ) - | ParseErrorType::Lexical(LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - )) => "unterminated triple-quoted f-string literal".into(), - - ParseErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - ) - | ParseErrorType::Lexical(LexicalErrorType::TStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, - )) => "unterminated triple-quoted t-string literal".into(), - - // ruff already prefixes these with `f-string: ` / `t-string: `, matching CPython. - // It quotes braces with backticks where CPython uses single quotes. - ParseErrorType::FStringError(_) - | ParseErrorType::TStringError(_) - | ParseErrorType::Lexical( - LexicalErrorType::FStringError(_) | LexicalErrorType::TStringError(_), - ) => self.msg.replace('`', "'"), - - ParseErrorType::UnexpectedExpressionToken => "invalid syntax".into(), - - ParseErrorType::ExpectedToken { expected, found } => { - Self::handle_expected_token(*expected, *found).into() - } - - ParseErrorType::InvalidStarredExpressionUsage => { - self.narrow_caret = true; - "invalid syntax".into() - } - - ParseErrorType::InvalidDeleteTarget => "invalid syntax".into(), - - ParseErrorType::Lexical(LexicalErrorType::LineContinuationError) => { - "unexpected character after line continuation character".into() - } - - ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError) => { - format!( - "unterminated string literal (detected at line {})", - location.line - ) - } - - ParseErrorType::EmptyTypeParams => "Type parameter list cannot be empty".into(), - - ParseErrorType::InvalidStarPatternUsage => { - self.narrow_caret = true; - "cannot use starred expression here".into() - } - - ParseErrorType::ExpectedKeywordParam => "named arguments must follow bare *".into(), - - ParseErrorType::EmptyImportNames => "Expected one or more names after 'import'".into(), - - ParseErrorType::UnparenthesizedGeneratorExpression => { - "Generator expression must be parenthesized".into() - } - - ParseErrorType::NonDefaultParamAfterDefaultParam => { - "parameter without a default follows parameter with a default".into() - } - - ParseErrorType::VarParameterWithDefault => { - "var-positional argument cannot have default value".into() - } - - ParseErrorType::PositionalAfterKeywordArgument => { - "positional argument follows keyword argument".into() - } - - ParseErrorType::PositionalAfterKeywordUnpacking => { - "positional argument follows keyword argument unpacking".into() - } - - ParseErrorType::InvalidArgumentUnpackingOrder => { - "iterable argument unpacking follows keyword argument unpacking".into() - } - - ParseErrorType::ParamAfterVarKeywordParam => { - "arguments cannot follow var-keyword argument".into() - } - - ParseErrorType::InvalidAnnotatedAssignmentTarget => { - "illegal target for annotation".into() - } - - ParseErrorType::Lexical(LexicalErrorType::UnrecognizedToken { .. }) - | ParseErrorType::SimpleStatementsOnSameLine - | ParseErrorType::SimpleAndCompoundStatementOnSameLine - | ParseErrorType::ExpectedExpression => "invalid syntax".into(), - - ParseErrorType::OtherError(s) if s.starts_with("Expected an identifier") => { - "invalid syntax".into() - } - - // What the parser says when it cannot continue the list it is - // recovering; each of these situations is a plain "invalid syntax". - ParseErrorType::OtherError(s) - if matches!( - s.as_str(), - "Expected a statement" - | "Expected an `elif` or `else` clause, or the end of the `if` statement." - | "Expected an `except` or `finally` clause or the end of the `try` statement." - | "The keyword is not allowed as a variable declaration name" - | "Expected an assignment target" - | "Expected a type parameter or the end of the type parameter list" - | "Expected an import name or a ')'" - | "Expected an import name" - | "Expected an expression or the end of the slice list" - | "Expected an expression or a ']'" - | "Expected an expression or a '}'" - | "Expected an expression or a ')'" - | "Expected an expression" - | "Expected a pattern or the end of the sequence pattern" - | "Expected a mapping pattern or the end of the mapping pattern" - | "Expected a pattern or a ')'" - | "Expected a delete target" - | "Expected a parameter or the end of the parameter list" - | "Expected an expression or the end of the with item list" - ) => - { - "invalid syntax".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - "bytes literal cannot be mixed with non-bytes literals", - ) => - { - "cannot mix bytes and nonbytes literals".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("positional patterns cannot follow keyword patterns") => - { - "positional patterns follow keyword patterns".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("boolean 'not' expression cannot be used here") => - { - "'not' after an operator must be parenthesized".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("trailing comma not allowed") => - { - "trailing comma not allowed without surrounding parentheses".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - "multiple exception types must be parenthesized when using `as`", - ) => - { - "multiple exception types must be parenthesized when using 'as'".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - "position-only parameter separator not allowed as first parameter", - ) => - { - "at least one argument must precede /".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("only one '/' separator allowed") => - { - "/ may appear only once".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("'/' parameter must appear before '*' parameter") => - { - "/ must be ahead of *".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("expected `except` or `finally` after `try` block") => - { - "expected 'except' or 'finally' block".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case("only one '*' parameter allowed") => - { - "* argument may appear only once".into() - } - - ParseErrorType::OtherError(s) - if s.eq_ignore_ascii_case( - r#"cannot have both 'except' and 'except*' on the same 'try'"#, - ) => - { - r#"cannot have both 'except' and 'except*' on the same 'try'"#.into() - } - - _ => return, - }; - - self.msg = msg; - } -} - /// Collection of object creation helpers impl VirtualMachine { /// Create a new python object @@ -703,38 +445,6 @@ impl VirtualMachine { .expect("UnicodeEncodeError constructor") } - #[cfg(feature = "parser")] - fn source_has_mixed_tabs_and_spaces(source: Option<&str>, error_line: usize) -> bool { - source.is_some_and(|source| { - let mut has_space_indent = false; - let mut has_tab_indent = false; - for (i, line) in source.lines().enumerate() { - if i + 1 > error_line { - break; - } - let rest = line.trim_start_matches([' ', '\t']); - // Blank and comment-only lines do not participate in indent. - if rest.is_empty() || rest.starts_with('#') { - continue; - } - let indent = &line.as_bytes()[..line.len() - rest.len()]; - if indent.is_empty() { - continue; - } - if indent.contains(&b' ') && indent.contains(&b'\t') { - return true; - } - if indent.contains(&b' ') { - has_space_indent = true; - } - if indent.contains(&b'\t') { - has_tab_indent = true; - } - } - has_space_indent && has_tab_indent - }) - } - // TODO: don't take ownership should make the success path faster pub fn new_key_error(&self, obj: PyObjectRef) -> PyBaseExceptionRef { let key_error = self.ctx.exceptions.key_error.to_owned(); @@ -767,27 +477,29 @@ impl VirtualMachine { }; let syntax_error_type = match &error { + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::TabError), + .. + }) => self.ctx.exceptions.tab_error, #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: - ruff_python_parser::ParseErrorType::Lexical( - ruff_python_parser::LexicalErrorType::IndentationError, + ParseErrorType::Lexical( + LexicalErrorType::IndentationError | LexicalErrorType::TooDeepIndentation, ) - | ruff_python_parser::ParseErrorType::UnexpectedIndentation, - location, + | ParseErrorType::UnexpectedIndentation, .. - }) => { - if Self::source_has_mixed_tabs_and_spaces(source, location.line.get()) { - self.ctx.exceptions.tab_error - } else { - self.ctx.exceptions.indentation_error - } - } + }) => self.ctx.exceptions.indentation_error, #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::Lexical( - ruff_python_parser::LexicalErrorType::Eof, + ruff_python_parser::LexicalErrorType::Eof + | ruff_python_parser::LexicalErrorType::UnclosedBracket { + incomplete: true, + .. + }, ), .. }) => incomplete_or_syntax(allow_incomplete), @@ -807,26 +519,31 @@ impl VirtualMachine { error: ruff_python_parser::ParseErrorType::Lexical( ruff_python_parser::LexicalErrorType::FStringError( - ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, ) | ruff_python_parser::LexicalErrorType::TStringError( - ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, ), ), .. }) => incomplete_or_syntax(allow_incomplete), #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { - error: - ruff_python_parser::ParseErrorType::Lexical( - ruff_python_parser::LexicalErrorType::UnclosedStringError, - ), + error: ruff_python_parser::ParseErrorType::ExpectedIndentedBlock { .. }, + raw_location, .. }) => { - if allow_incomplete { - incomplete_or_syntax(source.is_some_and(unclosed_string_is_incomplete)) + // The block can still follow when the error is found at whitespace, that is, at + // the end of the input. + let at_end = source.is_some_and(|source| { + source + .get(raw_location.start().to_usize()..raw_location.end().to_usize()) + .is_some_and(|text| text.chars().all(|c| c.is_ascii_whitespace())) + }); + if allow_incomplete && at_end { + self.ctx.exceptions.incomplete_input_error } else { - self.ctx.exceptions.syntax_error + self.ctx.exceptions.indentation_error } } #[cfg(feature = "parser")] @@ -835,39 +552,7 @@ impl VirtualMachine { raw_location, .. }) => { - if s.starts_with("Expected an indented block after") - || s.starts_with("expected an indented block after") - { - if allow_incomplete { - // Check that all chars in the error are whitespace, if so, the source is - // incomplete. Otherwise, we've found code that might violates - // indentation rules. - let mut is_incomplete = true; - if let Some(source) = source { - let start = raw_location.start().to_usize(); - let end = raw_location.end().to_usize(); - let mut iter = source.chars(); - iter.nth(start); - for _ in start..end { - if let Some(c) = iter.next() { - if !c.is_ascii_whitespace() { - is_incomplete = false; - } - } else { - break; - } - } - } - - if is_incomplete { - self.ctx.exceptions.incomplete_input_error - } else { - self.ctx.exceptions.indentation_error // not syntax_error - } - } else { - self.ctx.exceptions.indentation_error - } - } else if allow_incomplete + if allow_incomplete && (s == "incomplete input" || (s == "unexpected EOF while parsing" && source.is_some_and(|source| { @@ -908,52 +593,12 @@ impl VirtualMachine { source.and_then(|src| get_statement(src, error.location())) }; - let mut msg = error.to_string(); - if !msg.starts_with("Exceeds the limit ") - && !msg.starts_with("Did you mean ") - && !msg.starts_with("Invalid star expression") - && !msg.starts_with("Function parameters cannot be parenthesized") - && !msg.starts_with("Lambda expression parameters cannot be parenthesized") - && !msg.starts_with("Cannot have two type comments on def") - && !msg.starts_with("Variable annotation syntax is") - && !msg.starts_with("The '@' operator is") - && !msg.starts_with("Async functions are") - && !msg.starts_with("Async comprehensions are") - && !msg.starts_with("Async for loops are") - && !msg.starts_with("Async with statements are") - && !msg.starts_with("Exception groups are") - && !msg.starts_with("Positional-only parameters are") - && !msg.starts_with("Pattern matching is") - && !msg.starts_with("Type statement is") - && !msg.starts_with("Type parameter lists are") - && !msg.starts_with("Type parameter defaults are") - && !msg.starts_with("Assignment expressions are") - && !msg.starts_with("Await expressions are") - && !msg.starts_with("Underscores in numeric literals are") - && !msg.starts_with("Missing parentheses") - && let Some(msg) = msg.get_mut(..1) - { - msg.make_ascii_lowercase(); - } - - cfg_select! { - feature = "parser" => { - let mut syntax_error_info = SyntaxErrorInfo::new(msg, false); - syntax_error_info.analyze_compile_error(error); - } - _ => { - let syntax_error_info = SyntaxErrorInfo::new(msg, false); - } + let msg = if syntax_error_type.is(self.ctx.exceptions.incomplete_input_error) { + String::from("incomplete input") + } else { + error.to_string() }; - if syntax_error_type.is(self.ctx.exceptions.tab_error) { - syntax_error_info.msg = - String::from("inconsistent use of tabs and spaces in indentation"); - } else if syntax_error_type.is(self.ctx.exceptions.incomplete_input_error) { - syntax_error_info.msg = String::from("incomplete input"); - } - - let SyntaxErrorInfo { msg, narrow_caret } = syntax_error_info; let unterminated_triple_quoted_string = msg.starts_with("unterminated triple-quoted"); let unexpected_eof_error = msg == "unexpected EOF while parsing"; if unterminated_triple_quoted_string @@ -970,6 +615,50 @@ impl VirtualMachine { || msg.starts_with("except expressions without parentheses are") || msg.starts_with("Pattern matching is"); let line_end_binary_operator_error = msg.starts_with("The '@' operator is"); + // Tokenizer errors other than unexpected EOF end where the tokenizer stopped, which is + // reported as end offset 0 or -1, or where they start. + let tokenizer_end_offset = cfg_select! { + feature = "parser" => { + match error { + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::Lexical( + LexicalErrorType::TabError + | LexicalErrorType::TooDeepIndentation + | LexicalErrorType::LineContinuationError + | LexicalErrorType::UnclosedBracket { .. }, + ), + .. + }) => Some(0), + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::IndentationError) + | ParseErrorType::UnexpectedIndentation, + .. + }) => Some(-1), + // The span runs from the last character of the key to the end of the line. + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::ExpectedColonAfterDictionaryKey, + .. + }) => Some(0), + // Other tokenizer errors end where they start, except those reported + // over a range. + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ParseErrorType::Lexical(lexical), + .. + }) if lexical.is_tokenizer_error() + && !matches!( + lexical, + LexicalErrorType::Eof + | LexicalErrorType::LeadingZerosInDecimalInteger + | LexicalErrorType::IncompatibleStringPrefixes { .. } + ) => + { + Some(error.python_location().1 as isize) + } + _ => None, + } + } + _ => None, + }; let unclosed_bracket_error = cfg_select! { feature = "parser" => { matches!( @@ -1011,15 +700,14 @@ impl VirtualMachine { .is_some_and(|ch| ch.is_ascii_whitespace())); let (end_lineno, end_offset) = if no_end_offset { (end_lineno, -1) + } else if let Some(end_offset) = tokenizer_end_offset { + (end_lineno, end_offset) } else if unclosed_bracket_error { // The bracket that was never closed is marked where it opened, // and the span stops there. (end_lineno, 0) } else if line_end_binary_operator_error && end_offset == offset_raw { (end_lineno, (end_offset + 1) as isize) - } else if narrow_caret { - let (l, o) = error.python_location(); - (l, (o + 1) as isize) } else { (end_lineno, end_offset as isize) }; @@ -1207,119 +895,3 @@ impl VirtualMachine { .upcast() } } - -#[cfg(feature = "parser")] -enum QuotedStringScan { - Closed(usize), - Unclosed { - triple: bool, - unescaped_newline: bool, - }, -} - -/// An unclosed string is incomplete when more input could still finish it. -/// A non-triple string that already hit an unescaped newline is a hard error. -#[cfg(feature = "parser")] -fn unclosed_string_is_incomplete(source: &str) -> bool { - let bytes = source.as_bytes(); - let mut index = 0; - while index < bytes.len() { - match bytes[index] { - b'#' => { - while index < bytes.len() && bytes[index] != b'\n' { - index += 1; - } - } - b'\'' | b'"' => match scan_quoted_string_for_incomplete(bytes, index) { - QuotedStringScan::Closed(end) => index = end, - QuotedStringScan::Unclosed { - triple, - unescaped_newline, - } => { - // Single-quoted f/t-strings never set E_EOLS. - if !triple && interpolated_string_prefix_at(bytes, index) { - return false; - } - return triple || !unescaped_newline; - } - }, - _ => index += 1, - } - } - false -} - -#[cfg(feature = "parser")] -fn interpolated_string_prefix_at(bytes: &[u8], quote: usize) -> bool { - let Some(&prev) = quote.checked_sub(1).and_then(|index| bytes.get(index)) else { - return false; - }; - let lower = prev.to_ascii_lowercase(); - let marker_index = if matches!(lower, b'f' | b't') { - if quote >= 2 && bytes[quote - 2].eq_ignore_ascii_case(&b'r') { - quote - 2 - } else { - quote - 1 - } - } else if lower == b'r' - && quote >= 2 - && matches!(bytes[quote - 2].to_ascii_lowercase(), b'f' | b't') - { - quote - 2 - } else { - return false; - }; - marker_index == 0 || !identifier_continue_before(bytes, marker_index) -} - -#[cfg(feature = "parser")] -fn identifier_continue_before(bytes: &[u8], index: usize) -> bool { - if index == 0 { - return false; - } - if bytes[index - 1].is_ascii() { - return bytes[index - 1] == b'_' || bytes[index - 1].is_ascii_alphanumeric(); - } - let mut start = index - 1; - while start > 0 && bytes[start] & 0b1100_0000 == 0b1000_0000 { - start -= 1; - } - ::core::str::from_utf8(&bytes[start..index]) - .ok() - .and_then(|text| text.chars().next_back()) - .is_some_and(|ch| ch == '_' || ch.is_alphanumeric()) -} - -#[cfg(feature = "parser")] -fn scan_quoted_string_for_incomplete(bytes: &[u8], quote_index: usize) -> QuotedStringScan { - let quote = bytes[quote_index]; - let triple = - bytes.get(quote_index + 1) == Some("e) && bytes.get(quote_index + 2) == Some("e); - let mut index = quote_index + if triple { 3 } else { 1 }; - while index < bytes.len() { - if bytes[index] == b'\\' { - index = (index + 2).min(bytes.len()); - continue; - } - if triple { - if bytes.get(index) == Some("e) - && bytes.get(index + 1) == Some("e) - && bytes.get(index + 2) == Some("e) - { - return QuotedStringScan::Closed(index + 3); - } - } else if bytes[index] == quote { - return QuotedStringScan::Closed(index + 1); - } else if bytes[index] == b'\n' { - return QuotedStringScan::Unclosed { - triple: false, - unescaped_newline: true, - }; - } - index += 1; - } - QuotedStringScan::Unclosed { - triple, - unescaped_newline: false, - } -} diff --git a/extra_tests/snippets/syntax_invalid.py b/extra_tests/snippets/syntax_invalid.py index 37a3805a295..00acf6cfb16 100644 --- a/extra_tests/snippets/syntax_invalid.py +++ b/extra_tests/snippets/syntax_invalid.py @@ -77,3 +77,46 @@ def valid_func(): from __future__ import print_function """ compile(src, 'test.py', 'exec') + + +# CPython reports the unparenthesized `except ... as` error with a range that starts at the +# first exception type and stops just before the `:` closing the clause. `end_offset` is an +# exclusive 1-based *character* column, so that end lands on the column the `:` sits on. +# +# The cases below pin the parts that are easy to get wrong: the column counts characters +# rather than UTF-8 bytes, and an explicit line join may sit anywhere between the exception +# types and the `:`. Values verified against CPython 3.14. +except_as_ranges = [ + # (clause, (lineno, offset), (end_lineno, end_offset)) + ("except A, B as e:", (3, 8), (3, 17)), + ("except A, B as e :", (3, 8), (3, 18)), + ("except A, B as e :", (3, 8), (3, 20)), + ("except A, B as e\t:", (3, 8), (3, 18)), + ("except A, B, C as blech:", (3, 8), (3, 24)), + ("except* A, B, C as e:", (3, 9), (3, 21)), + # A non-ASCII name must not push the column right by its extra UTF-8 bytes. + ("except Ä, B as e:", (3, 8), (3, 17)), + ("except 사과, B as e:", (3, 8), (3, 18)), + ("except A, B as 오:", (3, 8), (3, 17)), + # An explicit line join may precede or follow `as`, or close the clause on its own line. + ("except A, B as exc\\\n :", (3, 8), (4, 5)), + ("except A, B \\\nas exc:", (3, 8), (4, 7)), + ("except A, \\\nB as exc:", (3, 8), (4, 9)), + ("except A, B as \\\nexc:", (3, 8), (4, 4)), +] + +for clause, start, end in except_as_ranges: + src = "try:\n pass\n%s\n pass\n" % clause + with assert_raises(SyntaxError) as ae: + compile(src, "test.py", "exec") + exc = ae.exception + assert exc.msg == "multiple exception types must be parenthesized when using 'as'", ( + clause, + exc.msg, + ) + assert (exc.lineno, exc.offset) == start, (clause, (exc.lineno, exc.offset), start) + assert (exc.end_lineno, exc.end_offset) == end, ( + clause, + (exc.end_lineno, exc.end_offset), + end, + ) diff --git a/src/shell.rs b/src/shell.rs index d727003e6d6..a899370ac47 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -60,29 +60,18 @@ fn shell_exec( error: ParseErrorType::Lexical( LexicalErrorType::Eof | LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString, + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, ) + | LexicalErrorType::UnclosedStringError { + triple_quoted: true, + .. + } ), .. })) ) { return ShellExecResult::ContinueLine; } - if let VmCompileError::Compile(CompileError::Parse(ParseError { - error: ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError), - raw_location, - .. - })) = &err - { - let loc = raw_location.start().to_usize(); - let mut iter = source.chars(); - if let Some(quote) = iter.nth(loc) - && iter.next() == Some(quote) - && iter.next() == Some(quote) - { - return ShellExecResult::ContinueLine; - } - } // An unfinished suite is _IncompleteInputError, not IndentationError. let exc = err.into_pyexception_maybe_incomplete(vm, Some(source), !empty_line_given); From ef5c651958b23ccb7f205fe52f5c28b80f1791e4 Mon Sep 17 00:00:00 2001 From: Jiwoo Ahn Date: Wed, 7 Oct 2026 13:57:13 +0900 Subject: [PATCH 033/229] fix: signal.SIGINT and signal() raise NotImplementedError on wasi (#8986) * fix: signal.SIGINT and signal() raise NotImplementedError on wasi - Problem: signal.SIGINT and friends don't exist on wasi, and signal.signal() is hard-stubbed to raise NotImplementedError there - Cause: wasi-libc emulates signal()/raise() in a separate opt-in library that Rust's own wasm32-wasip1 target distribution doesn't ship - Fix: vendor wasi-sdk's prebuilt libwasi-emulated-signal.a, link it for wasm32-wasip1, and wire SIGINT/SIGABRT/SIGFPE/SIGILL/SIGSEGV/SIGTERM, signal()/getsignal()/raise_signal() through it. CPython's own WASI build links the same library Assisted-by: Claude Code:claude-sonnet-5 Signed-off-by: Jiwoo Ahn * Group WASI signal definitions in mod wasm Move WASI-only types, constants, FFI, and handlers into `mod wasm` and re-export them with `#[cfg(target_os = "wasi")]`. Assisted-by: Grok 4.6 --------- Signed-off-by: Jiwoo Ahn Co-authored-by: Jeong YunWon --- .cargo/config.toml | 3 + crates/host_env/src/lib.rs | 2 +- crates/host_env/src/signal.rs | 79 +++++++++++++++++- .../host_env/vendor/wasm32-wasip1/README.md | 16 ++++ .../wasm32-wasip1/libwasi-emulated-signal.a | Bin 0 -> 12594 bytes crates/vm/src/signal.rs | 4 + crates/vm/src/stdlib/_signal.rs | 26 +++--- extra_tests/snippets/stdlib_signal.py | 4 +- 8 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 crates/host_env/vendor/wasm32-wasip1/README.md create mode 100644 crates/host_env/vendor/wasm32-wasip1/libwasi-emulated-signal.a diff --git a/.cargo/config.toml b/.cargo/config.toml index 4e54d1d3b18..79a36b71742 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -10,9 +10,12 @@ rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] # Enforce a 1 MB stack limit for WASI targets. # (Runaway heap growth is fixed by the smaller DataStack chunk # and the size‑optimized wasm‑release profile.) + [target.wasm32-wasip1] rustflags = [ "-C", "link-arg=-zstack-size=1048576", + "-L", "crates/host_env/vendor/wasm32-wasip1", + "-l", "wasi-emulated-signal", ] [target.wasm32-wasip2] diff --git a/crates/host_env/src/lib.rs b/crates/host_env/src/lib.rs index ee9c5238e5d..33b5ad60b99 100644 --- a/crates/host_env/src/lib.rs +++ b/crates/host_env/src/lib.rs @@ -74,7 +74,7 @@ pub mod pwd; pub mod resource; #[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] pub mod shm; -#[cfg(any(unix, windows))] +#[cfg(any(unix, windows, target_os = "wasi"))] pub mod signal; pub mod time; diff --git a/crates/host_env/src/signal.rs b/crates/host_env/src/signal.rs index 090269123fc..70d32a9fdf0 100644 --- a/crates/host_env/src/signal.rs +++ b/crates/host_env/src/signal.rs @@ -1,3 +1,4 @@ +#[cfg(any(unix, windows))] use std::io; #[cfg(windows)] use std::sync::Once; @@ -13,13 +14,83 @@ pub use libc::sighandler_t; #[cfg(unix)] pub use libc::{SIG_DFL, SIG_ERR, SIG_IGN}; -#[cfg(not(unix))] +#[cfg(not(any(unix, target_os = "wasi")))] pub const SIG_DFL: libc::sighandler_t = 0; -#[cfg(not(unix))] +#[cfg(not(any(unix, target_os = "wasi")))] pub const SIG_IGN: libc::sighandler_t = 1; -#[cfg(not(unix))] +#[cfg(not(any(unix, target_os = "wasi")))] pub const SIG_ERR: libc::sighandler_t = -1 as _; +/// wasi-libc's userspace emulation of `signal()`/`raise()`, linked from +/// `libwasi-emulated-signal` (see `crates/host_env/vendor/wasm32-wasip1`). +/// WebAssembly has no asynchronous signal delivery, so this is a +/// synchronous, in-process handler table: a handler only runs when this +/// process's own code calls `raise()`, never from an outside interrupt. +/// Numbering matches wasi-libc's `bits/signal.h`, the same values CPython's +/// WASI build gets by linking the same library. +#[cfg(target_os = "wasi")] +mod wasm { + use std::io; + + #[allow(non_camel_case_types)] + pub type sighandler_t = usize; + + pub const SIG_DFL: sighandler_t = 0; + pub const SIG_IGN: sighandler_t = 1; + pub const SIG_ERR: sighandler_t = -1isize as usize; + + pub const SIGINT: i32 = 2; + pub const SIGILL: i32 = 4; + pub const SIGABRT: i32 = 6; + pub const SIGFPE: i32 = 8; + pub const SIGSEGV: i32 = 11; + pub const SIGTERM: i32 = 15; + + unsafe extern "C" { + fn signal(signum: i32, handler: sighandler_t) -> sighandler_t; + fn raise(signum: i32) -> i32; + } + + /// # Safety + /// + /// The caller must ensure `signalnum` is a valid platform signal number. + pub unsafe fn probe_handler(signalnum: i32) -> Option { + let handler = unsafe { signal(signalnum, SIG_IGN) }; + if handler == SIG_ERR { + None + } else { + unsafe { signal(signalnum, handler) }; + Some(handler) + } + } + + /// # Safety + /// + /// The caller must ensure `signalnum` is a valid platform signal number and + /// `handler` is accepted by the platform signal ABI. + pub unsafe fn install_handler( + signalnum: i32, + handler: sighandler_t, + ) -> io::Result { + let old = unsafe { signal(signalnum, handler) }; + if old == SIG_ERR { + return Err(io::Error::last_os_error()); + } + Ok(old) + } + + pub fn raise_signal(signalnum: i32) -> io::Result<()> { + if unsafe { raise(signalnum) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } +} + +#[cfg(target_os = "wasi")] +pub use wasm::*; + #[cfg(unix)] pub use libc::{SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK}; @@ -320,7 +391,7 @@ pub fn notify_signal( } } -#[cfg(unix)] +#[cfg(any(unix, target_os = "wasi"))] pub fn notify_signal(signum: i32, wakeup_fd: i32) { if wakeup_fd == -1 { return; diff --git a/crates/host_env/vendor/wasm32-wasip1/README.md b/crates/host_env/vendor/wasm32-wasip1/README.md new file mode 100644 index 00000000000..b8d201c915e --- /dev/null +++ b/crates/host_env/vendor/wasm32-wasip1/README.md @@ -0,0 +1,16 @@ +# Vendored wasi-libc emulation library + +`libwasi-emulated-signal.a` is a prebuilt static library from +[wasi-sdk](https://github.com/WebAssembly/wasi-sdk), providing wasi-libc's +userspace emulation of `signal()`/`raise()` for `wasm32-wasip1` (WebAssembly +has no asynchronous signal delivery, so this is a synchronous, in-process +handler table, not a real OS signal mechanism). + +CPython's own WASI build links the same library +(`configure.ac`'s `_WASI_EMULATED_SIGNAL` case). + +- Source: https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-34/wasi-sysroot-34.0.tar.gz + (path: `lib/wasm32-wasip1/libwasi-emulated-signal.a`) +- License: Apache-2.0 WITH LLVM-exception (same as wasi-libc) +- sha256: see below +becc35b856608fb9282d94c70166796e4d7b21e4e1057e625725306e765eb035 crates/host_env/vendor/wasm32-wasip1/libwasi-emulated-signal.a diff --git a/crates/host_env/vendor/wasm32-wasip1/libwasi-emulated-signal.a b/crates/host_env/vendor/wasm32-wasip1/libwasi-emulated-signal.a new file mode 100644 index 0000000000000000000000000000000000000000..25ba3e0515c5322aaadfcbcab514d1e92b253643 GIT binary patch literal 12594 zcmeHN3y@q@nLdwuZr{^Acb<7nhmhPnfW$EG2a^yWoe*%q!UzOKw3?poJ2Rc>>F(-B z9%?Zmkgx>BkVOO(W*jeoOA#4zs`TpJ$>$fzI&FeO%#&jCx^S7nSb4NO`oo>9jB+Kx1w=jm@#Id+&O!h z`#59AJ{pgoGkkh{`1G@wPi;K=Y*t95i!Ljs$FhkG8!4vZ6*3+#?kH{zTKNjMSSkdR zXVIDL%#CcJxZf()!Tx?P#(};H;ULyTagrlYe*Mm!JDIwWPcaleb)lG|T26Tv#>DF+ zH>I21+7ypx(pfiAh)=qcxxx;?II*MgcsiR(7u;kiK3dKuOX*xTUP_E)T%NBQ7E6ib zL_D8MXG?BDPI-p9b|+ptG@36!KFTDyRf&;Yp~MV%IR$`m7u+3sDws25DZ6}LOHAOsuC@tkHeBM%FGlw~7 z?%M^G7EjUFW=#=D#0*v>&G}n6_d=l#ye_tFX^b!){4pM>kE5cB5D^KhM6yIs9F<(}C94~0bo$SOw{5c-(o7DqG^v`kf z%&bl#*X%|(3gJdB^`0a9zy257Cn`fh<3R=UcGgla59s=<0Xv0a9#)7S_zd)k7tkbo zNBqUEHX?*R#H53D#7%E?Qw|o2$#gbB%ZcMJcx#+vidbZ0_Zg98N2re`N{P&2 ziq9C4rAH_hOS$|^y&uv~CES^EN6KUIOfLB*{xnABc$7{QgS{x&G*KV&TTD!Y@cu}5 z6x0PMX%y(zN!h>Af?`)(Cq_mJ?pEHyk7;aEvaxyoQH?Ep^L#v!CRoZ&NLtNx?VnyB zm5pm6Cq|Boo)U#e#r)=Q8=mtlO2ciWW-2DyYl@+MCg#_g)+~0HahU1LcX8I!)`E9I zxRVF{4!5oK`i>|@y*4y?7kfD(IXM!EFxJ?Ph?`FE*v<2B@hmY5TiT$r7LHqw^@MG+ zbj;YCerOH1(M<|?n#(zLGj9n@PJs9MbNO*_(N6tEA}50r?T08(XY3HICXvdprn95D zhcKyWZlk8R8atY0>4Lx8{EJr$8O;6Ai{9*?{us1*v+Luvtvp`rVEZmM-TyP@lh<7&) z>pHEqGAA&08@qo~%R7+ZKtAFfTiQ8nyE)k^NH zR#dSdwgYJJ1$D74sK(X}kr&$;AXwYEsA-sj(Yco_SNdbW(u>x5KUlHA^LI(Uyw3CT zs^>C9p65@3U~^)qd*GV8%6sy2X$S3eB0oWfE-O zsbn+gxk!Bi+g9=uy@I5Ul5=pKO9S9}Udf_2li(;NNvkGU#-jI;;20$@oFQ0>P!G{+ z6f7BqM0U(ALdTokG&L#1r1|Hf&}8}^3e5!i1=FaXgF?f29ty>OElTz870Qus1R@vUZ&YsThj%|rzx}%tya8`_$O8MfNuCB3kX}mnrnaoXgWing+sv}>> zZ9&fG46PnWBzluWs|ORQ#0fpC`v-bP++MeD@PxkPP+uxBluC{E^z`?vWaJor;c#nT ze`j}Bv5@TYI?JR-l3i4FjO0qC++@diA~V|MFZz{BXYvU1k#ahd>OzWK$mL2kM-=~L zU$1u?`JN6>3&Fah>FqOnoK?|NXSK{UIi4smWTh-fPUFMxjh{CBo{h|_#uM382E#|$ zdxegdklM%2K09_kQvDBZjGue{rj7A9OSz+oa;6loTDpni$yDVftcdF|E?svCeT?s9>-5FSl&>(ylvhsuyu$bU@I2CMO1>6&Y_wWoc#)&#nQ|wG&CV%n zR7FgAF%~{9b)9%OI4!R#egJ`Uj@W2@l8d-n7q(7P@S&t}0sPRm>f^LzrsEv&042|Ef~NnQKR8Ahc+8*oD@Zx7C! zm0Kueb6i-}pDDXkaM67Dua|?Z=(TT{rvJd02(=vXixtgyS0ceM@wDNY3$yb*otOoJ ztz>wUGz%uIhWcMMdiFIsdnH@OglTbs$~>^+jzhpB%Z zcBL#_aFJbiZg}sh8`V;|P%_zVFnNk%=0-N>4AcAooAY5)?!cr0KyWk1^^2U#dpMN7 zgDUg`u0#L9HG7_iLz>rrkLP2wh$TTs4;ay!abQSO55 z9+Yo_egM~pP##5j4CM)wLwL4Ni5Bs+Xca$0c@E_Tl$XGN1vbA$c@27hK-ndk*dr~m z7v)OX#6ClhrP&{?+O{MlEJHA>8#(1;9!@bsfwEwIk~Zfv-4JEEHTn|M^A<3@AbK0u zOWT>X0$E<4)mP+R zq`xGh*9v`5L~j!MVG;d;(BBi$FA4psh~6vo8zOp#u=e5R^$jw5KE zrXlM)vW3}CNUfig(R-zS+7rJZTbcfyjDA<@OAONAZ?rD5K4Y{F>CYL_gF@YA7)(E4 zL?0FUkP&@S=zlk&|0UG(*i@;@R0C80pqiQby24!eF%5d#j4}1IuW&QvT$X{~Dl%xR zflCWNTx*W_3A5Tm%vu`8?tmW204098+j&B0d-NLN?7TkF&v(~mAWb9t*pk+n} z^eS$!tlKfwt@|bC))P`l>t-V)t(OdVyi&p2S5yn6s+F-TX_{&zUa4{tNr$i~eJdx% zTrM~C^sojC-H7q3Zd93RlqxllyiK{GHfIHx(BV0tb22bd=f|gdn(YM1!`+J85J&?2> z8@+~}m_d*z^TE*yEDEqxyiBq`hKm9&{ZmlSFi_Q8a$+={aeX|s3WExQK^x-na<-}< zr3dSA=7XLxe27M$e;-Le9Zo2gBetC^Kna(~7++$DbycX4;Koy9?A?TV`f%aKB?fTL z6=K~IMVIYMOh-68_II8eeiU#URkj>l_yII~P6z>2(vUftm!T znuB6HE^1J$gMs0Nfj2gG^ysB|iywU?@>ya@2FTDlrbzz?`>`S%wT{R&4r6!F-3Qwl z3{nSvr(GyMU_;dR{TBA*AgX^SfFhu##$5zXpetwS-dNG)Rqh#ex~j^3Ume%1axbgn z);zF#-xd~XMTRG4!A=P30Yp)c?HDtm9!1y+K_vvKm^FAObR@iE`S6bR;hk^|-ig#h zIt^{Wg(4AhiC2hSMQ(=BY@7?^hOnBZss*^sKK9cx;6XsGgl@G3xX>Izj^_Hv(Y#tn zKHfCH9_DHHQK4vn3N5I?L5zLtGMX=S2+#o}$p&1b#tf-PfarS+Wzh6|y;RzJ&;-+M zLmtj;xg3IvJ@Qwix(U@~UUN9`IYrGT!cUAfknW~t^UpyUv<3He|536vK8EUQlC~dB zw)@eXrse{Ys#}R{CDNoA78CJ1RO^XY0fWyHaSn(_K(NqabSN&Mek?L#`gv60mEf8j zXdOHnpG6 zH|PVzUva#BKWGYBuI#oStkV7VqoC=2Ae*Th{Q~Jwz`j3^@}@>p7*X8@N9w`w=jmdFpQMY?LgH4v z4C1zK7={SoGN-&4PXW_971bOzm+p2hO?Z74s%DyL^Ui~++~Civ`IBJtg+6-LPU0vs zb-+;zk`zv8{wFGON|tJJ&gS1z5$G8LFxdkHO9bopI}wfoqT*vwljUN@Ly!E;URO1} z_#46vv!POT$-`(SF;-;dEJcXVCxlrQ^0_*W2W1tIYq4Y|hY!x%h}=dh+eQ*-7u-au zW>3i(u!xViz_Yd$(j|PLcHId^Y{!%3Mn@qUDUVXYN9vEb3@F=HmPIDUMpMvCP9$?> zgyE@A=2EW5W0{drAX&MIw9DXwA#coN0c>4@<5O*V$Y(gsqcznKBQst?3M4b_Z00xmk&xwP^2>@wIA>DWi^RFRG)r zE7N>ftz4^)-lbNKD&S+S=GT7$`wcSPuaSix-hBD|TW*DK$-5!QV0fVZ+M?Hit*5D_>WDO3hz*e-}W9!>2r7%4x zNdrn!U-RLILR>1SIihQnK6kkp_c%L zFg^*Cfqj~5`31Dtw|O}9IBx!KLj7j$uL%^f&eUcDKoJ|Dh)tkKBR~-#85VMW>o~mc z`t9Et`ozd`eTsdt(q}{-$1BjjOzHC@On=yZ)6nCQH9BMOR(f0HIi^2qU!n9K9(jZ5 zz1+T9=}+;uy1Zr`dbWIPKke7icuLSAznw;to{A+CP_4WVCvhR`o_`*xv! z&+V@Yy4>|YA~gs@){`e|YR8XkoGd!c_T z>^}&-OWM~0tdaJOQeQ9aJEXov+ILI+RcYTZ^*5#cpwy2_`%$S6NgF81&!qhmsb7-z zPo;iM+RsV7$FN_JdcR@6EcK0s{Tr#jXxMK`eV1YHHUOR2R~Y&+!@kPUPZ{ca{y^uLU2h(bEb|RbG7Mvkr?4@~8h<3ARshn?e&O}g2Vf40Sg*k@JjW$O#M2{-HH(|MQ0GKs>Z zgVZfK!OpFmjX@9R(8)T-cd}s|Uqqb>3h3ZnTq+mYhRO#7QqAS_h+~Bx$jX2bT%8rA z(hdwkzFexxd|s|x@|c@}@?}fTaMybspmnLVcdHvVZFWlON!LN%mM(c?-~!KMrvcK! zJ%8i~!FlOIsf@uUqXP8vI6RnQn+iFAU;y?G=mEFQ;hmX&RR_4fFawI z+XfG#iF5|2BU_)q81E>STnrOl(CqAU`+BMBqpF{(0jdV6I)SPos#c@w@20AUDzfM& zi+-}`CyRcv=qHPQvgjv^0kRk%ivhA2Ad3OA7$A!QvKSzX0kRk%i$Ss&B#S|^7$l3q zer%Iz$_AStJYB4I3>XTs3<9VKs#=X=(ix(VxJdixQ`iTK zA~TN!HTd9z6rlyLCD^P{0lSn^A zp98`1p%WL@KqC8gRMUbr(#aVsBonwOg#F4c+)gk9u>QQ`PGuaqr$ChWrFP*GyldWK zOYm1Yp5g6eXTE3rfQqvoy{JY>`@3v;(mT&l1m;KM9!w+3HeSZ_F&I6i|EX=8H%vzi z&7Zh!tL(LE`+olvfxjCKb|^HDvFD(Rd=($-Tf<*g;&xN+p6XQXrhJ|4x(EA=wP>$Z zP1MFt#5qH?bStq-P|DAKL<>-D)9@8)YG#Pq`D`vi!T7Rig)iFJIQUxtY$&Agb^(7+ zX1po*99+>inEx@eKEXE(lQZKeOn#tg>m_@IEN7;EUfw=On+kb*2zh&m@^)m63qp5! zdAmLypL(r6{G?7c^@b3V<+Jde@9fZ1LVws8)QJ#|7UbE3Om7X{BJ@=}bf-`^;8yhk k9>T}QyLjlQ;*X@9-#~2d$dq#rEqsT6YzzIbd~6H { + any(unix, windows, target_os = "wasi") => { use crate::convert::{IntoPyException, TryFromBorrowedObject}; use rustpython_host_env::signal as host_signal; } @@ -73,11 +73,11 @@ pub(crate) mod _signal { } } - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] #[allow(unused_imports)] pub use host_signal::SIG_ERR; - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] #[pyattr] pub use host_signal::{SIG_DFL, SIG_IGN}; @@ -86,22 +86,22 @@ pub(crate) mod _signal { #[pyattr] use host_signal::{SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK}; - #[cfg(not(any(unix, windows)))] + #[cfg(not(any(unix, windows, target_os = "wasi")))] #[pyattr] pub const SIG_DFL: sighandler_t = 0; - #[cfg(not(any(unix, windows)))] + #[cfg(not(any(unix, windows, target_os = "wasi")))] #[pyattr] pub const SIG_IGN: sighandler_t = 1; - #[cfg(not(any(unix, windows)))] + #[cfg(not(any(unix, windows, target_os = "wasi")))] #[allow(dead_code)] pub const SIG_ERR: sighandler_t = -1 as _; #[pyattr] use crate::signal::NSIG; - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] #[pyattr] pub use host_signal::{SIGABRT, SIGFPE, SIGILL, SIGINT, SIGSEGV, SIGTERM}; @@ -160,7 +160,7 @@ pub(crate) mod _signal { const _: () = assert!(SignalNum::VALID_RANGE.start.is_positive()); const _: () = assert!(SignalNum::VALID_RANGE.end.is_positive()); - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] pub(super) fn init_signal_handlers( module: &Py, vm: &VirtualMachine, @@ -199,7 +199,7 @@ pub(crate) mod _signal { } } - #[cfg(not(any(unix, windows)))] + #[cfg(not(any(unix, windows, target_os = "wasi")))] #[pyfunction] pub fn signal( _signalnum: i32, @@ -209,7 +209,7 @@ pub(crate) mod _signal { Err(vm.new_not_implemented_error("signal is not implemented on this platform")) } - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] #[pyfunction] pub fn signal( signalnum: SignalNum, @@ -401,7 +401,7 @@ pub(crate) mod _signal { host_signal::siginterrupt(signalnum.into(), flag).map_err(|_| vm.new_last_errno_error()) } - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] #[pyfunction] fn raise_signal(signalnum: i32, vm: &VirtualMachine) -> PyResult<()> { let signalnum = SignalNum::try_from(signalnum).map_err(cfg_select! { @@ -506,7 +506,7 @@ pub(crate) mod _signal { sigset_to_pyset(old_mask, vm) } - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] pub extern "C" fn run_signal(signum: i32) { signal::TRIGGERS[signum as usize].store(true, Ordering::Relaxed); signal::set_triggered(); @@ -535,7 +535,7 @@ pub(crate) mod _signal { ) -> PyResult<()> { __module_exec(vm, module); - #[cfg(any(unix, windows))] + #[cfg(any(unix, windows, target_os = "wasi"))] init_signal_handlers(module, vm); Ok(()) diff --git a/extra_tests/snippets/stdlib_signal.py b/extra_tests/snippets/stdlib_signal.py index 50a025a0604..8e00b5d885b 100644 --- a/extra_tests/snippets/stdlib_signal.py +++ b/extra_tests/snippets/stdlib_signal.py @@ -21,8 +21,8 @@ def handler(signum, frame): assert signal.getsignal(signal.SIGILL) is signal.SIG_DFL -# unix -if "win" not in sys.platform: +# unix, and not WASI +if "win" not in sys.platform and sys.platform != "wasi": signal.signal(signal.SIGALRM, handler) assert signal.getsignal(signal.SIGALRM) is handler From 0e67b82f523ec146e3c60dc9f604a7aa92d8d560 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:32:20 +0000 Subject: [PATCH 034/229] build(deps): bump cc from 1.2.66 to 1.5.1 (#8994) Bumps [cc](https://github.com/rust-lang/cc-rs) from 1.2.66 to 1.5.1. - [Release notes](https://github.com/rust-lang/cc-rs/releases) - [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.66...cc-v1.5.1) --- updated-dependencies: - dependency-name: cc dependency-version: 1.5.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b15f9af776..2dd343efd5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -504,9 +504,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", "jobserver", @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1469,7 +1469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1502,9 +1502,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flame" @@ -2048,7 +2048,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3581,7 +3581,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4683,10 +4683,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5180,7 +5180,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From a39d6c9676d9c408e882fec443f9093b91a39af8 Mon Sep 17 00:00:00 2001 From: Jeongseop Lim Date: Wed, 7 Oct 2026 23:18:55 +0900 Subject: [PATCH 035/229] Pass round() ndigits to __round__ unchanged (#8997) round(x, ndigits) ran ndigits through __index__ before calling __round__, so a user-defined __round__ never saw a non-integer ndigits and an __index__ object reached it already converted to int. CPython's builtin_round_impl() forwards ndigits as-is and leaves the conversion to float.__round__ and int.__round__. Forward the object, and take ndigits as ArgIndex in both slots so they apply __index__ themselves. Calling the slots directly with an __index__ object now works as in CPython instead of raising TypeError. Assisted-by: Claude Code:claude-opus-5 Assisted-by: Claude Code:claude-opus-5-5 --- crates/vm/src/builtins/float.rs | 8 +++++--- crates/vm/src/builtins/int.rs | 8 ++++---- crates/vm/src/stdlib/builtins.rs | 7 +++---- extra_tests/snippets/builtin_round.py | 29 +++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 3d21596f658..de38a89fae4 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -8,7 +8,9 @@ use crate::{ class::{PyClassDef, PyClassImpl}, common::{float_ops, hash, wtf8::Wtf8Buf}, convert::{ToPyObject, ToPyResult}, - function::{ArgBytesLike, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, + function::{ + ArgBytesLike, ArgIndex, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue, + }, protocol::PyNumberMethods, types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; @@ -273,7 +275,7 @@ pub fn float_from_string(val: &PyObject, vm: &VirtualMachine) -> PyResult { #[derive(FromArgs)] struct RoundArgs { #[pyarg(positional, optional)] - ndigits: Option, + ndigits: Option, } #[pyclass( @@ -340,7 +342,7 @@ impl Py { fn __round__(&self, args: RoundArgs, vm: &VirtualMachine) -> PyResult { let ndigits = args.ndigits; let value = if let Some(ndigits) = ndigits { - let ndigits = ndigits.as_bigint(); + let ndigits = ndigits.as_ref().as_bigint(); let ndigits = match ndigits.to_i32() { Some(n) => n, None if ndigits.is_positive() => i32::MAX, diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 76522e14b62..0096fced15a 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -12,8 +12,8 @@ use crate::{ }, convert::{ToPyObject, ToPyResult}, function::{ - ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue, - PySsize, + ArgByteOrder, ArgIndex, ArgIntoBool, FuncArgs, OptionalArg, PyArithmeticValue, + PyComparisonValue, PySsize, }, protocol::{PyNumberMethods, handle_bytes_to_int_err, numeric_literal_from_str}, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, @@ -526,7 +526,7 @@ impl Py { #[derive(FromArgs)] struct RoundArgs { #[pyarg(positional, optional)] - ndigits: Option, + ndigits: Option, } #[pyclass( @@ -538,7 +538,7 @@ impl Py { #[pymethod] fn __round__(zelf: PyRef, args: RoundArgs, vm: &VirtualMachine) -> PyRef { if let Some(ndigits) = args.ndigits { - let ndigits = ndigits.as_bigint(); + let ndigits = ndigits.as_ref().as_bigint(); // round(12345, -2) == 12300 // If precision >= 0, then any integer is already rounded correctly if let Some(ndigits) = ndigits.neg().to_u32() diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index d899f1ffca4..65f64299443 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -1163,10 +1163,9 @@ mod builtins { )) })?; match ndigits { - Some(obj) => { - let ndigits = obj.try_index(vm)?; - meth.invoke((ndigits,), vm) - } + // Like CPython, pass ndigits through as-is; the __round__ implementation + // decides what it accepts (float and int apply __index__ themselves). + Some(obj) => meth.invoke((obj,), vm), None => { // without a parameter, the result type is coerced to int meth.invoke((), vm) diff --git a/extra_tests/snippets/builtin_round.py b/extra_tests/snippets/builtin_round.py index 83725420a40..e41277fb589 100644 --- a/extra_tests/snippets/builtin_round.py +++ b/extra_tests/snippets/builtin_round.py @@ -111,3 +111,32 @@ class MyInt(int): assert type(round(MyInt(5), 2)) is int # A negative ndigits already produced a fresh exact int. assert type(round(MyInt(15), -1)) is int + + +# round() passes ndigits to __round__ unchanged, like CPython's builtin_round_impl(). +class Echo: + def __round__(self, *args): + return args + + +assert round(Echo(), []) == ([],) +assert round(Echo(), 1.5) == (1.5,) +assert type(round(Echo(), True)[0]) is bool + + +class Index: + def __index__(self): + return -1 + + +assert type(round(Echo(), Index())[0]) is Index + +# int.__round__ and float.__round__ apply __index__ to ndigits themselves. +assert round(15, Index()) == 20 +assert round(1.25, Index()) == 0.0 +assert (15).__round__(Index()) == 20 +assert (1.25).__round__(Index()) == 0.0 +with assert_raises(TypeError): + (15).__round__(1.0) +with assert_raises(TypeError): + (1.25).__round__([]) From 95686c08e4f514380524f8905a8a1ca51568c3bf Mon Sep 17 00:00:00 2001 From: Jeongseop Lim Date: Wed, 7 Oct 2026 23:32:59 +0900 Subject: [PATCH 036/229] Accept __index__ for the remaining int-bound arguments (#8999) #8828 made bin(), chr() and several other arguments that were bound as PyIntRef go through __index__, but these still downcast to int and reject an object that only defines __index__: the start and end of the str/bytes/bytearray find family, enumerate(start=), itertools.combinations/combinations_with_replacement/batched, _thread.stack_size, resource.getrlimit/setrlimit, pwd.getpwuid, grp.getgrgid, fcntl.fcntl/lockf and mmap.move/madvise. CPython converts all of them with _PyNumber_Index or _PyEval_SliceIndex. Bind them as ArgIndex, as #8828 did for the others. test_mmap.test_madvise and test_mmap_closed_by_int_scenarios now pass, so drop their expectedFailure markers. Assisted-by: Claude Code:claude-opus-5-5 --- Lib/test/test_mmap.py | 2 - crates/stdlib/src/fcntl.rs | 15 +++-- crates/stdlib/src/grp.rs | 7 ++- crates/stdlib/src/mmap.rs | 20 +++--- crates/stdlib/src/resource.rs | 11 ++-- crates/vm/src/anystr.rs | 12 ++-- crates/vm/src/builtins/enumerate.rs | 6 +- crates/vm/src/builtins/str.rs | 14 +++-- crates/vm/src/bytes_inner.rs | 12 ++-- crates/vm/src/stdlib/_thread.rs | 8 +-- crates/vm/src/stdlib/itertools.rs | 13 ++-- crates/vm/src/stdlib/pwd.rs | 10 ++- .../snippets/protocol_index_argument.py | 62 +++++++++++++++++++ 13 files changed, 132 insertions(+), 60 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 09b44c9ca1b..b5a2a89d865 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -897,7 +897,6 @@ def test_repr(self): match = closed_mmap_repr_pat.match(repr(mm)) self.assertIsNotNone(match) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(mmap.mmap, 'madvise'), 'needs madvise') def test_madvise(self): size = 2 * PAGESIZE @@ -1039,7 +1038,6 @@ def test_resize_succeeds_with_error_for_second_named_mapping(self): self.assertEqual(m1[:data_length], data) self.assertEqual(m2[:data_length], data) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_mmap_closed_by_int_scenarios(self): """ gh-103987: Test that mmap objects raise ValueError diff --git a/crates/stdlib/src/fcntl.rs b/crates/stdlib/src/fcntl.rs index a6dfdeb63b1..f04304de059 100644 --- a/crates/stdlib/src/fcntl.rs +++ b/crates/stdlib/src/fcntl.rs @@ -8,9 +8,8 @@ mod fcntl { use crate::vm::{ PyResult, VirtualMachine, - builtins::PyIntRef, convert::ToPyException, - function::{ArgMemoryBuffer, ArgStrOrBytesLike, Either, OptionalArg}, + function::{ArgIndex, ArgMemoryBuffer, ArgStrOrBytesLike, Either, OptionalArg}, stdlib::_io, }; use rustpython_host_env::io::is_interrupted_error; @@ -86,7 +85,7 @@ mod fcntl { #[derive(FromArgs)] struct FcntlArg { #[pyarg(positional, default = 0)] - arg: Either, + arg: Either, } #[pyfunction] @@ -114,7 +113,7 @@ mod fcntl { )?; return Ok(vm.ctx.new_bytes(buf[..arg_len].to_vec()).into()); } - Either::B(i) => i.as_u32_mask(), + Either::B(i) => i.as_ref().as_u32_mask(), }; let ret = retry_on_eintr( vm, @@ -213,9 +212,9 @@ mod fcntl { #[derive(FromArgs)] struct LockfArgs { #[pyarg(positional, default = 0)] - len: PyIntRef, + len: ArgIndex, #[pyarg(positional, default = 0)] - start: PyIntRef, + start: ArgIndex, #[pyarg(positional, default)] whence: i32, } @@ -229,8 +228,8 @@ mod fcntl { LockfArgs { len, start, whence }: LockfArgs, vm: &VirtualMachine, ) -> PyResult { - let start = start.try_to_primitive(vm)?; - let len = len.try_to_primitive(vm)?; + let start = start.as_ref().try_to_primitive(vm)?; + let len = len.as_ref().try_to_primitive(vm)?; // F_LOCK and F_TLOCK differ in exactly this: the first one waits. let ret = retry_on_eintr( vm, diff --git a/crates/stdlib/src/grp.rs b/crates/stdlib/src/grp.rs index 5e73f5648d4..39e5285f6e2 100644 --- a/crates/stdlib/src/grp.rs +++ b/crates/stdlib/src/grp.rs @@ -5,9 +5,10 @@ pub(crate) use grp::module_def; mod grp { use crate::vm::{ PyObjectRef, PyResult, VirtualMachine, - builtins::{PyIntRef, PyListRef, PyUtf8StrRef}, + builtins::{PyListRef, PyUtf8StrRef}, convert::{IntoPyException, ToPyObject}, exceptions, + function::ArgIndex, types::PyStructSequence, }; use core::hint::cold_path; @@ -44,12 +45,12 @@ mod grp { #[derive(FromArgs)] struct GetGrGidArgs { #[pyarg(any)] - id: PyIntRef, + id: ArgIndex, } #[pyfunction] fn getgrgid(id: GetGrGidArgs, vm: &VirtualMachine) -> PyResult { - let gid = id.id; + let gid = id.id.into_int_ref(); let gr_gid = gid.as_bigint(); let gid = host_grp::gid_t::try_from(gr_gid).ok(); let group = gid diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index 317176c452b..c33c21fcdb0 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -14,7 +14,7 @@ mod mmap { builtins::{PyBytes, PyBytesRef, PyInt, PyIntRef, PyType, PyTypeRef}, byte::{bytes_from_object, value_from_object}, convert::ToPyException, - function::ArgBytesLike, + function::{ArgBytesLike, ArgIndex}, protocol::{ BufferDescriptor, BufferMethods, PyBuffer, PyMappingMethods, PySequenceMethods, }, @@ -330,9 +330,9 @@ mod mmap { option: core::ffi::c_int, // None means 0. #[pyarg(positional, default, py_default = "0")] - start: Option, + start: Option, #[pyarg(positional, optional)] - length: Option, + length: Option, } #[cfg(all(unix, not(target_os = "redox")))] @@ -345,7 +345,8 @@ mod mmap { let start = self .start .map(|s| { - s.try_to_primitive::(vm) + s.as_ref() + .try_to_primitive::(vm) .ok() .filter(|s| *s < len) .ok_or_else(|| vm.new_value_error("madvise start out of bounds")) @@ -355,7 +356,8 @@ mod mmap { let length = self .length .map(|s| { - s.try_to_primitive::(vm) + s.as_ref() + .try_to_primitive::(vm) .map_err(|_| vm.new_value_error("madvise length invalid")) }) .transpose()? @@ -920,9 +922,9 @@ mod mmap { #[pymethod(name = "move")] fn move_( zelf: &Py, - dest: PyIntRef, - src: PyIntRef, - count: PyIntRef, + dest: ArgIndex, + src: ArgIndex, + count: ArgIndex, vm: &VirtualMachine, ) -> PyResult<()> { fn args( @@ -948,7 +950,7 @@ mod mmap { } let size = zelf.__len__(); - let (dest, src, cnt) = args(&dest, &src, &count, size, vm) + let (dest, src, cnt) = args(dest.as_ref(), src.as_ref(), count.as_ref(), size, vm) .ok_or_else(|| vm.new_value_error("source, destination, or count out of range"))?; let dest_end = dest + cnt; diff --git a/crates/stdlib/src/resource.rs b/crates/stdlib/src/resource.rs index 0e2a777e6e2..0750a3bf107 100644 --- a/crates/stdlib/src/resource.rs +++ b/crates/stdlib/src/resource.rs @@ -6,8 +6,9 @@ pub(crate) use resource::module_def; mod resource { use crate::vm::{ Py, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, - builtins::{PyInt, PyIntRef}, + builtins::PyInt, convert::{ToPyException, ToPyObject}, + function::ArgIndex, types::PyStructSequence, }; use num_traits::Signed; @@ -185,8 +186,8 @@ mod resource { } #[pyfunction] - fn getrlimit(resource: PyIntRef, vm: &VirtualMachine) -> PyResult { - let resource = py2rlim(&resource, vm)?; + fn getrlimit(resource: ArgIndex, vm: &VirtualMachine) -> PyResult { + let resource = py2rlim(resource.as_ref(), vm)?; if resource >= RLIM_NLIMITS as host_resource::rlim_t { return Err(vm.new_value_error("invalid resource specified")); @@ -197,8 +198,8 @@ mod resource { } #[pyfunction] - fn setrlimit(resource: PyIntRef, limits: Limits, vm: &VirtualMachine) -> PyResult<()> { - let resource = py2rlim(&resource, vm)?; + fn setrlimit(resource: ArgIndex, limits: Limits, vm: &VirtualMachine) -> PyResult<()> { + let resource = py2rlim(resource.as_ref(), vm)?; if resource >= RLIM_NLIMITS as host_resource::rlim_t { return Err(vm.new_value_error("invalid resource specified")); diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs index f1ea7d7354d..b9ce612500b 100644 --- a/crates/vm/src/anystr.rs +++ b/crates/vm/src/anystr.rs @@ -5,9 +5,9 @@ use rustpython_unicode::case; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyInt, PyIntRef, PyTuple}, + builtins::{PyInt, PyTuple}, convert::TryFromBorrowedObject, - function::PySsize, + function::{ArgIndex, PySsize}, }; #[derive(FromArgs)] @@ -41,17 +41,17 @@ pub(crate) struct StartsEndsWithArgs { #[pyarg(positional)] affix: PyObjectRef, #[pyarg(positional, default)] - start: Option, + start: Option, #[pyarg(positional, default)] - end: Option, + end: Option, } impl StartsEndsWithArgs { pub(crate) fn get_value(self, len: usize) -> (PyObjectRef, Option>) { let range = if self.start.is_some() || self.end.is_some() { Some(adjust_indices( - self.start.as_deref(), - self.end.as_deref(), + self.start.as_ref().map(AsRef::as_ref), + self.end.as_ref().map(AsRef::as_ref), len, )) } else { diff --git a/crates/vm/src/builtins/enumerate.rs b/crates/vm/src/builtins/enumerate.rs index 76ee3cca2b7..f7a3b8d257d 100644 --- a/crates/vm/src/builtins/enumerate.rs +++ b/crates/vm/src/builtins/enumerate.rs @@ -1,11 +1,12 @@ use super::{ - IterStatus, PositionIterInternal, PyGenericAlias, PyIntRef, PyTupleRef, PyType, PyTypeRef, + IterStatus, PositionIterInternal, PyGenericAlias, PyTupleRef, PyType, PyTypeRef, iter::builtins_reversed, locked_rev_next, }; use crate::common::lock::{PyMutex, PyRwLock}; use crate::{ AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, + function::ArgIndex, protocol::{PyIter, PyIterReturn}, raise_if_stop, types::{Constructor, IterNext, Iterable, SelfIter}, @@ -56,7 +57,7 @@ pub struct EnumerateArgs { #[pyarg(any)] iterable: PyIter, #[pyarg(any, default = 0)] - start: PyIntRef, + start: ArgIndex, } impl Constructor for PyEnumerate { @@ -67,6 +68,7 @@ impl Constructor for PyEnumerate { Self::Args { iterable, start }: Self::Args, _vm: &VirtualMachine, ) -> PyResult { + let start = start.into_int_ref(); let counter = match start.as_bigint().to_usize() { Some(n) => Counter::Small(n), None => Counter::Big(start.as_bigint().clone()), diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index fa4e905cdd9..91f99b22c86 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -1,6 +1,6 @@ use super::{ PositionIterInternal, PyBytesRef, PyDict, PyList, PyTuple, PyTupleRef, PyType, PyTypeRef, - int::{PyInt, PyIntRef}, + int::PyInt, iter::{IterStatus, builtins_iter}, }; use crate::{ @@ -17,7 +17,7 @@ use crate::{ }, convert::{ToPyException, ToPyObject, ToPyResult}, format::{format, format_map}, - function::{ArgIterable, FuncArgs, OptionalArg, PyComparisonValue, PySsize}, + function::{ArgIndex, ArgIterable, FuncArgs, OptionalArg, PyComparisonValue, PySsize}, intern::PyInterned, object::{MaybeTraverse, Traverse, TraverseFn}, protocol::{ @@ -1871,14 +1871,18 @@ pub(crate) struct FindArgs { #[pyarg(positional)] sub: PyStrRef, #[pyarg(positional, default)] - start: Option, + start: Option, #[pyarg(positional, default)] - end: Option, + end: Option, } impl FindArgs { fn get_value(self, len: usize) -> (PyStrRef, core::ops::Range) { - let range = adjust_indices(self.start.as_deref(), self.end.as_deref(), len); + let range = adjust_indices( + self.start.as_ref().map(AsRef::as_ref), + self.end.as_ref().map(AsRef::as_ref), + len, + ); (self.sub, range) } } diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index f38b4dbd408..ecda0a42ce6 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -10,7 +10,7 @@ use crate::{ cformat::cformat_bytes, common::wtf8::is_py_ascii_whitespace, common::{borrow::BorrowedValue, hash}, - function::{ArgIterable, Either, OptionalArg, PyComparisonValue}, + function::{ArgIndex, ArgIterable, Either, OptionalArg, PyComparisonValue}, literal::escape::Escape, protocol::{BufferFlags, PyBuffer}, sequence::{SequenceExt, SequenceMutExt}, @@ -315,9 +315,9 @@ pub struct ByteInnerFindOptions { #[pyarg(positional)] sub: ByteInnerSub, #[pyarg(positional, default)] - start: Option, + start: Option, #[pyarg(positional, default)] - end: Option, + end: Option, } impl ByteInnerFindOptions { @@ -327,7 +327,11 @@ impl ByteInnerFindOptions { vm: &VirtualMachine, ) -> PyResult<(Vec, core::ops::Range)> { let sub = self.sub.into_vec(vm)?; - let range = anystr::adjust_indices(self.start.as_deref(), self.end.as_deref(), len); + let range = anystr::adjust_indices( + self.start.as_ref().map(AsRef::as_ref), + self.end.as_ref().map(AsRef::as_ref), + len, + ); Ok((sub, range)) } } diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 984a0865560..7ac20af9273 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -27,8 +27,8 @@ pub(crate) mod _thread { convert::ToPyException, frame::FrameObjectRef, function::{ - ArgCallable, ArgumentError, DefaultRepr, FromArgs, FuncArgs, KwArgs, NameExcInfo, - OptionalArg, Param, ParamKind, PosArgs, PySetterValue, TimeoutSeconds, + ArgCallable, ArgIndex, ArgumentError, DefaultRepr, FromArgs, FuncArgs, KwArgs, + NameExcInfo, OptionalArg, Param, ParamKind, PosArgs, PySetterValue, TimeoutSeconds, }, object::{Traverse, TraverseFn}, types::{Constructor, GetAttr, PyStructSequence, Representable, SetAttr}, @@ -847,7 +847,7 @@ pub(crate) mod _thread { #[derive(FromArgs)] struct StackSizeArgs { #[pyarg(positional, default = 0)] - size: PyIntRef, + size: ArgIndex, } #[pyfunction] @@ -855,7 +855,7 @@ pub(crate) mod _thread { const MIN_SIZE: usize = PY_OS_MIN_STACK_SIZE + SYSTEM_PAGE_SIZE; let StackSizeArgs { size } = args; - let Ok(size) = size.try_to_primitive(vm) else { + let Ok(size) = size.as_ref().try_to_primitive(vm) else { return Err(vm.new_value_error(format!("size must be at least {MIN_SIZE} bytes"))); }; diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 371e3ee68c2..f016f486c3a 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -5,13 +5,11 @@ mod decl { use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, TryFromObject, VirtualMachine, - builtins::{ - PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyTupleRef, PyType, PyTypeRef, int, - }, + builtins::{PyGenericAlias, PyInt, PyList, PyTuple, PyTupleRef, PyType, PyTypeRef, int}, class::PyClassDef, common::lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, convert::ToPyObject, - function::{FuncArgs, NameIterables, PosArgs}, + function::{ArgIndex, FuncArgs, NameIterables, PosArgs}, protocol::{PyIter, PyIterReturn, PyNumber}, raise_if_stop, stdlib::sys, @@ -1329,7 +1327,7 @@ mod decl { #[pyarg(any)] iterable: PyObjectRef, #[pyarg(any)] - r: PyIntRef, + r: ArgIndex, } impl Constructor for PyItertoolsCombinations { @@ -1342,6 +1340,7 @@ mod decl { ) -> PyResult { let pool: Vec<_> = iterable.try_to_value(vm)?; + let r = r.into_int_ref(); let r = r.as_bigint(); if r.is_negative() { return Err(vm.new_value_error("r must be non-negative")); @@ -1454,6 +1453,7 @@ mod decl { vm: &VirtualMachine, ) -> PyResult { let pool: Vec<_> = iterable.try_to_value(vm)?; + let r = r.into_int_ref(); let r = r.as_bigint(); if r.is_negative() { return Err(vm.new_value_error("r must be non-negative")); @@ -1800,7 +1800,7 @@ mod decl { #[pyarg(any)] iterable: PyObjectRef, #[pyarg(any)] - n: PyIntRef, + n: ArgIndex, #[pyarg(named, default)] strict: bool, } @@ -1817,6 +1817,7 @@ mod decl { }: Self::Args, vm: &VirtualMachine, ) -> PyResult { + let n = n.into_int_ref(); let n = n.as_bigint(); if n.lt(&BigInt::one()) { return Err(vm.new_value_error("n must be at least one")); diff --git a/crates/vm/src/stdlib/pwd.rs b/crates/vm/src/stdlib/pwd.rs index bfa7eb034e5..90a37d6d36d 100644 --- a/crates/vm/src/stdlib/pwd.rs +++ b/crates/vm/src/stdlib/pwd.rs @@ -5,11 +5,8 @@ pub(crate) use pwd::module_def; #[pymodule] mod pwd { use crate::{ - PyResult, VirtualMachine, - builtins::{PyIntRef, PyUtf8StrRef}, - convert::IntoPyException, - exceptions, - types::PyStructSequence, + PyResult, VirtualMachine, builtins::PyUtf8StrRef, convert::IntoPyException, exceptions, + function::ArgIndex, types::PyStructSequence, }; use core::hint::cold_path; use rustpython_host_env::pwd as host_pwd; @@ -68,7 +65,8 @@ mod pwd { } #[pyfunction] - fn getpwuid(uidobj: PyIntRef, vm: &VirtualMachine) -> PyResult { + fn getpwuid(uidobj: ArgIndex, vm: &VirtualMachine) -> PyResult { + let uidobj = uidobj.into_int_ref(); let uid_t = libc::uid_t::try_from(uidobj.as_bigint()).ok(); let user = uid_t .map(host_pwd::getpwuid) diff --git a/extra_tests/snippets/protocol_index_argument.py b/extra_tests/snippets/protocol_index_argument.py index c67a078ef05..ada8be24085 100644 --- a/extra_tests/snippets/protocol_index_argument.py +++ b/extra_tests/snippets/protocol_index_argument.py @@ -312,3 +312,65 @@ def __length_hint__(self): with assert_raises(TypeError): operator.length_hint(HintIndex()) + + +# Arguments bound as `int` objects accept `__index__` as well. +assert "abcabc".find("a", Index(1)) == 3 +assert "abcabc".rfind("a", 0, Index(3)) == 0 +assert "abcabc".index("c", Index(3)) == 5 +assert "abcabc".rindex("a", Index(1), Index(4)) == 3 +assert "aaaaaa".count("a", Index(2), Index(-1)) == 3 +assert "abcdef".startswith("d", Index(3)) +assert "abcdef".endswith("c", 0, Index(3)) +assert b"abcabc".find(b"a", Index(1)) == 3 +assert b"abcabc".count(b"b", Index(2)) == 1 +assert b"abcdef".startswith(b"d", Index(3)) +assert bytearray(b"abcabc").rindex(b"a", 0, Index(3)) == 0 +assert bytearray(b"abcdef").endswith(b"c", 0, Index(3)) + +assert list(enumerate("ab", Index(3))) == [(3, "a"), (4, "b")] +assert list(enumerate("ab", start=Index(3))) == [(3, "a"), (4, "b")] + +import itertools + +assert list(itertools.combinations("abc", Index(2))) == [ + ("a", "b"), + ("a", "c"), + ("b", "c"), +] +assert list(itertools.combinations_with_replacement("ab", Index(2))) == [ + ("a", "a"), + ("a", "b"), + ("b", "b"), +] +assert list(itertools.batched("abcde", Index(2))) == [("a", "b"), ("c", "d"), ("e",)] + +assert _thread.stack_size(Index(0)) == 0 + +import mmap + +mapped = mmap.mmap(-1, 6) +mapped.write(b"abcdef") +mapped.move(Index(0), Index(3), Index(3)) +assert mapped[:] == b"defdef" +if hasattr(mapped, "madvise"): + mapped.madvise(mmap.MADV_NORMAL, Index(0), Index(6)) + +if sys.platform != "win32": + import fcntl + import grp + import pwd + import resource + import tempfile + + assert pwd.getpwuid(Index(0)).pw_uid == 0 + assert grp.getgrgid(Index(0)).gr_gid == 0 + + nofile = resource.getrlimit(resource.RLIMIT_NOFILE) + assert resource.getrlimit(Index(resource.RLIMIT_NOFILE)) == nofile + resource.setrlimit(Index(resource.RLIMIT_NOFILE), nofile) + + with tempfile.TemporaryFile() as f: + assert fcntl.fcntl(f, fcntl.F_SETFD, Index(fcntl.FD_CLOEXEC)) == 0 + assert fcntl.fcntl(f, fcntl.F_GETFD) == fcntl.FD_CLOEXEC + fcntl.lockf(f, fcntl.LOCK_SH | fcntl.LOCK_NB, Index(0), Index(0)) From a38bf3b54f78e3346f5a64e8b1201caeea4cc14b Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:33:55 +0300 Subject: [PATCH 037/229] Build host_env on Windows without the ctypes feature (#9000) Assisted-by: Claude Code:claude-opus-5-5 Assisted-by: Codex:gpt-6 --- .github/workflows/ci.yaml | 4 ++++ crates/host_env/src/ctypes.rs | 24 +----------------------- crates/host_env/src/windows.rs | 21 +++++++++++++++++++++ crates/host_env/src/wmi.rs | 23 ++++++++++++----------- 4 files changed, 38 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 61d097997a3..2f62d91990f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -295,6 +295,10 @@ jobs: working-directory: crates/capi run: cargo test --doc --locked + - name: Check host_env without ctypes + if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Windows' + run: cargo check --locked -p rustpython-host_env --no-default-features --lib + - name: check compilation without host_env (sandbox mode) if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux' run: | diff --git a/crates/host_env/src/ctypes.rs b/crates/host_env/src/ctypes.rs index b0d15679b1f..440d2078eac 100644 --- a/crates/host_env/src/ctypes.rs +++ b/crates/host_env/src/ctypes.rs @@ -2809,29 +2809,7 @@ pub fn free_library(module: windows_sys::Win32::Foundation::HMODULE) -> std::io: } #[cfg(windows)] -#[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn sys_string_len(bstr: *const u16) -> usize { - unsafe { windows_sys::Win32::Foundation::SysStringLen(bstr) as usize } -} - -#[cfg(windows)] -#[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn sys_free_string(bstr: *const u16) { - unsafe { windows_sys::Win32::Foundation::SysFreeString(bstr) }; -} - -#[cfg(windows)] -pub fn sys_alloc_string_len(units: &[u16]) -> Option<*mut u16> { - let len = u32::try_from(units.len()).ok()?; - let bstr = unsafe { windows_sys::Win32::Foundation::SysAllocStringLen(units.as_ptr(), len) }; - (!bstr.is_null()).then_some(bstr as *mut u16) -} - -#[cfg(windows)] -#[allow(clippy::not_unsafe_ptr_arg_deref)] -pub fn co_task_mem_free(ptr: *mut core::ffi::c_void) { - unsafe { windows_sys::Win32::System::Com::CoTaskMemFree(ptr) }; -} +pub use crate::windows::{co_task_mem_free, sys_alloc_string_len, sys_free_string, sys_string_len}; /// `GetErrorInfo(0)`. `None` when the thread has no error object (`S_FALSE`). #[cfg(windows)] diff --git a/crates/host_env/src/windows.rs b/crates/host_env/src/windows.rs index 4cba8e3579e..5c34b34d1ee 100644 --- a/crates/host_env/src/windows.rs +++ b/crates/host_env/src/windows.rs @@ -426,3 +426,24 @@ impl ToWideString for Wtf8 { WideCString::from_vec(buf).map_err(|_| io::Error::other(NUL_ERROR)) } } + +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn sys_string_len(bstr: *const u16) -> usize { + unsafe { windows_sys::Win32::Foundation::SysStringLen(bstr) as usize } +} + +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn sys_free_string(bstr: *const u16) { + unsafe { windows_sys::Win32::Foundation::SysFreeString(bstr) }; +} + +pub fn sys_alloc_string_len(units: &[u16]) -> Option<*mut u16> { + let len = u32::try_from(units.len()).ok()?; + let bstr = unsafe { windows_sys::Win32::Foundation::SysAllocStringLen(units.as_ptr(), len) }; + (!bstr.is_null()).then_some(bstr as *mut u16) +} + +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn co_task_mem_free(ptr: *mut core::ffi::c_void) { + unsafe { windows_sys::Win32::System::Com::CoTaskMemFree(ptr) }; +} diff --git a/crates/host_env/src/wmi.rs b/crates/host_env/src/wmi.rs index ceccf4d79fb..bf6d53264e6 100644 --- a/crates/host_env/src/wmi.rs +++ b/crates/host_env/src/wmi.rs @@ -7,7 +7,7 @@ use core::ffi::c_void; use core::ptr::{NonNull, null, null_mut}; -use widestring::WideCString; +use widestring::{WideCStr, WideCString}; use windows_sys::Win32::Foundation::{ CloseHandle, ERROR_BROKEN_PIPE, ERROR_INVALID_NAME, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, GetLastError, HANDLE, WAIT_OBJECT_0, WAIT_TIMEOUT, @@ -19,7 +19,7 @@ use windows_sys::Win32::System::Threading::{ }; use windows_sys::w; -use crate::ctypes::wcslen; +use crate::windows::{sys_alloc_string_len, sys_free_string}; pub const BUFFER_SIZE: usize = 8192; @@ -122,21 +122,20 @@ fn alloc_bstr(wide: &[u16]) -> *mut u16 { .iter() .position(|&unit| unit == 0) .unwrap_or(wide.len()); - crate::ctypes::sys_alloc_string_len(&wide[..len]).unwrap_or(null_mut()) + sys_alloc_string_len(&wide[..len]).unwrap_or(null_mut()) } fn alloc_bstr_ptr(ptr: *const u16) -> *mut u16 { let Some(ptr) = NonNull::new(ptr.cast_mut()) else { return null_mut(); }; - let len = unsafe { crate::ctypes::wcslen(ptr) }; - crate::ctypes::sys_alloc_string_len(unsafe { core::slice::from_raw_parts(ptr.as_ptr(), len) }) - .unwrap_or(null_mut()) + let wide = unsafe { WideCStr::from_ptr_str(ptr.as_ptr()) }; + sys_alloc_string_len(wide.as_slice()).unwrap_or(null_mut()) } fn free_bstr(bstr: *mut u16) { if !bstr.is_null() { - crate::ctypes::sys_free_string(bstr); + sys_free_string(bstr); } } @@ -478,9 +477,9 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { } if succeeded(hr) && (flavor & WBEM_FLAVOR_MASK_ORIGIN) != WBEM_FLAVOR_ORIGIN_SYSTEM { - let Some(cb_str1) = NonNull::new(prop_name) - .map(|prop_name| (unsafe { wcslen(prop_name) } * 2) as u32) - else { + let Some(cb_str1) = NonNull::new(prop_name).map(|prop_name| { + (unsafe { WideCStr::from_ptr_str(prop_name.as_ptr()) }.len() * 2) as u32 + }) else { free_bstr(prop_name); break; }; @@ -490,7 +489,9 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { VariantToString(&prop_value, prop_str.as_mut_ptr(), BUFFER_SIZE as u32) }; let cb_str2 = NonNull::new(prop_str.as_ptr().cast_mut()) - .map(|prop_str| (unsafe { wcslen(prop_str) } * 2) as u32) + .map(|prop_str| { + (unsafe { WideCStr::from_ptr_str(prop_str.as_ptr()) }.len() * 2) as u32 + }) .expect("prop_str is never null"); if succeeded(hr) From 3d11c8b0534eb0f614772e44b27c0a31ad0da76d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:39:09 +0000 Subject: [PATCH 038/229] build(deps): bump rustls-pki-types from 1.15.0 to 1.15.1 (#9001) Bumps [rustls-pki-types](https://github.com/rustls/pki-types) from 1.15.0 to 1.15.1. - [Release notes](https://github.com/rustls/pki-types/releases) - [Commits](https://github.com/rustls/pki-types/compare/v/1.15.0...v/1.15.1) --- updated-dependencies: - dependency-name: rustls-pki-types dependency-version: 1.15.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2dd343efd5d..0641dd859a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3631,9 +3631,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", From f9ebb234479f32477b944f3eede31446de6f1013 Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:39:26 +0300 Subject: [PATCH 039/229] Preserve Unix signal dispositions when probing handlers (#8961) Assisted-by: Codex:gpt-6 --- crates/host_env/src/signal.rs | 72 ++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/crates/host_env/src/signal.rs b/crates/host_env/src/signal.rs index 70d32a9fdf0..67336eb039e 100644 --- a/crates/host_env/src/signal.rs +++ b/crates/host_env/src/signal.rs @@ -175,12 +175,26 @@ mod ffi { /// The caller must ensure `signalnum` is a valid platform signal number. #[cfg(any(unix, windows))] pub unsafe fn probe_handler(signalnum: i32) -> Option { - let handler = unsafe { libc::signal(signalnum, libc::SIG_IGN) }; - if handler == libc::SIG_ERR as sighandler_t { - None - } else { - unsafe { libc::signal(signalnum, handler) }; - Some(handler) + #[cfg(unix)] + { + // Query without changing disposition: even temporarily ignoring SIGCHLD + // can discard a child's exit status, and signal() also replaces masks. + let mut action = core::mem::MaybeUninit::::uninit(); + if unsafe { libc::sigaction(signalnum, core::ptr::null(), action.as_mut_ptr()) } == 0 { + Some(unsafe { action.assume_init() }.sa_sigaction) + } else { + None + } + } + #[cfg(windows)] + { + let handler = unsafe { libc::signal(signalnum, libc::SIG_IGN) }; + if handler == libc::SIG_ERR as sighandler_t { + None + } else { + unsafe { libc::signal(signalnum, handler) }; + Some(handler) + } } } @@ -450,3 +464,49 @@ pub fn sigset_contains(mask: libc::sigset_t, signum: i32) -> bool { pub fn valid_signals(_max_signum: usize) -> io::Result> { Ok(VALID_SIGNALS.to_vec()) } + +#[cfg(test)] +#[cfg(unix)] +mod tests { + #[test] + fn probing_preserves_signal_flags_and_mask() { + unsafe extern "C" fn handler(_: libc::c_int) {} + + struct Restore(libc::sigaction); + impl Drop for Restore { + fn drop(&mut self) { + unsafe { libc::sigaction(libc::SIGUSR2, &self.0, core::ptr::null_mut()) }; + } + } + + // No other host_env test uses SIGUSR2; retain the host disposition even + // when an assertion fails. Do not deliver any signal to the process. + unsafe { + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = handler as *const () as libc::sighandler_t; + action.sa_flags = libc::SA_NODEFER; + assert_eq!(libc::sigemptyset(&mut action.sa_mask), 0); + assert_eq!(libc::sigaddset(&mut action.sa_mask, libc::SIGUSR1), 0); + let mut original = core::mem::MaybeUninit::uninit(); + assert_eq!( + libc::sigaction(libc::SIGUSR2, &action, original.as_mut_ptr()), + 0 + ); + let _restore = Restore(original.assume_init()); + + assert_eq!( + super::probe_handler(libc::SIGUSR2), + Some(action.sa_sigaction) + ); + let mut observed = core::mem::MaybeUninit::::uninit(); + assert_eq!( + libc::sigaction(libc::SIGUSR2, core::ptr::null(), observed.as_mut_ptr()), + 0 + ); + let observed = observed.assume_init(); + assert_eq!(observed.sa_sigaction, action.sa_sigaction); + assert_ne!(observed.sa_flags & libc::SA_NODEFER, 0); + assert_eq!(libc::sigismember(&observed.sa_mask, libc::SIGUSR1), 1); + } + } +} From e053f7d5b6b47f730b9d3b1c69f13566f95bf6d3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:58:39 +0900 Subject: [PATCH 040/229] vm: drop trailing space from RecursionError text (#8992) * vm: drop trailing space from RecursionError text Format Python-limit and native-stack RecursionError messages through one helper so empty context is not a trailing space or an empty string. Assisted-by: Grok 4.6 * test: deepen nested hash snippets for release stack frames The cold RecursionError helper thins with_recursion's hot frame, so 100_000 nested hashes no longer trip the native stack guard in release. Assisted-by: Grok 4.6 * vm: let RecursionError callers own the where-clause space The helper concatenates an empty suffix or one that already starts with a space. Call sites supply that space. Assisted-by: Grok 4.6 --- crates/stdlib/src/json.rs | 4 ++-- crates/stdlib/src/pickle.rs | 2 +- crates/vm/src/builtins/genericalias.rs | 2 +- crates/vm/src/builtins/iter.rs | 2 +- crates/vm/src/exception_group.rs | 4 ++-- crates/vm/src/protocol/object.rs | 16 ++++++------- crates/vm/src/stdlib/_ast/node.rs | 4 ++-- crates/vm/src/types/slot.rs | 4 ++-- crates/vm/src/vm/crossinterp.rs | 2 +- crates/vm/src/vm/mod.rs | 31 +++++++++++++++++--------- extra_tests/snippets/builtin_hash.py | 6 ++--- extra_tests/snippets/recursion.py | 29 ++++++++++++++++++++++++ extra_tests/snippets/stdlib_types.py | 5 +++-- 13 files changed, 76 insertions(+), 35 deletions(-) diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index 8b9da96cc8e..abd46a42216 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -546,7 +546,7 @@ mod _json { // `'[' * 50000 + ']' * 50000` overflows the native Rust stack and // crashes the process with SIGSEGV. Matches CPython's // _Py_EnterRecursiveCall in Modules/_json.c. - vm.with_recursion("while decoding a JSON object from a string", || { + vm.with_recursion(" while decoding a JSON object from a string", || { let bytes = pystr.as_bytes(); let wtf8 = pystr.as_wtf8(); @@ -1254,7 +1254,7 @@ mod _json { out: &mut Wtf8Buf, vm: &VirtualMachine, ) -> PyResult<()> { - vm.with_recursion("while encoding a JSON object", || { + vm.with_recursion(" while encoding a JSON object", || { if let Some(s) = obj.downcast_ref::() { self.write_str_obj(obj, s, out, vm) } else if vm.is_none(obj) { diff --git a/crates/stdlib/src/pickle.rs b/crates/stdlib/src/pickle.rs index 8badf9a85ad..031c700fdf9 100644 --- a/crates/stdlib/src/pickle.rs +++ b/crates/stdlib/src/pickle.rs @@ -2896,7 +2896,7 @@ mod _pickle { impl SaveCtx<'_> { fn save(&mut self, obj: &PyObject, pers_save: bool, vm: &VirtualMachine) -> PyResult<()> { - vm.with_recursion("while pickling an object", || { + vm.with_recursion(" while pickling an object", || { self.save_inner(obj, pers_save, vm) }) } diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index 31e1b4ef1c7..9c6a2485c5c 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -346,7 +346,7 @@ fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyRe let list = arg.downcast_ref::().unwrap(); list.borrow_vec().to_vec() }; - let sub = vm.with_recursion("while computing __parameters__", || { + let sub = vm.with_recursion(" while computing __parameters__", || { make_parameters_from_slice(&items, vm) })?; for sub_param in sub.as_slice() { diff --git a/crates/vm/src/builtins/iter.rs b/crates/vm/src/builtins/iter.rs index 85f40c25977..c3eca4dc7f2 100644 --- a/crates/vm/src/builtins/iter.rs +++ b/crates/vm/src/builtins/iter.rs @@ -250,7 +250,7 @@ impl PySequenceIterator { impl Py { #[pymethod] fn __length_hint__(&self, vm: &VirtualMachine) -> PyResult { - vm.with_recursion("in __length_hint__", || { + vm.with_recursion(" in __length_hint__", || { let (obj, position) = { let internal = self.internal.lock(); match &internal.status { diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index 2c61d8ba7c1..0f1b4dca994 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -132,7 +132,7 @@ pub(super) mod types { // frame, so a deep enough group runs off the native stack // unless this guard is here. let subgroup_result = vm - .with_recursion("in exception group subgroup", || { + .with_recursion(" in exception group subgroup", || { vm.call_method(&exc, "subgroup", (matcher_value.clone(),)) })?; if !vm.is_none(&subgroup_result) { @@ -182,7 +182,7 @@ pub(super) mod types { if is_base_exception_group(&exc, vm) { // Same as in subgroup: nothing else bounds this recursion // against the native stack. - let result = vm.with_recursion("in exception group split", || { + let result = vm.with_recursion(" in exception group split", || { vm.call_method(&exc, "split", (matcher_value.clone(),)) })?; let result_tuple: PyTupleRef = result.try_into_value(vm)?; diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index 472f33c26f5..e5e625043ec 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -327,7 +327,7 @@ impl PyObject { ) -> PyResult> { // Single recursion guard for the entire comparison // (do_richcompare in Objects/object.c). - vm.with_recursion("in comparison", || self._cmp_inner(other, op, vm)) + vm.with_recursion(" in comparison", || self._cmp_inner(other, op, vm)) } fn _cmp_inner( @@ -419,7 +419,7 @@ impl PyObject { } pub fn repr(&self, vm: &VirtualMachine) -> PyResult> { - vm.with_recursion("while getting the repr of an object", || { + vm.with_recursion(" while getting the repr of an object", || { self.class().slots().repr.load().map_or_else( || { Err(vm.new_runtime_error(format!( @@ -542,7 +542,7 @@ impl PyObject { debug_assert!(n >= 2); for i in 0..n { - let result = vm.with_recursion("in __issubclass__", || { + let result = vm.with_recursion(" in __issubclass__", || { bases.as_slice()[i].abstract_issubclass(cls, vm) })?; @@ -611,7 +611,7 @@ impl PyObject { // Check if cls is a tuple if let Some(tuple) = cls.downcast_ref::() { for item in tuple { - if vm.with_recursion("in __subclasscheck__", || derived.is_subclass(item, vm))? { + if vm.with_recursion(" in __subclasscheck__", || derived.is_subclass(item, vm))? { return Ok(true); } } @@ -620,7 +620,7 @@ impl PyObject { // Check for __subclasscheck__ method using lookup_special if let Some(checker) = cls.lookup_special(identifier!(vm, __subclasscheck__), vm)? { - let res = vm.with_recursion("in __subclasscheck__", || { + let res = vm.with_recursion(" in __subclasscheck__", || { checker.call((derived.to_owned(),), vm) })?; return res.try_to_bool(vm); @@ -698,7 +698,7 @@ impl PyObject { // Check if cls is a tuple if let Some(tuple) = cls.downcast_ref::() { for item in tuple { - if vm.with_recursion("in __instancecheck__", || { + if vm.with_recursion(" in __instancecheck__", || { self.object_recursive_isinstance(item, vm) })? { return Ok(true); @@ -709,7 +709,7 @@ impl PyObject { // Check for __instancecheck__ method using lookup_special if let Some(checker) = cls.lookup_special(identifier!(vm, __instancecheck__), vm)? { - let res = vm.with_recursion("in __instancecheck__", || { + let res = vm.with_recursion(" in __instancecheck__", || { checker.call((self.to_owned(),), vm) })?; return res.try_to_bool(vm); @@ -721,7 +721,7 @@ impl PyObject { pub fn hash(&self, vm: &VirtualMachine) -> PyResult { if let Some(hash) = self.class().slots().hash.load() { - return vm.with_recursion("while hashing", || hash(self, vm)); + return vm.with_recursion(" while hashing", || hash(self, vm)); } Err(vm.new_type_error(format!("unhashable type: '{}'", self.class().slot_name()))) diff --git a/crates/vm/src/stdlib/_ast/node.rs b/crates/vm/src/stdlib/_ast/node.rs index 7737ae076bb..56be51e00fa 100644 --- a/crates/vm/src/stdlib/_ast/node.rs +++ b/crates/vm/src/stdlib/_ast/node.rs @@ -50,7 +50,7 @@ impl Node for Vec { } items[i].clone() }; - result.push(vm.with_recursion("while traversing AST node", || { + result.push(vm.with_recursion(" while traversing AST node", || { Node::ast_from_object(vm, source_file, item) })?); if list.borrow_vec().len() != len { @@ -91,7 +91,7 @@ impl Node for Box { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult { - vm.with_recursion("while traversing AST node", || { + vm.with_recursion(" while traversing AST node", || { T::ast_from_object(vm, source_file, object).map(Self::new) }) } diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 50d3a7a730b..31f8bba6bb3 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -589,7 +589,7 @@ pub fn hash_not_implemented(zelf: &PyObject, vm: &VirtualMachine) -> PyResult PyResult { // `__call__` can name the object being called, and dispatching it pushes no // Python frame, so nothing else counts the nesting. - vm.with_recursion("while calling a Python object", || { + vm.with_recursion(" while calling a Python object", || { vm.call_special_method(zelf, identifier!(vm, __call__), args) }) } @@ -776,7 +776,7 @@ fn descr_get_wrapper( ) -> PyResult { // A descriptor whose `__get__` is the descriptor itself resolves it by // fetching `__get__` again, and none of that pushes a Python frame. - vm.with_recursion("while calling a Python object", || { + vm.with_recursion(" while calling a Python object", || { vm.call_special_method( zelf, identifier!(vm, __get__), diff --git a/crates/vm/src/vm/crossinterp.rs b/crates/vm/src/vm/crossinterp.rs index 2b799bf0031..fbab68f153d 100644 --- a/crates/vm/src/vm/crossinterp.rs +++ b/crates/vm/src/vm/crossinterp.rs @@ -227,7 +227,7 @@ impl SharedValue { t.as_slice() .iter() .map(|item| { - vm.with_recursion("while sharing a tuple", || { + vm.with_recursion(" while sharing a tuple", || { Self::from_object(item, fallback, vm) }) }) diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 4a74479f43d..238b19a2b50 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2592,9 +2592,7 @@ impl VirtualMachine { let counted_too_deep = false; if counted_too_deep || self.check_c_stack_overflow() { - return Err( - self.new_recursion_error(format!("maximum recursion depth exceeded {_where}")) - ); + return Err(self.new_recursion_depth_error(_where)); } #[cfg(any(miri, target_env = "musl"))] @@ -2628,9 +2626,7 @@ impl VirtualMachine { let counted_too_deep = false; if counted_too_deep || self.check_c_stack_overflow() { - return Err( - self.new_recursion_error(format!("maximum recursion depth exceeded {_where}")) - ); + return Err(self.new_recursion_depth_error(_where)); } #[cfg(any(miri, target_env = "musl"))] @@ -2657,7 +2653,7 @@ impl VirtualMachine { // code -- an `__add__` chain, a sort key that sorts -- takes more than // the margin in that many. if self.check_c_stack_overflow() { - return Err(self.new_recursion_error(String::new())); + return Err(self.new_recursion_depth_error("")); } self.recursion_depth.update(|d| d + 1); @@ -2764,7 +2760,7 @@ impl VirtualMachine { iframe: &mut crate::frame::InterpreterFrame, ) -> PyResult { if self.check_c_stack_overflow() { - return Err(self.new_recursion_error(String::new())); + return Err(self.new_recursion_depth_error("")); } self.recursion_depth.update(|d| d + 1); @@ -2905,7 +2901,7 @@ impl VirtualMachine { ) -> PyResult { self.check_recursive_call("")?; if self.check_c_stack_overflow() { - return Err(self.new_recursion_error(String::new())); + return Err(self.new_recursion_depth_error("")); } self.recursion_depth.update(|d| d + 1); @@ -3084,10 +3080,25 @@ impl VirtualMachine { self.tracing_depth.get() != 0 } + /// `where_` is empty, or a suffix that already includes the leading space. + #[cold] + fn new_recursion_depth_error(&self, where_: &str) -> PyBaseExceptionRef { + debug_assert!( + where_.is_empty() || where_.starts_with(' '), + "recursion where-clause must be empty or start with a space: {where_:?}" + ); + let msg = if where_.is_empty() { + "maximum recursion depth exceeded".to_string() + } else { + format!("maximum recursion depth exceeded{where_}") + }; + self.new_recursion_error(msg) + } + // To be called right before raising the recursion depth. fn check_recursive_call(&self, _where: &str) -> PyResult<()> { if self.recursion_depth.get() >= self.recursion_limit.get() { - Err(self.new_recursion_error(format!("maximum recursion depth exceeded {_where}"))) + Err(self.new_recursion_depth_error(_where)) } else { Ok(()) } diff --git a/extra_tests/snippets/builtin_hash.py b/extra_tests/snippets/builtin_hash.py index d4bf1168323..2426babef5a 100644 --- a/extra_tests/snippets/builtin_hash.py +++ b/extra_tests/snippets/builtin_hash.py @@ -35,10 +35,10 @@ def __hash__(self): # slot dispatch is what recurses, so that is where the depth is checked. if sys.implementation.name == "rustpython": - # Deep enough to reach the native stack guard; CPython, which also runs - # this snippet, dies on the same value. + # Deep enough to reach the native stack guard in release; CPython, which + # also runs this snippet, dies on the same value. deep_tuple = () - for _ in range(100_000): + for _ in range(500_000): deep_tuple = (deep_tuple,) with assert_raises(RecursionError): hash(deep_tuple) diff --git a/extra_tests/snippets/recursion.py b/extra_tests/snippets/recursion.py index 4b61a74b438..fe7daa5d69a 100644 --- a/extra_tests/snippets/recursion.py +++ b/extra_tests/snippets/recursion.py @@ -44,3 +44,32 @@ class Descr: pass else: raise AssertionError("descr.x should not resolve") + + +def recurse(): + recurse() + + +try: + recurse() +except RecursionError as e: + msg = str(e) + assert msg, f"empty RecursionError message: {e!r}" + assert not msg.endswith(" "), f"trailing space in RecursionError: {e!r}" + +import sys + +prev_limit = sys.getrecursionlimit() +try: + sys.setrecursionlimit(50) + try: + recurse() + except RecursionError as e: + assert str(e) == "maximum recursion depth exceeded", ( + f"unexpected message: {e!r}" + ) + assert e.args == ("maximum recursion depth exceeded",), ( + f"unexpected args: {e.args!r}" + ) +finally: + sys.setrecursionlimit(prev_limit) diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index d913095f2f9..bffc9f82445 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -59,9 +59,10 @@ def _run_missing_type_params_regression(): with assert_raises(RecursionError): list[nested] - # hashing an alias walks the same shape + # hashing an alias walks the same shape. Release inlining of the hash + # slot needs more depth than debug to trip the native stack guard. deep_alias = int - for _ in range(100_000): + for _ in range(500_000): deep_alias = list[deep_alias] with assert_raises(RecursionError): hash(deep_alias) From 945469f2940ffd0da025f300ab61f9e326b60f27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:04:36 +0900 Subject: [PATCH 041/229] build(deps): bump dns-lookup from 4.0.1 to 4.0.2 (#9002) Bumps [dns-lookup](https://github.com/keeperofdakeys/dns-lookup) from 4.0.1 to 4.0.2. - [Commits](https://github.com/keeperofdakeys/dns-lookup/compare/4.0.1...4.0.2) --- updated-dependencies: - dependency-name: dns-lookup dependency-version: 4.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0641dd859a9..23f6a1dbfab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1334,9 +1334,9 @@ dependencies = [ [[package]] name = "dns-lookup" -version = "4.0.1" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6120e7e5dcd1c90098e349def389e2797377cdbe6b465ad8fdd115c0c99d5cf2" +checksum = "263c0ccfc6b3b6a93e959edad4f985779e76c0b36dbb2e13e3d27c690974adf1" dependencies = [ "cfg-if", "libc", @@ -1469,7 +1469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2048,7 +2048,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3581,7 +3581,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4686,7 +4686,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5180,7 +5180,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 1dcdec4dd59bbb7749525ba5f8176e2f9d6ad157 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:11:26 +0900 Subject: [PATCH 042/229] Port the remaining runtime fixes from #7993 (#8995) * Take re pos/endpos as signed integers Negative values clamp to 0, and values beyond Py_ssize_t raise OverflowError. Assisted-by: Claude Code:claude-opus-5-5 * Port the remaining runtime fixes from #7993 - bytearray: keep the buffer exported while converting the needle of find/count/index/rfind/rindex/__contains__/split/rsplit and the hex separator, so resizing from there raises BufferError - str(): report non-str encoding/errors as "str() argument 'X' must be str, not T", and check argument counts and name/position duplicates for keyword calls - _json.scanstring: take end as a signed integer and raise ValueError when it is out of bounds - lzma: reject filter specs with keys the filter does not take - mmap: raise ValueError when growing a shared anonymous mapping on Linux/NetBSD - _sqlite3: check that the backup progress argument is callable - Remove the expectedFailure markers of the tests these fix Assisted-by: Claude Code:claude-opus-5-5 * Reject arguments given by name and position in FuncArgs::bind_for bind_for reports a keyword that fills a parameter already given by position before other binding errors, using the parameter list of the FromArgs type. getargs shares the message, str() drops its own check, and blake2b/blake2s name themselves in binding errors. Assisted-by: Claude Code:claude-opus-5-5 --------- Co-authored-by: James David Clarke --- Lib/test/test_bytes.py | 2 - Lib/test/test_enum.py | 1 - Lib/test/test_hashlib.py | 1 - Lib/test/test_json/test_scanstring.py | 1 - Lib/test/test_lzma.py | 2 - Lib/test/test_mmap.py | 1 - Lib/test/test_sqlite3/test_backup.py | 1 - Lib/test/test_str.py | 1 - crates/stdlib/src/_sqlite3.rs | 10 +++- crates/stdlib/src/blake2.rs | 4 +- crates/stdlib/src/json.rs | 19 +++----- crates/stdlib/src/lzma.rs | 59 +++++++++++++++++++++++- crates/stdlib/src/mmap.rs | 30 +++++++++++- crates/vm/src/builtins/bytearray.rs | 60 +++++++++++++++++------- crates/vm/src/builtins/str.rs | 42 +++++++++++++++-- crates/vm/src/bytes_inner.rs | 6 ++- crates/vm/src/function/argument.rs | 66 ++++++++++++++++++++++++++- crates/vm/src/function/getargs.rs | 9 ++-- crates/vm/src/function/mod.rs | 4 +- crates/vm/src/stdlib/_sre.rs | 34 +++++++++----- 20 files changed, 283 insertions(+), 70 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 57dec4e04a8..ffc59dab9e8 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -2013,7 +2013,6 @@ def __index__(self): self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: unexpected type Evil def test_search_methods_reentrancy_raises_buffererror(self): # gh-142560: Raise BufferError if buffer mutates during search arg conversion. class Evil: @@ -2063,7 +2062,6 @@ def __length_hint__(self): self.assertRaises(ValueError, float, bytearray()) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: BufferError not raised by hex def test_hex_use_after_free(self): # Prevent UAF in bytearray.hex(sep) with re-entrant sep.__len__. # Regression test for https://github.com/python/cpython/issues/143195. diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 57981ccc71b..dc7628945d3 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -3036,7 +3036,6 @@ class ThirdFailedStrEnum(StrEnum): one = '1' two = b'2', 'ascii', 9 - @unittest.expectedFailure # TODO: RUSTPYTHON; fails on encoding testing : TypeError: Expected type 'str' but 'builtin_function_or_method' found def test_custom_strenum(self): class CustomStrEnum(str, Enum): pass diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 6652b829b5a..c7d7b801ac7 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -275,7 +275,6 @@ def test_clinic_signature(self): self._hashlib.new(digest_name, data=b'') self._hashlib.new(digest_name, string=b'') - @unittest.expectedFailure # TODO: RUSTPYTHON; duplicate positional/keyword arg error message differs @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") def test_clinic_signature_errors(self): nomsg = b'' diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index e77ec152280..baa43f93624 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -144,7 +144,6 @@ def test_bad_escapes(self): with self.assertRaises(self.JSONDecodeError, msg=s): scanstring(s, 1, True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_overflow(self): with self.assertRaises(OverflowError): self.json.decoder.scanstring("xxx", sys.maxsize+1) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index c7f2662ab0b..90556525dc9 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -63,7 +63,6 @@ def test_simple_bad_args(self): lzd.decompress(empty) self.assertRaises(EOFError, lzd.decompress, b"quux") - @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder def test_bad_filter_spec(self): self.assertRaises(TypeError, LZMACompressor, filters=[b"wobsite"]) self.assertRaises(ValueError, LZMACompressor, filters=[{"xyzzy": 3}]) @@ -674,7 +673,6 @@ def test_init_bad_preset(self): with self.assertRaises(ValueError): LZMAFile(BytesIO(COMPRESSED_XZ), preset=3) - @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder def test_init_bad_filter_spec(self): with self.assertRaises(TypeError): LZMAFile(BytesIO(), "w", filters=[b"wobsite"]) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index b5a2a89d865..bd8744a0ca8 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -924,7 +924,6 @@ def __index__(self): self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, Number()), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None) - @unittest.expectedFailureIf(sys.platform in ("linux", "win32"), "TODO: RUSTPYTHON") def test_resize_up_anonymous_mapping(self): """If the mmap is backed by the pagefile ensure a resize up can happen and that the original data is still in place diff --git a/Lib/test/test_sqlite3/test_backup.py b/Lib/test/test_sqlite3/test_backup.py index bc24831a0c7..c7400d8b216 100644 --- a/Lib/test/test_sqlite3/test_backup.py +++ b/Lib/test/test_sqlite3/test_backup.py @@ -103,7 +103,6 @@ def progress(status, remaining, total): self.assertEqual(len(journal), 1) self.assertEqual(journal[0], 0) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_callable_progress(self): with self.assertRaises(TypeError) as cm: with memory_database() as bck: diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 4869c5ca9b0..f10f41df8b9 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -2672,7 +2672,6 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "str expected at most 3 arguments, got 4" does not match "expected at most 3 arguments, got 4" def test_str_invalid_call(self): # too many args with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs index 75c228fa3e6..e5953aef6ae 100644 --- a/crates/stdlib/src/_sqlite3.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -397,7 +397,7 @@ mod _sqlite3 { #[pyarg(named, default = -1)] pages: c_int, #[pyarg(named, optional)] - progress: Option, + progress: Option, #[pyarg(named, optional)] name: Option, #[pyarg(named, default = 0.250)] @@ -1253,6 +1253,12 @@ mod _sqlite3 { return Err(vm.new_value_error("target cannot be the same connection instance")); } + if let Some(progress) = &progress + && !progress.is_callable() + { + return Err(vm.new_type_error("progress argument must be a callable")); + } + let pages = if pages == 0 { -1 } else { pages }; let name_cstring; @@ -1284,7 +1290,7 @@ mod _sqlite3 { if let Some(progress) = &progress { let remaining = unsafe { sqlite3_backup_remaining(handle) }; let pagecount = unsafe { sqlite3_backup_pagecount(handle) }; - if let Err(err) = progress.invoke((ret, remaining, pagecount), vm) { + if let Err(err) = progress.call((ret, remaining, pagecount), vm) { unsafe { sqlite3_backup_finish(handle) }; return Err(err); } diff --git a/crates/stdlib/src/blake2.rs b/crates/stdlib/src/blake2.rs index a7dad88d099..04d3f2e8708 100644 --- a/crates/stdlib/src/blake2.rs +++ b/crates/stdlib/src/blake2.rs @@ -149,7 +149,7 @@ mod _blake2 { #[pyslot] fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let args: BlakeHashArgs = args.bind(vm)?; + let args: BlakeHashArgs = args.bind_for(vm, "blake2b")?; Ok(Self { inner: local_blake2b(args, vm)?, } @@ -246,7 +246,7 @@ mod _blake2 { #[pyslot] fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let args: BlakeHashArgs = args.bind(vm)?; + let args: BlakeHashArgs = args.bind_for(vm, "blake2s")?; Ok(Self { inner: local_blake2s(args, vm)?, } diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index abd46a42216..e553dbb652b 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -1323,30 +1323,25 @@ mod _json { #[pyfunction] fn scanstring( s: PyStrRef, - end: usize, + end: isize, strict: OptionalArg, vm: &VirtualMachine, ) -> PyResult<(Wtf8Buf, usize)> { flame_guard!("_json::scanstring"); let wtf8 = s.as_wtf8(); + if end < 0 || end as usize > s.char_len() { + return Err(vm.new_value_error("end is out of bounds")); + } + let end = end as usize; + // Convert char index `end` to byte index let byte_idx = if end == 0 { 0 } else { wtf8.code_point_indices() .nth(end) - .map(|(i, _)| i) - .ok_or_else(|| { - py_decode_error( - json::DecodeError { - msg: "Unterminated string starting at".to_owned(), - pos: end - 1, - }, - s.clone(), - vm, - ) - })? + .map_or(wtf8.len(), |(i, _)| i) }; let (result, end_char_idx, _bytes_consumed) = diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index 1e539c821d5..068307a4920 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -8,7 +8,7 @@ mod _lzma { use alloc::fmt; use core::sync::atomic::{AtomicBool, AtomicI32, Ordering}; use rustpython_common::{compression::lzma as backend, lock::PyMutex}; - use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytes, PyDict, PyType, PyTypeRef}; + use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytes, PyDict, PyStr, PyType, PyTypeRef}; use rustpython_vm::function::{ArgBytesLike, ItemDoc}; use rustpython_vm::object::PyAtomicRef; use rustpython_vm::types::Constructor; @@ -129,6 +129,30 @@ mod _lzma { }) } + /// A key the filter does not take makes the whole specifier invalid, as + /// the spec dict is read as keyword arguments. + fn check_filter_spec_keys( + spec: &PyObject, + allowed: &[&str], + error: &str, + vm: &VirtualMachine, + ) -> PyResult<()> { + let dict = spec.downcast_ref::().ok_or_else(|| { + vm.new_type_error("Filter specifier must be a dict or dict-like object") + })?; + for key in dict.keys_vec() { + let ok = key.downcast_ref::().is_some_and(|k| { + allowed + .iter() + .any(|a| k.as_wtf8().as_bytes() == a.as_bytes()) + }); + if !ok { + return Err(vm.new_value_error(error.to_owned())); + } + } + Ok(()) + } + fn parse_filter_chain_item( spec: &PyObject, vm: &VirtualMachine, @@ -136,6 +160,23 @@ mod _lzma { let mut parsed = filter_spec_with_id(spec, vm)?; match parsed.id { FILTER_LZMA1 | FILTER_LZMA2 => { + check_filter_spec_keys( + spec, + &[ + "id", + "preset", + "dict_size", + "lc", + "lp", + "pb", + "mode", + "nice_len", + "mf", + "depth", + ], + "Invalid filter specifier for LZMA filter", + vm, + )?; parsed.preset = get_dict_opt_u32(spec, "preset", vm)?; parsed.dict_size = get_dict_opt_u32(spec, "dict_size", vm)?; parsed.lc = get_dict_opt_u32(spec, "lc", vm)?; @@ -146,9 +187,23 @@ mod _lzma { parsed.mf = get_dict_opt_u32(spec, "mf", vm)?; parsed.depth = get_dict_opt_u32(spec, "depth", vm)?; } - FILTER_DELTA => parsed.dist = get_dict_opt_u32(spec, "dist", vm)?, + FILTER_DELTA => { + check_filter_spec_keys( + spec, + &["id", "dist"], + "Invalid filter specifier for delta filter", + vm, + )?; + parsed.dist = get_dict_opt_u32(spec, "dist", vm)?; + } FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB | FILTER_SPARC => { + check_filter_spec_keys( + spec, + &["id", "start_offset"], + "Invalid filter specifier for BCJ filter", + vm, + )?; parsed.start_offset = get_dict_opt_u32(spec, "start_offset", vm)?; } _ => {} diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index c33c21fcdb0..a39c1278455 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -188,6 +188,9 @@ mod mmap { mmap: PyMutex>, #[cfg(unix)] fd: AtomicCell, + // only read by the linux/netbsd mremap expansion check + #[cfg(any(target_os = "linux", target_os = "netbsd"))] + flags: core::ffi::c_int, #[cfg(windows)] handle: AtomicCell, // host_mmap::Handle is isize on Windows offset: i64, @@ -402,7 +405,11 @@ mod mmap { } // TODO: memmap2 doesn't support mapping with prot and flags right now - let (_flags, _prot, access) = match access { + #[cfg_attr( + not(any(target_os = "linux", target_os = "netbsd")), + allow(unused_variables) + )] + let (flags, _prot, access) = match access { AccessMode::Read => (MAP_SHARED, PROT_READ, access), AccessMode::Write => (MAP_SHARED, PROT_READ | PROT_WRITE, access), AccessMode::Copy => (MAP_PRIVATE, PROT_READ | PROT_WRITE, access), @@ -476,6 +483,8 @@ mod mmap { } else { -1 }), + #[cfg(any(target_os = "linux", target_os = "netbsd"))] + flags, offset, size: AtomicCell::new(map_size), pos: AtomicCell::new(0), @@ -1041,9 +1050,26 @@ mod mmap { #[cfg(unix)] #[pymethod] - fn resize(zelf: &Py, _newsize: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + fn resize(zelf: &Py, newsize: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { zelf.check_resizeable(vm)?; + + let new_size: isize = newsize.try_to_primitive(vm).map_err(|_| { + vm.new_overflow_error("Python int too large to convert to C ssize_t") + })?; + + // Linux mremap() refuses to grow a shared anonymous mapping, and NetBSD + // mremap() returns a mapping whose grown region is not backed. + #[cfg(any(target_os = "linux", target_os = "netbsd"))] + if zelf.fd.load() == -1 + && zelf.flags & host_mmap::MAP_PRIVATE == 0 + && new_size > zelf.size.load() as isize + { + return Err(vm.new_value_error("mmap: can't expand a shared anonymous mapping")); + } + // TODO: implement using mremap on Linux + #[cfg(not(any(target_os = "linux", target_os = "netbsd")))] + let _ = new_size; Err(vm.new_system_error("mmap: resizing not available--no mremap()")) } diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index 9e166ca24ca..5afeec62833 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -23,7 +23,7 @@ use crate::{ }, }, convert::{ToPyObject, ToPyResult}, - function::{ArgBytesLike, PyComparisonValue, PySsize}, + function::{ArgBytesLike, FuncArgs, PyComparisonValue, PySsize}, protocol::{ BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods, @@ -187,6 +187,9 @@ impl PyByteArray { } fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Converting the needle runs Python, which must not resize this bytearray. + self.exports.fetch_add(1, Ordering::Release); + scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } let needle = ByteInnerSub::from_contains_arg(needle, vm)?; self.inner().contains(needle, vm) } @@ -469,8 +472,13 @@ impl Py { #[pymethod] fn hex(&self, options: ByteInnerHexOptions, vm: &VirtualMachine) -> PyResult { // Measuring the separator runs Python, so it happens before the buffer - // is borrowed. - let (sep, bytes_per_sep) = options.resolve(vm)?; + // is borrowed, with the buffer exported so that resizing this + // bytearray from there raises BufferError. + let (sep, bytes_per_sep) = { + self.exports.fetch_add(1, Ordering::Release); + scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } + options.resolve(vm)? + }; Ok(self.inner().hex(sep, bytes_per_sep)) } @@ -511,6 +519,8 @@ impl Py { #[pymethod] fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.exports.fetch_add(1, Ordering::Release); + scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } self.inner().count(options, vm) } @@ -565,27 +575,43 @@ impl Py { ) } + /// Converting the needle runs Python, which must not resize this + /// bytearray, so the buffer stays exported while it does. + fn find_exported( + &self, + options: ByteInnerFindOptions, + find: F, + vm: &VirtualMachine, + ) -> PyResult> + where + F: Fn(&[u8], &[u8]) -> Option, + { + self.exports.fetch_add(1, Ordering::Release); + scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } + self.inner().find(options, find, vm) + } + #[pymethod] fn find(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { - let index = self.inner().find(options, |h, n| h.find(n), vm)?; + let index = self.find_exported(options, |h, n| h.find(n), vm)?; Ok(index.map_or(-1, |v| v as isize)) } #[pymethod] fn index(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { - let index = self.inner().find(options, |h, n| h.find(n), vm)?; + let index = self.find_exported(options, |h, n| h.find(n), vm)?; index.ok_or_else(|| vm.new_value_error("substring not found")) } #[pymethod] fn rfind(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { - let index = self.inner().find(options, |h, n| h.rfind(n), vm)?; + let index = self.find_exported(options, |h, n| h.rfind(n), vm)?; Ok(index.map_or(-1, |v| v as isize)) } #[pymethod] fn rindex(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { - let index = self.inner().find(options, |h, n| h.rfind(n), vm)?; + let index = self.find_exported(options, |h, n| h.rfind(n), vm)?; index.ok_or_else(|| vm.new_value_error("substring not found")) } @@ -614,21 +640,21 @@ impl Py { } #[pymethod] - fn split( - &self, - options: ByteInnerSplitOptions, - vm: &VirtualMachine, - ) -> PyResult> { + fn split(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult> { + // Converting the separator runs Python, which must not resize this + // bytearray, so the arguments are bound with the buffer exported. + self.exports.fetch_add(1, Ordering::Release); + scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } + let options: ByteInnerSplitOptions = args.bind_for(vm, "split")?; self.inner() .split(options, |s, vm| vm.ctx.new_bytearray(s.to_vec()).into(), vm) } #[pymethod] - fn rsplit( - &self, - options: ByteInnerSplitOptions, - vm: &VirtualMachine, - ) -> PyResult> { + fn rsplit(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult> { + self.exports.fetch_add(1, Ordering::Release); + scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } + let options: ByteInnerSplitOptions = args.bind_for(vm, "rsplit")?; self.inner() .rsplit(options, |s, vm| vm.ctx.new_bytearray(s.to_vec()).into(), vm) } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index 91f99b22c86..7d339ad1b42 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -391,15 +391,49 @@ pub struct StrArgs { #[pyarg(any, optional)] object: OptionalArg, #[pyarg(any, optional)] - encoding: OptionalArg, + encoding: OptionalArg, #[pyarg(any, optional)] - errors: OptionalArg, + errors: OptionalArg, +} + +/// The `encoding` or `errors` argument of `str()`, which must be a str. +fn str_new_str_arg( + arg: OptionalArg, + name: &str, + vm: &VirtualMachine, +) -> PyResult> { + let OptionalArg::Present(arg) = arg else { + return Ok(None); + }; + let arg = arg.downcast::().map_err(|arg| { + vm.new_type_error(format!( + "str() argument '{name}' must be str, not {}", + arg.class().name() + )) + })?; + Ok(Some(arg.try_into_utf8(vm)?)) } impl Constructor for PyStr { type Args = StrArgs; fn slot_new(cls: PyTypeRef, func_args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // A call with keyword arguments is parsed by unicode_new, which counts + // every argument. + if !func_args.kwargs.is_empty() { + let total = func_args.args.len() + func_args.kwargs.len(); + if total > 3 { + let keyword = if func_args.args.is_empty() { + "keyword " + } else { + "" + }; + return Err(vm.new_type_error(format!( + "str() takes at most 3 {keyword}arguments ({total} given)" + ))); + } + } + // Optimization: return exact str as-is (only when no encoding/errors provided) if cls.is(vm.ctx.types.str_type) && func_args.args.len() == 1 @@ -433,8 +467,8 @@ impl Constructor for PyStr { fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { match args.object { OptionalArg::Present(input) => { - let encoding = args.encoding.into_option(); - let errors = args.errors.into_option(); + let encoding = str_new_str_arg(args.encoding, "encoding", vm)?; + let errors = str_new_str_arg(args.errors, "errors", vm)?; // CPython parity: presence of `encoding` OR `errors` triggers // decode mode. When `errors` is given alone, the encoding // defaults to UTF-8. diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index ecda0a42ce6..f577e302085 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -313,7 +313,7 @@ impl ByteInnerSub { #[derive(FromArgs)] pub struct ByteInnerFindOptions { #[pyarg(positional)] - sub: ByteInnerSub, + sub: PyObjectRef, #[pyarg(positional, default)] start: Option, #[pyarg(positional, default)] @@ -326,12 +326,14 @@ impl ByteInnerFindOptions { len: usize, vm: &VirtualMachine, ) -> PyResult<(Vec, core::ops::Range)> { - let sub = self.sub.into_vec(vm)?; let range = anystr::adjust_indices( self.start.as_ref().map(AsRef::as_ref), self.end.as_ref().map(AsRef::as_ref), len, ); + // The needle is converted after the slice indices, and only here, so + // that a caller can guard its buffer while the conversion runs Python. + let sub = ByteInnerSub::try_from_object(vm, self.sub)?.into_vec(vm)?; Ok((sub, range)) } } diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index 2106a668f77..0df326c71ff 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -1,4 +1,4 @@ -use super::signature::Param; +use super::signature::{Param, ParamKind}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyTupleRef, PyType}, @@ -307,6 +307,15 @@ impl FuncArgs { let arity = arity.start().saturating_sub(instance)..=arity.end().saturating_sub(instance); let num_given = self.args.len().saturating_sub(instance); + // A keyword that fills a parameter already given by position is + // reported before leftover-keyword and conversion errors. + if let Some(params) = T::PARAMS + && !self.kwargs.is_empty() + && let Some((name, pos)) = duplicate_name_and_position(params, num_given, &self.kwargs) + { + return Err(callee.given_by_name_and_position(name, pos, vm)); + } + let bound = T::from_args(vm, &mut self) .map_err(|e| e.into_exception(&arity, num_given, callee, vm))?; @@ -408,6 +417,15 @@ impl Callee { vm.new_type_error(unexpected_keyword_message(self.name, keyword)) } + fn given_by_name_and_position( + self, + keyword: &str, + pos: usize, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + vm.new_type_error(given_by_name_and_position_message(self.name, keyword, pos)) + } + /// The branch of _PyArg_UnpackKeywords that names a parameter it didn't get. fn missing_argument( self, @@ -464,6 +482,52 @@ pub(crate) fn unexpected_keyword_message(name: Option<&str>, keyword: &str) -> S format!("{name}{parens} got an unexpected keyword argument '{keyword}'") } +pub(crate) fn given_by_name_and_position_message( + name: Option<&str>, + keyword: &str, + pos: usize, +) -> String { + let name = name.map_or("function", short_name); + format!("argument for {name}() given by name ('{keyword}') and position ({pos})") +} + +/// First positional-or-keyword parameter that was also passed by keyword. +fn duplicate_name_and_position<'a>( + params: &'a [Param], + nargs: usize, + kwargs: &KwArgs, +) -> Option<(&'a str, usize)> { + fn walk<'a>( + params: &'a [Param], + nargs: usize, + kwargs: &KwArgs, + pos: &mut usize, + ) -> Option<(&'a str, usize)> { + for param in params { + match param.kind { + ParamKind::Flatten(Some(inner)) => { + if let Some(hit) = walk(inner, nargs, kwargs, pos) { + return Some(hit); + } + } + ParamKind::Flatten(None) | ParamKind::PositionalOnly => { + *pos += 1; + } + ParamKind::PositionalOrKeyword => { + *pos += 1; + if nargs >= *pos && !param.name.is_empty() && kwargs.contains_key(param.name) { + return Some((param.name, *pos)); + } + } + ParamKind::VarPositional => return None, + ParamKind::KeywordOnly | ParamKind::VarKeyword => {} + } + } + None + } + walk(params, nargs, kwargs, &mut 0) +} + /// The branch of _PyArg_UnpackKeywords that names a parameter it didn't get. pub(crate) fn missing_argument_message(name: Option<&str>, keyword: &str, pos: usize) -> String { let (name, parens) = call_form(name, "function"); diff --git a/crates/vm/src/function/getargs.rs b/crates/vm/src/function/getargs.rs index cc335727b7f..371b99137b6 100644 --- a/crates/vm/src/function/getargs.rs +++ b/crates/vm/src/function/getargs.rs @@ -1,6 +1,6 @@ //! `PyArg_ParseTupleAndKeywords` argument matching. -use super::FuncArgs; +use super::{FuncArgs, given_by_name_and_position_message}; use crate::{PyObject, PyObjectRef, PyResult, VirtualMachine, common::wtf8::Wtf8}; /// The keyword list of a `PyArg_ParseTupleAndKeywords` call together with the @@ -103,9 +103,10 @@ impl ArgSpec<'_> { if !exhausted && nkwargs > 0 { for (i, name) in self.keywords.iter().enumerate().take(nargs) { if args.kwargs.contains_key(name) { - return Err(vm.new_type_error(format!( - "argument for {fname}() given by name ('{name}') and position ({})", - i + 1 + return Err(vm.new_type_error(given_by_name_and_position_message( + Some(fname), + name, + i + 1, ))); } } diff --git a/crates/vm/src/function/mod.rs b/crates/vm/src/function/mod.rs index 0fcbffc96bb..707afed0966 100644 --- a/crates/vm/src/function/mod.rs +++ b/crates/vm/src/function/mod.rs @@ -19,7 +19,9 @@ pub use argument::{ NameKeywords, NameKwargs, NameKwds, NameKws, NameObjs, NameOthers, OptionalArg, OptionalArgDefault, OptionalOption, PosArgs, PositionalIterable, }; -pub(crate) use argument::{arity_message, unexpected_keyword_message}; +pub(crate) use argument::{ + arity_message, given_by_name_and_position_message, unexpected_keyword_message, +}; pub use arithmetic::{PyArithmeticValue, PyComparisonValue}; pub use buffer::{ ArgAsciiBuffer, ArgBytesLike, ArgContiguousBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike, diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs index af5b75fc8a7..9fe57a86955 100644 --- a/crates/vm/src/stdlib/_sre.rs +++ b/crates/vm/src/stdlib/_sre.rs @@ -332,14 +332,26 @@ mod _sre { default: OptionalArg, } + /// A position in the searched string, taken as a Py_ssize_t; a negative + /// one means the start of the string. + #[derive(Clone, Copy, Default)] + struct StringPos(usize); + + impl TryFromObject for StringPos { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let pos: isize = obj.try_into_value(vm)?; + Ok(Self(pos.max(0) as usize)) + } + } + #[derive(FromArgs)] struct StringArgs { string: PyObjectRef, #[pyarg(any, default)] - pos: usize, + pos: StringPos, // Platform ssize maximum, shown as sys.maxsize. - #[pyarg(any, default = sys::MAXSIZE as usize, py_default = "sys.maxsize")] - endpos: usize, + #[pyarg(any, default = StringPos(sys::MAXSIZE as usize), py_default = "sys.maxsize")] + endpos: StringPos, } #[derive(FromArgs)] @@ -533,7 +545,7 @@ mod _sre { endpos, } = string_args; with_sre_str!(zelf, &string.clone(), vm, |x| { - let req = x.create_request(&zelf, pos, endpos); + let req = x.create_request(&zelf, pos.0, endpos.0); let mut state = State::default(); Ok(state .py_match(&req) @@ -548,7 +560,7 @@ mod _sre { vm: &VirtualMachine, ) -> PyResult>> { with_sre_str!(zelf, &string_args.string.clone(), vm, |x| { - let mut req = x.create_request(&zelf, string_args.pos, string_args.endpos); + let mut req = x.create_request(&zelf, string_args.pos.0, string_args.endpos.0); req.match_all = true; let mut state = State::default(); Ok(state.py_match(&req).then(|| { @@ -564,7 +576,7 @@ mod _sre { vm: &VirtualMachine, ) -> PyResult>> { with_sre_str!(zelf, &string_args.string.clone(), vm, |x| { - let req = x.create_request(&zelf, string_args.pos, string_args.endpos); + let req = x.create_request(&zelf, string_args.pos.0, string_args.endpos.0); let mut state = State::default(); Ok(state.search(req).then(|| { Match::new(&mut state, zelf.clone(), string_args.string).into_ref(&vm.ctx) @@ -579,7 +591,7 @@ mod _sre { vm: &VirtualMachine, ) -> PyResult> { with_sre_str!(zelf, &string_args.string, vm, |s| { - let req = s.create_request(&zelf, string_args.pos, string_args.endpos); + let req = s.create_request(&zelf, string_args.pos.0, string_args.endpos.0); let state = State::default(); let mut match_list: Vec = Vec::new(); let mut iter = SearchIter { req, state }; @@ -627,8 +639,8 @@ mod _sre { let scanner = SreScanner { pattern: zelf, string: string_args.string, - start: AtomicCell::new(string_args.pos), - end: string_args.endpos, + start: AtomicCell::new(string_args.pos.0), + end: string_args.endpos.0, must_advance: AtomicCell::new(false), } .into_ref(&vm.ctx); @@ -647,8 +659,8 @@ mod _sre { SreScanner { pattern: zelf, string: string_args.string, - start: AtomicCell::new(string_args.pos), - end: string_args.endpos, + start: AtomicCell::new(string_args.pos.0), + end: string_args.endpos.0, must_advance: AtomicCell::new(false), } .into_ref(&vm.ctx) From 1e72b22470df9748e74527542676ef9064ecb3ef Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:23:35 +0900 Subject: [PATCH 043/229] Update malachite crates to 0.13.0 (#9003) Assisted-by: Claude Code:claude-opus-5-5 --- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 6 +++--- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23f6a1dbfab..0ee20a4ca9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1469,7 +1469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2048,7 +2048,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2384,9 +2384,9 @@ dependencies = [ [[package]] name = "malachite-base" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f368e032195e6f046f237a982cd541cac7d4e17d5c680cac5534dfa71cd8879e" +checksum = "60f9f06976ea22081565c6f4ddc40e699982c1c6b9bd1bd4871b92eb986f2307" dependencies = [ "hashbrown 0.16.1", "itertools 0.14.0", @@ -2396,9 +2396,9 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d673b71d4521ba5ca045cd23d62eda6b321474ff231d6a1698cc9b892851f2" +checksum = "5a0ea1859fa74e642919ea67bb7afe2e663f88f2e59afdeafda4a0b5ec6855fb" dependencies = [ "malachite-base", "malachite-nz", @@ -2409,9 +2409,9 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b3ad1a8d2a722fbfad2e1fc7c5044f7ab38e027240cfbcb086a62ca95c9c2" +checksum = "d5eef5870b4c0072ab10e28ed6e8228ae224f188bdbb65c739fcc189248381f5" dependencies = [ "itertools 0.14.0", "libm", @@ -2421,9 +2421,9 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7841d36b3f5d39f1c65d4bcc45cb104fde66cf7eafc10da3922483fa2847cb94" +checksum = "11ff775589824d11de83716ee96d70b7a4fbd70794415ba363b6aeed272010f7" dependencies = [ "itertools 0.14.0", "libm", @@ -3581,7 +3581,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4686,7 +4686,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5180,7 +5180,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 111ba98687e..792f2b981af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -246,9 +246,9 @@ log = "0.4.30" lz4_flex = "0.14" nix = { version = "0.31", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } mac_address = "1.1.3" -malachite-bigint = "0.12.0" -malachite-q = "0.12.0" -malachite-base = "0.12.0" +malachite-bigint = "0.13.0" +malachite-q = "0.13.0" +malachite-base = "0.13.0" md-5 = "0.11" memchr = { version = "2.8.1", default-features = false, features = ["alloc"] } memmap2 = "0.9.10" From ca9dbb40f89044072a051ed3a12cea40745d0ad3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:01:52 +0900 Subject: [PATCH 044/229] Give builtin_method its own type (#9004) Split `builtin_method` out as a subclass of `builtin_function_or_method`. Ordinary descriptor bind and `__new__` wrappers allocate a CFunction; only `build_bound_method` uses the METH_METHOD payload. Keep `types.BuiltinMethodType` as an alias of `BuiltinFunctionType`. Assisted-by: Grok 4.6 --- crates/vm/src/builtins/builtin_func.rs | 70 ++++++++++++++++++++++---- crates/vm/src/builtins/descriptor.rs | 10 ++-- crates/vm/src/class.rs | 6 +-- crates/vm/src/function/method.rs | 16 ++---- crates/vm/src/types/zoo.rs | 4 +- extra_tests/snippets/builtin_type.py | 5 ++ 6 files changed, 80 insertions(+), 31 deletions(-) diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index 79e2c1ca721..17e3e342485 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -233,16 +233,20 @@ impl PyNativeFunction { } } -// PyCMethodObject in CPython -#[pyclass(name = "builtin_function_or_method", module = false, base = PyNativeFunction, ctx = "builtin_function_or_method_type")] +// Bound METH_METHOD object. The payload starts with PyNativeFunction so it can +// be read as a builtin function. The Python type is `builtin_method`. +#[repr(C)] +#[pyclass( + name = "builtin_method", + module = false, + base = PyNativeFunction, + ctx = "builtin_method_type" +)] pub struct PyNativeMethod { pub(crate) func: PyNativeFunction, - pub(crate) class: &'static Py, // TODO: the actual life is &'self + pub(crate) class: &'static Py, } -// All Python-visible behavior (getters, slots) is registered by PyNativeFunction::extend_class. -// PyNativeMethod only extends the Rust-side struct with the defining class reference. -// The func field at offset 0 (#[repr(C)]) allows NativeFunctionOrMethod to read it safely. #[pyclass(flags(HAS_WEAKREF, DISALLOW_INSTANTIATION))] impl PyNativeMethod {} @@ -296,6 +300,7 @@ pub(crate) fn init(context: &'static Context) { .slots .vectorcall .store(Some(vectorcall_native_function)); + PyNativeMethod::extend_class(context, context.types.builtin_method_type); } /// Wrapper that provides access to the common PyNativeFunction data @@ -306,12 +311,59 @@ impl TryFromObject for NativeFunctionOrMethod { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let class = vm.ctx.types.builtin_function_or_method_type; if obj.fast_isinstance(class) { - // Both PyNativeFunction and PyNativeMethod share the same type now. - // PyNativeMethod has `func: PyNativeFunction` as its first field, - // so we can safely treat the data pointer as PyNativeFunction for reading. + // `builtin_method` is a subclass; the payload starts with PyNativeFunction. Ok(Self(unsafe { obj.downcast_unchecked() })) } else { Err(vm.new_downcast_type_error(class, &obj)) } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + Interpreter, PyObjectRef, + function::{PyMethodDef, PyMethodFlags}, + }; + + #[test] + fn builtin_method_is_subclass_of_builtin_function() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let function_type = vm.ctx.types.builtin_function_or_method_type; + let method_type = vm.ctx.types.builtin_method_type; + assert!(!function_type.is(method_type)); + assert_eq!(&*function_type.name(), "builtin_function_or_method"); + assert_eq!(&*method_type.name(), "builtin_method"); + assert!(method_type.fast_issubclass(function_type)); + }); + } + + #[test] + fn bound_instance_method_uses_function_type() { + fn identity(value: PyObjectRef) -> PyObjectRef { + value + } + const DEF: PyMethodDef = PyMethodDef::new_const( + "identity", + identity, + PyMethodFlags::METHOD, + crate::function::ItemDoc::NONE, + ); + + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let bound = DEF.build_bound_function(&vm.ctx, vm.ctx.none()); + assert!( + bound + .class() + .is(vm.ctx.types.builtin_function_or_method_type) + ); + assert!(!bound.as_object().downcastable::()); + + let method = DEF.build_bound_method(&vm.ctx, vm.ctx.none(), vm.ctx.types.object_type); + assert!(method.class().is(vm.ctx.types.builtin_method_type)); + assert!(method.as_object().downcastable::()); + assert!(method.as_object().downcastable::()); + }); + } +} diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index b3189e592a8..301448654a6 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -1,7 +1,7 @@ use super::{PyStr, PyStrInterned, PyTuple, PyType}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyTypeRef, builtin_func::PyNativeMethod, type_}, + builtins::{PyTypeRef, builtin_func::PyNativeFunction, type_}, class::PyClassImpl, common::hash::PyHash, convert::{ToPyObject, ToPyResult}, @@ -124,8 +124,8 @@ impl Callable for PyMethodDescriptor { } impl PyMethodDescriptor { - pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef { - self.method.build_bound_method(ctx, obj, self.common.typ) + pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef { + self.method.build_bound_function(ctx, obj) } } @@ -195,8 +195,8 @@ impl PyClassMethodDescriptor { } } - pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef { - self.method.build_bound_method(ctx, obj, self.common.typ) + pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef { + self.method.build_bound_function(ctx, obj) } } diff --git a/crates/vm/src/class.rs b/crates/vm/src/class.rs index 8a7a8306d16..4afdf6329fb 100644 --- a/crates/vm/src/class.rs +++ b/crates/vm/src/class.rs @@ -375,9 +375,9 @@ pub trait PyClassImpl: PyClassDef { && object_new.is_some_and(|obj_new| fn_addr(slot_new) == fn_addr(obj_new)); if !is_inherited_from_object { - let bound_new = - ctx.slot_new_wrapper - .build_bound_method(ctx, class.to_owned().into(), class); + let bound_new = ctx + .slot_new_wrapper + .build_bound_function(ctx, class.to_owned().into()); class.set_attr(identifier!(ctx, __new__), bound_new.into()); } } diff --git a/crates/vm/src/function/method.rs b/crates/vm/src/function/method.rs index 0a80efc939e..c9953f8a7e1 100644 --- a/crates/vm/src/function/method.rs +++ b/crates/vm/src/function/method.rs @@ -249,7 +249,7 @@ impl PyMethodDef { ) -> PyRef { PyRef::new_ref( self.to_bound_method(obj, class), - ctx.types.builtin_function_or_method_type.to_owned(), + ctx.types.builtin_method_type.to_owned(), None, ) } @@ -267,18 +267,10 @@ impl PyMethodDef { &'static self, ctx: &Context, class: &'static Py, - ) -> PyRef { + ) -> PyRef { debug_assert!(self.flags.contains(PyMethodFlags::STATIC)); - // Set zelf to the class (m_self = type for static methods). - // Callable::call skips prepending when STATIC flag is set. - let func = PyNativeFunction { - zelf: Some(class.to_owned().into()), - value: self, - module_object: None, - module: crate::object::PyAtomicRef::new_empty(), - _method_def_owner: None, - }; - PyNativeMethod { func, class }.into_ref(ctx) + // m_self is the type; STATIC skips prepending it on call. + self.build_bound_function(ctx, class.to_owned().into()) } /// Concatenate method groups. A pending body is copied from `docs`, then cleared. diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 9672e4cf3d2..fce1a1e9eb6 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -121,7 +121,7 @@ impl TypeZoo { let weakref_type = weakref::PyWeak::init_manually(hierarchy.weakref_type); let int_type = int::PyInt::init_builtin_type(); - // builtin_function_or_method and builtin_method share the same type (CPython behavior) + // `builtin_method` is a subclass and is initialized once this base exists. let builtin_function_or_method_type = builtin_func::PyNativeFunction::init_builtin_type(); let types = Self { @@ -164,7 +164,7 @@ impl TypeZoo { anext_awaitable: asyncgenerator::PyAnextAwaitable::init_builtin_type(), bound_method_type: function::PyBoundMethod::init_builtin_type(), builtin_function_or_method_type, - builtin_method_type: builtin_function_or_method_type, + builtin_method_type: builtin_func::PyNativeMethod::init_builtin_type(), bytearray_iterator_type: bytearray::PyByteArrayIterator::init_builtin_type(), bytes_iterator_type: bytes::PyBytesIterator::init_builtin_type(), callable_iterator: iter::PyCallableIterator::init_builtin_type(), diff --git a/extra_tests/snippets/builtin_type.py b/extra_tests/snippets/builtin_type.py index 06b40fe41f6..f00baaa57c7 100644 --- a/extra_tests/snippets/builtin_type.py +++ b/extra_tests/snippets/builtin_type.py @@ -654,7 +654,12 @@ def my_repr_func(): # https://github.com/RustPython/RustPython/issues/3100 +assert types.BuiltinMethodType is types.BuiltinFunctionType assert issubclass(types.BuiltinMethodType, types.BuiltinFunctionType) +assert type(len) is types.BuiltinFunctionType +assert type([].append) is types.BuiltinFunctionType +assert type([].append).__name__ == "builtin_function_or_method" +assert type(len).__name__ == "builtin_function_or_method" assert type.__dict__["__dict__"].__objclass__ is type assert ( From bb9ef5d0c290aed3ed6d5cb75a712719bcdeaf1d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:43:30 +0900 Subject: [PATCH 045/229] Infer native calling convention flags from argument metadata (#9005) `PyMethodDef::new_const`, `PyMethodDef::new_raw_const` and `Context::new_method_def` add the calling convention (`NOARGS`, `O`, `FASTCALL`, or `FASTCALL | KEYWORDS`) inferred from the `FromArgs::PARAMS` of each argument, exposed as `IntoPyNativeFn::ARGS`. Flags that already contain a calling convention, such as C-API `ml_flags`, are kept. For `METHOD` and `CLASS` the first argument is the bound receiver. Raw functions get `FASTCALL | KEYWORDS`. The derive macros pass only binding flags and no longer infer call flags from Rust type names. Definitions built without the macros, such as the `__new__` wrapper and `vm.new_function`, get calling convention flags the same way. Assisted-by: Claude Code:claude-opus-5-5 --- crates/derive-impl/src/pyclass.rs | 12 +-- crates/derive-impl/src/pymodule.rs | 10 +-- crates/derive-impl/src/util.rs | 72 ---------------- crates/vm/src/function/builtin.rs | 25 +++++- crates/vm/src/function/method.rs | 26 ++++-- crates/vm/src/function/signature.rs | 129 +++++++++++++++++++++++++++- crates/vm/src/vm/context.rs | 2 +- 7 files changed, 180 insertions(+), 96 deletions(-) diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index be59d7a339d..b935420591f 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -1,8 +1,8 @@ use super::Diagnostic; use crate::util::{ ALL_ALLOWED_NAMES, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ExceptionItemMeta, - ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, infer_native_call_flags, - internal_doc_tokens, pyclass_ident_and_attrs, pyexception_ident_and_attrs, + ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, internal_doc_tokens, + pyclass_ident_and_attrs, pyexception_ident_and_attrs, }; use core::str::FromStr; use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree}; @@ -1307,9 +1307,6 @@ where _ => None, } }; - let drop_first_typed = usize::from(implicit_self.is_some()); - let call_flags = infer_native_call_flags(func.sig(), drop_first_typed); - // Add #[allow(non_snake_case)] for setter methods like set___name__ let method_name = ident.to_string(); if method_name.starts_with("set_") && method_name.contains("__") { @@ -1341,7 +1338,6 @@ where raw, coexist, attr_name: self.inner.attr_name, - call_flags, }); Ok(()) } @@ -1538,7 +1534,6 @@ struct MethodNurseryItem { doc: TokenStream, doc_body_pending: TokenStream, attr_name: AttrName, - call_flags: TokenStream, } impl MethodNursery { @@ -1578,7 +1573,6 @@ impl ToTokens for MethodNursery { } _ => unreachable!(), }; - let call_flags = &item.call_flags; let coexist_flags = if item.coexist { quote! { | rustpython_vm::function::PyMethodFlags::COEXIST.bits() } } else { @@ -1586,7 +1580,7 @@ impl ToTokens for MethodNursery { }; let flags = quote! { rustpython_vm::function::PyMethodFlags::from_bits_retain( - (#binding_flags).bits() | (#call_flags).bits() #coexist_flags + (#binding_flags).bits() #coexist_flags ) }; // TODO: intern diff --git a/crates/derive-impl/src/pymodule.rs b/crates/derive-impl/src/pymodule.rs index 14ad5ea3cfb..9a2ab6878e7 100644 --- a/crates/derive-impl/src/pymodule.rs +++ b/crates/derive-impl/src/pymodule.rs @@ -2,8 +2,8 @@ use crate::error::Diagnostic; use crate::pystructseq::PyStructSequenceMeta; use crate::util::{ ALL_ALLOWED_NAMES, AttrItemMeta, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, - ErrorVec, ItemMeta, ItemNursery, ModuleItemMeta, SimpleItemMeta, infer_native_call_flags, - internal_doc_tokens, iter_use_idents, pyclass_ident_and_attrs, + ErrorVec, ItemMeta, ItemNursery, ModuleItemMeta, SimpleItemMeta, internal_doc_tokens, + iter_use_idents, pyclass_ident_and_attrs, }; use core::str::FromStr; use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; @@ -536,7 +536,6 @@ struct FunctionNurseryItem { ident: Ident, /// One internal doc per [`py_names`](Self::py_names) entry. docs: Vec, - call_flags: TokenStream, } impl FunctionNursery { @@ -566,14 +565,13 @@ impl ToTokens for ValidatedFunctionNursery { let ident = &item.ident; let cfgs = &item.cfgs; let cfgs = quote!(#(#cfgs)*); - let flags = &item.call_flags; for (py_name, doc) in item.py_names.iter().zip(&item.docs) { inner_tokens.extend(quote![ #cfgs rustpython_vm::function::PyMethodDef::new_const( #py_name, #ident, - #flags, + rustpython_vm::function::PyMethodFlags::empty(), #doc, ), ]); @@ -711,14 +709,12 @@ impl ModuleItem for FunctionItem { internal_doc_tokens(func.sig(), py_name, None, doc, None, Some("$module")) }) .collect(); - let call_flags = infer_native_call_flags(func.sig(), 0); args.context.function_items.add_item(FunctionNurseryItem { ident: ident.to_owned(), py_names, cfgs: args.cfgs.to_vec(), docs, - call_flags, }); Ok(()) } diff --git a/crates/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs index 985a8312301..6a14dd11688 100644 --- a/crates/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -936,75 +936,3 @@ pub(crate) fn internal_doc_tokens( } } } - -pub(crate) fn infer_native_call_flags(sig: &Signature, drop_first_typed: usize) -> TokenStream { - // Best-effort mapping of Rust function signatures to CPython-style - // METH_* calling convention flags used by CALL specialization. - let mut typed_args = Vec::new(); - for arg in &sig.inputs { - let FnArg::Typed(typed) = arg else { - continue; - }; - let ty_tokens = &typed.ty; - let ty = quote!(#ty_tokens).to_string().replace(' ', ""); - // The interpreter supplies `vm` and `callee`; a Python call never - // passes them. - if (ty.starts_with('&') && ty.ends_with("VirtualMachine")) || ty.ends_with("Callee") { - continue; - } - typed_args.push(ty); - } - - let mut user_args = typed_args.into_iter(); - for _ in 0..drop_first_typed { - if user_args.next().is_none() { - break; - } - } - - let mut has_keywords = false; - let mut variable_arity = false; - let mut fixed_positional = 0usize; - - for ty in user_args { - let is_named = |name: &str| { - ty == name - || ty.starts_with(&format!("{name}<")) - || ty.contains(&format!("::{name}<")) - || ty.ends_with(&format!("::{name}")) - }; - - if is_named("FuncArgs") { - has_keywords = true; - variable_arity = true; - continue; - } - if is_named("KwArgs") { - has_keywords = true; - variable_arity = true; - continue; - } - if is_named("PosArgs") || is_named("OptionalArg") || is_named("OptionalOption") { - variable_arity = true; - continue; - } - fixed_positional += 1; - } - - if has_keywords { - quote! { - rustpython_vm::function::PyMethodFlags::from_bits_retain( - rustpython_vm::function::PyMethodFlags::FASTCALL.bits() - | rustpython_vm::function::PyMethodFlags::KEYWORDS.bits() - ) - } - } else if variable_arity { - quote! { rustpython_vm::function::PyMethodFlags::FASTCALL } - } else { - match fixed_positional { - 0 => quote! { rustpython_vm::function::PyMethodFlags::NOARGS }, - 1 => quote! { rustpython_vm::function::PyMethodFlags::O }, - _ => quote! { rustpython_vm::function::PyMethodFlags::FASTCALL }, - } - } -} diff --git a/crates/vm/src/function/builtin.rs b/crates/vm/src/function/builtin.rs index b4bd610776d..bda24f87944 100644 --- a/crates/vm/src/function/builtin.rs +++ b/crates/vm/src/function/builtin.rs @@ -1,4 +1,4 @@ -use super::{Callee, FromArgs, FuncArgs}; +use super::{Callee, FromArgs, FuncArgs, SigArg}; use crate::{ Py, PyPayload, PyRef, PyResult, VirtualMachine, convert::ToPyResult, object::PyThreadingConstraint, @@ -37,6 +37,10 @@ impl PyNativeFn for F where /// just pass an unconstrained generic type, e.g. /// `fn foo(f: F) where F: IntoPyNativeFn` pub trait IntoPyNativeFn: Sized + PyThreadingConstraint + 'static { + /// Binding metadata of each argument, from which the calling convention + /// is inferred. A `&self` receiver is the `$self` marker. + const ARGS: &'static [SigArg] = &[SigArg::from_arg::("args")]; + fn call(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult; /// `IntoPyNativeFn::into_func()` generates a PyNativeFn that performs the @@ -89,6 +93,8 @@ impl IntoPyNativeFn<(T, R, VM)> for F where F: PyNativeFnInternal, { + const ARGS: &'static [SigArg] = F::ARGS; + #[inline(always)] fn call(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult { self.call_(vm, args, callee) @@ -98,6 +104,7 @@ where mod sealed { use super::*; pub trait PyNativeFnInternal: Sized + PyThreadingConstraint + 'static { + const ARGS: &'static [SigArg]; fn call_(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult; } } @@ -124,6 +131,8 @@ macro_rules! into_py_native_fn_tuple { $($T: FromArgs,)* R: ToPyResult, { + const ARGS: &'static [SigArg] = &[$(SigArg::from_arg::<$T>("")),*]; + fn call_(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult { let ($($n,)*) = args.bind_for::<($($T,)*)>(vm, callee)?; @@ -138,6 +147,9 @@ macro_rules! into_py_native_fn_tuple { $($T: FromArgs,)* R: ToPyResult, { + const ARGS: &'static [SigArg] = + &[SigArg::marker("$self") $(, SigArg::from_arg::<$T>(""))*]; + fn call_(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult { let (zelf, $($n,)*) = args.bind_for::<(PyRef, $($T,)*)>(vm, callee)?; @@ -152,6 +164,9 @@ macro_rules! into_py_native_fn_tuple { $($T: FromArgs,)* R: ToPyResult, { + const ARGS: &'static [SigArg] = + &[SigArg::marker("$self") $(, SigArg::from_arg::<$T>(""))*]; + fn call_(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult { let (zelf, $($n,)*) = args.bind_for::<(PyRef, $($T,)*)>(vm, callee)?; @@ -165,6 +180,8 @@ macro_rules! into_py_native_fn_tuple { $($T: FromArgs,)* R: ToPyResult, { + const ARGS: &'static [SigArg] = &[$(SigArg::from_arg::<$T>("")),*]; + fn call_(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult { let ($($n,)*) = args.bind_for::<($($T,)*)>(vm, callee)?; @@ -179,6 +196,9 @@ macro_rules! into_py_native_fn_tuple { $($T: FromArgs,)* R: ToPyResult, { + const ARGS: &'static [SigArg] = + &[SigArg::marker("$self") $(, SigArg::from_arg::<$T>(""))*]; + fn call_(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult { let (zelf, $($n,)*) = args.bind_for::<(PyRef, $($T,)*)>(vm, callee)?; @@ -193,6 +213,9 @@ macro_rules! into_py_native_fn_tuple { $($T: FromArgs,)* R: ToPyResult, { + const ARGS: &'static [SigArg] = + &[SigArg::marker("$self") $(, SigArg::from_arg::<$T>(""))*]; + fn call_(&self, vm: &VirtualMachine, args: FuncArgs, callee: Callee) -> PyResult { let (zelf, $($n,)*) = args.bind_for::<(PyRef, $($T,)*)>(vm, callee)?; diff --git a/crates/vm/src/function/method.rs b/crates/vm/src/function/method.rs index c9953f8a7e1..b33da287710 100644 --- a/crates/vm/src/function/method.rs +++ b/crates/vm/src/function/method.rs @@ -6,7 +6,7 @@ use crate::{ descriptor::{PyClassMethodDescriptor, PyMethodDescriptor}, }, class::PyClassDef, - function::{IntoPyNativeFn, PyNativeFn}, + function::{FuncArgs, IntoPyNativeFn, PyNativeFn, SigArg}, }; bitflags::bitflags! { @@ -49,6 +49,22 @@ bitflags::bitflags! { impl PyMethodFlags { // FIXME: macro temp pub const EMPTY: Self = Self::empty(); + + const CALL_CONVENTION: Self = Self::VARARGS + .union(Self::KEYWORDS) + .union(Self::NOARGS) + .union(Self::O) + .union(Self::FASTCALL); + + /// Adds the calling convention inferred from `args` unless one is set. + /// For METHOD and CLASS the first argument is the bound receiver. + pub(crate) const fn with_call_convention(self, args: &[SigArg]) -> Self { + if self.intersects(Self::CALL_CONVENTION) { + return self; + } + let receiver = self.intersects(Self::METHOD.union(Self::CLASS)); + self.union(super::signature::native_call_flags(args, receiver)) + } } #[macro_export] @@ -115,16 +131,16 @@ impl PyMethodDef { } #[inline] - pub const fn new_const( + pub const fn new_const>( name: &'static str, - func: impl IntoPyNativeFn, + func: F, flags: PyMethodFlags, doc: super::ItemDoc, ) -> Self { Self { name, func: super::static_func(func), - flags, + flags: flags.with_call_convention(F::ARGS), #[cfg(feature = "doc")] doc_off: doc.offset, #[cfg(feature = "doc")] @@ -145,7 +161,7 @@ impl PyMethodDef { Self { name, func: super::static_raw_func(func), - flags, + flags: flags.with_call_convention(&[SigArg::from_arg::("args")]), #[cfg(feature = "doc")] doc_off: doc.offset, #[cfg(feature = "doc")] diff --git a/crates/vm/src/function/signature.rs b/crates/vm/src/function/signature.rs index b43595ee358..74cf058d007 100644 --- a/crates/vm/src/function/signature.rs +++ b/crates/vm/src/function/signature.rs @@ -135,6 +135,81 @@ impl SigArg { } } +/// Calling-convention bits from each argument's binding metadata. +/// +/// Inspect arguments separately: a tuple's `PARAMS` can erase leaf metadata. +/// With `receiver`, the first argument is the bound `$self` / `$type`; it +/// contributes only the parameters its type supplies. +/// A keyword-capable parameter uses `FASTCALL | KEYWORDS`. +#[must_use] +pub(crate) const fn native_call_flags(args: &[SigArg], receiver: bool) -> super::PyMethodFlags { + let mut has_keywords = false; + let mut variable = false; + let mut fixed = 0usize; + let mut i = 0; + while i < args.len() { + match args[i].params { + Some(params) => count_params(params, &mut has_keywords, &mut variable, &mut fixed), + None if receiver && i == 0 => {} + None => fixed += 1, + } + i += 1; + } + if has_keywords { + super::PyMethodFlags::FASTCALL.union(super::PyMethodFlags::KEYWORDS) + } else if variable { + super::PyMethodFlags::FASTCALL + } else { + match fixed { + 0 => super::PyMethodFlags::NOARGS, + 1 => super::PyMethodFlags::O, + _ => super::PyMethodFlags::FASTCALL, + } + } +} + +const fn count_params( + params: &[Param], + has_keywords: &mut bool, + variable: &mut bool, + fixed: &mut usize, +) { + let mut i = 0; + while i < params.len() { + count_param(¶ms[i], has_keywords, variable, fixed); + i += 1; + } +} + +const fn count_param( + param: &Param, + has_keywords: &mut bool, + variable: &mut bool, + fixed: &mut usize, +) { + match param.kind { + ParamKind::PositionalOrKeyword => { + *has_keywords = true; + if param.default.is_some() { + *variable = true; + } else { + *fixed += 1; + } + } + ParamKind::KeywordOnly | ParamKind::VarKeyword => *has_keywords = true, + ParamKind::VarPositional => *variable = true, + ParamKind::PositionalOnly => { + if param.default.is_some() { + *variable = true; + } else { + *fixed += 1; + } + } + ParamKind::Flatten(Some(inner)) => count_params(inner, has_keywords, variable, fixed), + ParamKind::Flatten(None) => {} + } +} + const fn name_eq(name: &str, bytes: &[u8]) -> bool { let got = name.as_bytes(); if got.len() != bytes.len() { @@ -631,7 +706,8 @@ const fn write_args(buf: &mut [u8], mut st: St, args: &[SigArg]) -> St { #[cfg(test)] mod tests { - use super::{DefaultRepr, Param, ParamKind, St, write_one}; + use super::{DefaultRepr, Param, ParamKind, SigArg, St, native_call_flags, write_one}; + use crate::function::PyMethodFlags; fn rendered(default: DefaultRepr) -> String { let mut buf = [0u8; 64]; @@ -674,4 +750,55 @@ mod tests { assert_eq!(rendered(DefaultRepr::Raw("sys.maxsize")), "sys.maxsize"); assert_eq!(rendered(DefaultRepr::Unrepresentable), ""); } + + #[test] + fn native_call_flags_from_params() { + const fn arg(params: Option<&'static [Param]>) -> SigArg { + SigArg { name: "", params } + } + let keywords = PyMethodFlags::FASTCALL.union(PyMethodFlags::KEYWORDS); + assert_eq!(native_call_flags(&[], false), PyMethodFlags::NOARGS); + assert_eq!(native_call_flags(&[arg(None)], false), PyMethodFlags::O); + assert_eq!( + native_call_flags(&[arg(None), arg(None)], false), + PyMethodFlags::FASTCALL + ); + // The receiver is not a Python argument. + assert_eq!( + native_call_flags(&[SigArg::marker("$self"), arg(None)], true), + PyMethodFlags::O + ); + assert_eq!(native_call_flags(&[arg(None)], true), PyMethodFlags::NOARGS); + + const OPTIONAL: &[Param] = &[Param { + name: "", + kind: ParamKind::PositionalOnly, + default: Some(DefaultRepr::Unrepresentable), + }]; + assert_eq!( + native_call_flags(&[arg(Some(OPTIONAL))], false), + PyMethodFlags::FASTCALL + ); + + const KW: &[Param] = &[Param::positional_or_keyword("exp")]; + assert_eq!(native_call_flags(&[arg(Some(KW))], false), keywords); + + const FUNCARGS: &[Param] = &[Param::var_positional("args"), Param::var_keyword("kwargs")]; + assert_eq!(native_call_flags(&[arg(Some(FUNCARGS))], false), keywords); + // A receiver that binds the whole call keeps its parameters. + assert_eq!(native_call_flags(&[arg(Some(FUNCARGS))], true), keywords); + + const POSARGS: &[Param] = &[Param::var_positional("args")]; + assert_eq!( + native_call_flags(&[arg(Some(POSARGS))], false), + PyMethodFlags::FASTCALL + ); + + const INNER: &[Param] = &[Param::keyword_only( + "reverse", + Some(DefaultRepr::Bool(false)), + )]; + const FLAT: &[Param] = &[Param::flatten(Some(INNER))]; + assert_eq!(native_call_flags(&[arg(Some(FLAT))], false), keywords); + } } diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 78e6bc6fef8..42da0551cae 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -662,7 +662,7 @@ impl Context { let def = PyMethodDef { name, func: Box::leak(Box::new(f.into_func())), - flags, + flags: flags.with_call_convention(F::ARGS), #[cfg(feature = "doc")] doc_off: doc.offset, #[cfg(feature = "doc")] From b6c60a5939625557159117ffe17b7d4871f0e990 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:12:16 +0900 Subject: [PATCH 046/229] build(deps): bump mac_address (#9007) Bumps the unix group with 1 update in the / directory: [mac_address](https://github.com/repnop/mac_address). Updates `mac_address` from 1.1.8 to 1.2.0 - [Commits](https://github.com/repnop/mac_address/commits) --- updated-dependencies: - dependency-name: mac_address dependency-version: 1.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: unix ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ee20a4ca9a..9a0b3b46c9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2365,12 +2365,12 @@ dependencies = [ [[package]] name = "mac_address" -version = "1.1.8" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" +checksum = "b367b50a9be9a5d3b5718c1da74e5d6b9c17647f89752ee2562a470cc3c4eb1a" dependencies = [ - "nix 0.29.0", - "winapi", + "nix 0.30.1", + "windows-sys 0.61.2", ] [[package]] @@ -2523,9 +2523,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.29.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags 2.13.2", "cfg-if", From 54e47cda213e483e7f477dbce73d2331d0c5daa7 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Thu, 8 Oct 2026 08:13:00 -0700 Subject: [PATCH 047/229] Fix os.closerange and immediate process exit (#9006) * Ignore invalid descriptors in os.closerange Close raw descriptor numbers through the host CRT layer rather than constructing OwnedFd values for possibly closed descriptors. Preserve the half-open range, ignored close errors and Windows invalid-parameter handling. Assisted-by: Codex:model-version-unavailable * Use immediate exit for host process termination Python os._exit, failed child exec and fatal/watchdog exits must bypass C exit handlers. Call the host CRT _exit on Unix, Windows and WASI while retaining the existing non-host fallback. Normal interpreter termination is unchanged. Assisted-by: Codex:model-version-unavailable --- crates/host_env/src/crt_fd.rs | 7 +++++++ crates/host_env/src/crt_fd_unsupported.rs | 2 ++ crates/host_env/src/os.rs | 6 +++++- crates/vm/src/stdlib/os.rs | 6 +----- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/host_env/src/crt_fd.rs b/crates/host_env/src/crt_fd.rs index f06c5aad984..2390dc2aa83 100644 --- a/crates/host_env/src/crt_fd.rs +++ b/crates/host_env/src/crt_fd.rs @@ -348,6 +348,13 @@ pub fn close(fd: Owned) -> io::Result<()> { _close(fd.into_raw()) } +pub fn closerange(fd_low: Raw, fd_high: Raw) { + for fd in fd_low..fd_high { + // The range can contain closed descriptors, so it cannot use Owned. + let _ = _close(fd); + } +} + pub fn ftruncate(fd: Borrowed<'_>, len: Offset) -> io::Result<()> { let ret = unsafe { suppress_iph!(c::ftruncate(fd.as_raw(), len)) }; // On Windows, _chsize_s returns 0 on success, or a positive error code (errno value) on failure. diff --git a/crates/host_env/src/crt_fd_unsupported.rs b/crates/host_env/src/crt_fd_unsupported.rs index 7fb8a406e32..7b5336a9b98 100644 --- a/crates/host_env/src/crt_fd_unsupported.rs +++ b/crates/host_env/src/crt_fd_unsupported.rs @@ -153,6 +153,8 @@ pub fn close(_fd: Owned) -> io::Result<()> { Err(ebadf()) } +pub fn closerange(_fd_low: Raw, _fd_high: Raw) {} + pub fn ftruncate(_fd: Borrowed<'_>, _len: Offset) -> io::Result<()> { Err(ebadf()) } diff --git a/crates/host_env/src/os.rs b/crates/host_env/src/os.rs index 6eda6030c83..7d6879984bd 100644 --- a/crates/host_env/src/os.rs +++ b/crates/host_env/src/os.rs @@ -520,7 +520,11 @@ pub fn device_encoding(_fd: i32) -> Option { } pub fn exit(code: i32) -> ! { - std::process::exit(code) + // These callers require immediate termination without C exit handlers. + cfg_select! { + any(unix, windows, target_os = "wasi") => unsafe { libc::_exit(code) }, + _ => std::process::exit(code), + } } /// Wrapper around the C `abort()` call: terminates the process abnormally. diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 527643822b8..6f1dcca1556 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -272,11 +272,7 @@ pub(super) mod _os { #[pyfunction] fn closerange(fd_low: i32, fd_high: i32) { - for fileno in fd_low..fd_high { - if let Ok(fd) = unsafe { crt_fd::Owned::try_from_raw(fileno) } { - drop(fd); - } - } + crt_fd::closerange(fd_low, fd_high); } #[cfg(any(unix, windows, target_os = "wasi"))] From 06389eb2ac27bcf2e10c5c36df4fa6b3b77bdf7a Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:03:19 +0300 Subject: [PATCH 048/229] Make the PySubclass layout contract unsafe (#9012) Assisted-by: Codex:gpt-6 --- crates/derive-impl/src/pyclass.rs | 4 ++- crates/derive-impl/src/pystructseq.rs | 3 ++- crates/vm/src/class.rs | 37 ++++++++++++++++++++++----- crates/vm/src/exception_group.rs | 3 ++- crates/vm/src/exceptions.rs | 24 +++++++++++------ 5 files changed, 54 insertions(+), 17 deletions(-) diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index b935420591f..b82082bad5d 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -649,7 +649,9 @@ fn generate_class_def( let subclass_impl = if !is_pystruct && is_repr_transparent { base.as_ref().map(|typ| { quote! { - impl ::rustpython_vm::class::PySubclass for #ident { + // SAFETY: the transparent base field has the same layout as Self; + // pyclass also checks the object payload offsets and alignment. + unsafe impl ::rustpython_vm::class::PySubclass for #ident { type Base = #typ; #[inline] diff --git a/crates/derive-impl/src/pystructseq.rs b/crates/derive-impl/src/pystructseq.rs index aa89b3717bb..d4352735790 100644 --- a/crates/derive-impl/src/pystructseq.rs +++ b/crates/derive-impl/src/pystructseq.rs @@ -602,7 +602,8 @@ pub(crate) fn impl_pystruct_sequence( } // PySubclass for proper inheritance - impl ::rustpython_vm::class::PySubclass for #pytype_ident { + // SAFETY: the generated payload is a transparent wrapper around PyTuple. + unsafe impl ::rustpython_vm::class::PySubclass for #pytype_ident { type Base = ::rustpython_vm::builtins::PyTuple; #[inline] diff --git a/crates/vm/src/class.rs b/crates/vm/src/class.rs index 4afdf6329fb..fe655968665 100644 --- a/crates/vm/src/class.rs +++ b/crates/vm/src/class.rs @@ -451,13 +451,38 @@ pub trait PyClassImpl: PyClassDef { /// Trait for Python subclasses that can provide a reference to their base type. /// -/// This trait is automatically implemented by the `#[pyclass]` macro when -/// `base = SomeType` is specified. It provides safe reference access to the -/// base type's payload. +/// The `#[pyclass]` macro implements this trait for `#[repr(transparent)]` +/// subclasses with `base = SomeType`. Other native subclasses can implement +/// it manually when their base is a compatible physical prefix. /// -/// For subclasses with `#[repr(transparent)]` -/// which enables ownership transfer via `into_base()`. -pub trait PySubclass: crate::PyPayload { +/// # Safety +/// +/// Every valid `Self` must contain a valid `Base` at offset zero, returned by +/// `as_base`. Their payload offsets in `Py` and `Py` must match, +/// and `Py` must meet `Py`'s alignment requirement. The base must +/// remain valid for the lifetime of `Self`, independently of the object's +/// Python class or MRO. These guarantees allow safe borrowing and ownership +/// transfer of the same allocation as `Py`. +/// +/// A manual implementation must explicitly acknowledge this contract: +/// +/// ```compile_fail,E0200 +/// use rustpython_vm::{builtins::PyInt, class::PySubclass, pyclass, PyPayload}; +/// +/// #[pyclass(module = false, name = "IntWrapper")] +/// #[derive(Debug, PyPayload)] +/// #[repr(transparent)] +/// struct IntWrapper(PyInt); +/// +/// #[pyclass] +/// impl IntWrapper {} +/// +/// impl PySubclass for IntWrapper { +/// type Base = PyInt; +/// fn as_base(&self) -> &PyInt { &self.0 } +/// } +/// ``` +pub unsafe trait PySubclass: crate::PyPayload { type Base: crate::PyPayload; /// Returns a reference to the base type's payload. diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index 0f1b4dca994..8c182d98158 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -65,7 +65,8 @@ pub(super) mod types { excs_str: PyAtomicRef>, } - impl crate::class::PySubclass for PyBaseExceptionGroup { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PyBaseExceptionGroup { type Base = PyBaseException; fn as_base(&self) -> &Self::Base { &self.base diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 9562ad7a8b5..dbbdb920422 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -1750,7 +1750,8 @@ pub(super) mod types { code: PyAtomicRef>, } - impl crate::class::PySubclass for PySystemExit { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PySystemExit { type Base = PyBaseException; fn as_base(&self) -> &Self::Base { &self.base @@ -1830,7 +1831,8 @@ pub(super) mod types { value: PyAtomicRef>, } - impl crate::class::PySubclass for PyStopIteration { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PyStopIteration { type Base = PyException; fn as_base(&self) -> &Self::Base { &self.base @@ -1925,7 +1927,8 @@ pub(super) mod types { obj: PyAtomicRef>, } - impl crate::class::PySubclass for PyAttributeError { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PyAttributeError { type Base = PyException; fn as_base(&self) -> &Self::Base { &self.base @@ -2051,7 +2054,8 @@ pub(super) mod types { name_from: PyAtomicRef>, } - impl crate::class::PySubclass for PyImportError { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PyImportError { type Base = PyException; fn as_base(&self) -> &Self::Base { &self.base @@ -2263,7 +2267,8 @@ pub(super) mod types { name: PyAtomicRef>, } - impl crate::class::PySubclass for PyNameError { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PyNameError { type Base = PyException; fn as_base(&self) -> &Self::Base { &self.base @@ -2357,7 +2362,8 @@ pub(super) mod types { written: AtomicCell, } - impl crate::class::PySubclass for PyOSError { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PyOSError { type Base = PyException; fn as_base(&self) -> &Self::Base { &self.base @@ -2927,7 +2933,8 @@ pub(super) mod types { metadata: PyAtomicRef>, } - impl crate::class::PySubclass for PySyntaxError { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PySyntaxError { type Base = PyException; fn as_base(&self) -> &Self::Base { &self.base @@ -3230,7 +3237,8 @@ pub(super) mod types { reason: PyAtomicRef>, } - impl crate::class::PySubclass for PyUnicodeError { + // SAFETY: pyexception checks the base field offset, payload offset, and alignment. + unsafe impl crate::class::PySubclass for PyUnicodeError { type Base = PyValueError; fn as_base(&self) -> &Self::Base { &self.base From 0943b4813ba35344c680da6ecb362ae3f3162e3a Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:04:25 +0300 Subject: [PATCH 049/229] Guard untracked frames against recursive deallocation (#9013) Assisted-by: Codex:gpt-6 --- crates/vm/src/object/core.rs | 15 +++---- .../stdlib_threading_current_frames.py | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index eb7edc93cca..1eed9b712a0 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -173,17 +173,14 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { return; // resurrected by __del__ } - // Only tracked objects take the trashcan recursion guard and untrack path. - // Untracked objects either own no children (int, float, str, ...) or, like - // non-escaped frames, are released at interpreter depth with at most one - // unguarded link before their tracked children (dicts, functions, code) - // re-enter guarded deallocation, so recursion stays bounded. A frame stored - // in an object graph is forced to escape, becoming tracked and guarded here. - // Read once and reuse for both gates below. + // Untracked payloads can still own recursive chains of references, such as + // detached frames returned by sys._current_frames(). Guard traversable + // payloads regardless of tracking, while keeping scalar deallocation cheap. let tracked = obj_ref.is_gc_tracked(); + let guarded = tracked || T::HAS_TRAVERSE; // Trashcan: limit recursive deallocation depth to prevent stack overflow - if tracked && !unsafe { trashcan::begin(obj, default_dealloc::) } { + if guarded && !unsafe { trashcan::begin(obj, default_dealloc::) } { return; // deferred to queue } @@ -262,7 +259,7 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { } // Trashcan: decrement depth and process deferred objects at outermost level - if tracked { + if guarded { unsafe { trashcan::end() }; } } diff --git a/extra_tests/snippets/stdlib_threading_current_frames.py b/extra_tests/snippets/stdlib_threading_current_frames.py index e93a222148e..332c29eb5e7 100644 --- a/extra_tests/snippets/stdlib_threading_current_frames.py +++ b/extra_tests/snippets/stdlib_threading_current_frames.py @@ -97,4 +97,48 @@ def f123(): leave.set() t.join() + +def release_deep_snapshot(): + # Detached snapshots can own a deep chain of untracked frames. Releasing + # one on a small native stack must not recurse through the entire chain. + depth = 5000 + ready = threading.Event() + release = threading.Event() + disposed = threading.Event() + + def recurse(n): + if n: + recurse(n - 1) + else: + ready.set() + release.wait() + + def inspect(): + frame = sys._current_frames()[worker.ident] + del frame + disposed.set() + + old_limit = sys.getrecursionlimit() + old_stack = threading.stack_size() + worker = threading.Thread(target=recurse, args=(depth,), daemon=True) + try: + sys.setrecursionlimit(depth + 100) + worker.start() + assert ready.wait(30), "worker did not reach the bottom of its stack" + threading.stack_size(256 * 1024) + observer = threading.Thread(target=inspect, daemon=True) + observer.start() + observer.join(30) + assert not observer.is_alive(), "snapshot disposal did not finish" + assert disposed.is_set(), "snapshot disposal failed" + finally: + threading.stack_size(old_stack) + release.set() + worker.join(30) + sys.setrecursionlimit(old_limit) + assert not worker.is_alive(), "worker did not finish" + + +release_deep_snapshot() + print("ok") From 98b280109fc344dbac3e6cd3bcf00e82dda007cc Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 02:27:10 -0700 Subject: [PATCH 050/229] Return EBADF for negative stat file descriptors (#9008) * Reject negative file descriptors before fstat Return EBADF for negative ordinary descriptors before calling rustix, avoiding debug-build assertions on canonical invalid-fd tests. Keep directory-fd sentinel and absolute-path handling unchanged. Assisted-by: Codex:model-version-unavailable * Cover negative stat descriptors through the interpreter Exercise ordinary negative descriptors including values accepted by rustix as directory sentinels, without changing canonical tests or directory-fd parsing. Assisted-by: Codex:model-version-unavailable * Move negative stat regression into the os snippet Preserve the original Unix RustPython-only coverage and every regression assertion while removing the redundant Rust interpreter wrapper. CPython uses -1 as a path sentinel, so keep this regression scoped to its original runtime. Assisted-by: Codex:model-version-unavailable --- crates/host_env/src/fileutils.rs | 4 ++++ extra_tests/snippets/stdlib_os.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/crates/host_env/src/fileutils.rs b/crates/host_env/src/fileutils.rs index 8db9af89501..1674c90743b 100644 --- a/crates/host_env/src/fileutils.rs +++ b/crates/host_env/src/fileutils.rs @@ -11,6 +11,10 @@ pub use windows::{StatStruct, fstat}; #[cfg(not(windows))] pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result { + // Unlike a directory fd, fstat accepts no negative descriptor sentinels. + if fd.as_raw() < 0 { + return Err(std::io::Error::from_raw_os_error(libc::EBADF)); + } rustix::fs::fstat(fd).map_err(Into::into) } diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index a1f40ef4c45..2c673e3dee3 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -8,6 +8,26 @@ assert os.name == "posix" or os.name == "nt" + +def stat_rejects_negative_file_descriptors(): + import errno + import os + + for stat in (os.stat, os.fstat): + for fd in (-1, -5, -9, -100, -(2**31)): + try: + stat(fd) + except OSError as exc: + assert exc.errno == errno.EBADF, (stat, fd, exc) + else: + raise AssertionError((stat, fd, "accepted a negative file descriptor")) + + +# Keep the original RustPython-only coverage: CPython treats -1 as a path sentinel. +if os.name == "posix" and sys.implementation.name == "rustpython": + stat_rejects_negative_file_descriptors() + + fd = os.open("README.md", os.O_RDONLY) assert fd > 0 From cd623a9216da4b666aca53f06fdea063d34737d7 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 02:27:43 -0700 Subject: [PATCH 051/229] Prevent ContextVar cache hits after Context address reuse (#9009) * Prevent ContextVar cache hits after Context address reuse Give each Context, including copies, a non-reused cache identity instead of using its heap address. Disable cache hits if the identity counter saturates. This prevents an empty Context from reading a destroyed Context's cached value while preserving existing locking and drop order. Assisted-by: Codex:model-version-unavailable * Cover ContextVar reads after context destruction Exercise fresh and copied contexts with variable defaults, explicit defaults and missing values. The repeated lifecycle check complements canonical context tests without relying on a fixed allocator address-reuse count. Assisted-by: Codex:model-version-unavailable * Move ContextVar cache regression into the contextvars snippet Preserve the fresh and copied Context lifetime checks, defaults and missing-value assertions while removing the redundant Rust interpreter wrapper. Assisted-by: Codex:model-version-unavailable --- crates/stdlib/src/contextvars.rs | 22 ++++++++++++--- extra_tests/snippets/stdlib_contextvars.py | 32 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 3c8cf9fa169..4d01ede3704 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -44,8 +44,19 @@ mod _contextvars { } } + // Unlike an object address, a cache identity must not be reused after a + // context is destroyed. Zero disables cache hits if the counter is exhausted. + static CONTEXT_CACHE_ID: AtomicUsize = AtomicUsize::new(0); + + fn next_context_cache_id() -> usize { + CONTEXT_CACHE_ID + .try_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1)) + .map_or(0, |id| id + 1) + } + #[derive(Debug)] struct ContextInner { + cache_id: usize, idx: AtomicUsize, vars: PyRef, // PyObject *ctx_weakreflist; @@ -72,6 +83,7 @@ mod _contextvars { fn empty(vm: &VirtualMachine) -> Self { Self { inner: ContextInner { + cache_id: next_context_cache_id(), idx: AtomicUsize::new(usize::MAX), vars: HamtObject::default().into_ref(&vm.ctx), entered: AtomicBool::new(false), @@ -192,6 +204,7 @@ mod _contextvars { }; Self { inner: ContextInner { + cache_id: next_context_cache_id(), idx: AtomicUsize::new(usize::MAX), vars: vars_copy.into_ref(&vm.ctx), entered: AtomicBool::new(false), @@ -356,7 +369,7 @@ mod _contextvars { #[pytraverse(skip)] cached: PyMutex>, #[pytraverse(skip)] - cached_id: AtomicUsize, // cached_tsid in CPython + cached_id: AtomicUsize, // non-reused Context cache identity #[pytraverse(skip)] hash: AtomicI64, } @@ -413,7 +426,7 @@ mod _contextvars { idx: ctx.inner.idx.load(Ordering::Relaxed), }; let mut cached = zelf.cached.lock(); - zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); + zelf.cached_id.store(ctx.inner.cache_id, Ordering::SeqCst); let replaced = cached.replace(cache); drop(cached); drop(replaced); @@ -444,7 +457,8 @@ mod _contextvars { let ctx = ctx_obj.downcast_ref::().unwrap(); let mut cached = zelf.cached.lock(); if let Some(cached) = &*cached - && zelf.cached_id.load(Ordering::SeqCst) == ctx.get_id() + && ctx.inner.cache_id != 0 + && zelf.cached_id.load(Ordering::SeqCst) == ctx.inner.cache_id && cached.idx + 1 == ctxs.len() { return (Some(cached.object.clone()), None); @@ -452,7 +466,7 @@ mod _contextvars { let Some(obj) = ctx.borrow_vars().get(zelf).map(|obj| obj.to_owned()) else { return (None, None); }; - zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); + zelf.cached_id.store(ctx.inner.cache_id, Ordering::SeqCst); let replaced = cached.replace(ContextVarCache { object: obj.clone(), diff --git a/extra_tests/snippets/stdlib_contextvars.py b/extra_tests/snippets/stdlib_contextvars.py index 915f732d188..a54d6a3a618 100644 --- a/extra_tests/snippets/stdlib_contextvars.py +++ b/extra_tests/snippets/stdlib_contextvars.py @@ -84,3 +84,35 @@ def __eq__(self, other): for var, value in copy_context().items() if var.name == "child_marker" ] == [] + + +def context_cache_does_not_outlive_its_context(): + import contextvars + + for factory_name in ("new", "copy"): + for _ in range(100): + var = contextvars.ContextVar("var", default="default") + empty = contextvars.Context() + ctx = contextvars.Context() if factory_name == "new" else empty.copy() + ctx.run(var.set, "cached") + del ctx + ctx = contextvars.Context() if factory_name == "new" else empty.copy() + assert list(ctx.items()) == [] + assert ctx.run(var.get) == "default" + assert ctx.run(var.get, "explicit default") == "explicit default" + + missing = contextvars.ContextVar("missing") + ctx.run(missing.set, "cached") + del ctx + ctx = contextvars.Context() if factory_name == "new" else empty.copy() + try: + ctx.run(missing.get) + except LookupError: + pass + else: + raise AssertionError( + "an empty context returned a destroyed context value" + ) + + +context_cache_does_not_outlive_its_context() From d0dfcb54e356d8043bea9ee45b18a7fdc3839999 Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:29:43 +0300 Subject: [PATCH 052/229] Synchronize mmap buffer exports with close and resize (#9015) * Synchronize mmap buffer exports with close and resize Assisted-by: Codex:gpt-6 * Keep mmap export checks ahead of data locks Assisted-by: Codex:gpt-6 --- crates/stdlib/src/mmap.rs | 147 ++++++++++++++++++++++++---- extra_tests/snippets/stdlib_mmap.py | 38 +++++++ 2 files changed, 165 insertions(+), 20 deletions(-) diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index a39c1278455..a242f8f1d29 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -26,7 +26,10 @@ mod mmap { use num_traits::Signed; use std::io::Write; #[cfg(windows)] - use {core::hint::cold_path, memchr::memchr, rustpython_vm::exceptions, std::io}; + use { + core::hint::cold_path, memchr::memchr, num_traits::ToPrimitive, rustpython_vm::exceptions, + std::io, + }; #[cfg(unix)] use rustpython_host_env::crt_fd; @@ -664,7 +667,9 @@ mod mmap { impl AsBuffer for PyMmap { const RELEASE_BUFFER: bool = true; - fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + fn as_buffer(zelf: &Py, vm: &VirtualMachine) -> PyResult { + // Publish the export while close/resize cannot invalidate the mapping. + let _mmap = zelf.check_valid(vm)?; let readonly = matches!(zelf.access, AccessMode::Read); let buf = PyBuffer::new( zelf.to_owned().into(), @@ -781,6 +786,7 @@ mod mmap { let m = self.mmap.lock(); if m.is_none() { + drop(m); return Err(vm.new_value_error("mmap closed or invalid")); } @@ -789,20 +795,31 @@ mod mmap { /// TODO: impl resize #[allow(dead_code)] - fn check_resizeable(&self, vm: &VirtualMachine) -> PyResult<()> { + fn check_resizeable( + &self, + vm: &VirtualMachine, + ) -> PyResult>> { + // Borrowed buffers can hold the data lock across blocking I/O. if self.exports.load() > 0 { return Err(vm.new_buffer_error("mmap can't resize with extant buffers exported.")); } + let mmap = self.check_valid(vm)?; + if self.exports.load() > 0 { + drop(mmap); + return Err(vm.new_buffer_error("mmap can't resize with extant buffers exported.")); + } #[cfg(unix)] if !self.trackfd { + drop(mmap); return Err(vm.new_value_error("mmap can't resize with trackfd=False.")); } if self.access == AccessMode::Write || self.access == AccessMode::Default { - return Ok(()); + return Ok(mmap); } + drop(mmap); Err(vm.new_type_error("mmap can't resize a readonly or copy-on-write memory map.")) } @@ -813,15 +830,20 @@ mod mmap { #[pymethod] fn close(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { + // Reject an existing export without waiting for its borrowed-data guard. + if zelf.exports.load() > 0 { + return Err(vm.new_buffer_error("cannot close exported pointers exist.")); + } + let mut mmap = zelf.mmap.lock(); if Self::closed(zelf) { return Ok(()); } if zelf.exports.load() > 0 { + drop(mmap); return Err(vm.new_buffer_error("cannot close exported pointers exist.")); } - let mut mmap = zelf.mmap.lock(); zelf.closed.store(true); *mmap = None; @@ -1051,7 +1073,7 @@ mod mmap { #[cfg(unix)] #[pymethod] fn resize(zelf: &Py, newsize: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.check_resizeable(vm)?; + drop(zelf.check_resizeable(vm)?); let new_size: isize = newsize.try_to_primitive(vm).map_err(|_| { vm.new_overflow_error("Python int too large to convert to C ssize_t") @@ -1076,23 +1098,24 @@ mod mmap { #[cfg(windows)] #[pymethod] fn resize(zelf: &Py, newsize: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.check_resizeable(vm)?; - - let newsize: usize = newsize - .try_to_primitive(vm) - .map_err(|_| vm.new_value_error("new size out of range"))?; + let newsize = newsize.as_bigint().to_usize(); + // Get the lock on mmap + // Keep the export check valid until resize ends. + let mut mmap_guard = zelf.check_resizeable(vm)?; + let Some(newsize) = newsize else { + drop(mmap_guard); + return Err(vm.new_value_error("new size out of range")); + }; if newsize == 0 { + drop(mmap_guard); return Err(vm.new_value_error("new size must be positive")); } - let handle = zelf.handle.load(); - // Get the lock on mmap - let mut mmap_guard = zelf.mmap.lock(); - // Check if this is a Named mmap - these cannot be resized if let Some(MmapObj::Named(_)) = mmap_guard.as_ref() { + drop(mmap_guard); return Err(vm.new_os_error("mmap: cannot resize a named memory mapping")); } @@ -1108,8 +1131,13 @@ mod mmap { let copy_size = core::cmp::min(old_size, newsize); // Create new anonymous mmap - let mut new_mmap = - host_mmap::map_anon(newsize).map_err(|e| e.to_pyexception(vm))?; + let mut new_mmap = match host_mmap::map_anon(newsize) { + Ok(mmap) => mmap, + Err(err) => { + drop(mmap_guard); + return Err(err.to_pyexception(vm)); + } + }; // Copy data from old mmap to new mmap if let Some(old_mmap) = mmap_guard.as_ref() { @@ -1134,17 +1162,23 @@ mod mmap { handle as host_mmap::Handle, zelf.size.load(), ); + drop(mmap_guard); return Err(err.to_pyexception(vm)); } // Create new mmap with the new size - let new_mmap = Self::create_mmap_windows( + let new_mmap = match Self::create_mmap_windows( handle as host_mmap::Handle, zelf.offset, newsize, &zelf.access, - ) - .map_err(|e| e.to_pyexception(vm))?; + ) { + Ok(mmap) => mmap, + Err(err) => { + drop(mmap_guard); + return Err(err.to_pyexception(vm)); + } + }; *mmap_guard = Some(new_mmap); zelf.size.store(newsize); @@ -1480,6 +1514,79 @@ mod mmap { } } + #[cfg(test)] + #[cfg(feature = "threading")] + mod tests { + use super::*; + use core::time::Duration; + use std::{sync::mpsc, thread}; + + fn rejects_borrowed_buffer(resize: bool) { + let builder = crate::vm::Interpreter::builder(Default::default()); + let defs = crate::stdlib_module_defs(&builder.ctx); + let interp = builder + .add_native_modules(&defs) + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + .build(); + interp.enter(|vm| { + let module = vm.import("mmap", 0).unwrap(); + let mapped = module + .get_attr("mmap", vm) + .unwrap() + .call((-1, 1), vm) + .unwrap() + .downcast::() + .unwrap(); + let buffer = PyMmap::as_buffer(&mapped, vm).unwrap(); + thread::scope(|scope| { + let (ready_tx, ready_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let buffer = &buffer; + let borrower = scope.spawn(move || { + // A native I/O operation can keep this guard while detached. + let bytes = buffer.as_contiguous().unwrap(); + assert!(bytes.is_locked()); + ready_tx.send(()).unwrap(); + // Release even on a regression, so the test fails rather than hangs. + let rejected_before_release = + done_rx.recv_timeout(Duration::from_secs(5)).is_ok(); + drop(bytes); + rejected_before_release + }); + ready_rx.recv().unwrap(); + let result = if resize { + mapped.check_resizeable(vm).map(drop) + } else { + PyMmap::close(&mapped, vm) + }; + let _ = done_tx.send(()); + assert!( + borrower.join().unwrap(), + "export rejection waited for the borrowed buffer" + ); + assert!( + result + .unwrap_err() + .class() + .is(vm.ctx.exceptions.buffer_error) + ); + }); + drop(buffer); + PyMmap::close(&mapped, vm).unwrap(); + }); + } + + #[test] + fn close_rejects_borrowed_buffer() { + rejects_borrowed_buffer(false); + } + + #[test] + fn resize_rejects_borrowed_buffer() { + rejects_borrowed_buffer(true); + } + } + impl Representable for PyMmap { #[inline] fn repr_str(zelf: &Py, _vm: &VirtualMachine) -> PyResult { diff --git a/extra_tests/snippets/stdlib_mmap.py b/extra_tests/snippets/stdlib_mmap.py index 9d4414e5552..2c4efdf58bf 100644 --- a/extra_tests/snippets/stdlib_mmap.py +++ b/extra_tests/snippets/stdlib_mmap.py @@ -39,3 +39,41 @@ mapped.find(b"") with assert_raises(ValueError): mapped.rfind(b"") + + +# Buffer acquisition must reject a closed mapping before returning a view. +with assert_raises(ValueError): + memoryview(mapped) + +mapped = mmap.mmap(-1, 4) +mapped[:] = b"data" +view = memoryview(mapped) +child = view[1:] +view.release() +with assert_raises(BufferError): + mapped.close() +assert not mapped.closed +assert child.tobytes() == b"ata" +child.release() +mapped.close() +mapped.close() + +# On Windows resize must neither invalidate an export nor revive a closed map. +import sys + +if sys.platform == "win32": + mapped = mmap.mmap(-1, 4) + mapped[:] = b"data" + with memoryview(mapped) as view: + with assert_raises(BufferError): + mapped.resize(8) + with assert_raises(BufferError): + mapped.resize(0) + assert view.tobytes() == b"data" + mapped.resize(8) + with memoryview(mapped) as view: + assert view.tobytes() == b"data" + b"\0" * 4 + mapped.close() + with assert_raises(ValueError): + mapped.resize(4) + assert mapped.closed From e6cbf3628d5e963034a23799e001bfbdd3ec529b Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers <7943856+bschoenmaeckers@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:00:59 +0200 Subject: [PATCH 053/229] Add buffer support to c-api (#8071) * Add buffer support to c-api * Review * Refactor PyBuffer * Manual implement `PyBuffer_GetPointer` * Review * Store in 1 Box --- crates/capi/src/lib.rs | 1 + crates/capi/src/pybuffer.rs | 334 ++++++++++++++++++++++ crates/vm/src/builtins/memory.rs | 93 +++--- crates/vm/src/protocol/buffer.rs | 241 ++++++++++++---- crates/vm/src/stdlib/_ctypes/array.rs | 12 +- crates/vm/src/stdlib/_ctypes/function.rs | 9 +- crates/vm/src/stdlib/_ctypes/pointer.rs | 9 +- crates/vm/src/stdlib/_ctypes/simple.rs | 9 +- crates/vm/src/stdlib/_ctypes/structure.rs | 16 +- crates/vm/src/stdlib/_ctypes/union.rs | 16 +- 10 files changed, 597 insertions(+), 143 deletions(-) create mode 100644 crates/capi/src/pybuffer.rs diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index 90ef71da7a8..ea99242005f 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -32,6 +32,7 @@ pub mod moduleobject; pub mod object; pub mod objimpl; pub mod osmodule; +pub mod pybuffer; pub mod pycapsule; pub mod pyerrors; pub mod pyframe; diff --git a/crates/capi/src/pybuffer.rs b/crates/capi/src/pybuffer.rs new file mode 100644 index 00000000000..b20f07c6d1b --- /dev/null +++ b/crates/capi/src/pybuffer.rs @@ -0,0 +1,334 @@ +use crate::PyObject; +use crate::pystate::with_vm; +use crate::util::FfiPtrExt; +use alloc::borrow::Cow; +use alloc::ffi::CString; +use core::ffi::{c_char, c_int, c_void}; +use core::ptr::{self, NonNull}; +use rustpython_vm::protocol::{BufferFlags, PyBuffer}; + +#[repr(C)] +#[derive(Default)] +pub struct Py_buffer { + pub buf: *mut c_void, + pub obj: *mut PyObject, + pub len: isize, + pub itemsize: isize, + pub readonly: c_int, + pub ndim: c_int, + pub format: *mut c_char, + pub shape: *mut isize, + pub strides: *mut isize, + pub suboffsets: *mut isize, + pub internal: *mut c_void, +} + +impl Py_buffer { + fn is_contiguous_for_order(&self, order: u8) -> bool { + let desc = unsafe { &(&*self.internal.cast::()).buffer.desc }; + if desc.len == 0 || desc.ndim() <= 1 { + return true; + } + + if desc.has_suboffsets() { + return false; + } + + match order { + b'C' => desc.is_contiguous(), + b'F' => desc.is_fortran_contiguous(), + b'A' => desc.is_contiguous() || desc.is_fortran_contiguous(), + _ => false, + } + } +} + +#[allow(dead_code)] +struct BufferInternal { + buffer: PyBuffer, + format: Option, +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyObject_GetBuffer( + obj: *mut PyObject, + view: *mut Py_buffer, + flags: c_int, +) -> c_int { + with_vm(|vm| { + let obj = unsafe { obj.assume_borrowed() }; + let flags = BufferFlags::from_bits_retain(flags as u32); + let buffer = PyBuffer::from_object(vm, obj, flags)?; + + if flags.contains(BufferFlags::WRITABLE) && buffer.desc.readonly { + return Err(vm.new_buffer_error("Object is not writable")); + } + + let format = flags + .contains(BufferFlags::FORMAT) + .then(|| { + CString::new(&*buffer.desc.format) + .map_err(|_| vm.new_system_error("buffer format contains NUL")) + }) + .transpose()?; + + let shape = if flags.contains(BufferFlags::ND) { + buffer.desc.shape().as_ptr().cast_mut() + } else { + ptr::null_mut() + }; + + let strides = if flags.contains(BufferFlags::STRIDES) { + buffer.desc.strides().as_ptr().cast_mut() + } else { + ptr::null_mut() + }; + + let suboffsets = if flags.contains(BufferFlags::INDIRECT) { + buffer.desc.suboffsets().as_ptr().cast_mut() + } else { + ptr::null_mut() + }; + + let mut buffer_view = Py_buffer { + len: buffer.desc.len as _, + itemsize: buffer.desc.itemsize as _, + readonly: buffer.desc.readonly.into(), + ndim: buffer.desc.ndim() as _, + format: format.as_ref().map_or_default(|s| s.as_ptr().cast_mut()), + shape, + strides, + suboffsets, + ..Default::default() + }; + + let c_contig = buffer.desc.is_contiguous(); + let f_contig = buffer.desc.is_fortran_contiguous(); + + if flags.contains(BufferFlags::C_CONTIGUOUS) && !c_contig { + return Err(vm.new_buffer_error("Object is not C-contiguous")); + } + if flags.contains(BufferFlags::F_CONTIGUOUS) && !f_contig { + return Err(vm.new_buffer_error("Object is not Fortran-contiguous")); + } + if flags.contains(BufferFlags::ANY_CONTIGUOUS) && !(c_contig || f_contig) { + return Err(vm.new_buffer_error("Object is not contiguous")); + } + + buffer_view.obj = obj.as_raw().cast_mut(); + buffer_view.buf = unsafe { + buffer + .obj_bytes() + .as_ptr() + .cast_mut() + .offset(buffer.desc.offset) + } + .cast(); + buffer_view.internal = Box::into_raw(Box::new(BufferInternal { buffer, format })).cast(); + + unsafe { + ptr::replace(view, buffer_view); + } + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyBuffer_Release(view: *mut Py_buffer) { + let view = unsafe { &mut *view }; + let internal = unsafe { + NonNull::new(view.internal.cast::()).map(|ptr| Box::from_non_null(ptr)) + }; + drop(internal); + core::mem::take(view); +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyBuffer_IsContiguous(view: *const Py_buffer, fort: c_char) -> c_int { + unsafe { &*view } + .is_contiguous_for_order((fort as u8).to_ascii_uppercase()) + .into() +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyBuffer_GetPointer( + view: *const Py_buffer, + indices: *const isize, +) -> *mut c_void { + with_vm(|vm| { + let view = unsafe { &*view }; + + if view.strides.is_null() && view.shape.is_null() { + let i0 = unsafe { *indices }; + let delta = i0 + .checked_mul(view.itemsize) + .ok_or_else(|| vm.new_system_error("index * itemsize overflow"))?; + return unsafe { Ok(view.buf.cast::().offset(delta).cast()) }; + } + + let ndim: usize = view + .ndim + .try_into() + .map_err(|_| vm.new_system_error("view.ndim does not fit usize"))?; + + let idx = unsafe { core::slice::from_raw_parts(indices, ndim) }; + + let strides: Cow<'_, _> = unsafe { view.strides.as_ref() }.map_or_else( + || { + let shape = unsafe { core::slice::from_raw_parts(view.shape, ndim) }; + let mut strides = vec![0; shape.len()]; + let mut stride = view.itemsize; + for (ix, dim) in shape.iter().copied().enumerate().rev() { + strides[ix] = stride; + stride = stride + .checked_mul(dim) + .ok_or_else(|| vm.new_system_error("stride overflow"))?; + } + Ok(strides.into()) + }, + |strides| unsafe { Ok(core::slice::from_raw_parts(strides, ndim).into()) }, + )?; + + let suboffsets = unsafe { + view.suboffsets + .as_ref() + .map(|suboffsets| core::slice::from_raw_parts(suboffsets, ndim)) + }; + + let mut ptr_u8 = view.buf.cast::(); + for (dim, index) in idx.iter().enumerate() { + let delta = index + .checked_mul(strides[dim]) + .ok_or_else(|| vm.new_system_error("index * stride overflow"))?; + ptr_u8 = unsafe { ptr_u8.offset(delta) }; + if let Some(suboffsets) = suboffsets { + let suboffset = suboffsets[dim]; + if suboffset >= 0 { + let inner = unsafe { core::ptr::read_unaligned(ptr_u8.cast::<*mut u8>()) }; + ptr_u8 = unsafe { inner.offset(suboffset) }; + } + } + } + + Ok(ptr_u8.cast()) + }) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyBuffer_ToContiguous( + buf: *mut c_void, + view: *const Py_buffer, + len: isize, + order: c_char, +) -> c_int { + with_vm(|vm| { + let view = unsafe { &*view }; + + if !view.is_contiguous_for_order((order as u8).to_ascii_uppercase()) { + return Err(vm.new_buffer_error( + "PyBuffer_ToContiguous only supports contiguous exported buffers", + )); + } + + if len != view.len { + return Err(vm.new_buffer_error("len must match view->len")); + } + + let len: usize = len + .try_into() + .map_err(|_| vm.new_system_error("buffer len does not fit usize"))?; + + let src = unsafe { core::slice::from_raw_parts(view.buf.cast::(), len) }; + let dst = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), len) }; + + dst.copy_from_slice(src); + Ok(()) + }) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyBuffer_FromContiguous( + view: *const Py_buffer, + buf: *const c_void, + len: isize, + order: c_char, +) -> c_int { + with_vm(|vm| { + let view = unsafe { &*view }; + + if view.readonly != 0 { + return Err(vm.new_buffer_error("cannot write into readonly buffer")); + } + + if !view.is_contiguous_for_order((order as u8).to_ascii_uppercase()) { + return Err(vm.new_buffer_error( + "PyBuffer_FromContiguous only supports contiguous exported buffers", + )); + } + + let len: usize = len + .min(view.len) + .try_into() + .map_err(|_| vm.new_system_error("len does not fit usize"))?; + + let src = unsafe { core::slice::from_raw_parts(buf.cast::(), len) }; + let dst = unsafe { core::slice::from_raw_parts_mut(view.buf.cast::(), len) }; + dst.copy_from_slice(src); + Ok(()) + }) +} + +#[cfg(test)] +mod tests { + use pyo3::buffer::PyBuffer; + use pyo3::prelude::*; + use pyo3::types::{PyByteArray, PyBytes}; + + #[test] + fn object_getbuffer_basic_and_release() { + Python::attach(|py| { + let bytes = PyBytes::new(py, b"hello"); + let buffer = PyBuffer::::get(&bytes).unwrap(); + assert_eq!(buffer.dimensions(), 1); + assert_eq!(buffer.item_count(), 5); + assert_eq!(buffer.to_vec(py).unwrap(), b"hello"); + }); + } + + #[test] + fn contiguous_copy_roundtrip() { + Python::attach(|py| { + let src = PyBytes::new(py, b"abcde"); + let buffer = PyBuffer::::get(&src).unwrap(); + let mut out = [0u8; 5]; + buffer.copy_to_slice(py, &mut out).unwrap(); + assert_eq!(&out, b"abcde"); + }); + } + + #[test] + fn is_contiguous_and_get_pointer() { + Python::attach(|py| { + let bytes = PyBytes::new(py, b"xyz"); + let buffer = PyBuffer::::get(&bytes).unwrap(); + assert!(buffer.is_c_contiguous()); + + let p = buffer.get_ptr(&[1]); + assert!(!p.is_null()); + unsafe { assert_eq!(*(p.cast::()), b'y') }; + }); + } + + #[test] + fn writable_bytearray() { + Python::attach(|py| { + let bytearray = PyByteArray::new(py, b"hello"); + let buffer = PyBuffer::::get(&bytearray).unwrap(); + assert_eq!(buffer.dimensions(), 1); + assert_eq!(buffer.item_count(), 5); + buffer.as_mut_slice(py).unwrap()[0].replace(b'H'); + drop(buffer); + assert_eq!(bytearray.to_vec(), b"Hello"); + }); + } +} diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index 2c9e8c62809..33481ce22a5 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -301,7 +301,7 @@ impl PyMemoryView { vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") ); } - let (shape, _, _) = self.desc.dim_desc[0]; + let shape = self.desc.shape()[0] as usize; // ptr_from_index let index = i .wrapped_at(shape) @@ -327,7 +327,7 @@ impl PyMemoryView { if self.desc.ndim() != 1 { return Err(vm.new_not_implemented_error("sub-views are not implemented")); } - let (shape, _, _) = self.desc.dim_desc[0]; + let shape = self.desc.shape()[0] as usize; // ptr_from_index let index = i .wrapped_at(shape) @@ -412,7 +412,7 @@ impl PyMemoryView { } fn init_len(&mut self) { - let product: usize = self.desc.dim_desc.iter().map(|x| x.0).product(); + let product: usize = self.desc.shape().iter().map(|&x| x as usize).product(); self.desc.len = product * self.desc.itemsize; } @@ -420,34 +420,36 @@ impl PyMemoryView { /// outside `dim` is reached through a pointer, in which case its suboffset /// does. fn adjust_position(&mut self, dim: usize, delta: isize) { - match self.desc.dim_desc[..dim] + match self.desc.suboffsets()[..dim] .iter() - .rposition(|&(_, _, suboffset)| suboffset != 0) + .rposition(|&suboffset| suboffset != -1) { - Some(n) => self.desc.dim_desc[n].2 += delta, + Some(n) => self.desc.suboffsets_mut()[n] += delta, None => self.desc.offset += delta, } } fn init_range(&mut self, range: Range, dim: usize) { - let (shape, stride, _) = self.desc.dim_desc[dim]; + let shape = self.desc.shape()[dim] as usize; + let stride = self.desc.strides()[dim]; debug_assert!(shape >= range.len()); self.adjust_position(dim, stride * range.start as isize); - self.desc.dim_desc[dim].0 = range.len(); + self.desc.shape_mut()[dim] = range.len() as isize; } // init_slice fn init_slice(&mut self, slice: &Py, dim: usize, vm: &VirtualMachine) -> PyResult<()> { - let (shape, stride, _) = self.desc.dim_desc[dim]; + let shape = self.desc.shape()[dim] as usize; + let stride = self.desc.strides()[dim]; let slice = slice.to_saturated(vm)?; let (start, slice_len) = slice.adjust_indices_start(shape); // Repeated slicing multiplies the stride by the step every time, which // overflows after about twenty rounds; C wraps there and so does this. self.adjust_position(dim, stride.wrapping_mul(start)); - self.desc.dim_desc[dim].0 = slice_len; - self.desc.dim_desc[dim].1 = stride.wrapping_mul(slice.step()); + self.desc.shape_mut()[dim] = slice_len as isize; + self.desc.strides_mut()[dim] = stride.wrapping_mul(slice.step()); Ok(()) } @@ -459,7 +461,13 @@ impl PyMemoryView { dim: usize, vm: &VirtualMachine, ) -> PyResult { - let (shape, stride, suboffset) = self.desc.dim_desc[dim]; + let shape = self.desc.shape()[dim] as usize; + let stride = self.desc.strides()[dim]; + let suboffset = if self.desc.suboffsets()[dim] == -1 { + 0 + } else { + self.desc.suboffsets()[dim] + }; if dim + 1 == self.desc.ndim() { let mut v = Vec::with_capacity(shape); for _ in 0..shape { @@ -731,7 +739,7 @@ impl PyMemoryView { Err(vm.new_type_error("0-dim memory has no length")) } else { // shape for dim[0] - Ok(self.desc.dim_desc[0].0) + Ok(self.desc.shape()[0] as usize) } } @@ -760,14 +768,14 @@ impl PyMemoryView { released: AtomicCell::new(false), restricted: AtomicCell::new(false), format_spec, - desc: BufferDescriptor { - len: self.desc.len, - offset: self.desc.offset, - readonly: self.desc.readonly, + desc: BufferDescriptor::from_dim_desc( + self.desc.len, + self.desc.offset, + self.desc.readonly, itemsize, - format: format_str.to_owned().into(), - dim_desc: vec![(self.desc.len / itemsize, itemsize as isize, 0)], - }, + format_str.to_owned().into(), + vec![(self.desc.len / itemsize, itemsize as isize, 0)], + ), hash: OnceCell::new(), exports: AtomicCell::new(0), }; @@ -903,9 +911,9 @@ impl Py { self.try_not_released(vm)?; Ok(vm.ctx.new_tuple( self.desc - .dim_desc + .shape() .iter() - .map(|(shape, _, _)| shape.to_pyobject(vm)) + .map(|shape| (*shape as usize).to_pyobject(vm)) .collect(), )) } @@ -915,9 +923,9 @@ impl Py { self.try_not_released(vm)?; Ok(vm.ctx.new_tuple( self.desc - .dim_desc + .strides() .iter() - .map(|(_, stride, _)| stride.to_pyobject(vm)) + .map(|stride| stride.to_pyobject(vm)) .collect(), )) } @@ -927,15 +935,15 @@ impl Py { self.try_not_released(vm)?; let has_suboffsets = self .desc - .dim_desc + .suboffsets() .iter() - .any(|(_, _, suboffset)| *suboffset != 0); + .any(|suboffset| *suboffset != -1); if has_suboffsets { Ok(vm.ctx.new_tuple( self.desc - .dim_desc + .suboffsets() .iter() - .map(|(_, _, suboffset)| suboffset.to_pyobject(vm)) + .map(|suboffset| suboffset.to_pyobject(vm)) .collect(), )) } else { @@ -1055,7 +1063,7 @@ impl Py { vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") ); } - let len = self.desc.dim_desc[0].0; + let len = self.desc.shape()[0] as usize; let mut count = 0; for i in 0..len { let item = self.getitem_by_idx(i as isize, vm)?; @@ -1074,7 +1082,7 @@ impl Py { vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") ); } - let len = self.desc.dim_desc[0].0; + let len = self.desc.shape()[0] as usize; let MemoryIndexArgs { value, start, stop } = args; let start = if start < 0 { @@ -1148,12 +1156,14 @@ impl Py { vm.new_type_error("memoryview: product(shape) * itemsize != buffer size") ); } - other.desc.dim_desc = vec![]; + other + .desc + .set_dimensions(Box::new([]), Box::new([]), Box::new([])); return Ok(other.into_ref(&vm.ctx)); } let mut product_shape = itemsize; - let mut dim_descriptor = Vec::with_capacity(shape_ndim); + let mut shape_descriptor = Vec::with_capacity(shape_ndim); for x in shape { let x = x @@ -1174,12 +1184,13 @@ impl Py { return Err(vm.new_value_error("memoryview.cast(): product(shape) > SSIZE_MAX")); } product_shape *= x; - dim_descriptor.push((x, 0, 0)); + shape_descriptor.push(x as isize); } - dim_descriptor.last_mut().unwrap().1 = itemsize as isize; - for i in (0..dim_descriptor.len() - 1).rev() { - dim_descriptor[i].1 = dim_descriptor[i + 1].1 * dim_descriptor[i + 1].0 as isize; + let mut strides = vec![0isize; shape_descriptor.len()]; + strides[shape_descriptor.len() - 1] = itemsize as isize; + for i in (0..strides.len() - 1).rev() { + strides[i] = strides[i + 1] * shape_descriptor[i + 1]; } if product_shape != other.desc.len { @@ -1188,7 +1199,11 @@ impl Py { ); } - other.desc.dim_desc = dim_descriptor; + other.desc.set_dimensions( + shape_descriptor.into_boxed_slice(), + strides.into_boxed_slice(), + vec![-1; shape_ndim].into_boxed_slice(), + ); Ok(other.into_ref(&vm.ctx)) } else { @@ -1670,8 +1685,8 @@ fn is_equiv_shape(a: &BufferDescriptor, b: &BufferDescriptor) -> bool { return false; } - let a_iter = a.dim_desc.iter().map(|x| x.0); - let b_iter = b.dim_desc.iter().map(|x| x.0); + let a_iter = a.shape().iter().copied(); + let b_iter = b.shape().iter().copied(); for (a_shape, b_shape) in a_iter.zip(b_iter) { if a_shape != b_shape { return false; diff --git a/crates/vm/src/protocol/buffer.rs b/crates/vm/src/protocol/buffer.rs index 500ea5992f7..e8f13e22a51 100644 --- a/crates/vm/src/protocol/buffer.rs +++ b/crates/vm/src/protocol/buffer.rs @@ -403,25 +403,110 @@ pub struct BufferDescriptor { pub readonly: bool, pub itemsize: usize, pub format: Cow<'static, str>, - /// (shape, stride, suboffset) for each dimension. A non-zero suboffset means - /// the dimension is reached through a pointer; slicing never introduces one. - pub dim_desc: Vec<(usize, isize, isize)>, + /// Number of dimensions. + ndim: usize, + /// Shape, strides and suboffsets packed as 3 contiguous slices of length `ndim`. + dims: Box<[isize]>, // TODO: flags } impl BufferDescriptor { #[must_use] - pub fn simple(bytes_len: usize, readonly: bool) -> Self { + pub fn from_dim_desc( + len: usize, + offset: isize, + readonly: bool, + itemsize: usize, + format: Cow<'static, str>, + dim_desc: Vec<(usize, isize, isize)>, + ) -> Self { + let ndim = dim_desc.len(); + let mut dims = Vec::with_capacity(ndim * 3); + dims.extend(dim_desc.iter().map(|(shape, _, _)| *shape as isize)); + dims.extend(dim_desc.iter().map(|(_, stride, _)| *stride)); + dims.extend( + dim_desc + .iter() + .map(|(_, _, suboffset)| if *suboffset == 0 { -1 } else { *suboffset }), + ); Self { - len: bytes_len, - offset: 0, + len, + offset, readonly, - itemsize: 1, - format: Cow::Borrowed("B"), - dim_desc: vec![(bytes_len, 1, 0)], + itemsize, + format, + ndim, + dims: dims.into_boxed_slice(), } } + #[must_use] + pub fn shape(&self) -> &[isize] { + debug_assert_eq!(self.dims.len(), self.ndim * 3); + &self.dims[..self.ndim] + } + + #[must_use] + pub fn strides(&self) -> &[isize] { + debug_assert_eq!(self.dims.len(), self.ndim * 3); + &self.dims[self.ndim..self.ndim * 2] + } + + #[must_use] + pub fn suboffsets(&self) -> &[isize] { + debug_assert_eq!(self.dims.len(), self.ndim * 3); + &self.dims[self.ndim * 2..] + } + + pub fn shape_mut(&mut self) -> &mut [isize] { + debug_assert_eq!(self.dims.len(), self.ndim * 3); + &mut self.dims[..self.ndim] + } + + pub fn strides_mut(&mut self) -> &mut [isize] { + debug_assert_eq!(self.dims.len(), self.ndim * 3); + &mut self.dims[self.ndim..self.ndim * 2] + } + + pub fn suboffsets_mut(&mut self) -> &mut [isize] { + debug_assert_eq!(self.dims.len(), self.ndim * 3); + &mut self.dims[self.ndim * 2..] + } + + pub fn set_dimensions( + &mut self, + shape: Box<[isize]>, + strides: Box<[isize]>, + suboffsets: Box<[isize]>, + ) { + let ndim = shape.len(); + debug_assert_eq!(strides.len(), ndim); + debug_assert_eq!(suboffsets.len(), ndim); + + let shape = Vec::from(shape); + let strides = Vec::from(strides); + let suboffsets = Vec::from(suboffsets); + let mut dims = Vec::with_capacity(ndim * 3); + dims.extend(shape); + dims.extend(strides); + dims.extend(suboffsets); + + self.ndim = ndim; + self.dims = dims.into_boxed_slice(); + } + + #[must_use] + pub fn simple(bytes_len: usize, readonly: bool) -> Self { + Self::from_dim_desc( + bytes_len, + 0, + readonly, + 1, + Cow::Borrowed("B"), + vec![(bytes_len, 1, 0)], + ) + } + #[must_use] pub fn format( bytes_len: usize, @@ -429,14 +514,14 @@ impl BufferDescriptor { itemsize: usize, format: Cow<'static, str>, ) -> Self { - Self { - len: bytes_len, - offset: 0, + Self::from_dim_desc( + bytes_len, + 0, readonly, itemsize, format, - dim_desc: vec![(bytes_len / itemsize, itemsize as isize, 0)], - } + vec![(bytes_len / itemsize, itemsize as isize, 0)], + ) } /// The descriptor an exporter hands to a consumer that asked for `flags`: @@ -460,14 +545,19 @@ impl BufferDescriptor { // A request this flat is refused unless the layout is C-contiguous, so // one dimension addresses the same bytes. let shape = desc.len.checked_div(desc.itemsize).unwrap_or(0); - desc.dim_desc = vec![(shape, desc.itemsize as isize, 0)]; + desc.set_dimensions( + Box::new([shape as isize]), + Box::new([desc.itemsize as isize]), + Box::new([-1]), + ); } else if !flags.contains(BufferFlags::STRIDES) { // Shape survives but strides do not, which means C order. let mut stride = desc.itemsize as isize; - for (shape, dim_stride, suboffset) in desc.dim_desc.iter_mut().rev() { - *dim_stride = stride; - *suboffset = 0; - stride *= *shape as isize; + for i in (0..desc.ndim()).rev() { + let dim_shape = desc.shape()[i]; + desc.strides_mut()[i] = stride; + desc.suboffsets_mut()[i] = -1; + stride *= dim_shape; } } desc @@ -490,10 +580,9 @@ impl BufferDescriptor { debug_assert_eq!(self.itemsize, self.len); } else { let mut shape_product = 1; - let has_zero_dim = self.dim_desc.iter().any(|(s, _, _)| *s == 0); - for (shape, stride, suboffset) in self.dim_desc.iter().copied() { - shape_product *= shape; - debug_assert!(suboffset >= 0); + let has_zero_dim = self.shape().contains(&0); + for (&shape, &stride) in self.shape().iter().zip(self.strides().iter()) { + shape_product *= shape as usize; // For empty arrays (any dimension is 0), strides can be 0 if !has_zero_dim { debug_assert_ne!(stride, 0); @@ -506,7 +595,8 @@ impl BufferDescriptor { #[must_use] pub fn ndim(&self) -> usize { - self.dim_desc.len() + debug_assert_eq!(self.dims.len(), self.ndim * 3); + self.ndim } /// Whether the elements are laid out in row-major order. _IsCContiguous @@ -516,11 +606,11 @@ impl BufferDescriptor { return true; } let mut sd = self.itemsize; - for (shape, stride, _) in self.dim_desc.iter().copied().rev() { + for (&shape, &stride) in self.shape().iter().zip(self.strides().iter()).rev() { if shape > 1 && stride != sd as isize { return false; } - sd *= shape; + sd *= shape as usize; } true } @@ -534,11 +624,11 @@ impl BufferDescriptor { return true; } let mut sd = self.itemsize; - for (shape, stride, _) in self.dim_desc.iter().copied() { + for (&shape, &stride) in self.shape().iter().zip(self.strides().iter()) { if shape > 1 && stride != sd as isize { return false; } - sd *= shape; + sd *= shape as usize; } true } @@ -561,23 +651,27 @@ impl BufferDescriptor { /// The same shape, format and item size, laid out in C order from byte zero. #[must_use] pub fn contiguous(&self) -> Self { - let itemsize = self.itemsize; - let mut dim_desc = self.dim_desc.clone(); - if let Some((_, stride, suboffset)) = dim_desc.last_mut() { - *stride = itemsize as isize; - *suboffset = 0; - } - for i in (1..dim_desc.len()).rev() { - dim_desc[i - 1].1 = dim_desc[i].1 * dim_desc[i].0 as isize; - dim_desc[i - 1].2 = 0; + let mut strides = vec![0isize; self.ndim()]; + let mut stride = self.itemsize as isize; + for i in (0..self.ndim()).rev() { + strides[i] = stride; + stride *= self.shape()[i]; } + let shape = self.shape().to_vec().into_boxed_slice(); Self { len: self.len, offset: 0, readonly: self.readonly, itemsize: self.itemsize, format: self.format.clone(), - dim_desc, + ndim: self.ndim, + dims: { + let mut dims = Vec::with_capacity(self.ndim * 3); + dims.extend(shape.iter().copied()); + dims.extend(strides.iter().copied()); + dims.extend(vec![-1; self.ndim]); + dims.into_boxed_slice() + }, } } @@ -585,9 +679,7 @@ impl BufferDescriptor { /// stepping, the layout `PyBUF_INDIRECT` describes. #[must_use] pub fn has_suboffsets(&self) -> bool { - self.dim_desc - .iter() - .any(|(_, _, suboffset)| *suboffset != 0) + self.suboffsets().iter().any(|&suboffset| suboffset != -1) } /// this function do not check the bound @@ -595,12 +687,13 @@ impl BufferDescriptor { #[must_use] pub fn fast_position(&self, indices: &[usize]) -> isize { let mut pos = self.offset; - for (i, (_, stride, suboffset)) in indices - .iter() - .copied() - .zip_eq(self.dim_desc.iter().copied()) - { - pos += i as isize * stride + suboffset; + for (dim, i) in indices.iter().copied().enumerate() { + let suboffset = if self.suboffsets()[dim] == -1 { + 0 + } else { + self.suboffsets()[dim] + }; + pos += i as isize * self.strides()[dim] + suboffset; } pos } @@ -611,7 +704,19 @@ impl BufferDescriptor { for (dim, (i, (shape, stride, suboffset))) in indices .iter() .copied() - .zip_eq(self.dim_desc.iter().copied()) + .zip_eq( + self.shape() + .iter() + .zip(self.strides().iter()) + .zip(self.suboffsets().iter()) + .map(|((&shape, &stride), &suboffset)| { + ( + shape as usize, + stride, + if suboffset == -1 { 0 } else { suboffset }, + ) + }), + ) .enumerate() { // The dimension is named the way a person counts it. lookup_dimension @@ -664,15 +769,17 @@ impl BufferDescriptor { let pos = self.offset + indices .iter() - .zip_eq(self.dim_desc.iter()) - .map(|(&i, &(_, stride, suboffset))| i as isize * stride + suboffset) + .zip_eq(self.strides().iter().zip(self.suboffsets().iter())) + .map(|(&i, (&stride, &suboffset))| { + i as isize * stride + if suboffset == -1 { 0 } else { suboffset } + }) .sum::(); f(pos..pos + self.itemsize as isize); let mut dim = 0; loop { indices[dim] += 1; - if indices[dim] < self.dim_desc[dim].0 { + if (indices[dim] as isize) < self.shape()[dim] { break; } indices[dim] = 0; @@ -688,7 +795,13 @@ impl BufferDescriptor { where F: FnMut(Range), { - let (shape, stride, suboffset) = self.dim_desc[dim]; + let shape = self.shape()[dim] as usize; + let stride = self.strides()[dim]; + let suboffset = if self.suboffsets()[dim] == -1 { + 0 + } else { + self.suboffsets()[dim] + }; if dim + 1 == self.ndim() { if CONTIGUOUS { f(index..index + (shape * self.itemsize) as isize); @@ -743,9 +856,21 @@ impl BufferDescriptor { ) where F: FnMut(Range, Range) -> bool, { - let (shape, a_stride, a_suboffset) = self.dim_desc[dim]; - let (_b_shape, b_stride, b_suboffset) = other.dim_desc[dim]; - debug_assert_eq!(shape, _b_shape); + let shape = self.shape()[dim] as usize; + let a_stride = self.strides()[dim]; + let a_suboffset = if self.suboffsets()[dim] == -1 { + 0 + } else { + self.suboffsets()[dim] + }; + let b_shape = other.shape()[dim] as usize; + let b_stride = other.strides()[dim]; + let b_suboffset = if other.suboffsets()[dim] == -1 { + 0 + } else { + other.suboffsets()[dim] + }; + debug_assert_eq!(shape, b_shape); if dim + 1 == self.ndim() { if CONTIGUOUS { if f( @@ -786,13 +911,13 @@ impl BufferDescriptor { #[must_use] fn is_last_dim_contiguous(&self) -> bool { - let (_, stride, suboffset) = self.dim_desc[self.ndim() - 1]; - suboffset == 0 && stride == self.itemsize as isize + let i = self.ndim() - 1; + self.suboffsets()[i] == -1 && self.strides()[i] == self.itemsize as isize } #[must_use] pub fn is_zero_in_shape(&self) -> bool { - self.dim_desc.iter().any(|(shape, _, _)| *shape == 0) + self.shape().contains(&0) } // TODO: support column-major order diff --git a/crates/vm/src/stdlib/_ctypes/array.rs b/crates/vm/src/stdlib/_ctypes/array.rs index 0504688d85c..1ce52e47cbf 100644 --- a/crates/vm/src/stdlib/_ctypes/array.rs +++ b/crates/vm/src/stdlib/_ctypes/array.rs @@ -1060,14 +1060,14 @@ impl AsBuffer for PyCArray { } dim_desc.reverse(); - BufferDescriptor { - offset: 0, - len: buffer_len, - readonly: false, + BufferDescriptor::from_dim_desc( + buffer_len, + 0, + false, itemsize, - format: alloc::borrow::Cow::Owned(fmt), + alloc::borrow::Cow::Owned(fmt), dim_desc, - } + ) } else { // Fallback to simple buffer if no format/shape info BufferDescriptor::simple(buffer_len, false) diff --git a/crates/vm/src/stdlib/_ctypes/function.rs b/crates/vm/src/stdlib/_ctypes/function.rs index 6066310d534..4c9b46a9313 100644 --- a/crates/vm/src/stdlib/_ctypes/function.rs +++ b/crates/vm/src/stdlib/_ctypes/function.rs @@ -1663,14 +1663,7 @@ impl AsBuffer for PyCFuncPtr { } else { (Cow::Borrowed(pointer_format()), pointer_size()) }; - let desc = BufferDescriptor { - offset: 0, - len: itemsize, - readonly: false, - itemsize, - format, - dim_desc: vec![], - }; + let desc = BufferDescriptor::from_dim_desc(itemsize, 0, false, itemsize, format, vec![]); let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); Ok(buf) } diff --git a/crates/vm/src/stdlib/_ctypes/pointer.rs b/crates/vm/src/stdlib/_ctypes/pointer.rs index 6a8ebe660a0..27778a01c0f 100644 --- a/crates/vm/src/stdlib/_ctypes/pointer.rs +++ b/crates/vm/src/stdlib/_ctypes/pointer.rs @@ -781,14 +781,7 @@ impl AsBuffer for PyCPointer { .map_or(Cow::Borrowed("&B"), Cow::Owned); let itemsize = stg_info.size; // Pointer types are scalars with ndim=0, shape=() - let desc = BufferDescriptor { - offset: 0, - len: itemsize, - readonly: false, - itemsize, - format, - dim_desc: vec![], - }; + let desc = BufferDescriptor::from_dim_desc(itemsize, 0, false, itemsize, format, vec![]); let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); Ok(buf) } diff --git a/crates/vm/src/stdlib/_ctypes/simple.rs b/crates/vm/src/stdlib/_ctypes/simple.rs index 1a0e12ba457..18b67fca7c8 100644 --- a/crates/vm/src/stdlib/_ctypes/simple.rs +++ b/crates/vm/src/stdlib/_ctypes/simple.rs @@ -1288,14 +1288,7 @@ impl AsBuffer for PyCSimple { .map_or(Cow::Borrowed("B"), Cow::Owned); let itemsize = stg_info.size; // Simple types are scalars with ndim=0, shape=() - let desc = BufferDescriptor { - offset: 0, - len: itemsize, - readonly: false, - itemsize, - format, - dim_desc: vec![], - }; + let desc = BufferDescriptor::from_dim_desc(itemsize, 0, false, itemsize, format, vec![]); let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); Ok(buf) } diff --git a/crates/vm/src/stdlib/_ctypes/structure.rs b/crates/vm/src/stdlib/_ctypes/structure.rs index bc4b3d54c8f..ed4cd254883 100644 --- a/crates/vm/src/stdlib/_ctypes/structure.rs +++ b/crates/vm/src/stdlib/_ctypes/structure.rs @@ -823,14 +823,14 @@ impl AsBuffer for PyCStructure { // Structure: ndim=0, shape=(), itemsize=struct_size let buf = PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor { - offset: 0, - len: buffer_len, - readonly: false, - itemsize: buffer_len, - format: Cow::Owned(format), - dim_desc: vec![], // ndim=0 means empty dim_desc - }, + BufferDescriptor::from_dim_desc( + buffer_len, + 0, + false, + buffer_len, + Cow::Owned(format), + vec![], // ndim=0 means empty dim_desc + ), &CDATA_BUFFER_METHODS, ); Ok(buf) diff --git a/crates/vm/src/stdlib/_ctypes/union.rs b/crates/vm/src/stdlib/_ctypes/union.rs index 2322d652479..8bd0125990d 100644 --- a/crates/vm/src/stdlib/_ctypes/union.rs +++ b/crates/vm/src/stdlib/_ctypes/union.rs @@ -684,14 +684,14 @@ impl AsBuffer for PyCUnion { // Union: ndim=0, shape=(), itemsize=union_size let buf = PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor { - offset: 0, - len: buffer_len, - readonly: false, - itemsize: buffer_len, - format: Cow::Owned(format), - dim_desc: vec![], // ndim=0 means empty dim_desc - }, + BufferDescriptor::from_dim_desc( + buffer_len, + 0, + false, + buffer_len, + Cow::Owned(format), + vec![], // ndim=0 means empty dim_desc + ), &CDATA_BUFFER_METHODS, ); Ok(buf) From 48419bd11402221e2c9be45d19e56beb204d1216 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 04:55:36 -0700 Subject: [PATCH 054/229] Skip overridden items for empty dictionaries in JSON encoding (#9018) Split the Python 3.14-compatible empty native dictionary fast path from #8954. Extend the existing JSON snippet with root, nested, sorting, and nonempty subclass regression coverage. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/json.rs | 4 ++++ extra_tests/snippets/stdlib_json.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index e553dbb652b..1716d673f15 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -1160,6 +1160,10 @@ mod _json { out: &mut Wtf8Buf, vm: &VirtualMachine, ) -> PyResult<()> { + if dict.is_empty() { + out.push_str("{}"); + return Ok(()); + } let mut items = self.dict_items(obj, dict, vm)?; if items.is_empty() { out.push_str("{}"); diff --git a/extra_tests/snippets/stdlib_json.py b/extra_tests/snippets/stdlib_json.py index e55382c8223..8366a19cd66 100644 --- a/extra_tests/snippets/stdlib_json.py +++ b/extra_tests/snippets/stdlib_json.py @@ -67,6 +67,24 @@ def json_load(obj): assert "{}" == json_dump({}) assert round_trip_test({"a": "b"}) + +class EmptyDictWithItems(dict): + def items(self): + raise AssertionError("items() must not be called for an empty dictionary") + + +for sort_keys in (False, True): + assert json.dumps(EmptyDictWithItems(), sort_keys=sort_keys) == "{}" + assert json.dumps([EmptyDictWithItems()], sort_keys=sort_keys) == "[{}]" + + +class NonemptyDictWithItems(dict): + def items(self): + return [("replacement", 2)] + + +assert json.dumps(NonemptyDictWithItems(original=1)) == '{"replacement": 2}' + # should reject non-str keys in jsons assert_raises(json.JSONDecodeError, lambda: json.loads('{3: "abc"}')) assert_raises(json.JSONDecodeError, lambda: json_load('{3: "abc"}')) From bd8fdb7e5ca3e4e9f78f4f34b122a84c9dd2e2d0 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 04:56:40 -0700 Subject: [PATCH 055/229] Handle math.log bases without premature float conversion (#9019) Use the natural logarithm path for both operands, evaluating x first. Preserve large integer bases and raise ZeroDivisionError for base one while retaining noninteger conversion errors. Split the Python 3.14-compatible behavior from #8954 without its additional __index__ fallback or changes to log2 and log10. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/math.rs | 53 ++++++++++++++------------- extra_tests/snippets/stdlib_math.py | 57 +++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 26 deletions(-) diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index 4b23b7b7b4a..3f5dab6b9b0 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -94,35 +94,36 @@ mod math { } #[pyfunction] - fn log(x: PyObjectRef, base: OptionalArg, vm: &VirtualMachine) -> PyResult { - let base = base.into_option().map(|v| v.into_float()); - // Check base first for proper error messages - if let Some(b) = base { - if b <= 0.0 { - return Err(vm.new_value_error(format!( - "expected a positive input, got {}", - super::float_repr(b) - ))); - } - if b == 1.0 { - return Err(vm.new_value_error("math domain error")); + fn log(x: PyObjectRef, base: OptionalArg, vm: &VirtualMachine) -> PyResult { + let natural_log = |x: &PyObject| { + // Handle BigInt specially for large values (only for actual int type, not float) + if let Some(i) = x.downcast_ref::() { + return pymath::math::log_bigint(i.as_bigint(), None).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input"), + _ => pymath_exception(err, vm), + }); } - } - // Handle BigInt specially for large values (only for actual int type, not float) - if let Some(i) = x.downcast_ref::() { - return pymath::math::log_bigint(i.as_bigint(), base).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error("expected a positive input"), + let val = x.try_float(vm)?.to_f64(); + pymath::math::log(val, None).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), _ => pymath_exception(err, vm), - }); + }) + }; + let numerator = natural_log(&x)?; + match base { + OptionalArg::Missing => Ok(numerator), + OptionalArg::Present(base) => { + let denominator = natural_log(&base)?; + if denominator == 0.0 { + Err(vm.new_zero_division_error("division by zero")) + } else { + Ok(numerator / denominator) + } + } } - let val = x.try_float(vm)?.to_f64(); - pymath::math::log(val, base).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error(format!( - "expected a positive input, got {}", - super::float_repr(val) - )), - _ => pymath_exception(err, vm), - }) } #[pyfunction] diff --git a/extra_tests/snippets/stdlib_math.py b/extra_tests/snippets/stdlib_math.py index 00d6c5f3ab1..29824043aa2 100644 --- a/extra_tests/snippets/stdlib_math.py +++ b/extra_tests/snippets/stdlib_math.py @@ -17,6 +17,63 @@ # # assert math.log(1.1111) == math.log(Conversible()) +# A logarithm can fit in a float even when its integer base cannot. +assert math.isclose(math.log(2**4000, 2**2000), 2.0) +assert math.isclose(math.log(2.0, 2**2000), 1.0 / 2000) + +for value in (1, 8.0, INF, NAN): + assert_raises(ZeroDivisionError, math.log, value, 1) +assert_raises(TypeError, math.log, object(), 1) +assert_raises(ValueError, math.log, 8, -(2**2000)) + +log_conversions = [] + + +class LogFloat: + def __init__(self, name, value): + self.name = name + self.value = value + + def __float__(self): + log_conversions.append(self.name) + return self.value + + +assert math.isclose(math.log(LogFloat("x", 8.0), LogFloat("base", 2.0)), 3.0) +assert log_conversions == ["x", "base"] +log_conversions.clear() +assert_raises(ValueError, math.log, -1, LogFloat("base", 2.0)) +assert log_conversions == [] + + +class LogFloatOverflow: + def __float__(self): + raise OverflowError("float conversion failed") + + def __index__(self): + raise AssertionError("must not retry __index__ after __float__ fails") + + +assert_raises(OverflowError, math.log, LogFloatOverflow()) +assert_raises(OverflowError, math.log, 8, LogFloatOverflow()) + + +class LogIndexOnly: + def __index__(self): + return 2**2000 + + +assert_raises(OverflowError, math.log, LogIndexOnly()) +assert_raises(OverflowError, math.log, 8, LogIndexOnly()) + + +class LogInt(int): + def __float__(self): + raise AssertionError("integer logarithms use the native integer value") + + +assert math.isclose(math.log(LogInt(2**4000), LogInt(2**2000)), 2.0) + # roundings assert int.__trunc__ assert int.__floor__ From fe984cec5da6215c50843dcc8bb4f24f423cb895 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 07:25:59 -0700 Subject: [PATCH 056/229] select: validate and expand Windows descriptor capacity (#9021) * Cover Windows select capacity above the SDK default Add local-socket readiness coverage for 65 entries and the Windows 512-entry limit, preserving 513-entry rejection. This regression-only commit intentionally leaves the current Windows 64-entry defect visible before the implementation is added. AI assistance covered regression design, CPython 3.14 source comparison, and automated Linux baseline/reference testing. No manual Windows validation is claimed. Assisted-by: Codex:model-version-unavailable * Match Windows select capacity to Python 3.14 Use a 512-entry WinSock fd_set with checked C layout and scan its returned socket array. Keep the SDK default-size declaration at the FFI boundary only. The preceding regression-only commit passed CPython 3.14.7 on hosted Windows and failed RustPython at the first 65-socket selection. Windows-GNU host_env cargo check and Clippy with warnings denied pass. The fixed native Windows runtime result remains pending CI. AI assistance covered source comparison, implementation, regression design, and automated verification. No manual validation is claimed. Assisted-by: Codex:model-version-unavailable --- crates/host_env/src/select.rs | 58 +++++++++++++++++++++++++-- extra_tests/snippets/stdlib_select.py | 19 +++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/crates/host_env/src/select.rs b/crates/host_env/src/select.rs index 287f64b56bf..814eb9900df 100644 --- a/crates/host_env/src/select.rs +++ b/crates/host_env/src/select.rs @@ -36,10 +36,62 @@ pub mod platform { #[allow(non_snake_case)] #[cfg(windows)] pub mod platform { - pub use WinSock::{FD_SET as fd_set, FD_SETSIZE, SOCKET as RawFd, TIMEVAL as timeval, select}; + pub use WinSock::{SOCKET as RawFd, TIMEVAL as timeval}; use std::io; use windows_sys::Win32::Networking::WinSock; + // Match CPython's Windows capacity. WinSock uses fd_count to determine + // the array length; the windows-sys declaration has the SDK default of 64. + pub const FD_SETSIZE: u32 = 512; + + #[allow(non_camel_case_types)] + #[repr(C)] + pub struct fd_set { + fd_count: u32, + fd_array: [RawFd; FD_SETSIZE as usize], + } + + const _: () = { + assert!(core::mem::align_of::() == core::mem::align_of::()); + assert!( + core::mem::offset_of!(fd_set, fd_count) + == core::mem::offset_of!(WinSock::FD_SET, fd_count) + ); + assert!( + core::mem::offset_of!(fd_set, fd_array) + == core::mem::offset_of!(WinSock::FD_SET, fd_array) + ); + assert!( + core::mem::size_of::() + == core::mem::offset_of!(fd_set, fd_array) + + FD_SETSIZE as usize * core::mem::size_of::() + ); + }; + + /// # Safety + /// + /// Every non-null set must point to an initialized fd_set, and timeout + /// must be null or point to an initialized timeval. + pub unsafe fn select( + nfds: i32, + readfds: *mut fd_set, + writefds: *mut fd_set, + errorfds: *mut fd_set, + timeout: *const timeval, + ) -> i32 { + // fd_set is a count followed by a caller-sized SOCKET array. Both + // layouts have the same alignment and offset of the array. + unsafe { + WinSock::select( + nfds, + readfds.cast(), + writefds.cast(), + errorfds.cast(), + timeout, + ) + } + } + /// # Safety /// /// `set` must be a valid mutable pointer to an initialized WinSock fd_set. @@ -71,8 +123,8 @@ pub mod platform { /// /// `set` must be a valid mutable pointer to an initialized WinSock fd_set. pub unsafe fn FD_ISSET(fd: RawFd, set: *mut fd_set) -> bool { - use WinSock::__WSAFDIsSet; - unsafe { __WSAFDIsSet(fd as _, set) != 0 } + let set = unsafe { &*set }; + set.fd_array[..set.fd_count as usize].contains(&fd) } #[must_use] diff --git a/extra_tests/snippets/stdlib_select.py b/extra_tests/snippets/stdlib_select.py index 6b214cfc374..733e8421ff4 100644 --- a/extra_tests/snippets/stdlib_select.py +++ b/extra_tests/snippets/stdlib_select.py @@ -83,6 +83,25 @@ def fileno(self): b.close() +# Windows supports up to 512 entries, above the Windows SDK default of 64. +for count in (65, 512) if sys.platform == "win32" else (65,): + pairs = [] + try: + for _ in range(count): + pairs.append(socket.socketpair()) + readers = [reader for reader, writer in pairs] + assert select.select(readers, [], [], 0) == ([], [], []) + for reader, writer in pairs: + writer.send(b"x") + assert select.select(readers, [], [], 0) == (readers, [], []) + if sys.platform == "win32": + assert_raises(ValueError, select.select, [readers[0]] * 513, [], [], 0) + finally: + for pair in pairs: + for sock in pair: + sock.close() + + # fileno() runs while the sequence is being read, and it can mutate the very # list it was handed. mutable_pair, other_end = socket.socketpair() From 8986de27cc8b93125c973f5b0f54523a837f14c4 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 07:28:56 -0700 Subject: [PATCH 057/229] Preserve CSV dialect conversion errors and ordering (#9020) Collect raw dialect fields and keyword overrides before converting them in CPython 3.14 order. Preserve truth-conversion exceptions, read classes without instantiating them, and retain defaulting for failed attribute lookup. Use one shared parser for reader, writer, registry, and native dialect construction. Extend the existing CSV snippet with class, instance, direct-construction, lookup, override, conversion-order, and argument-binding regressions. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/csv.rs | 525 ++++++++--------------------- extra_tests/snippets/stdlib_csv.py | 243 +++++++++++++ 2 files changed, 390 insertions(+), 378 deletions(-) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index b07f9c644ad..51bba58c91e 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -6,7 +6,7 @@ mod _csv { use crate::vm::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyBaseExceptionRef, PyInt, PyNone, PyStr, PyType, PyTypeRef, PyUtf8StrRef}, + builtins::{PyBaseExceptionRef, PyInt, PyStr, PyType, PyTypeRef, PyUtf8StrRef}, function::{ArgIterable, ArgumentError, FromArgs, FuncArgs, OptionalArg, Param}, protocol::{PyIter, PyIterReturn, PyNumber}, types::{Callable, Constructor, IterNext, Iterable, SelfIter}, @@ -88,12 +88,10 @@ mod _csv { } impl Constructor for PyDialect { - type Args = PyObjectRef; + type Args = FormatOptions; - fn py_new(_cls: &Py, obj: Self::Args, vm: &VirtualMachine) -> PyResult { - let dialect = Self::try_from_object(vm, obj)?; - validate_dialect(vm, &dialect)?; - Ok(dialect) + fn py_new(_cls: &Py, opts: Self::Args, vm: &VirtualMachine) -> PyResult { + opts.result(vm) } } @@ -140,96 +138,32 @@ mod _csv { } } - /// Parses the delimiter from a Python object and returns its ASCII value. - /// - /// This function attempts to extract the 'delimiter' attribute from the given Python object and ensures that the attribute is a single-character string. If successful, it returns the ASCII value of the character. If the attribute is not a single-character string, an error is returned. - /// - /// # Arguments - /// - /// * `vm` - A reference to the VirtualMachine, used for executing Python code and manipulating Python objects. - /// * `obj` - A reference to the PyObjectRef from which the 'delimiter' attribute is to be parsed. - /// - /// # Returns - /// - /// If successful, returns a `PyResult` representing the ASCII value of the 'delimiter' attribute. If unsuccessful, returns a `PyResult` containing an error message. - /// - /// # Errors - /// - /// This function can return the following errors: - /// - /// * If the 'delimiter' attribute is not a single-character string, a type error is returned. - /// * If the 'obj' is not of string type and does not have a 'delimiter' attribute, a type error is returned. - fn parse_delimiter_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult { - if let Ok(attr) = obj.get_attr("delimiter", vm) { - parse_delimiter_from_obj(vm, &attr) - } else { - match_class!(match obj.to_owned() { - s @ PyStr => { - parse_single_char(&s, |len| { - vm.new_type_error(format!( - r#""delimiter" must be a unicode character, not a string of length {len}"# - )) - }) - } - attr => { - Err(vm.new_type_error(format!( - r#""delimiter" must be a unicode character, not {}"#, - attr.class().name() - ))) - } - }) + fn parse_char( + vm: &VirtualMachine, + obj: &PyObject, + name: &str, + allow_none: bool, + ) -> PyResult> { + if allow_none && vm.is_none(obj) { + return Ok(None); } - } - - fn parse_quotechar_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult> { - match_class!(match obj.get_attr("quotechar", vm)? { - s @ PyStr => { - Ok(Some(parse_single_char(&s, |len| { - new_csv_error( - vm, - format!( - r#""quotechar" must be a unicode character or None, not a string of length {len}"# - ), - ) - })?)) - } - _n @ PyNone => { - Ok(None) - } - attr => { - Err(new_csv_error( - vm, - format!( - r#""quotechar" must be a unicode character or None, not {}"#, - attr.class().name() - ), - )) - } - }) - } - - fn parse_escapechar_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult> { - match_class!(match obj.get_attr("escapechar", vm)? { - s @ PyStr => { - Ok(Some(parse_single_char(&s, |len| { - new_csv_error( - vm, - format!( - r#""escapechar" must be a unicode character or None, not a string of length {len}"# - ), - ) - })?)) - } - _n @ PyNone => { - Ok(None) - } - attr => { - Err(vm.new_type_error(format!( - r#""escapechar" must be a unicode character or None, not {}"#, - attr.class().name() - ))) - } + let expected = if allow_none { + "a unicode character or None" + } else { + "a unicode character" + }; + let value = obj.downcast_ref::().ok_or_else(|| { + vm.new_type_error(format!( + r#""{name}" must be {expected}, not {}"#, + obj.class().name() + )) + })?; + parse_single_char(value, |len| { + vm.new_type_error(format!( + r#""{name}" must be {expected}, not a string of length {len}"# + )) }) + .map(Some) } fn parse_lineterminator<'a>(vm: &VirtualMachine, s: &'a Py) -> PyResult<&'a str> { @@ -237,25 +171,6 @@ mod _csv { .ok_or_else(|| new_csv_error(vm, r#""lineterminator" must be a string"#)) } - fn prase_lineterminator_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult { - match_class!(match obj.get_attr("lineterminator", vm)? { - s @ PyStr => { - // Store the full line terminator string. CPython accepts an - // arbitrary-length terminator; the manual writer paths emit it - // verbatim and the csv-core writer path appends it after a - // sentinel terminator (see `writerow`). - let value = parse_lineterminator(vm, &s)?; - Ok(value.to_owned()) - } - attr => { - Err(vm.new_type_error(format!( - r#""lineterminator" must be a string, not {}"#, - attr.class().name() - ))) - } - }) - } - fn parse_single_char( s: &Py, error: impl Fn(usize) -> PyBaseExceptionRef, @@ -268,51 +183,6 @@ mod _csv { u8::try_from(ch.to_u32()).map_err(|_| error(s.char_len())) } - fn prase_quoting_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult { - match_class!(match obj.get_attr("quoting", vm)? { - i @ PyInt => { - Ok(i.try_to_primitive::(vm)? - .try_into() - .map_err(|_| vm.new_type_error(r#"bad "quoting" value"#))?) - } - attr => { - Err(vm.new_type_error(format!( - r#""quoting" must be string or None, not {}"#, - attr.class().name() - ))) - } - }) - } - - impl TryFromObject for PyDialect { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - let delimiter = parse_delimiter_from_obj(vm, &obj)?; - let quotechar = parse_quotechar_from_obj(vm, &obj)?; - let escapechar = parse_escapechar_from_obj(vm, &obj)?; - let doublequote = obj.get_attr("doublequote", vm)?.try_to_bool(vm)?; - let skipinitialspace = obj.get_attr("skipinitialspace", vm)?.try_to_bool(vm)?; - let lineterminator = prase_lineterminator_from_obj(vm, &obj)?; - let quoting = prase_quoting_from_obj(vm, &obj)?; - - let strict = if let Ok(t) = obj.get_attr("strict", vm) { - t.try_to_bool(vm).unwrap_or(false) - } else { - false - }; - - Ok(Self { - delimiter, - quotechar, - escapechar, - doublequote, - skipinitialspace, - lineterminator, - quoting, - strict, - }) - } - } - #[derive(FromArgs)] struct DialectName { #[pyarg(any)] @@ -492,79 +362,17 @@ mod _csv { } } - #[derive(Default)] - enum DialectItem { - Str(String), - Obj(PyDialect), - #[default] - None, - } - #[derive(Default)] struct FormatOptions { - dialect: DialectItem, - delimiter: Option, - quotechar: Option>, - escapechar: Option>, - doublequote: Option, - skipinitialspace: Option, - lineterminator: Option, - quoting: Option, - strict: Option, - } - - /// prase a dialect item from a Python argument and returns a `DialectItem` or an `ArgumentError`. - /// - /// This function takes a reference to the VirtualMachine and a PyObjectRef as input and attempts to parse a dialect item from the provided Python argument. It returns a `DialectItem` if successful, or an `ArgumentError` if unsuccessful. - /// - /// # Arguments - /// - /// * `vm` - A reference to the VirtualMachine, used for executing Python code and manipulating Python objects. - /// * `obj` - The PyObjectRef from which the dialect item is to be parsed. - /// - /// # Returns - /// - /// If successful, returns a `Result` representing the parsed dialect item. If unsuccessful, returns an `ArgumentError`. - /// - /// # Errors - /// - /// This function can return the following errors: - /// - /// * If the provided object is a PyStr, it returns a `DialectItem::Str` containing the string value. - /// * If the provided object is PyNone, it returns an `ArgumentError` with the message "InvalidKeywordArgument('dialect')". - /// * If the provided object is a PyType, it attempts to create a PyDialect from the object and returns a `DialectItem::Obj` containing the PyDialect if successful. If unsuccessful, it returns an `ArgumentError` with the message "InvalidKeywordArgument('dialect')". - /// * If the provided object is none of the above types, it attempts to create a PyDialect from the object and returns a `DialectItem::Obj` containing the PyDialect if successful. If unsuccessful, it returns an `ArgumentError` with the message "InvalidKeywordArgument('dialect')". - fn prase_dialect_item_from_arg( - vm: &VirtualMachine, - obj: PyObjectRef, - ) -> Result { - match_class!(match obj { - s @ PyStr => { - let s = s.try_into_utf8(vm).map_err(ArgumentError::Exception)?; - Ok(DialectItem::Str(s.as_str().to_owned())) - } - PyNone => { - Err(ArgumentError::InvalidKeywordArgument("dialect".to_string())) - } - t @ PyType => { - let temp = t - .as_object() - .call(vec![], vm) - .map_err(|_e| ArgumentError::InvalidKeywordArgument("dialect".to_string()))?; - Ok(DialectItem::Obj( - PyDialect::try_from_object(vm, temp).map_err(|_| { - ArgumentError::InvalidKeywordArgument("dialect".to_string()) - })?, - )) - } - obj => { - if let Ok(cur_dialect_item) = PyDialect::try_from_object(vm, obj) { - Ok(DialectItem::Obj(cur_dialect_item)) - } else { - Err(ArgumentError::InvalidKeywordArgument("dialect".to_string())) - } - } - }) + dialect: Option, + delimiter: Option, + doublequote: Option, + escapechar: Option, + lineterminator: Option, + quotechar: Option, + quoting: Option, + skipinitialspace: Option, + strict: Option, } impl FromArgs for FormatOptions { @@ -578,107 +386,16 @@ mod _csv { ]); fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result { - let dialect = if let Some(dialect) = args.take_positional_keyword("dialect") { - prase_dialect_item_from_arg(vm, dialect)? - } else { - DialectItem::None - }; - - let mut res = Self { - dialect, - ..Default::default() - }; - - if let Some(delimiter) = args.kwargs.swap_remove("delimiter") { - res.delimiter = Some(parse_delimiter_from_obj(vm, &delimiter)?); - } - - if let Some(escapechar) = args.kwargs.swap_remove("escapechar") { - res.escapechar = match_class!(match escapechar { - s @ PyStr => Some(Some(parse_single_char(&s, |_| { - vm.new_type_error(r#""escapechar" must be a 1-character string"#) - })?)), - PyNone => Some(None), - _ => { - return Err(ArgumentError::Exception( - vm.new_type_error(r#""escapechar" must be a 1-character string"#), - )); - } - }) - }; - - if let Some(lineterminator) = args.kwargs.swap_remove("lineterminator") { - let s = lineterminator.downcast_ref::().ok_or_else(|| { - vm.new_type_error(format!( - r#""lineterminator" must be a string, not {}"#, - lineterminator.class().name() - )) - })?; - let value = parse_lineterminator(vm, s)?; - res.lineterminator = Some(value.to_owned()); - }; - - if let Some(doublequote) = args.kwargs.swap_remove("doublequote") { - res.doublequote = Some( - doublequote - .try_to_bool(vm) - .map_err(|_| vm.new_type_error(r#""doublequote" must be a bool"#))?, - ) - }; - - if let Some(skipinitialspace) = args.kwargs.swap_remove("skipinitialspace") { - res.skipinitialspace = Some( - skipinitialspace - .try_to_bool(vm) - .map_err(|_| vm.new_type_error(r#""skipinitialspace" must be a bool"#))?, - ) - }; - - if let Some(quoting) = args.kwargs.swap_remove("quoting") { - res.quoting = match_class!(match quoting { - i @ PyInt => - Some(i.try_to_primitive::(vm)?.try_into().map_err(|_e| { - ArgumentError::InvalidKeywordArgument("quoting".to_string()) - })?), - _ => { - // let msg = r#""quoting" must be a int enum"#; - return Err(ArgumentError::InvalidKeywordArgument("quoting".to_string())); - } - }); - }; - - if let Some(quotechar) = args.kwargs.swap_remove("quotechar") { - res.quotechar = match_class!(match quotechar { - s @ PyStr => Some(Some(parse_single_char(&s, |_| { - vm.new_type_error(r#""quotechar" must be a 1-character string"#) - })?)), - PyNone => { - if res - .quoting - .is_some_and(|quoting| quoting != QuoteStyle::None) - { - return Err(ArgumentError::Exception( - vm.new_type_error("quotechar must be set if quoting enabled"), - )); - } - Some(None) - } - _o => { - return Err( - rustpython_vm::function::ArgumentError::InvalidKeywordArgument( - "quotechar".to_string(), - ), - ); - } - }) - }; - - if let Some(strict) = args.kwargs.swap_remove("strict") { - res.strict = Some( - strict - .try_to_bool(vm) - .map_err(|_| vm.new_type_error(r#""strict" must be a int enum"#))?, - ) + let res = Self { + dialect: args.take_positional_keyword("dialect"), + delimiter: args.kwargs.swap_remove("delimiter"), + doublequote: args.kwargs.swap_remove("doublequote"), + escapechar: args.kwargs.swap_remove("escapechar"), + lineterminator: args.kwargs.swap_remove("lineterminator"), + quotechar: args.kwargs.swap_remove("quotechar"), + quoting: args.kwargs.swap_remove("quoting"), + skipinitialspace: args.kwargs.swap_remove("skipinitialspace"), + strict: args.kwargs.swap_remove("strict"), }; if let Some(last_arg) = args.kwargs.pop() { @@ -688,7 +405,6 @@ mod _csv { vm.new_unexpected_keyword_type_error(None, &last_arg.0.to_string()), )); } - Ok(res) } } @@ -748,60 +464,113 @@ mod _csv { } impl FormatOptions { - fn update_py_dialect(&self, mut res: PyDialect) -> PyDialect { - macro_rules! check_and_fill { - ($res:ident, $e:ident) => {{ - if let Some(t) = self.$e { - $res.$e = t; - } - }}; + fn result(mut self, vm: &VirtualMachine) -> PyResult { + if let Some(obj) = self.dialect.take() { + let obj = if obj.downcast_ref::().is_some() { + get_dialect(DialectName { name: obj }, vm)? + .into_ref(&vm.ctx) + .into() + } else { + obj + }; + // CPython ignores attribute lookup errors, but not errors converting + // the retrieved values. Read all non-overridden attributes first. + macro_rules! fill_from_dialect { + ($($name:ident),* $(,)?) => { + $(if self.$name.is_none() { + self.$name = obj.get_attr(stringify!($name), vm).ok(); + })* + }; + } + fill_from_dialect!( + delimiter, + doublequote, + escapechar, + lineterminator, + quotechar, + quoting, + skipinitialspace, + strict, + ); + self.dialect = Some(obj); } - check_and_fill!(res, delimiter); - // check_and_fill!(res, quotechar); - check_and_fill!(res, delimiter); - check_and_fill!(res, doublequote); - check_and_fill!(res, skipinitialspace); - - if let Some(t) = self.escapechar { - res.escapechar = t; + let delimiter = match &self.delimiter { + Some(obj) => parse_char(vm, obj, "delimiter", false)?.unwrap(), + None => b',', }; - - if let Some(t) = self.quotechar { - res.quotechar = t; + let doublequote = match &self.doublequote { + Some(obj) => obj.try_to_bool(vm)?, + None => true, }; - - check_and_fill!(res, quoting); - if let Some(t) = &self.lineterminator { - res.lineterminator.clone_from(t); + let escapechar = match &self.escapechar { + Some(obj) => parse_char(vm, obj, "escapechar", true)?, + None => None, }; - check_and_fill!(res, strict); - res - } - - fn result(&self, vm: &VirtualMachine) -> PyResult { - let dialect = match &self.dialect { - DialectItem::Str(name) => { - let g = GLOBAL_HASHMAP.lock(); - if let Some(dialect) = g.get(name) { - Ok(self.update_py_dialect(dialect.clone())) - } else { - Err(new_csv_error(vm, format!("{name} is not registered."))) + let lineterminator = match &self.lineterminator { + Some(obj) => { + let value = obj.downcast_ref::().ok_or_else(|| { + vm.new_type_error(format!( + r#""lineterminator" must be a string, not {}"#, + obj.class().name() + )) + })?; + // Store the full line terminator string. CPython accepts an + // arbitrary-length terminator; the manual writer paths emit it + // verbatim and the csv-core writer path appends it after a + // sentinel terminator (see `writerow`). + parse_lineterminator(vm, value)?.to_owned() + } + None => "\r\n".to_owned(), + }; + let quotechar = match &self.quotechar { + Some(obj) => parse_char(vm, obj, "quotechar", true)?, + None => Some(b'"'), + }; + let quoting = match &self.quoting { + Some(obj) => { + if !obj.class().is(vm.ctx.types.int_type) { + return Err(vm.new_type_error(format!( + r#""quoting" must be an integer, not {}"#, + obj.class().name() + ))); } - // TODO: Maybe need to update the obj from HashMap + obj.downcast_ref::() + .unwrap() + .try_to_primitive::(vm)? } - DialectItem::Obj(o) => Ok(self.update_py_dialect(o.clone())), - DialectItem::None => Ok(self.update_py_dialect(PyDialect { - delimiter: b',', - quotechar: Some(b'"'), - escapechar: None, - doublequote: true, - skipinitialspace: false, - lineterminator: "\r\n".to_owned(), - quoting: QuoteStyle::Minimal, - strict: false, - })), - }?; + None => QuoteStyle::Minimal as i32, + }; + let skipinitialspace = match &self.skipinitialspace { + Some(obj) => obj.try_to_bool(vm)?, + None => false, + }; + let strict = match &self.strict { + Some(obj) => obj.try_to_bool(vm)?, + None => false, + }; + + // Validate the quoting value only after all field conversions. + let quoting = QuoteStyle::try_from(quoting as isize) + .map_err(|_| vm.new_type_error(r#"bad "quoting" value"#))?; + let quoting = if quotechar.is_none() && self.quoting.is_none() { + QuoteStyle::None + } else { + quoting + }; + if quotechar.is_none() && quoting != QuoteStyle::None { + return Err(vm.new_type_error("quotechar must be set if quoting enabled")); + } + let dialect = PyDialect { + delimiter, + quotechar, + escapechar, + doublequote, + skipinitialspace, + lineterminator, + quoting, + strict, + }; validate_dialect(vm, &dialect)?; Ok(dialect) } diff --git a/extra_tests/snippets/stdlib_csv.py b/extra_tests/snippets/stdlib_csv.py index c80847bc741..08adeaf5294 100644 --- a/extra_tests/snippets/stdlib_csv.py +++ b/extra_tests/snippets/stdlib_csv.py @@ -361,3 +361,246 @@ def __str__(self): test_quote_nonnumeric_writer() + + +def test_dialect_truth_conversion(): + def registered(dialect, **kwargs): + name = "truth_conversion" + try: + csv.register_dialect(name, dialect, **kwargs) + return csv.get_dialect(name) + finally: + if name in csv.list_dialects(): + csv.unregister_dialect(name) + + constructors = ( + lambda dialect, **kwargs: csv.reader([], dialect, **kwargs).dialect, + lambda dialect, **kwargs: csv.writer(io.StringIO(), dialect, **kwargs).dialect, + _csv.Dialect, + registered, + ) + + class TruthError: + def __init__(self, error): + self.error = error + self.calls = 0 + + def __bool__(self): + self.calls += 1 + raise self.error + + class InvalidTruth: + def __bool__(self): + return 1 + + for construct in constructors: + for field in ("strict", "doublequote", "skipinitialspace"): + for error_type in (RuntimeError, AttributeError, TypeError): + for use_class in (False, True): + error = error_type("dialect truth conversion") + value = TruthError(error) + + class Dialect(csv.excel): + def __new__(cls): + raise AssertionError( + "dialect classes must not be instantiated" + ) + + dialect = Dialect if use_class else csv.excel() + setattr(dialect, field, value) + try: + construct(dialect) + except error_type as caught: + assert caught is error + else: + raise AssertionError("dialect truth error was suppressed") + assert value.calls == 1 + + # An explicit option replaces the attribute before conversion. + result = construct(dialect, **{field: True}) + assert getattr(result, field) is True + assert value.calls == 1 + + value = TruthError(error) + try: + construct(None, **{field: value}) + except error_type as caught: + assert caught is error + else: + raise AssertionError("keyword truth error was replaced") + assert value.calls == 1 + + for value in (None, False, True, 0, 1, [], [1]): + dialect = csv.excel() + setattr(dialect, field, value) + assert getattr(construct(dialect), field) is bool(value) + assert getattr(construct(None, **{field: value}), field) is bool(value) + + dialect = csv.excel() + setattr(dialect, field, InvalidTruth()) + with assert_raises(TypeError): + construct(dialect) + with assert_raises(TypeError): + construct(None, **{field: InvalidTruth()}) + + # csv.Dialect's Python validation wrapper translates TypeError to csv.Error. + for error_type in (RuntimeError, AttributeError, TypeError): + error = error_type("direct Python dialect validation") + value = TruthError(error) + + class Dialect(csv.excel): + strict = value + + expected = csv.Error if error_type is TypeError else error_type + try: + Dialect() + except expected as caught: + assert str(caught) == str(error) + if error_type is not TypeError: + assert caught is error + else: + raise AssertionError("Python dialect validation swallowed the error") + assert value.calls == 1 + + +test_dialect_truth_conversion() + + +def test_dialect_lookup_and_conversion_order(): + fields = ( + "delimiter", + "doublequote", + "escapechar", + "lineterminator", + "quotechar", + "quoting", + "skipinitialspace", + "strict", + ) + defaults = (",", True, None, "\r\n", '"', csv.QUOTE_MINIMAL, False, False) + constructors = ( + lambda dialect, **kwargs: csv.reader([], dialect, **kwargs).dialect, + lambda dialect, **kwargs: csv.writer(io.StringIO(), dialect, **kwargs).dialect, + _csv.Dialect, + ) + + for construct in constructors: + for error_type in (AttributeError, RuntimeError, KeyboardInterrupt): + events = [] + + class Missing: + def __getattribute__(self, name): + events.append(name) + raise error_type(name) + + result = construct(Missing()) + assert tuple(getattr(result, field) for field in fields) == defaults + assert events == list(fields) + for dialect in (None, object(), _csv.Dialect(csv.excel)): + result = construct(dialect) + assert tuple(getattr(result, field) for field in fields) == defaults + + events = [] + + class Truth: + def __init__(self, name): + self.name = name + + def __bool__(self): + events.append("bool:" + self.name) + return False + + values = dict(zip(fields, defaults)) + for field in ("doublequote", "skipinitialspace", "strict"): + values[field] = Truth(field) + + class Dialect: + def __getattribute__(self, name): + events.append(name) + return values[name] + + construct(Dialect()) + assert events == list(fields) + [ + "bool:doublequote", + "bool:skipinitialspace", + "bool:strict", + ] + + events.clear() + construct(Dialect(), strict=True, doublequote=True) + assert events == [ + field for field in fields if field not in ("strict", "doublequote") + ] + ["bool:skipinitialspace"] + + events.clear() + construct( + None, + strict=Truth("strict"), + skipinitialspace=Truth("skipinitialspace"), + doublequote=Truth("doublequote"), + ) + assert events == ["bool:doublequote", "bool:skipinitialspace", "bool:strict"] + + # Collect every attribute before starting conversions, even on failure. + events.clear() + values["delimiter"] = "too long" + with assert_raises(TypeError): + construct(Dialect()) + assert events == list(fields) + + error = RuntimeError("truth conversion wins") + + class BadTruth: + def __bool__(self): + raise error + + # Conversion order differs from option-validation order. + for options in ( + {"doublequote": BadTruth(), "escapechar": "too long"}, + {"strict": BadTruth(), "quoting": 42}, + {"strict": BadTruth(), "quotechar": None, "quoting": csv.QUOTE_ALL}, + {"strict": BadTruth(), "delimiter": "\n"}, + ): + try: + construct(None, **options) + except RuntimeError as caught: + assert caught is error + else: + raise AssertionError("conversion order did not preserve truth error") + for options in ( + {"delimiter": "too long", "doublequote": BadTruth()}, + {"lineterminator": None, "skipinitialspace": BadTruth()}, + {"quoting": None, "strict": BadTruth()}, + {"unknown_option": None, "strict": BadTruth()}, + ): + with assert_raises(TypeError): + construct(None, **options) + + class IntegerSubclass(int): + pass + + class Index: + def __index__(self): + raise AssertionError("quoting must not call __index__") + + for value in (True, IntegerSubclass(0), Index()): + with assert_raises(TypeError): + construct(None, quoting=value) + with assert_raises(OverflowError): + construct(None, quoting=2**40, strict=BadTruth()) + + # None quotechar disables quoting only when quoting was not supplied. + assert construct(None, quotechar=None).quoting == csv.QUOTE_NONE + with assert_raises(TypeError): + construct(csv.excel, quotechar=None) + with assert_raises(TypeError): + construct(None, quotechar=None, quoting=csv.QUOTE_MINIMAL) + + assert _csv.Dialect().strict is False + assert _csv.Dialect(strict=True).strict is True + for constructor, arg in ((csv.reader, []), (csv.writer, io.StringIO())): + with assert_raises(TypeError): + constructor(arg, None, None, strict=BadTruth()) + + +test_dialect_lookup_and_conversion_order() From 59b36dd4264661ccfe4f624fd8912de37d2607f3 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 07:29:16 -0700 Subject: [PATCH 058/229] _zoneinfo: reject empty quoted POSIX TZ abbreviations (#9017) * Reject empty quoted POSIX timezone abbreviations Split the Python 3.14-compatible parser guard from #8954. Cover empty standard and daylight-saving abbreviations and preserve valid quoted names. Assisted-by: OpenAI Codex:model identifier unavailable * Keep zoneinfo regression in module snippet Rename the snippet as requested during review without changing test contents. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/_zoneinfo.rs | 3 +++ extra_tests/snippets/stdlib_zoneinfo.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 extra_tests/snippets/stdlib_zoneinfo.py diff --git a/crates/stdlib/src/_zoneinfo.rs b/crates/stdlib/src/_zoneinfo.rs index 9ab94cbefcd..aea7db10b4a 100644 --- a/crates/stdlib/src/_zoneinfo.rs +++ b/crates/stdlib/src/_zoneinfo.rs @@ -552,6 +552,9 @@ mod _zoneinfo { parser.bump(); } let end = parser.i; + if end == start { + return None; + } parser.bump(); return Some(String::from_utf8_lossy(&parser.bytes[start..end]).into_owned()); } diff --git a/extra_tests/snippets/stdlib_zoneinfo.py b/extra_tests/snippets/stdlib_zoneinfo.py new file mode 100644 index 00000000000..c8004585f81 --- /dev/null +++ b/extra_tests/snippets/stdlib_zoneinfo.py @@ -0,0 +1,19 @@ +import io +import struct +from zoneinfo import ZoneInfo + +from testutils import assert_raises + +# Minimal TZif v3 data with no transitions and a POSIX rule in the footer. +header = (b"TZif3" + b"\0" * 15 + struct.pack(">6l", 0, 0, 0, 0, 0, 0)) * 2 + + +def zone_from_rule(rule): + return ZoneInfo.from_file(io.BytesIO(header + b"\n" + rule + b"\n")) + + +for rule in (b"<>5", b"AAA4<>,M3.2.0/2,M11.1.0/3"): + assert_raises(ValueError, zone_from_rule, rule) + +# Quoting still permits nonempty numeric abbreviations. +assert zone_from_rule(b"<+05>-5").tzname(None) == "+05" From bf984b465ebbe62e1584764808a6360e631c07d2 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 07:58:08 -0700 Subject: [PATCH 059/229] Fix itertools.compress iterator consumption order (#9025) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/itertools.rs | 10 +-- extra_tests/snippets/stdlib_itertools.py | 79 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index f016f486c3a..4433cf4704f 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -177,12 +177,12 @@ mod decl { impl IterNext for PyItertoolsCompress { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { loop { + // Match zip(data, selectors): consume data before the selector, + // and stop as soon as either input is exhausted. + let data_obj = raise_if_stop!(zelf.data.next(vm)?); let sel_obj = raise_if_stop!(zelf.selectors.next(vm)?); - let verdict = sel_obj.try_to_bool(vm)?; - let data_obj = zelf.data.next(vm)?; - - if verdict { - return Ok(data_obj); + if sel_obj.try_to_bool(vm)? { + return Ok(PyIterReturn::Return(data_obj)); } } } diff --git a/extra_tests/snippets/stdlib_itertools.py b/extra_tests/snippets/stdlib_itertools.py index ef06ce52985..04917fe994b 100644 --- a/extra_tests/snippets/stdlib_itertools.py +++ b/extra_tests/snippets/stdlib_itertools.py @@ -569,3 +569,82 @@ def __iter__(self): with assert_raises(OverflowError): itertools.product(Watched(), repeat=2**62) assert ran == [] + + +# compress consumes data, then the selector, then tests the selector's truth. +def check_compress_consumption(): + events = [] + + def tracked(name, values): + for value in values: + events.append(name) + yield value + events.append(name + " exhausted") + + class Selector: + def __init__(self, value): + self.value = value + + def __bool__(self): + events.append("bool") + return self.value + + assert list( + itertools.compress( + tracked("data", [1, 2]), + tracked("selectors", [Selector(False), Selector(True), Selector(True)]), + ) + ) == [2] + assert events == [ + "data", + "selectors", + "bool", + "data", + "selectors", + "bool", + "data exhausted", + ], events + + events.clear() + data = tracked("data", [1, 2, 3]) + assert list(itertools.compress(data, tracked("selectors", [True]))) == [1] + assert events == ["data", "selectors", "data", "selectors exhausted"], events + assert next(data) == 3 + + events.clear() + assert ( + list(itertools.compress(tracked("data", []), tracked("selectors", [True]))) + == [] + ) + assert events == ["data exhausted"], events + + def failing(name): + events.append(name) + raise ValueError(name) + yield + + events.clear() + with assert_raises(ValueError) as caught: + next(itertools.compress(failing("data"), tracked("selectors", [True]))) + assert str(caught.exception) == "data" + assert events == ["data"], events + + events.clear() + with assert_raises(ValueError) as caught: + next(itertools.compress(tracked("data", [1]), failing("selectors"))) + assert str(caught.exception) == "selectors" + assert events == ["data", "selectors"], events + + class BadSelector: + def __bool__(self): + events.append("bool") + raise ValueError("truth") + + events.clear() + with assert_raises(ValueError) as caught: + next(itertools.compress(tracked("data", [1]), [BadSelector()])) + assert str(caught.exception) == "truth" + assert events == ["data", "bool"], events + + +check_compress_consumption() From d4b43b860d03dc2771004f94a6f4b8e3e17905b7 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 07:59:43 -0700 Subject: [PATCH 060/229] Translate WASI signal dispositions through libc sentinels (#9024) Keep Python disposition values distinct from WASI function-table pointers. Recognize wasi-libc SIG_ERR before updating Python handlers and preserve ignored/default/custom dispositions. Add synchronous signal and rejected-registration regressions to the existing snippet. AI assistance covered source investigation, implementation, and automated reference/baseline tests. Isolated patched runtime validation is pending. No human validation is claimed. Assisted-by: Codex:model-version-unavailable --- crates/host_env/src/signal.rs | 64 +++++++++++++++++++-------- crates/vm/src/stdlib/_signal.rs | 31 ++++++++++--- extra_tests/snippets/stdlib_signal.py | 29 ++++++++++++ 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/crates/host_env/src/signal.rs b/crates/host_env/src/signal.rs index 67336eb039e..60b06d63e37 100644 --- a/crates/host_env/src/signal.rs +++ b/crates/host_env/src/signal.rs @@ -46,37 +46,63 @@ mod wasm { pub const SIGSEGV: i32 = 11; pub const SIGTERM: i32 = 15; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Handler { + Default, + Ignore, + Custom(sighandler_t), + } + unsafe extern "C" { - fn signal(signum: i32, handler: sighandler_t) -> sighandler_t; + #[link_name = "signal"] + fn c_signal(signum: i32, handler: sighandler_t) -> sighandler_t; + #[link_name = "__SIG_IGN"] + fn c_sig_ign(signum: i32); + #[link_name = "__SIG_ERR"] + fn c_sig_err(signum: i32); fn raise(signum: i32) -> i32; } + impl Handler { + fn as_raw(self) -> sighandler_t { + match self { + Self::Default => 0, + Self::Ignore => c_sig_ign as *const () as sighandler_t, + Self::Custom(handler) => handler, + } + } + + fn from_raw(handler: sighandler_t) -> io::Result { + // wasi-libc uses function pointers for SIG_IGN and SIG_ERR because + // table index 1 is also a valid custom Wasm function pointer. + if handler == c_sig_err as *const () as sighandler_t { + Err(io::Error::last_os_error()) + } else if handler == 0 { + Ok(Self::Default) + } else if handler == c_sig_ign as *const () as sighandler_t { + Ok(Self::Ignore) + } else { + Ok(Self::Custom(handler)) + } + } + } + /// # Safety /// /// The caller must ensure `signalnum` is a valid platform signal number. - pub unsafe fn probe_handler(signalnum: i32) -> Option { - let handler = unsafe { signal(signalnum, SIG_IGN) }; - if handler == SIG_ERR { - None - } else { - unsafe { signal(signalnum, handler) }; - Some(handler) - } + pub unsafe fn probe_handler(signalnum: i32) -> Option { + let handler = unsafe { install_handler(signalnum, Handler::Ignore) }.ok()?; + unsafe { install_handler(signalnum, handler) }.ok()?; + Some(handler) } /// # Safety /// /// The caller must ensure `signalnum` is a valid platform signal number and - /// `handler` is accepted by the platform signal ABI. - pub unsafe fn install_handler( - signalnum: i32, - handler: sighandler_t, - ) -> io::Result { - let old = unsafe { signal(signalnum, handler) }; - if old == SIG_ERR { - return Err(io::Error::last_os_error()); - } - Ok(old) + /// a custom `handler` is accepted by the platform signal ABI. + pub unsafe fn install_handler(signalnum: i32, handler: Handler) -> io::Result { + let old = unsafe { c_signal(signalnum, handler.as_raw()) }; + Handler::from_raw(old) } pub fn raise_signal(signalnum: i32) -> io::Result<()> { diff --git a/crates/vm/src/stdlib/_signal.rs b/crates/vm/src/stdlib/_signal.rs index 67a765a6230..1c2bc792539 100644 --- a/crates/vm/src/stdlib/_signal.rs +++ b/crates/vm/src/stdlib/_signal.rs @@ -175,9 +175,16 @@ pub(crate) mod _signal { let Some(handler) = (unsafe { host_signal::probe_handler(signum) }) else { continue; }; - let py_handler = if handler == SIG_DFL { + let (is_default, is_ignored) = cfg_select! { + target_os = "wasi" => ( + handler == host_signal::Handler::Default, + handler == host_signal::Handler::Ignore, + ), + _ => (handler == SIG_DFL, handler == SIG_IGN), + }; + let py_handler = if is_default { Some(sig_dfl.clone()) - } else if handler == SIG_IGN { + } else if is_ignored { Some(sig_ign.clone()) } else { None @@ -223,15 +230,29 @@ pub(crate) mod _signal { } let sig_handler = if handler.is_callable() { - run_signal as *const () as sighandler_t + let callback = run_signal as *const () as sighandler_t; + cfg_select! { + target_os = "wasi" => host_signal::Handler::Custom(callback), + _ => callback, + } } else { const MSG: &str = "signal handler must be signal.SIG_IGN, signal.SIG_DFL, or a callable object"; - usize::try_from_borrowed_object(vm, &handler) + let disposition = usize::try_from_borrowed_object(vm, &handler) .ok() .filter(|&v| matches!(v, SIG_DFL | SIG_IGN)) - .ok_or_else(|| vm.new_type_error(MSG))? + .ok_or_else(|| vm.new_type_error(MSG))?; + cfg_select! { + target_os = "wasi" => { + if disposition == SIG_DFL { + host_signal::Handler::Default + } else { + host_signal::Handler::Ignore + } + }, + _ => disposition, + } }; signal::check_signals(vm)?; diff --git a/extra_tests/snippets/stdlib_signal.py b/extra_tests/snippets/stdlib_signal.py index 8e00b5d885b..39f103157fa 100644 --- a/extra_tests/snippets/stdlib_signal.py +++ b/extra_tests/snippets/stdlib_signal.py @@ -21,6 +21,35 @@ def handler(signum, frame): assert signal.getsignal(signal.SIGILL) is signal.SIG_DFL +# Ignoring a synchronously raised signal must not invoke a callback, including +# on WASI where libc uses a function pointer rather than integer 1 for SIG_IGN. +synchronous_signals = [] + + +def synchronous_handler(signum, frame): + synchronous_signals.append(signum) + + +previous = signal.signal(signal.SIGILL, synchronous_handler) +try: + signal.raise_signal(signal.SIGILL) + assert synchronous_signals == [signal.SIGILL] + assert signal.signal(signal.SIGILL, signal.SIG_IGN) is synchronous_handler + signal.raise_signal(signal.SIGILL) + assert synchronous_signals == [signal.SIGILL] + assert signal.getsignal(signal.SIGILL) is signal.SIG_IGN +finally: + signal.signal(signal.SIGILL, previous) + +if sys.platform == "wasi": + # wasi-libc rejects SIGKILL (9) and SIGSTOP (19), even though they are not + # exported by RustPython's minimal WASI signal module. + for uncatchable in (9, 19): + previous = signal.getsignal(uncatchable) + assert_raises(OSError, signal.signal, uncatchable, signal.SIG_IGN) + assert signal.getsignal(uncatchable) is previous + + # unix, and not WASI if "win" not in sys.platform and sys.platform != "wasi": signal.signal(signal.SIGALRM, handler) From 87aedfb7c5064222ce2df23d375dbffae793d6c7 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 08:41:40 -0700 Subject: [PATCH 061/229] Preserve Windows time conversion error classes (#9022) * Cover Windows time conversion error classes Require OSError with EINVAL for representable timestamps rejected by the Windows C runtime, while preserving OverflowError for float(2**63). This initial regression-only commit intentionally exposes the current Windows error mapping before the implementation follows. AI assistance covered source comparison, regression design, and automated Linux baseline/reference controls. Native Windows results are pending. No human validation is claimed. Assisted-by: Codex:model-version-unavailable * Record native Windows time conversion outcomes The first hosted regression run failed on CPython 3.14.7 as well as RustPython because a presumed-invalid CRT timestamp succeeded. Add bounded diagnostics before the unchanged assertions to capture full version/build metadata and every conversion result, including negative half-day controls. No implementation or canonical test changes are included. The existing time snippet still passes the repository pytest runner on Linux CPython 3.14.7 and the current RustPython baseline. Native diagnostic results remain pending. Assisted-by: Codex:model-version-unavailable * Use the observed UCRT rejection boundary in time regression Hosted Windows diagnostics show that CPython 3.14.7 and RustPython both accept gmtime(-1) and gmtime(-43200), while -86400 is rejected by all three converters. Correct only the newly added shared negative test value. CPython raises OSError(errno 22) for every selected rejected value, while the RustPython baseline raises OverflowError. Canonical tests and the recorded diagnostics remain unchanged. The existing snippet passes both Linux reference and baseline controls. Assisted source investigation and automated verification used no manual validation. Assisted-by: Codex:model-version-unavailable * Preserve Windows CRT time conversion errors Return CRT errno failures from the Windows calendar helpers and translate them to Python OSError. Adapt the datetime caller while retaining its Unix behavior. Use an exclusive signed float upper bound so float(2**63) still raises OverflowError before reaching the CRT. Keep Windows negative-time FILETIME fallback and time64 ABI changes outside this Python 3.14 extraction. Existing datetime local-error coverage is retained and a UTC error control is added for the observed rejected timestamp -86400. Hosted native diagnostics prove CPython 3.14.7 raises OSError(errno 22) where the baseline raises OverflowError. Windows-GNU host_env check and warnings-denied Clippy pass. Selected canonical datetime and complete time/datetime snippet Linux controls pass, with six existing baseline canonical skips. Fixed native Windows verification remains pending CI. AI assistance covered source comparison, implementation, regression design, and automated verification. No manual validation is claimed. Assisted-by: Codex:model-version-unavailable --- crates/host_env/src/time.rs | 14 +++-- crates/stdlib/src/_datetime.rs | 5 ++ crates/vm/src/stdlib/time.rs | 13 ++-- extra_tests/snippets/stdlib_datetime.py | 6 ++ extra_tests/snippets/stdlib_time.py | 82 +++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 9 deletions(-) diff --git a/crates/host_env/src/time.rs b/crates/host_env/src/time.rs index 14d77fe7a51..5cac2ff4887 100644 --- a/crates/host_env/src/time.rs +++ b/crates/host_env/src/time.rs @@ -146,18 +146,24 @@ pub fn current_time_t() -> TimeT { #[cfg(windows)] #[cfg_attr(target_env = "musl", allow(deprecated))] -pub fn gmtime_from_timestamp(when: TimeT) -> Option { +pub fn gmtime_from_timestamp(when: TimeT) -> std::io::Result { let mut out = core::mem::MaybeUninit::::uninit(); let err = unsafe { _gmtime64_s(out.as_mut_ptr(), &when) }; - (err == 0).then(|| unsafe { out.assume_init() }) + if err != 0 { + return Err(crate::os::io_error_from_errno(err)); + } + Ok(unsafe { out.assume_init() }) } #[cfg(windows)] #[cfg_attr(target_env = "musl", allow(deprecated))] -pub fn localtime_from_timestamp(when: TimeT) -> Option { +pub fn localtime_from_timestamp(when: TimeT) -> std::io::Result { let mut out = core::mem::MaybeUninit::::uninit(); let err = unsafe { _localtime64_s(out.as_mut_ptr(), &when) }; - (err == 0).then(|| unsafe { out.assume_init() }) + if err != 0 { + return Err(crate::os::io_error_from_errno(err)); + } + Ok(unsafe { out.assume_init() }) } #[cfg(windows)] diff --git a/crates/stdlib/src/_datetime.rs b/crates/stdlib/src/_datetime.rs index e424d903b1d..8fb4cc29352 100644 --- a/crates/stdlib/src/_datetime.rs +++ b/crates/stdlib/src/_datetime.rs @@ -9,6 +9,8 @@ pub(crate) use _datetime::{PyTzInfo, datetime_type, module_def, timedelta_from_s #[pymodule] mod _datetime { + #[cfg(windows)] + use crate::vm::convert::ToPyException; use crate::vm::{ AsObject, Py, PyObject, PyObjectCell, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, @@ -1825,6 +1827,9 @@ mod _datetime { } else { rustpython_host_env::time::gmtime_from_timestamp(t_c) }; + #[cfg(windows)] + let tm = tm.map_err(|error| error.to_pyexception(vm))?; + #[cfg(unix)] let Some(tm) = tm else { return Err(vm.new_last_errno_error()); }; diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index 70b6479a5d3..7760b7ed8d2 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -914,7 +914,9 @@ mod decl { } let secs = float.floor(); #[cfg_attr(target_env = "musl", allow(deprecated))] - if secs < libc::time_t::MIN as f64 || secs > libc::time_t::MAX as f64 { + let minimum = libc::time_t::MIN as f64; + // The signed maximum rounds up when represented by f64. + if !(minimum..-minimum).contains(&secs) { return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); } #[cfg_attr(target_env = "musl", allow(deprecated))] @@ -1248,6 +1250,7 @@ mod platform { use crate::{ PyRef, PyResult, VirtualMachine, builtins::{PyNamespace, PyUtf8StrRef}, + convert::ToPyException, }; use core::time::Duration; use rustpython_host_env::time as host_time; @@ -1281,8 +1284,8 @@ mod platform { when: host_time::TimeT, vm: &VirtualMachine, ) -> PyResult { - let tm = host_time::gmtime_from_timestamp(when) - .ok_or_else(|| vm.new_overflow_error("timestamp out of range for platform time_t"))?; + let tm = + host_time::gmtime_from_timestamp(when).map_err(|error| error.to_pyexception(vm))?; Ok(struct_time_from_tm(vm, tm, "UTC", 0)) } @@ -1290,8 +1293,8 @@ mod platform { when: host_time::TimeT, vm: &VirtualMachine, ) -> PyResult { - let tm = host_time::localtime_from_timestamp(when) - .ok_or_else(|| vm.new_overflow_error("timestamp out of range for platform time_t"))?; + let tm = + host_time::localtime_from_timestamp(when).map_err(|error| error.to_pyexception(vm))?; // Get timezone info from Windows API let info = get_tz_info(); diff --git a/extra_tests/snippets/stdlib_datetime.py b/extra_tests/snippets/stdlib_datetime.py index d6609528cca..41efd03a837 100644 --- a/extra_tests/snippets/stdlib_datetime.py +++ b/extra_tests/snippets/stdlib_datetime.py @@ -3949,6 +3949,12 @@ def test_timestamp_errno(): assert error.errno == errno.EINVAL else: raise AssertionError("Windows localtime accepted a negative timestamp") + try: + datetime_module.datetime.fromtimestamp(-86400, datetime_module.timezone.utc) + except OSError as error: + assert error.errno == errno.EINVAL + else: + raise AssertionError("Windows gmtime accepted an out-of-range timestamp") test_astimezone_subclass_state() diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index b74d5bbc638..dd43d3f0b5a 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -1,6 +1,88 @@ import sys import time +if sys.platform == "win32": + # Record the native runtime before evaluating the range regression below. + print("Windows time runtime:", sys.version, sys.implementation, sys.flags) + print("Windows time word size:", sys.maxsize) + if hasattr(sys, "getwindowsversion"): + print("Windows version:", sys.getwindowsversion()) + try: + import sysconfig + + build_keys = ( + "Py_DEBUG", + "Py_GIL_DISABLED", + "Py_ENABLE_SHARED", + "SIZEOF_TIME_T", + "CC", + "CFLAGS", + ) + print( + "Windows time build:", + dict(zip(build_keys, sysconfig.get_config_vars(*build_keys))), + ) + except Exception as error: + print("Windows time build unavailable:", type(error).__name__, str(error)) + for converter in (time.gmtime, time.localtime, time.ctime): + for timestamp in ( + 0, + -1, + -43200, + -43201, + -86400, + 2**40, + 2**63 - 1, + -(2**63), + -float(2**63), + float(2**63), + ): + try: + result = converter(timestamp) + except Exception as error: + print( + "Windows time result:", + converter.__name__, + repr(timestamp), + type(error).__name__, + str(error), + getattr(error, "errno", None), + getattr(error, "winerror", None), + ) + else: + print( + "Windows time result:", + converter.__name__, + repr(timestamp), + "success", + repr(result), + ) + +if sys.platform != "wasi": + for converter in (time.gmtime, time.localtime, time.ctime): + # 2**63 is outside signed time_t even though its maximum rounds up to + # that value when represented as a float. + try: + converter(float(2**63)) + except OverflowError: + pass + else: + raise AssertionError("out-of-range timestamp did not raise OverflowError") + +if sys.platform == "win32": + import errno + + for converter in (time.gmtime, time.localtime, time.ctime): + # Python 3.14 preserves UCRT conversion failures as OSError. Use a + # negative timestamp below gmtime's near-epoch allowance in UCRT. + for timestamp in (-86400, 2**40, 2**63 - 1, -(2**63), -float(2**63)): + try: + converter(timestamp) + except OSError as error: + assert error.errno == errno.EINVAL, (converter, timestamp, error) + else: + raise AssertionError("UCRT timestamp failure did not raise OSError") + x = time.gmtime(1000) assert x.tm_year == 1970 From 42778444a866526e2722b851846ae56e473e8352 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 08:47:49 -0700 Subject: [PATCH 062/229] Preserve defaultdict subclasses in union results (#9026) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_collections.rs | 40 +++++------ extra_tests/snippets/stdlib_collections.py | 82 ++++++++++++++++++++++ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/crates/vm/src/stdlib/_collections.rs b/crates/vm/src/stdlib/_collections.rs index 9307f7c67fc..e120e73cbc5 100644 --- a/crates/vm/src/stdlib/_collections.rs +++ b/crates/vm/src/stdlib/_collections.rs @@ -1061,18 +1061,10 @@ mod _collections { fn __or__(lhs: &PyObject, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { let not_implemented = || Ok(vm.ctx.not_implemented.clone().into()); - let (default_factory, dict) = if let Some(zelf) = lhs.downcast_ref::() { - if !rhs.fast_isinstance(vm.ctx.types.dict_type) { - return not_implemented(); - } - - (zelf.default_factory.load_owned(), zelf.dict.copy()) + let (zelf, other) = if let Some(zelf) = lhs.downcast_ref::() { + (zelf, &*rhs) } else if let Some(zelf) = rhs.downcast_ref::() { - let Some(dict) = lhs.downcast_ref::() else { - return not_implemented(); - }; - - (zelf.default_factory.load_owned(), dict.copy()) + (zelf, lhs) } else { return Err(vm.new_type_error(format!( "unsupported operand type(s) for |: '{}' and '{}'", @@ -1080,17 +1072,23 @@ mod _collections { rhs.class().name() ))); }; - - dict.update(rhs.into(), KwArgs::default(), vm)?; - - Ok(Self { - dict, - default_factory: match default_factory { - Some(factory) => PyAtomicRef::from(Some(factory)), - None => PyAtomicRef::new_empty(), - }, + if !other.fast_isinstance(vm.ctx.types.dict_type) { + return not_implemented(); } - .to_pyobject(vm)) + + let factory = zelf + .default_factory + .load_owned() + .unwrap_or_else(|| vm.ctx.none()); + let new = zelf + .class() + .as_object() + .call((factory, lhs.to_owned()), vm)?; + let dict = new + .downcast_ref::() + .ok_or_else(|| vm.new_system_error("bad argument to internal function"))?; + dict.merge_object(rhs, vm)?; + Ok(new) } } diff --git a/extra_tests/snippets/stdlib_collections.py b/extra_tests/snippets/stdlib_collections.py index e4602d1c9ce..2da8df87c56 100644 --- a/extra_tests/snippets/stdlib_collections.py +++ b/extra_tests/snippets/stdlib_collections.py @@ -116,3 +116,85 @@ def get(self, key, default=None): assert custom_get == {"a": 101, "b": 101} with assert_raises(AttributeError): _count_elements([], "a") + + +# defaultdict union constructs the subclass with its factory and left operand. +def check_defaultdict_subclass_union(): + calls = [] + + class DefaultSubclass(defaultdict): + def __init__(self, factory=None, initial=()): + calls.append((factory, initial)) + super().__init__(factory, initial) + + def update(self, *args, **kwargs): + raise AssertionError("union must use native dict update") + + def __setitem__(self, key, value): + raise AssertionError("union must not call overridden __setitem__") + + for factory in (None, list): + left = DefaultSubclass(factory, {"shared": 1, "left": 2}) + right = {"right": 3, "shared": 4} + calls.clear() + result = left | right + assert type(result) is DefaultSubclass + assert result.default_factory is factory + assert list(result.items()) == [("shared", 4), ("left", 2), ("right", 3)] + assert len(calls) == 1 and calls[0][0] is factory and calls[0][1] is left + + calls.clear() + result = right | left + assert type(result) is DefaultSubclass + assert result.default_factory is factory + assert list(result.items()) == [("right", 3), ("shared", 1), ("left", 2)] + assert len(calls) == 1 and calls[0][0] is factory and calls[0][1] is right + assert left == {"shared": 1, "left": 2} + assert right == {"right": 3, "shared": 4} + + calls.clear() + assert left.__or__([("x", 1)]) is NotImplemented + assert left.__ror__([("x", 1)]) is NotImplemented + assert calls == [] + + class FailingSubclass(defaultdict): + fail = False + + def __init__(self, *args): + if self.fail: + raise ValueError("union constructor") + super().__init__(*args) + + left = FailingSubclass(int, {"a": 1}) + FailingSubclass.fail = True + with assert_raises(ValueError) as caught: + left | {} + assert str(caught.exception) == "union constructor" + with assert_raises(ValueError) as caught: + {} | left + assert str(caught.exception) == "union constructor" + + class ReplacingSubclass(defaultdict): + replacement = None + + def __new__(cls, *args): + if cls.replacement is not None: + return cls.replacement + return super().__new__(cls) + + left = ReplacingSubclass(int, {"left": 1}) + replacement = {"replacement": 2} + ReplacingSubclass.replacement = replacement + assert left | {"right": 3} is replacement + assert replacement == {"replacement": 2, "right": 3} + replacement.clear() + assert {"right": 3} | left is replacement + assert replacement == {"left": 1} + ReplacingSubclass.replacement = object() + with assert_raises(SystemError): + left | {} + with assert_raises(SystemError): + {} | left + + +check_defaultdict_subclass_union() From a8e4bde885d9f8620727b5dbc123aab00cb80ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=B4=EA=B4=91=ED=9A=A8?= <87641474+widehyo1@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:49:44 +0900 Subject: [PATCH 063/229] Reject unsupported native keywords before argument binding (#8957) * Reject unsupported native keywords before argument binding Add explicit keyword dispatch metadata independently of inferred Rust calling flags. Reject nonempty keywords before binding and conversion, while preserving descriptor self checks and keyword-capable calls. Annotate 31 builtins and list.append/list.clear, derive C-API policies from ml_flags, and group dynamic registration metadata in MethodDefSpec. Add regression coverage and remove the id doctest expected-failure marker. Validation: workspace and C-API tests/clippy, VM without doc, keyword regressions, and test_call/test_extcall/test_builtin/test_list passed. The full snippet suite had 475 passes and 7 failures: six CPython-side expectation mismatches and one missing optional SQLite module. Assisted-by: Codex:GPT-6 * Infer native keyword dispatch from argument metadata Replace no_keywords annotations with registration-time inference from FromArgs::PARAMS, including nested parameter metadata. Forward arguments with keyword-capable parameters to the binder, preserving min/max body validation and descriptor self checks. Add regressions for raw arguments, keyword argument structs, and list.sort. Tested: C-API tests, workspace and C-API Clippy, VM without doc, both keyword regression snippets, and test_call/test_extcall/test_builtin/test_list. Workspace tests reported eight _opcode snapshot failures. The full snippet suite reported 477 passes and seven failures: six CPython-side failures and one RustPython failure due to the missing optional SQLite module. Assisted-by: Codex:GPT-6 * Simplify native function registration metadata Reuse SimpleItemMeta for pyfunction attributes and remove FunctionItemMeta. Remove MethodDefSpec and restore the original new_method_def arguments with an additional keyword_dispatch parameter, updating its callers. Consolidate keyword dispatch regressions in syntax_function_args.py, preserving their assertions and placing them before builtin shadowing. Validation: workspace and C-API tests/clippy, VM without doc, the combined snippet, and test_extcall/test_builtin/test_list/test_call passed. The full snippet suite reported 477 passes and seven existing failures: six CPython-side failures and one missing optional SQLite module. Assisted-by: Codex:GPT-6 * Reject native keywords by calling convention flags Remove `KeywordDispatch`, the `keyword_dispatch` field of `PyMethodDef`, `PyMethodDef::with_keyword_dispatch`, `function::keyword_dispatch` and the derive macros' `keyword_dispatch_tokens`. Native functions and method descriptors reject nonempty keywords when `flags` lacks `KEYWORDS`, and report the unqualified name when `flags` contains `VARARGS`. `Context::new_method_def` takes `(name, f, flags, doc)` again. Add a snippet for `int.__new__` called with a keyword argument. Assisted-by: Claude Code:claude-opus-5-5 --------- Co-authored-by: Jeong YunWon --- Lib/test/test_extcall.py | 2 +- crates/capi/src/methodobject.rs | 22 +++++ crates/vm/src/builtins/builtin_func.rs | 33 ++++++++ crates/vm/src/builtins/descriptor.rs | 54 +++++++++++-- extra_tests/snippets/syntax_function_args.py | 84 ++++++++++++++++++++ 5 files changed, 189 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index 483d5ad5f2b..c816d0135a1 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -442,7 +442,7 @@ ... False True - >>> id(1, **{'foo': 1}) # TODO: RUSTPYTHON # doctest:+EXPECTED_FAILURE + >>> id(1, **{'foo': 1}) Traceback (most recent call last): ... TypeError: id() takes no keyword arguments diff --git a/crates/capi/src/methodobject.rs b/crates/capi/src/methodobject.rs index 9868db36cac..2e82c5e9daa 100644 --- a/crates/capi/src/methodobject.rs +++ b/crates/capi/src/methodobject.rs @@ -385,4 +385,26 @@ mod tests { func.call0().unwrap(); }); } + + #[test] + fn no_keywords_before_arity() { + Python::attach(|py| { + unsafe extern "C" fn c_fn(_self: *mut PyObject, _args: *mut PyObject) -> *mut PyObject { + unsafe { PyLong_FromLong(42) } + } + let f = PyCFunction::new(py, c_fn, c"no_keywords", c"", None).unwrap(); + let kwargs = pyo3::types::PyDict::new(py); + kwargs.set_item("unexpected", 1).unwrap(); + let err = f.call((1,), Some(&kwargs)).unwrap_err(); + assert_eq!( + err.value(py).str().unwrap().to_str().unwrap(), + "no_keywords() takes no keyword arguments" + ); + kwargs.clear(); + assert_eq!( + f.call((), Some(&kwargs)).unwrap().extract::().unwrap(), + 42 + ); + }); + } } diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index 17e3e342485..e0399987049 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -87,6 +87,9 @@ impl Callable for PyNativeFunction { type Args = FuncArgs; #[inline] fn call(zelf: &Py, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.kwargs.is_empty() && !zelf.value.flags.contains(PyMethodFlags::KEYWORDS) { + return Err(native_no_keywords_error(zelf, vm)?); + } let mut callee = Callee::named(zelf.value.name); if let Some(z) = &zelf.zelf { // STATIC methods store the class in zelf for qualname/repr purposes, @@ -272,6 +275,12 @@ fn vectorcall_native_function( ) -> PyResult { let zelf: &Py = zelf_obj.downcast_ref().unwrap(); + if kwnames.is_some_and(|names| !names.is_empty()) + && !zelf.value.flags.contains(PyMethodFlags::KEYWORDS) + { + return Err(native_no_keywords_error(zelf, vm)?); + } + // Build FuncArgs with self already at position 0 (no insert(0) needed) let needs_self = zelf .zelf @@ -292,6 +301,30 @@ fn vectorcall_native_function( (zelf.value.func)(vm, func_args, callee) } +#[cold] +fn native_no_keywords_error( + zelf: &Py, + vm: &VirtualMachine, +) -> PyResult { + if zelf.value.flags.contains(PyMethodFlags::VARARGS) { + return Ok(vm.new_type_error(format!("{}() takes no keyword arguments", zelf.value.name))); + } + let qualname = PyNativeFunction::__qualname__(NativeFunctionOrMethod(zelf.to_owned()), vm)?; + let module = zelf + .module + .load_owned() + .filter(|module| !vm.is_none(module)); + let module = module.map(|module| module.str(vm)).transpose()?; + let name = if let Some(module) = module + && module.to_string() != "builtins" + { + format!("{module}.{qualname}") + } else { + qualname.to_string() + }; + Ok(vm.new_type_error(format!("{name}() takes no keyword arguments"))) +} + pub(crate) fn init(context: &'static Context) { PyNativeFunction::extend_class(context, context.types.builtin_function_or_method_type); context diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index 301448654a6..55c2275f5bb 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -112,8 +112,9 @@ impl Callable for PyMethodDescriptor { type Args = FuncArgs; #[inline] fn call(zelf: &Py, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - if let Some(obj) = args.args.first() { - method_descr_typecheck(zelf, obj, vm)?; + method_descr_check_self(zelf, args.args.first().map(|obj| &**obj), vm)?; + if !args.kwargs.is_empty() && !zelf.method.flags.contains(PyMethodFlags::KEYWORDS) { + return Err(method_descr_no_keywords_error(zelf, vm)); } (zelf.method.func)( vm, @@ -1556,6 +1557,40 @@ fn method_descr_typecheck( ))) } +#[inline] +fn method_descr_check_self( + descr: &PyMethodDescriptor, + obj: Option<&PyObject>, + vm: &VirtualMachine, +) -> PyResult<()> { + let obj = obj.ok_or_else(|| method_descr_missing_self_error(descr, vm))?; + method_descr_typecheck(descr, obj, vm) +} + +#[cold] +fn method_descr_missing_self_error( + descr: &PyMethodDescriptor, + vm: &VirtualMachine, +) -> crate::exceptions::types::PyBaseExceptionRef { + vm.new_type_error(format!( + "unbound method {}.{}() needs an argument", + descr.common.typ.name(), + descr.common.name + )) +} + +#[cold] +fn method_descr_no_keywords_error( + descr: &PyMethodDescriptor, + vm: &VirtualMachine, +) -> crate::exceptions::types::PyBaseExceptionRef { + vm.new_type_error(format!( + "{}.{}() takes no keyword arguments", + descr.common.typ.name(), + descr.common.name + )) +} + /// Vectorcall for method_descriptor: calls native method directly fn vectorcall_method_descriptor( zelf_obj: &PyObject, @@ -1565,10 +1600,19 @@ fn vectorcall_method_descriptor( vm: &VirtualMachine, ) -> PyResult { let zelf: &Py = zelf_obj.downcast_ref().unwrap(); - if nargs > 0 - && let Some(obj) = args.first() + method_descr_check_self( + zelf, + if nargs > 0 { + args.first().map(|obj| &**obj) + } else { + None + }, + vm, + )?; + if kwnames.is_some_and(|names| !names.is_empty()) + && !zelf.method.flags.contains(PyMethodFlags::KEYWORDS) { - method_descr_typecheck(zelf, obj, vm)?; + return Err(method_descr_no_keywords_error(zelf, vm)); } let func_args = FuncArgs::from_vectorcall_owned(args, nargs, kwnames); (zelf.method.func)( diff --git a/extra_tests/snippets/syntax_function_args.py b/extra_tests/snippets/syntax_function_args.py index eb4f94b8a36..96cbaf37272 100644 --- a/extra_tests/snippets/syntax_function_args.py +++ b/extra_tests/snippets/syntax_function_args.py @@ -1,6 +1,90 @@ from testutils import assert_raises +def error(call, expected): + try: + call() + except TypeError as exc: + assert str(exc) == expected, (str(exc), expected) + else: + raise AssertionError("expected TypeError") + + +error(lambda: id(obj=1), "id() takes no keyword arguments") +error(lambda: id(**{"obj": 1}), "id() takes no keyword arguments") +error(lambda: id(1, 2, obj=1), "id() takes no keyword arguments") +error(lambda: list.append(x=1), "unbound method list.append() needs an argument") +error(lambda: list.append(**{"x": 1}), "unbound method list.append() needs an argument") +error( + lambda: list.append(1, x=1), + "descriptor 'append' for 'list' objects doesn't apply to a 'int' object", +) +error(lambda: list.append([], x=1), "list.append() takes no keyword arguments") +error(lambda: list.append([], **{"x": 1}), "list.append() takes no keyword arguments") +error(lambda: [].append(x=1), "list.append() takes no keyword arguments") +error(lambda: [].clear(1, x=1), "list.clear() takes no keyword arguments") + + +class Index: + def __index__(self): + raise AssertionError("conversion ran before keyword check") + + +error(lambda: bin(Index(), x=1), "bin() takes no keyword arguments") +assert id(1, **{}) == id(1) +assert round(1.25, ndigits=1) == 1.2 +assert pow(2, exp=3) == 8 +assert sorted([2, 1], reverse=True) == [2, 1] +assert sum([1, 2], start=4) == 7 +values = [] +list.append(values, 1) +values.append(2) +assert values == [1, 2] +list.clear(values) +assert values == [] + +# Keyword metadata must forward raw arguments without changing body validation. +for choose in (min, max): + assert choose([], default=42) == 42 + assert choose([1, 2], key=lambda value: -value) == (2 if choose is min else 1) + try: + choose(bogus=1) + except TypeError as exc: + assert str(exc) == f"{choose.__name__} expected at least 1 argument, got 0" + else: + raise AssertionError("missing positional argument was accepted") + try: + choose(1, 2, default=0) + except TypeError as exc: + assert str(exc) == ( + f"Cannot specify a default for {choose.__name__}() " + "with multiple positional arguments" + ) + else: + raise AssertionError("default with multiple arguments was accepted") + +# Named fields in argument structs must override positional-only arguments. +assert eval("value", globals={"value": 42}) == 42 +namespace = {} +exec("value = 42", globals=namespace) +assert namespace["value"] == 42 +assert compile(source="42", filename="", mode="eval") +assert __import__(name="sys").__name__ == "sys" + +from io import StringIO + +output = StringIO() +print(1, 2, sep=":", end="!", file=output) +assert output.getvalue() == "1:2!" + +# Methods using a keyword-only argument struct must keep accepting keywords. +values = [1, 2] +values.sort(reverse=True) +assert values == [2, 1] + +# Definitions built without the macros infer their calling convention too. +assert int.__new__(int, "11", base=2) == 3 + def sum(x, y): return x+y From f5b6526c49fc06d74d485faf7911d88fa2da2746 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Fri, 9 Oct 2026 08:50:55 -0700 Subject: [PATCH 064/229] Support compile(module=...) across AST warning paths (#8947) * Support compile module context across AST warning paths Add Python 3.15 compile(module=...) warning filtering. Move format-spec diagnostics to a fallible, literal-aware AST boundary; preserve nested literal warnings while excluding raw text and comments, map function-type fragments to source locations, and suppress duplicate type-comment emissions with per-call tracking. Keep unrelated legacy warning-detection and exact diagnostic metadata gaps documented by differential probes. Add focused flag, source-location, exception, reentrancy, filtering and literal-boundary regression coverage. Assisted-by: Codex:model-version-unavailable * Adapt compile module context to the current parser APIs Convert the single-mode multiple statements error into the generic parse error type, and iterate f-string parts through FStringPartRef. Assisted-by: Claude Code:claude-opus-5-5 --------- Co-authored-by: Jeong YunWon --- crates/vm/src/stdlib/_ast.rs | 113 ++++-- crates/vm/src/stdlib/_ast/string.rs | 241 +++++++++++-- crates/vm/src/stdlib/builtins.rs | 25 +- crates/vm/src/vm/compile.rs | 217 +++++++++--- .../snippets/builtin_compile_module.py | 330 ++++++++++++++++++ 5 files changed, 810 insertions(+), 116 deletions(-) create mode 100644 extra_tests/snippets/builtin_compile_module.py diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index 241ff35eaef..497dec48b49 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -1788,7 +1788,7 @@ fn empty_arguments_object(vm: &VirtualMachine) -> PyObjectRef { #[cfg(feature = "parser")] #[allow(clippy::too_many_arguments)] -pub(crate) fn parse( +pub(crate) fn parse>( vm: &VirtualMachine, source: &str, filename: &str, @@ -1800,12 +1800,13 @@ pub(crate) fn parse( interactive: bool, explicit_future_features: crate::bytecode::CodeFlags, dont_imply_dedent: bool, -) -> Result { + mut emit_warning: impl FnMut(usize, char) -> Result<(), E>, +) -> Result { let source_file = SourceFileBuilder::new(filename.to_owned(), source.to_owned()).finish(); let mut options = parser::ParseOptions::from(mode); let target_version = target_version.unwrap_or(ast::PythonVersion::PY314); if let Some(error) = feature_version_syntax_error(source, &source_file, target_version) { - return Err(error); + return Err(error.into()); } options = options.with_target_version(target_version); let barry_source = rustpython_compiler::prepare_barry_as_flufl_source( @@ -1819,22 +1820,22 @@ pub(crate) fn parse( && let Some(error) = rustpython_compiler::single_mode_multiple_statements_error(&source_file, &parsed) { - return Err(error); + return Err(error.into()); } let type_comment_source = type_comments.then(|| TypeCommentSource::new(source, parsed.tokens())); if let Some(lines) = &type_comment_source && let Some(error) = invalid_type_comment_syntax_error(&source_file, lines) { - return Err(error); + return Err(error.into()); } if let Err(errors) = parsed.as_result() { let parse_error = errors[0].clone(); if let Some(error) = barry_source.diagnostic(Some(&parse_error), &source_file) { - return Err(error); + return Err(error.into()); } let range = text_range_to_source_range(&source_file, parse_error.location); - return Err(ParseError { + return Err(CompileError::from(ParseError { error: parse_error.error, raw_location: parse_error.location, location: range.start.to_source_location(), @@ -1842,17 +1843,17 @@ pub(crate) fn parse( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, - } + }) .into()); } if let Some(error) = barry_source.diagnostic(None, &source_file) { - return Err(error); + return Err(error.into()); } if dont_imply_dedent && interactive && let Some(error) = rustpython_compiler::dont_imply_dedent_source_error(&source_file) { - return Err(error); + return Err(error.into()); } if let Some(error) = parsed @@ -1861,10 +1862,10 @@ pub(crate) fn parse( .find(|error| should_report_unsupported_syntax_error(error)) { if let Some(error) = cpython_unsupported_syntax_error(error, source, &source_file) { - return Err(error); + return Err(error.into()); } let range = text_range_to_source_range(&source_file, error.range()); - return Err(ParseError { + return Err(CompileError::from(ParseError { error: parser::ParseErrorType::OtherError(error.to_string()), raw_location: error.range(), location: range.start.to_source_location(), @@ -1872,7 +1873,7 @@ pub(crate) fn parse( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, - } + }) .into()); } @@ -1881,11 +1882,11 @@ pub(crate) fn parse( parsed.tokens(), vm.state.int_max_str_digits.load(), ) { - return Err(error); + return Err(error.into()); } if let Some(error) = rustpython_compiler::leading_byte_order_mark_error(&source_file) { - return Err(error); + return Err(error.into()); } let mut top = parsed.into_syntax(); rustpython_compiler::too_deeply_nested_error( @@ -1894,18 +1895,34 @@ pub(crate) fn parse( rustpython_compiler::CompileOpts::default().recursion_limit, )?; if let Some(error) = rustpython_compiler::unsupported_grammar_error(&top, &source_file) { - return Err(error); + return Err(error.into()); } if let Some(error) = ipython_escape_command_syntax_error(&top, &source_file) { - return Err(error); + return Err(error.into()); } if let Some(error) = feature_version_ast_syntax_error(&top, &source_file, target_version) { - return Err(error); + return Err(error.into()); + } + #[cfg(feature = "codegen")] + let future_features = codegen::preprocess::checked_future_features(&top) + .map_err(|err| future_feature_compile_error(&source_file, err))?; + // Keep original literal spans: preprocessing can fold multiple source + // literals into one node, but each has its own lexical diagnostic. + match &top { + ast::Mod::Module(module) => { + string::emit_format_spec_warnings(&module.body, &[], &source_file, &mut emit_warning)?; + } + ast::Mod::Expression(expression) => { + string::emit_format_spec_warnings( + &[], + core::slice::from_ref(expression.body.as_ref()), + &source_file, + &mut emit_warning, + )?; + } } #[cfg(feature = "codegen")] { - let future_features = codegen::preprocess::checked_future_features(&top) - .map_err(|err| future_feature_compile_error(&source_file, err))?; let future_annotations = explicit_future_features .contains(crate::bytecode::CodeFlags::FUTURE_ANNOTATIONS) || future_features.contains(crate::bytecode::CodeFlags::FUTURE_ANNOTATIONS); @@ -1969,14 +1986,16 @@ pub(crate) fn wrap_interactive(vm: &VirtualMachine, module_obj: &PyObject) -> Py } #[cfg(feature = "parser")] -pub(crate) fn parse_func_type( +pub(crate) fn parse_func_type>( vm: &VirtualMachine, source: &str, filename: &str, optimize: u8, target_version: Option, -) -> Result { + mut emit_warning: impl FnMut(usize, char) -> Result<(), E>, +) -> Result { let _ = optimize; + let leading_space = source.len() - source.trim_start().len(); let source = source.trim(); let invalid_func_type = || -> CompileError { ParseError { @@ -2008,7 +2027,7 @@ pub(crate) fn parse_func_type( } let Some(split_at) = split_at else { - return Err(ParseError { + return Err(CompileError::from(ParseError { error: parser::ParseErrorType::OtherError("invalid func_type".to_owned()), raw_location: TextRange::default(), location: SourceLocation::default(), @@ -2016,7 +2035,7 @@ pub(crate) fn parse_func_type( source_path: filename.to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, - } + }) .into()); }; @@ -2029,7 +2048,7 @@ pub(crate) fn parse_func_type( .with_target_version(target_version.unwrap_or(ast::PythonVersion::PY314)); let parsed = parser::parse(expr_src, options).map_err(|parse_error| { let range = text_range_to_source_range(&source_file, parse_error.location); - ParseError { + CompileError::from(ParseError { error: parse_error.error, raw_location: parse_error.location, location: range.start.to_source_location(), @@ -2037,7 +2056,7 @@ pub(crate) fn parse_func_type( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, - } + }) })?; let ast::Mod::Expression(expression) = parsed.into_syntax() else { unreachable!(); @@ -2046,22 +2065,25 @@ pub(crate) fn parse_func_type( }; if !left.starts_with('(') || !left.ends_with(')') { - return Err(invalid_func_type()); + return Err(invalid_func_type().into()); } let inner = left[1..left.len() - 1].trim(); + let inner_start = left.len() - 1 - left[1..left.len() - 1].trim_start().len(); + const ARG_PREFIX: &str = "__rustpython_func_type__("; + let mut arg_source_file = None; let argtypes = if inner.is_empty() { Vec::new() } else { if inner.ends_with(',') { - return Err(invalid_func_type()); + return Err(invalid_func_type().into()); } - let call_source = format!("__rustpython_func_type__({inner})"); + let call_source = format!("{ARG_PREFIX}{inner})"); let source_file = SourceFileBuilder::new(filename.to_owned(), call_source.clone()).finish(); let options = parser::ParseOptions::from(parser::Mode::Expression) .with_target_version(target_version.unwrap_or(ast::PythonVersion::PY314)); let parsed = parser::parse(&call_source, options).map_err(|parse_error| { let range = text_range_to_source_range(&source_file, parse_error.location); - ParseError { + CompileError::from(ParseError { error: parse_error.error, raw_location: parse_error.location, location: range.start.to_source_location(), @@ -2069,13 +2091,13 @@ pub(crate) fn parse_func_type( source_path: source_file.name().to_owned(), is_unclosed_bracket: false, is_unclosed_string: false, - } + }) })?; let ast::Mod::Expression(expression) = parsed.into_syntax() else { unreachable!(); }; let ast::Expr::Call(call) = *expression.body else { - return Err(invalid_func_type()); + return Err(invalid_func_type().into()); }; let mut args = Vec::new(); let positional_len = call.arguments.args.len(); @@ -2084,7 +2106,7 @@ pub(crate) fn parse_func_type( match arg { ast::Expr::Starred(starred) => { if seen_star || index + 1 != positional_len { - return Err(invalid_func_type()); + return Err(invalid_func_type().into()); } seen_star = true; args.push(*starred.value); @@ -2095,22 +2117,38 @@ pub(crate) fn parse_func_type( let mut seen_kw_star = false; for keyword in call.arguments.keywords { if keyword.arg.is_some() || seen_kw_star { - return Err(invalid_func_type()); + return Err(invalid_func_type().into()); } seen_kw_star = true; args.push(keyword.value); } + arg_source_file = Some(source_file); args }; let returns = parse_expr(right)?; + // Func-type nodes are parsed from separate fragments. Keep warning offsets + // in those fragments until the boundary, then map them to the original + // source without changing the AST's existing location representation. + if let Some(source_file) = arg_source_file { + string::emit_format_spec_warnings(&[], &argtypes, &source_file, &mut |offset, ch| { + emit_warning(leading_space + inner_start + offset - ARG_PREFIX.len(), ch) + })?; + } + let return_source_file = SourceFileBuilder::new(filename.to_owned(), right.to_owned()).finish(); + string::emit_format_spec_warnings( + &[], + core::slice::from_ref(&returns), + &return_source_file, + &mut |offset, ch| emit_warning(leading_space + source.len() - right.len() + offset, ch), + )?; let func_type = ModFunctionType { argtypes: argtypes.into_boxed_slice(), returns, runtime_argtypes: None, }; - let source_file = SourceFileBuilder::new("".to_owned(), source.to_owned()).finish(); + let source_file = SourceFileBuilder::new(filename.to_owned(), source.to_owned()).finish(); Ok(func_type.ast_to_object(vm, &source_file)) } @@ -2498,7 +2536,10 @@ pub(crate) fn compile( filename: &str, mode: crate::compiler::Mode, mut opts: codegen::CompileOpts, + module: Option<&Py>, ) -> PyResult { + #[cfg(not(feature = "parser"))] + let _ = module; let text = synthetic_source_from_ast_object(vm, &object)?; let source_file = SourceFileBuilder::new(filename.to_owned(), text.clone()).finish(); let ast = Node::ast_from_object(vm, &source_file, object)?; @@ -2535,7 +2576,7 @@ pub(crate) fn compile( message.into(), fname, location.line.get(), - None, + module.map(|module| module.to_owned().into()), vm.ctx.none(), None, None, diff --git a/crates/vm/src/stdlib/_ast/string.rs b/crates/vm/src/stdlib/_ast/string.rs index e3b1c94754e..c3f0a10514f 100644 --- a/crates/vm/src/stdlib/_ast/string.rs +++ b/crates/vm/src/stdlib/_ast/string.rs @@ -1,6 +1,5 @@ use super::constant::{Constant, ConstantLiteral}; use super::*; -use crate::warn; use ast::str_prefix::StringLiteralPrefix; use rustpython_common::wtf8::Wtf8Buf; @@ -175,42 +174,41 @@ fn normalize_template_str_parts(values: Vec) -> Vec( source_file: &SourceFile, range: TextRange, -) { + emit_warning: &mut impl FnMut(usize, char) -> Result<(), E>, +) -> Result<(), E> { let source = source_file.source_text(); let start = range.start().to_usize(); let end = range.end().to_usize(); if start >= end || end > source.len() { - return; - } - let mut raw = &source[start..end]; - if raw.starts_with(':') { - raw = &raw[1..]; + return Ok(()); } - - let mut chars = raw.chars().peekable(); - while let Some(ch) = chars.next() { + let Some(raw) = source.get(start..end) else { + return Ok(()); + }; + let mut chars = raw.char_indices().peekable(); + while let Some((offset, ch)) = chars.next() { if ch != '\\' { continue; } - let Some(next) = chars.next() else { + let Some((_, next)) = chars.next() else { break; }; let valid = match next { '\\' | '\'' | '"' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' => true, '\n' => true, '\r' => { - if let Some('\n') = chars.peek().copied() { + if let Some((_, '\n')) = chars.peek().copied() { chars.next(); } true } '0'..='7' => { for _ in 0..2 { - if let Some('0'..='7') = chars.peek().copied() { + if let Some((_, '0'..='7')) = chars.peek().copied() { chars.next(); } else { break; @@ -220,7 +218,7 @@ fn warn_invalid_escape_sequences_in_format_spec( } 'x' => { for _ in 0..2 { - if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + if chars.peek().is_some_and(|(_, c)| c.is_ascii_hexdigit()) { chars.next(); } else { break; @@ -230,7 +228,7 @@ fn warn_invalid_escape_sequences_in_format_spec( } 'u' => { for _ in 0..4 { - if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + if chars.peek().is_some_and(|(_, c)| c.is_ascii_hexdigit()) { chars.next(); } else { break; @@ -240,7 +238,7 @@ fn warn_invalid_escape_sequences_in_format_spec( } 'U' => { for _ in 0..8 { - if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + if chars.peek().is_some_and(|(_, c)| c.is_ascii_hexdigit()) { chars.next(); } else { break; @@ -249,9 +247,9 @@ fn warn_invalid_escape_sequences_in_format_spec( true } 'N' => { - if let Some('{') = chars.peek().copied() { + if let Some((_, '{')) = chars.peek().copied() { chars.next(); - for c in chars.by_ref() { + for (_, c) in chars.by_ref() { if c == '}' { break; } @@ -262,18 +260,196 @@ fn warn_invalid_escape_sequences_in_format_spec( _ => false, }; if !valid { - let message = vm.ctx.new_str(format!( - "\"\\{next}\" is an invalid escape sequence. Such sequences will not work in the future. Did you mean \"\\\\{next}\"? A raw string is also an option." - )); - let _ = warn::warn( - message.into(), - Some(vm.ctx.exceptions.syntax_warning.to_owned()), - 1, - None, - vm, + return emit_warning(start + offset, next); + } + } + Ok(()) +} + +/// Keep AST parsing's existing format-spec diagnostics at a fallible boundary, +/// rather than emitting warnings from the infallible Python AST conversion. +#[cfg(feature = "parser")] +pub(super) fn emit_format_spec_warnings( + statements: &[ast::Stmt], + expressions: &[ast::Expr], + source_file: &SourceFile, + emit_warning: &mut impl FnMut(usize, char) -> Result<(), E>, +) -> Result<(), E> { + use ast::visitor::{self, Visitor}; + use std::collections::HashSet; + + struct LiteralVisitor<'a, F, E> { + source_file: &'a SourceFile, + emit_warning: F, + emitted: HashSet, + error: Option, + } + + impl LiteralVisitor<'_, F, E> + where + F: FnMut(usize, char) -> Result<(), E>, + { + fn check_literal(&mut self, mut range: TextRange, interpolated: bool) { + if self.error.is_some() { + return; + } + // Ruff leaves the interpolation delimiter outside the literal range. + // Include it so a trailing backslash can still diagnose \{ or \}. + if interpolated + && self + .source_file + .source_text() + .as_bytes() + .get(range.end().to_usize()) + .is_some_and(|byte| matches!(byte, b'{' | b'}')) + { + range = TextRange::new(range.start(), range.end() + TextSize::from(1)); + } + let result = warn_invalid_escape_sequences_in_format_spec( + self.source_file, + range, + &mut |offset, ch| { + if !self.emitted.contains(&offset) { + (self.emit_warning)(offset, ch)?; + self.emitted.insert(offset); + } + Ok(()) + }, ); + if let Err(error) = result { + self.error = Some(error); + } + } + + fn visit_elements<'a>(&mut self, elements: &'a ast::InterpolatedStringElements, raw: bool) + where + Self: Visitor<'a>, + { + for element in elements { + match element { + ast::InterpolatedStringElement::Literal(literal) => { + if !raw { + self.check_literal(literal.range, true); + } + } + ast::InterpolatedStringElement::Interpolation(interpolation) => { + self.visit_expr(&interpolation.expression); + if let Some(spec) = &interpolation.format_spec { + self.visit_elements(&spec.elements, raw); + } + } + } + } } } + + // Visit only literal spans within an existing format-spec warning region. + // An enclosing spec can include nested raw strings, nonraw expressions in a + // raw f-string, and comments; scanning its entire source range conflates them. + impl<'a, F, E> Visitor<'a> for LiteralVisitor<'_, F, E> + where + F: FnMut(usize, char) -> Result<(), E>, + { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if self.error.is_none() { + visitor::walk_expr(self, expr); + } + } + + fn visit_string_literal(&mut self, literal: &'a ast::StringLiteral) { + if !matches!( + literal.flags.prefix(), + ast::str_prefix::StringLiteralPrefix::Raw { .. } + ) { + self.check_literal(literal.range, false); + } + } + + fn visit_bytes_literal(&mut self, literal: &'a ast::BytesLiteral) { + if !literal.flags.prefix().is_raw() { + self.check_literal(literal.range, false); + } + } + + fn visit_f_string(&mut self, fstring: &'a ast::FString) { + self.visit_elements(&fstring.elements, fstring.flags.prefix().is_raw()); + } + + fn visit_t_string(&mut self, tstring: &'a ast::TString) { + self.visit_elements(&tstring.elements, tstring.flags.prefix().is_raw()); + } + } + + struct FormatSpecVisitor<'a, F, E> { + literals: LiteralVisitor<'a, F, E>, + } + + impl<'a, F, E> Visitor<'a> for FormatSpecVisitor<'_, F, E> + where + F: FnMut(usize, char) -> Result<(), E>, + { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if self.literals.error.is_some() { + return; + } + if let ast::Expr::FString(fstring) = expr { + // Conversion checked all enclosing specs before converting their + // expressions, including across concatenated f-string parts. + for part in &fstring.value { + if let ast::FStringPartRef::FString(part) = part { + self.visit_format_specs(&part.elements, part.flags.prefix().is_raw()); + } + } + } + visitor::walk_expr(self, expr); + } + + fn visit_t_string(&mut self, tstring: &'a ast::TString) { + let raw = tstring.flags.prefix().is_raw(); + for element in &tstring.elements { + if let ast::InterpolatedStringElement::Interpolation(interpolation) = element { + self.visit_expr(&interpolation.expression); + if let Some(spec) = &interpolation.format_spec { + // Template specs become JoinedStr nodes during conversion; + // their nested format specs already emitted warnings too. + self.visit_format_specs(&spec.elements, raw); + for element in &spec.elements { + self.visit_interpolated_string_element(element); + } + } + } + } + } + } + + impl FormatSpecVisitor<'_, F, E> + where + F: FnMut(usize, char) -> Result<(), E>, + { + fn visit_format_specs(&mut self, elements: &ast::InterpolatedStringElements, raw: bool) { + for element in elements { + if let ast::InterpolatedStringElement::Interpolation(interpolation) = element + && let Some(spec) = &interpolation.format_spec + { + self.literals.visit_elements(&spec.elements, raw); + } + } + } + } + + let mut visitor = FormatSpecVisitor { + literals: LiteralVisitor { + source_file, + emit_warning, + emitted: HashSet::new(), + error: None, + }, + }; + visitor.visit_body(statements); + for expr in expressions { + visitor.visit_expr(expr); + } + visitor.literals.error.map_or(Ok(()), Err) } fn ruff_format_spec_to_joined_str( @@ -763,13 +939,6 @@ pub(super) fn fstring_to_object( } } let values = normalize_joined_str_parts(values); - for part in &values { - if let JoinedStrPart::FormattedValue(value) = part - && let Some(format_spec) = &value.format_spec - { - warn_invalid_escape_sequences_in_format_spec(vm, source_file, format_spec.range()); - } - } let c = JoinedStr { range, values: values.into_boxed_slice(), diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 65f64299443..23e4c7ea7d7 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -118,6 +118,8 @@ mod builtins { dont_inherit: ArgIntoBool, #[pyarg(any, default = -1)] optimize: i32, + #[pyarg(named, optional)] + module: Option, #[pyarg(named, default = -1)] _feature_version: i32, } @@ -208,6 +210,17 @@ mod builtins { 0..=2 => optimize as u8, _ => return Err(vm.new_value_error("compile(): invalid optimize value")), }; + let module = args.module.filter(|module| !vm.is_none(module)); + let module = module + .map(|module| { + PyStrRef::try_from_object(vm, module.clone()).map_err(|_| { + vm.new_type_error(format!( + "compile() argument 'module' must be str or None, not {}", + module.class().name() + )) + }) + }) + .transpose()?; let dont_inherit = args.dont_inherit.into_bool(); let is_ast_only = cf.contains(CompilerFlags::ONLY_AST); let future_features = merge_compile_future_features(flags, dont_inherit, vm); @@ -297,7 +310,14 @@ mod builtins { opts.optimize = optimize; opts.allow_top_level_await = cf.contains(CompilerFlags::ALLOW_TOP_LEVEL_AWAIT); opts.future_features = future_features; - return _ast::compile(vm, args.source, &filename.to_string_lossy(), mode, opts); + return _ast::compile( + vm, + args.source, + &filename.to_string_lossy(), + mode, + opts, + module.as_deref(), + ); } } @@ -314,13 +334,14 @@ mod builtins { let mut compile_flags = flags | future_features.bits() as i32; #[cfg(feature = "rustpython-compiler")] let compile_source = |source: &[u8], compile_flags: i32| { - vm.compile_string_object_with_flags( + vm.compile_string_object_with_flags_and_module( source, &filename.to_string_lossy(), start.as_i32(), compile_flags, feature_version, optimize as i32, + module.as_deref(), ) }; match &source { diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index b5f7c77c772..2f13fb63a4c 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -5,8 +5,8 @@ use core::fmt; use crate::{ - AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyCode}, + AsObject, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyCode, PyStr}, compiler::{self, CompileError, CompileOpts}, vm::compile_mode::{CompileStart, CompilerFlags, compile_future_features_from_flags}, }; @@ -381,7 +381,29 @@ impl VirtualMachine { feature_version: i32, optimize: i32, ) -> PyResult { - use crate::convert::ToPyException; + self.compile_string_object_with_flags_and_module( + source, + filename, + start, + flags, + feature_version, + optimize, + None, + ) + } + + #[cfg(feature = "parser")] + #[allow(clippy::too_many_arguments)] + pub(crate) fn compile_string_object_with_flags_and_module( + &self, + source: &[u8], + filename: &str, + start: i32, + flags: i32, + feature_version: i32, + optimize: i32, + module: Option<&Py>, + ) -> PyResult { use crate::stdlib::_ast; let cf = CompilerFlags::from_bits_retain(flags); @@ -411,10 +433,29 @@ impl VirtualMachine { None }; + let mut emit_ast_warning = |offset, ch| { + escape_warnings::warn_invalid_escape_sequence( + source, + escape_warnings::InvalidEscape::Char { ch, offset }, + filename, + module, + self, + ) + .map_err(VmCompileError::Warning) + }; if is_ast_only { if start == CompileStart::FuncType { - return _ast::parse_func_type(self, source, filename, optimize, target_version) - .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(self)); + return _ast::parse_func_type( + self, + source, + filename, + optimize, + target_version, + &mut emit_ast_warning, + ) + .map_err(|e| { + e.into_pyexception_maybe_incomplete(self, Some(source), allow_incomplete) + }); } let (parser_mode, interactive) = match start { CompileStart::Single => (ruff_python_parser::Mode::Module, true), @@ -434,14 +475,18 @@ impl VirtualMachine { interactive, future_features, dont_imply_dedent, + &mut emit_ast_warning, ) - .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(self))?; + .map_err(|e| { + e.into_pyexception_maybe_incomplete(self, Some(source), allow_incomplete) + })?; if start == CompileStart::Single { return _ast::wrap_interactive(self, &parsed); } return Ok(parsed); } + let mut emitted_escapes = Vec::new(); if type_comments { let parser_mode = match start { CompileStart::Single | CompileStart::File => ruff_python_parser::Mode::Module, @@ -460,8 +505,20 @@ impl VirtualMachine { start == CompileStart::Single, future_features, dont_imply_dedent, + |offset, ch| { + emit_ast_warning(offset, ch)?; + // The ordinary compiler must not emit this same escape again. + emitted_escapes.push(offset); + Ok(()) + }, ) - .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(self))?; + .map_err(|e: VmCompileError| { + e.into_pyexception_maybe_incomplete(self, Some(source), allow_incomplete) + })?; + // AST traversal need not follow source order. The later compiler + // uses binary search to skip these already-considered offsets. + emitted_escapes.sort_unstable(); + emitted_escapes.dedup(); } let mode = match start { @@ -476,7 +533,7 @@ impl VirtualMachine { opts.future_features = future_features; opts.dont_imply_dedent = dont_imply_dedent; let code = self - .compile_with_opts(source, mode, filename, opts) + .compile_with_opts_and_module(source, mode, filename, opts, module, &emitted_escapes) .map_err(|err| { err.into_pyexception_maybe_incomplete(self, Some(source), allow_incomplete) })?; @@ -498,13 +555,27 @@ impl VirtualMachine { mode: compiler::Mode, source_path: impl Into, opts: CompileOpts, + ) -> Result, VmCompileError> { + self.compile_with_opts_and_module(source, mode, source_path, opts, None, &[]) + } + + pub(crate) fn compile_with_opts_and_module( + &self, + source: &str, + mode: compiler::Mode, + source_path: impl Into, + opts: CompileOpts, + module: Option<&Py>, + emitted_escapes: &[usize], ) -> Result, VmCompileError> { let source_path = source_path.into(); + #[cfg(not(feature = "parser"))] + let _ = (module, emitted_escapes); #[cfg(feature = "parser")] { - self.emit_tokenizer_syntax_warnings(source, &source_path) + self.emit_tokenizer_syntax_warnings(source, &source_path, module) .map_err(VmCompileError::Warning)?; - self.emit_string_escape_warnings(source, &source_path) + self.emit_string_escape_warnings(source, &source_path, module, emitted_escapes) .map_err(VmCompileError::Warning)?; } #[cfg(feature = "parser")] @@ -514,19 +585,25 @@ impl VirtualMachine { let escalated: core::cell::Cell> = core::cell::Cell::new(None); let mut syntax_warning_handler = |location, message| { - escape_warnings::warn_syntax_at_location(&source_path, location, message, self) - .map_err(|warning| { - escalated.set(Some(warning)); - // Recovered below via `escalated`, so this is never surfaced. - compiler::codegen::error::CodegenError { - location: Some(location), - end_location: None, - error: compiler::codegen::error::CodegenErrorType::SyntaxError( - String::new(), - ), - source_path: source_path.clone(), - } - }) + escape_warnings::warn_syntax_at_location( + &source_path, + module, + location, + message, + self, + ) + .map_err(|warning| { + escalated.set(Some(warning)); + // Recovered below via `escalated`, so this is never surfaced. + compiler::codegen::error::CodegenError { + location: Some(location), + end_location: None, + error: compiler::codegen::error::CodegenErrorType::SyntaxError( + String::new(), + ), + source_path: source_path.clone(), + } + }) }; let result = compiler::compile_with_syntax_warning_handler( source, @@ -618,7 +695,7 @@ mod escape_warnings { if cs <= ce { Some((cs, ce)) } else { None } } - enum InvalidEscape { + pub(super) enum InvalidEscape { Char { ch: char, offset: usize }, Octal { digits: [u8; 3], offset: usize }, } @@ -773,10 +850,11 @@ mod escape_warnings { /// Emit `SyntaxWarning` for an invalid escape sequence. /// /// `warn_invalid_escape_sequence()` in `Parser/string_parser.c` - fn warn_invalid_escape_sequence( + pub(super) fn warn_invalid_escape_sequence( source: &str, escape: InvalidEscape, filename: &str, + module: Option<&Py>, vm: &VirtualMachine, ) -> Result<(), CompileWarningError> { let (lineno, column) = line_offset_at(source, escape.offset()); @@ -788,7 +866,7 @@ mod escape_warnings { vm.ctx.new_str(warning).into(), fname, lineno, - None, + module.map(|module| module.to_owned().into()), vm.ctx.none(), None, None, @@ -809,6 +887,7 @@ mod escape_warnings { fn warn_syntax_at_offset( source: &str, filename: &str, + module: Option<&Py>, offset: usize, message: String, vm: &VirtualMachine, @@ -821,7 +900,7 @@ mod escape_warnings { message.into(), fname, lineno, - None, + module.map(|module| module.to_owned().into()), vm.ctx.none(), None, None, @@ -832,6 +911,7 @@ mod escape_warnings { pub(super) fn warn_syntax_at_location( filename: &str, + module: Option<&Py>, location: compiler::core::SourceLocation, message: String, vm: &VirtualMachine, @@ -843,7 +923,7 @@ mod escape_warnings { message.into(), fname, location.line.get(), - None, + module.map(|module| module.to_owned().into()), vm.ctx.none(), None, None, @@ -1084,13 +1164,14 @@ mod escape_warnings { end: usize, is_bytes: bool, filename: &str, + module: Option<&Py>, vm: &VirtualMachine, ) -> Result<(), CompileWarningError> { if let Some((start, content_end)) = content_bounds(source, text_range_from_bounds(quote_index, end)) && let Some(escape) = first_invalid_escape(source, start, content_end, is_bytes) { - warn_invalid_escape_sequence(source, escape, filename, vm)?; + warn_invalid_escape_sequence(source, escape, filename, module, vm)?; } Ok(()) } @@ -1100,13 +1181,14 @@ mod escape_warnings { start: usize, end: usize, filename: &str, + module: Option<&Py>, vm: &VirtualMachine, ) -> Result<(), CompileWarningError> { if start >= end || end > source.len() { return Ok(()); } if let Some(escape) = first_invalid_escape(source, start, end, false) { - return warn_invalid_escape_sequence(source, escape, filename, vm); + return warn_invalid_escape_sequence(source, escape, filename, module, vm); } let trailing_bs = source.as_bytes()[start..end] .iter() @@ -1124,6 +1206,7 @@ mod escape_warnings { offset: end - 1, }, filename, + module, vm, )?; } @@ -1185,6 +1268,7 @@ mod escape_warnings { end: usize, is_raw: bool, filename: &str, + module: Option<&Py>, vm: &VirtualMachine, ) -> Result<(), CompileWarningError> { let bytes = source.as_bytes(); @@ -1197,7 +1281,14 @@ mod escape_warnings { } b'{' => { if !is_raw { - warn_fstring_literal_part(source, literal_start, index, filename, vm)?; + warn_fstring_literal_part( + source, + literal_start, + index, + filename, + module, + vm, + )?; } let close = find_interpolation_end(bytes, index, end); let body_end = if close > index + 1 { close - 1 } else { close }; @@ -1207,6 +1298,7 @@ mod escape_warnings { body_end, Some(is_raw), filename, + module, vm, )?; index = close.max(index + 1); @@ -1220,7 +1312,7 @@ mod escape_warnings { } } if !is_raw { - warn_fstring_literal_part(source, literal_start, end, filename, vm)?; + warn_fstring_literal_part(source, literal_start, end, filename, module, vm)?; } Ok(()) } @@ -1230,9 +1322,10 @@ mod escape_warnings { fn emit_string_escape_warnings_unparsed( source: &str, filename: &str, + module: Option<&Py>, vm: &VirtualMachine, ) -> Result<(), CompileWarningError> { - emit_string_escape_warnings_in_range(source, 0, source.len(), None, filename, vm) + emit_string_escape_warnings_in_range(source, 0, source.len(), None, filename, module, vm) } fn emit_string_escape_warnings_in_range( @@ -1241,6 +1334,7 @@ mod escape_warnings { end: usize, format_spec_raw: Option, filename: &str, + module: Option<&Py>, vm: &VirtualMachine, ) -> Result<(), CompileWarningError> { let bytes = source.as_bytes(); @@ -1275,17 +1369,20 @@ mod escape_warnings { content_end.min(end), is_raw, filename, + module, vm, )?; } } else if !is_raw { warn_quoted_literal( - source, index, quote_end, is_bytes, filename, vm, + source, index, quote_end, is_bytes, filename, module, vm, )?; } } StringPrefix::None => { - warn_quoted_literal(source, index, quote_end, false, filename, vm)?; + warn_quoted_literal( + source, index, quote_end, false, filename, module, vm, + )?; } StringPrefix::Invalid => {} } @@ -1296,7 +1393,15 @@ mod escape_warnings { && bracket == 0 && brace == 0 => { - scan_interpolated_content(source, index + 1, end, is_raw, filename, vm)?; + scan_interpolated_content( + source, + index + 1, + end, + is_raw, + filename, + module, + vm, + )?; return Ok(()); } b'(' if format_spec_raw.is_some() => { @@ -1332,6 +1437,7 @@ mod escape_warnings { fn emit_numeric_literal_warnings( source: &str, filename: &str, + module: Option<&Py>, vm: &VirtualMachine, ) -> Result<(), CompileWarningError> { let bytes = source.as_bytes(); @@ -1363,6 +1469,7 @@ mod escape_warnings { warn_syntax_at_offset( source, filename, + module, index, format!("invalid {kind} literal"), vm, @@ -1379,8 +1486,10 @@ mod escape_warnings { struct EscapeWarningVisitor<'a> { source: &'a str, filename: &'a str, + module: Option<&'a Py>, vm: &'a VirtualMachine, error: Option, + emitted_escapes: &'a [usize], /// This pass runs before the compile that rejects an over-nested /// tree, so it has to stop itself. depth: usize, @@ -1401,9 +1510,18 @@ mod escape_warnings { fn check_quoted_literal(&mut self, range: TextRange, is_bytes: bool) { if let Some((start, end)) = content_bounds(self.source, range) && let Some(escape) = first_invalid_escape(self.source, start, end, is_bytes) + && self + .emitted_escapes + .binary_search(&escape.offset()) + .is_err() { - let result = - warn_invalid_escape_sequence(self.source, escape, self.filename, self.vm); + let result = warn_invalid_escape_sequence( + self.source, + escape, + self.filename, + self.module, + self.vm, + ); self.record_warning(result); } } @@ -1421,8 +1539,16 @@ mod escape_warnings { return; } if let Some(escape) = first_invalid_escape(self.source, start, end, false) { - let result = - warn_invalid_escape_sequence(self.source, escape, self.filename, self.vm); + if self.emitted_escapes.binary_search(&escape.offset()).is_ok() { + return; + } + let result = warn_invalid_escape_sequence( + self.source, + escape, + self.filename, + self.module, + self.vm, + ); self.record_warning(result); return; } @@ -1440,6 +1566,7 @@ mod escape_warnings { if trailing_bs % 2 == 1 && let Some(&after) = self.source.as_bytes().get(end) && (after == b'{' || after == b'}') + && self.emitted_escapes.binary_search(&(end - 1)).is_err() { let result = warn_invalid_escape_sequence( self.source, @@ -1448,6 +1575,7 @@ mod escape_warnings { offset: end - 1, }, self.filename, + self.module, self.vm, ); self.record_warning(result); @@ -1547,8 +1675,9 @@ mod escape_warnings { &self, source: &str, filename: &str, + module: Option<&Py>, ) -> Result<(), CompileWarningError> { - emit_numeric_literal_warnings(source, filename, self) + emit_numeric_literal_warnings(source, filename, module, self) } /// Walk all string literals in `source` and emit `SyntaxWarning` for @@ -1557,6 +1686,8 @@ mod escape_warnings { &self, source: &str, filename: &str, + module: Option<&Py>, + emitted_escapes: &[usize], ) -> Result<(), CompileWarningError> { // The compile that follows rejects this source; parsing it here // would build a tree that exhausts the stack when dropped. @@ -1567,14 +1698,16 @@ mod escape_warnings { let Ok(parsed) = ruff_python_parser::parse(source, ruff_python_parser::Mode::Module.into()) else { - return emit_string_escape_warnings_unparsed(source, filename, self); + return emit_string_escape_warnings_unparsed(source, filename, module, self); }; let ast = parsed.into_syntax(); let mut visitor = EscapeWarningVisitor { source, filename, + module, vm: self, error: None, + emitted_escapes, depth: 0, depth_limit: compiler::CompileOpts::default().recursion_limit, }; diff --git a/extra_tests/snippets/builtin_compile_module.py b/extra_tests/snippets/builtin_compile_module.py new file mode 100644 index 00000000000..8ff58583b41 --- /dev/null +++ b/extra_tests/snippets/builtin_compile_module.py @@ -0,0 +1,330 @@ +import ast +import warnings + +from testutils import assert_raises, skip_if_unsupported + + +def test_compile_module(): + class ModuleName(str): + pass + + for module in (None, "", "package.module", ModuleName("package.module"), "\ud800"): + code = compile("value = 42", "", "exec", module=module) + assert code.co_filename == "" + namespace = {} + exec(code, namespace) + assert namespace["value"] == 42 + + for module in (1, False, b"package.module", [], {}, object()): + with assert_raises(TypeError) as caught: + compile("", "", "exec", module=module) + assert "argument 'module' must be str or None" in str(caught.exception) + + # module is keyword-only; _feature_version stays keyword-only too. + with assert_raises(TypeError): + compile("", "", "exec", 0, False, -1, "package.module") + + lexical_sources = [ + "x = 1or 0", + r"'\z'", + r"'\400'", + r'f"{1:\z}"', + r'f"\{1}"', + ] + codegen_sources = [ + "assert (1, 2)", + "1 is 1", + "def f():\n try:\n pass\n finally:\n return 1\n", + ] + + # Default source compilation uses module for matching, while diagnostics retain + # the real source filename. Test both text and bytes compilation. + for source in lexical_sources + codegen_sources: + for input_source in (source, source.encode()): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("error", SyntaxWarning) + warnings.filterwarnings( + "always", category=SyntaxWarning, module=r"^package\.module$" + ) + compile( + input_source, "", "exec", module="package.module" + ) + assert len(caught) == 1, (source, caught) + assert caught[0].filename == "" + assert caught[0].category is SyntaxWarning + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + warnings.filterwarnings( + "error", category=SyntaxWarning, module=r"^package\.module$" + ) + with assert_raises(SyntaxError): + compile( + input_source, + "", + "exec", + module="package.module", + ) + + # Compiling a pre-existing AST must use the module too. + for source in codegen_sources: + tree = ast.parse(source) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("error", SyntaxWarning) + warnings.filterwarnings( + "always", category=SyntaxWarning, module=r"^package\.module$" + ) + compile(tree, "", "exec", module="package.module") + assert len(caught) == 1, (source, caught) + assert caught[0].filename == "" + + # A following compile, with omitted or None module, must not inherit the override. + for kwargs in ({}, {"module": None}): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("error", SyntaxWarning) + warnings.filterwarnings( + "always", category=SyntaxWarning, module=r"^$" + ) + compile("1 is 1", "", "exec", **kwargs) + assert len(caught) == 1 + + # Failed parses use the fallback string scanner, which must retain the module. + for source in (r"b'\z' )", r'f"{1:\z}" )'): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("error", SyntaxWarning) + warnings.filterwarnings( + "always", category=SyntaxWarning, module=r"^package\.module$" + ) + with assert_raises(SyntaxError): + compile(source, "", "exec", module="package.module") + assert len(caught) == 1, (source, caught) + assert caught[0].filename == "" + + # AST conversion must use the same fallible, source-located warning boundary. + # TYPE_COMMENTS validates through the AST parser before compiling the source; + # the later compiler must not filter or emit the same escape a second time. + for flags in ( + ast.PyCF_ONLY_AST, + ast.PyCF_TYPE_COMMENTS, + ast.PyCF_ONLY_AST | ast.PyCF_TYPE_COMMENTS, + ast.PyCF_ONLY_AST | ast.PyCF_OPTIMIZED_AST, + ): + source = '\nπ = f"{1:\\z}"\n' + for kwargs, expected_module in ( + ({}, ""), + ({"module": None}, ""), + ({"module": "package.module"}, "package.module"), + ): + modules = [] + + class ModuleMatcher: + def match(self, module): + modules.append(module) + return True + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", SyntaxWarning) + warnings.filters.insert( + 0, ("always", None, SyntaxWarning, ModuleMatcher(), 0) + ) + compile(source, "", "exec", flags, **kwargs) + assert modules == [expected_module], (flags, kwargs, modules) + assert len(caught) == 1, (flags, kwargs, caught) + assert (caught[0].filename, caught[0].lineno) == ("", 2) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + warnings.filterwarnings( + "error", category=SyntaxWarning, module=r"^package\.module$" + ) + with assert_raises(SyntaxError) as caught: + compile( + source, "", "exec", flags, module="package.module" + ) + error = caught.exception + assert error.filename == "" + assert (error.lineno, error.offset, error.end_lineno, error.end_offset) == ( + 2, + 10, + 2, + 12, + ) + assert error.text == 'π = f"{1:\\z}"\n' + assert "Such sequences will not work in the future" not in error.msg + + # Warning handlers may raise arbitrary exceptions, which must survive intact. + failure = RuntimeError("warning handler failed") + + def showwarning(*args, **kwargs): + raise failure + + with warnings.catch_warnings(): + warnings.simplefilter("always", SyntaxWarning) + warnings.showwarning = showwarning + with assert_raises(RuntimeError) as caught: + compile( + source, "", "exec", flags, module="package.module" + ) + assert caught.exception is failure + + # Nested compile calls must not share module overrides or duplicate tracking. + modules = [] + locations = [] + + class ModuleMatcher: + def match(self, module): + modules.append(module) + return True + + def showwarning(message, category, filename, lineno, *args, **kwargs): + locations.append((filename, lineno)) + if len(locations) == 1: + compile(r'f"{1:\z}"', "", "exec", flags, module="package.inner") + + with warnings.catch_warnings(): + warnings.simplefilter("always", SyntaxWarning) + warnings.filters.insert( + 0, ("always", None, SyntaxWarning, ModuleMatcher(), 0) + ) + warnings.showwarning = showwarning + compile( + 'f"{1:\\z}"\nf"{2:\\q}"', + "", + "exec", + flags, + module="package.outer", + ) + assert modules == ["package.outer", "package.inner", "package.outer"], modules + assert locations == [("", 1), ("", 1), ("", 2)], locations + + # These warnings already came from AST conversion. Retain them instead of + # replacing that path with a broader scanner before every AST parse. + for source in ('''rf"{1:{'\\z'}}"''', '''f"{1:{t'\\z'}}"'''): + for flags in ( + ast.PyCF_ONLY_AST, + ast.PyCF_TYPE_COMMENTS, + ast.PyCF_ONLY_AST | ast.PyCF_TYPE_COMMENTS, + ): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("error", SyntaxWarning) + warnings.filterwarnings( + "always", category=SyntaxWarning, module=r"^package\.module$" + ) + compile( + source, "", "exec", flags, module="package.module" + ) + assert len(caught) == 1, (source, flags, caught) + assert (caught[0].filename, caught[0].lineno) == ("", 1) + + for flags in (ast.PyCF_ONLY_AST, ast.PyCF_ONLY_AST | ast.PyCF_TYPE_COMMENTS): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", SyntaxWarning) + with assert_raises(SyntaxError): + compile( + '!!!\nf"{1:\\z}"', + "", + "exec", + flags, + module="package.module", + ) + assert not caught + + # A filter changed by showwarning must affect the next distinct diagnostic, + # rather than treating the already-considered format escape as a new warning. + with warnings.catch_warnings(): + warnings.simplefilter("always", SyntaxWarning) + + def showwarning(*args, **kwargs): + warnings.simplefilter("error", SyntaxWarning) + + warnings.showwarning = showwarning + with assert_raises(SyntaxError) as caught: + compile( + 'f"{1:\\z}"\n"\\q"', + "", + "exec", + ast.PyCF_TYPE_COMMENTS, + module="package.module", + ) + assert caught.exception.lineno == 2 + assert "\\q" in caught.exception.msg + + # Raw literal text is not an invalid escape. Nested nonraw expressions still + # retain the diagnostics previously emitted during AST conversion. + for flags in ( + ast.PyCF_ONLY_AST, + ast.PyCF_TYPE_COMMENTS, + ast.PyCF_ONLY_AST | ast.PyCF_TYPE_COMMENTS, + ): + for source in ( + r'rf"{1:\z}"', + r'rf"{1:{2:\z}}"', + r'tr"{1:{2:\z}}"', + "f\"{1:{r'\\z'}}\"", + ): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("error", SyntaxWarning) + compile( + source, "", "exec", flags, module="package.module" + ) + assert not caught, (source, flags, caught) + for source in ("rf\"{f'{2:\\z}'}\"", "rf\"{1:{'\\z'}}\""): + with warnings.catch_warnings(): + warnings.simplefilter("error", SyntaxWarning) + with assert_raises(SyntaxError): + compile( + source, + "", + "exec", + flags, + module="package.module", + ) + + # Func-type parsing trims its source before converting the parsed nodes. + # Already-detected escapes must still point into the original source. + for prefix in ("", "\n\n"): + source = prefix + '(int) -> f"{1:aaaaaaaaaaaa\\zbbbbbbbbbbbb}"' + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", SyntaxWarning) + compile( + source, + "", + "func_type", + ast.PyCF_ONLY_AST, + module="package.module", + ) + assert len(caught) == 1, (source, caught) + assert caught[0].filename == "" + assert caught[0].lineno == prefix.count("\n") + 1 + + # Warn for the first invalid escape in each original literal, including + # literals later folded together. Comments are not literal source text. + for source, count in ( + (r'f"{1:\z\q}"', 1), + (r'f"{1:\{2}}"', 1), + (r'f"{1:\\{2}}"', 0), + ('f"{1:{(\n1 # \\z\n)}}"', 0), + ("f\"{1:{'\\z' + '\\q'}}\"", 2), + ("f\"{1:{'\\z' '\\q'}}\"", 2), + ("f\"{1:{('\\z', '\\q')[0]}}\"", 2), + ): + for flags in ( + ast.PyCF_ONLY_AST, + ast.PyCF_TYPE_COMMENTS, + ast.PyCF_ONLY_AST | ast.PyCF_OPTIMIZED_AST, + ): + for optimize in (0, 2): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", SyntaxWarning) + compile( + source, + "", + "exec", + flags, + optimize=optimize, + module="package.module", + ) + assert len(caught) == count, (source, flags, optimize, caught) + assert all(w.filename == "" for w in caught) + + +skip_if_unsupported(3, 15, test_compile_module) From 5c40bb25af94b3854b25a3fbba0b8dd415a5a319 Mon Sep 17 00:00:00 2001 From: Joshua Megnauth <48846352+joshuamegnauth54@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:08:44 +0000 Subject: [PATCH 065/229] host_env: Use close_range syscall (FreeBSD/Linux) (#9016) FreeBSD and Linux implement a syscall for close_range. It's faster than looping through the file descriptors to close which would cause a context switch per descriptor. CPython also uses the syscall where available. See: * https://man.freebsd.org/cgi/man.cgi?close_range(2) * https://man7.org/linux/man-pages/man2/close_range.2.html * https://github.com/python/cpython/blob/5a22a62b96a68c2dd31784c5e3427ad7b365388a/Python/fileutils.c#L3193-L3231 --- crates/host_env/src/crt_fd.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/host_env/src/crt_fd.rs b/crates/host_env/src/crt_fd.rs index 2390dc2aa83..cdedf177b04 100644 --- a/crates/host_env/src/crt_fd.rs +++ b/crates/host_env/src/crt_fd.rs @@ -348,8 +348,41 @@ pub fn close(fd: Owned) -> io::Result<()> { _close(fd.into_raw()) } +#[cfg(not(any(target_os = "freebsd", target_os = "linux")))] pub fn closerange(fd_low: Raw, fd_high: Raw) { - for fd in fd_low..fd_high { + close_range_slow(fd_low, fd_high); +} + +// _Py_closerange +#[cfg(any(target_os = "freebsd", target_os = "linux"))] +pub fn closerange(fd_low: Raw, fd_high: Raw) { + // CPython clamps low to 0. + let fd_low = fd_low.max(0); + // close_range(2) is [low, high] whereas CPython's is [low, high). + if fd_high < fd_low { + return; + } + let high = fd_high - 1; + + if close_range(fd_low as _, high as _) == -1 + && io::Error::last_os_error().raw_os_error() == Some(libc::ENOSYS) + { + close_range_slow(fd_low, fd_high); + } +} + +#[cfg(target_os = "freebsd")] +fn close_range(low: ffi::c_uint, high: ffi::c_uint) -> ffi::c_int { + unsafe { libc::close_range(low, high, 0) as ffi::c_int } +} + +#[cfg(target_os = "linux")] +fn close_range(low: ffi::c_uint, high: ffi::c_uint) -> ffi::c_int { + unsafe { libc::syscall(libc::SYS_close_range, low, high, 0) as ffi::c_int } +} + +fn close_range_slow(low: Raw, high: Raw) { + for fd in low..high { // The range can contain closed descriptors, so it cannot use Owned. let _ = _close(fd); } From 11a8de1a870b90100c731f904078b47de2968dbb Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:24:37 +0300 Subject: [PATCH 066/229] Release array and list locks around fromlist conversions (#9030) Assisted-by: Codex:gpt-6 --- crates/stdlib/src/array.rs | 81 +++++++++++++++++++---- extra_tests/snippets/stdlib_array.py | 99 ++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 12 deletions(-) diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index fd3e1bba429..cb61711934c 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -18,8 +18,8 @@ pub mod array { atomic_func, builtins::{ PositionIterInternal, PyByteArray, PyBytes, PyBytesRef, PyDictRef, PyFloat, - PyGenericAlias, PyInt, PyList, PyListRef, PyStr, PyStrRef, PyTupleRef, PyType, - PyTypeRef, PyUtf8Str, PyUtf8StrRef, builtins_iter, locked_next, + PyGenericAlias, PyInt, PyListRef, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, + PyUtf8Str, PyUtf8StrRef, builtins_iter, locked_next, }, class_or_notimplemented, convert::{ToPyObject, ToPyResult, TryFromBorrowedObject, TryFromObject}, @@ -235,20 +235,29 @@ pub mod array { } } - fn fromlist(&mut self, list: &Py, vm: &VirtualMachine) -> PyResult<()> { + // The caller must rule out conversion callbacks before taking locks. + fn extend_builtin_list(&mut self, list: &[PyObjectRef], vm: &VirtualMachine) -> PyResult<()> { match self { $(ArrayContentType::$n(v) => { // convert list before modify self - let mut list: Vec<$t> = list - .borrow_vec() - .iter() - .cloned() + let mut items: Vec<$t> = list.iter().cloned() .map(|value| <$t>::try_into_from_object(vm, value)) .try_collect()?; - v.append(&mut list); - Ok(()) + v.append(&mut items); })* } + Ok(()) + } + + fn resize(&mut self, len: usize, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + v.try_reserve(len.saturating_sub(v.len())) + .map_err(|_| vm.no_memory_error())?; + v.resize(len, <$t>::default()); + })* + } + Ok(()) } fn get_bytes(&self) -> &[u8] { @@ -506,12 +515,12 @@ pub mod array { (Double, f64, 'd', "d"), ); - #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] + #[derive(Copy, Clone, Default, Ord, PartialOrd, Eq, PartialEq, Debug)] pub struct WideChar(wchar_t); /// Element type for the 'w' typecode: always a 4-byte unicode code point /// (Py_UCS4), unlike 'u' which is platform-dependent `wchar_t`. - #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] + #[derive(Copy, Clone, Default, Ord, PartialOrd, Eq, PartialEq, Debug)] pub struct Ucs4Char(u32); trait ArrayElement: Sized { @@ -1186,7 +1195,55 @@ pub mod array { #[pymethod] fn fromlist(zelf: &Py, list: PyListRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.try_resizable(vm)?.fromlist(&list, vm) + let n = list.borrow_vec().len(); + if n == 0 { + return Ok(()); + } + { + let items = list.borrow_vec(); + if items.iter().all(|item| { + let class = item.class(); + class.is(vm.ctx.types.int_type) + || class.is(vm.ctx.types.float_type) + || class.is(vm.ctx.types.bool_type) + || class.is(vm.ctx.types.str_type) + }) { + return zelf.try_resizable(vm)?.extend_builtin_list(&items, vm); + } + } + let (old_size, typecode) = { + let mut array = zelf.try_resizable(vm)?; + let old_size = array.len(); + let new_size = old_size + .checked_add(n) + .ok_or_else(|| vm.no_memory_error())?; + // Callbacks can observe the new size, so initialize the new elements. + array.resize(new_size, vm)?; + (old_size, array.typecode()) + }; + let result = (|| { + for i in 0..n { + let value = list.borrow_vec().get(i).cloned().ok_or_else(|| { + vm.new_runtime_error("list changed size during iteration") + })?; + let index = zelf.read().len() as isize - n as isize + i as isize; + let item = ArrayContentType::item_from_object(typecode, value, vm)?; + // Conversion can resize the array. Check the saved index afterwards; + // a negative index is the conversion-only sentinel used by CPython. + if index >= 0 { + zelf.write().setitem_by_item(index, item, vm)?; + } + if list.borrow_vec().len() != n { + return Err(vm.new_runtime_error("list changed size during iteration")); + } + } + Ok(()) + })(); + if result.is_err() && zelf.read().len() != old_size { + // A callback may have exported a buffer, preventing even the rollback. + zelf.try_resizable(vm)?.resize(old_size, vm)?; + } + result } #[pymethod] diff --git a/extra_tests/snippets/stdlib_array.py b/extra_tests/snippets/stdlib_array.py index 540bc0796c1..9d3d9a7ca90 100644 --- a/extra_tests/snippets/stdlib_array.py +++ b/extra_tests/snippets/stdlib_array.py @@ -346,3 +346,102 @@ def source(): test_array_search_sees_appended_items() test_array_insert_converts_without_lock() test_array_extend_callbacks_and_partial_progress() + + +def test_array_fromlist_callbacks(): + for code in ("i", "d"): + values = array(code, [1]) + source = [] + + class Number: + def __index__(self): + assert len(values) == 3 + values[0] = 9 + source[1] = 8 + return 7 + + def __float__(self): + return float(self.__index__()) + + source[:] = [Number(), 2] + values.fromlist(source) + assert list(values) == [9, 7, 8] + + values = array("i", [1]) + + class Append: + def __index__(self): + values.append(9) + return 7 + + values.fromlist([Append()]) + assert list(values) == [1, 7, 9] + + +def test_array_fromlist_rollback(): + for mutate in (lambda source: source.clear(), lambda source: source.append(3)): + values = array("i", [1]) + + class ChangeList: + def __index__(self): + values[0] = 9 + mutate(source) + return 7 + + source = [ChangeList(), 2] + with assert_raises(RuntimeError): + values.fromlist(source) + assert list(values) == [9] + + values = array("i", [1]) + + class Fail: + def __index__(self): + assert values[1] == 2 + raise ValueError("conversion failed") + + with assert_raises(ValueError): + values.fromlist([2, Fail()]) + assert list(values) == [1] + + class Clear: + def __index__(self): + values.clear() + return 7 + + with assert_raises(IndexError): + values.fromlist([Clear()]) + assert len(values) == 1 + + +def test_array_fromlist_exports(): + values = array("i", [1]) + with memoryview(values): + values.fromlist([]) + with assert_raises(BufferError): + values.fromlist([2]) + + views = [] + + class Export: + def __index__(self): + views.append(memoryview(values)) + return 7 + + values.fromlist([Export()]) + assert list(views.pop()) == [1, 7] + + class ExportAndFail: + def __index__(self): + views.append(memoryview(values)) + raise ValueError("conversion failed") + + with assert_raises(BufferError): + values.fromlist([ExportAndFail()]) + assert len(values) == len(views[0]) == 3 + views.pop().release() + + +test_array_fromlist_callbacks() +test_array_fromlist_rollback() +test_array_fromlist_exports() From 3af5f8a15314446d2b28a90e429c938dc7b8beeb Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:25:57 +0300 Subject: [PATCH 067/229] Run benchmark workloads with consistent script namespaces (#9027) Assisted-by: Codex:gpt-6 --- benches/README.md | 11 ++++++---- benches/execution.rs | 14 +++++++++++-- benches/microbenchmarks.rs | 43 ++++++++++++++++++++------------------ 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/benches/README.md b/benches/README.md index d31e1a0f271..cdea0b3b41b 100644 --- a/benches/README.md +++ b/benches/README.md @@ -42,9 +42,9 @@ Two things differ when the benchmarks run under CodSpeed: - The CPython comparison benchmarks are skipped. They are useful to compare RustPython against CPython locally, but CodSpeed tracks the evolution of RustPython itself, and running them would double the duration of an already slow instrumented run. -- The microbenchmarks using `ITERATIONS` run with a single value instead of five. The criterion - benchmark id does not include the iteration count, so all five sizes are reported under the same - name. +- The microbenchmarks using `ITERATIONS` run with a single value of 1,000 instead of five sizes. + CodSpeed retains the existing benchmark names; local runs include the iteration count in each + benchmark id. ### Adding a benchmark @@ -54,6 +54,9 @@ in two ways: 1. The time to parse the file to AST 2. The time it takes to execute the file +Execution includes compilation and runs each iteration with a fresh namespace whose `__name__` +is `"__main__"`, so script entry points run in both RustPython and CPython. + ### Adding a micro benchmark Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has @@ -73,7 +76,7 @@ len(a_list) ``` Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If -present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For +present then the benchmark runs at five sizes: 100, 300, 500, 700 and 900 iterations locally. For example: ```python diff --git a/benches/execution.rs b/benches/execution.rs index 56d4fc4fc19..a09ab9c7890 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -7,6 +7,7 @@ use criterion::{ Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, measurement::WallTime, }; +use pyo3::types::{PyCode, PyCodeInput, PyCodeMethods, PyDict, PyDictMethods}; use rustpython_compiler::Mode; use rustpython_vm::{Interpreter, PyResult, Settings}; use std::{collections::HashMap, path::Path}; @@ -25,9 +26,14 @@ fn bench_cpython_code(b: &mut Bencher<'_>, source: &str) { let c_str_source = c_str_source_head.as_c_str(); pyo3::Python::attach(|py| { b.iter(|| { - let module = pyo3::types::PyModule::from_code(py, c_str_source, c"", c"") + let code = PyCode::compile(py, c_str_source, c"", PyCodeInput::File) + .expect("Error compiling source"); + let globals = PyDict::new(py); + globals.set_item("__name__", "__main__").unwrap(); + let result = code + .run(Some(&globals), None) .expect("Error running source"); - black_box(module); + black_box(result); }) }) } @@ -46,6 +52,10 @@ fn bench_rustpython_code(b: &mut Bencher<'_>, name: &str, source: &str) { b.iter(|| { let code = vm.compile(source, Mode::Exec, name).unwrap(); let scope = vm.new_scope_with_builtins(); + scope + .globals + .set_item("__name__", vm.ctx.new_str("__main__").into(), vm) + .unwrap(); let res: PyResult = vm.run_code_obj(code, scope); vm.unwrap_pyresult(res); }) diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 4d48278e447..1c3bfea55a3 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -45,11 +45,8 @@ fn bench_cpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &MicroBen pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let exec = builtins.getattr("exec").expect("no exec in builtins"); - let bench_func = |(globals, locals): &mut ( - pyo3::Bound<'_, pyo3::types::PyDict>, - pyo3::Bound<'_, pyo3::types::PyDict>, - )| { - let res = exec.call((&code, &*globals, &*locals), None); + let bench_func = |globals: &mut pyo3::Bound<'_, pyo3::types::PyDict>| { + let res = exec.call((&code, &*globals), None); if let Err(e) = res { e.print(py); panic!("Error running microbenchmark") @@ -58,23 +55,24 @@ fn bench_cpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &MicroBen let bench_setup = |iterations| { let globals = pyo3::types::PyDict::new(py); - let locals = pyo3::types::PyDict::new(py); + globals.set_item("__name__", "__main__").unwrap(); if let Some(idx) = iterations { globals.set_item("ITERATIONS", idx).unwrap(); } - let res = exec.call((&setup_code, &globals, &locals), None); + let res = exec.call((&setup_code, &globals), None); if let Err(e) = res { e.print(py); panic!("Error running microbenchmark setup code") } - (globals, locals) + globals }; if bench.iterate { for idx in iteration_counts() { group.throughput(Throughput::Elements(idx as u64)); - group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| { + let id = BenchmarkId::new("cpython", format!("{}[{idx}]", bench.name)); + group.bench_with_input(id, &idx, |b, idx| { b.iter_batched_ref( || bench_setup(Some(*idx)), bench_func, @@ -128,6 +126,10 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &Micro let bench_setup = |iterations| { let scope = vm.new_scope_with_builtins(); + scope + .globals + .set_item("__name__", vm.ctx.new_str("__main__").into(), vm) + .unwrap(); if let Some(idx) = iterations { scope .locals @@ -145,17 +147,18 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &Micro if bench.iterate { for idx in iteration_counts() { group.throughput(Throughput::Elements(idx as u64)); - group.bench_with_input( - BenchmarkId::new("rustpython", &bench.name), - &idx, - |b, idx| { - b.iter_batched( - || bench_setup(Some(*idx)), - bench_func, - BatchSize::LargeInput, - ); - }, - ); + let name = if is_codspeed() { + bench.name.clone() + } else { + format!("{}[{idx}]", bench.name) + }; + group.bench_with_input(BenchmarkId::new("rustpython", name), &idx, |b, idx| { + b.iter_batched( + || bench_setup(Some(*idx)), + bench_func, + BatchSize::LargeInput, + ); + }); } } else { group.bench_function(BenchmarkId::new("rustpython", &bench.name), move |b| { From b7251889655c96f6df1ecffbc908f62ba15b366e Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:32:22 -0700 Subject: [PATCH 068/229] Represent OrderedDict views with their contents (#9038) Extract the Python 3.14-compatible view representation from #8954. Snapshot view elements before invoking their repr methods, preserve recursion handling, and cover each view in the existing collections snippet. Leave migration-specific mapping changes out. Assisted-by: OpenAI Codex:model identifier unavailable --- .../src/stdlib/_collections/ordered_dict.rs | 80 ++++++++++++++++++- extra_tests/snippets/stdlib_collections.py | 54 +++++++++++++ 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/crates/vm/src/stdlib/_collections/ordered_dict.rs b/crates/vm/src/stdlib/_collections/ordered_dict.rs index 4ba9777fef5..8c3079c3bb1 100644 --- a/crates/vm/src/stdlib/_collections/ordered_dict.rs +++ b/crates/vm/src/stdlib/_collections/ordered_dict.rs @@ -7,7 +7,7 @@ pub(crate) mod ordered_dict { AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, atomic_func, builtins::{ - PyDict, PyGenericAlias, PyMappingProxy, PyTuple, PyTypeRef, + PyDict, PyGenericAlias, PyMappingProxy, PyStrRef, PyTuple, PyTypeRef, dict::{ PyDictItems, set_inner_number_or, set_inner_number_subtract, set_inner_number_xor, set_item_view_number_xor, set_view_number_and, @@ -27,6 +27,7 @@ pub(crate) mod ordered_dict { Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, }; + use rustpython_common::wtf8::Wtf8Buf; use std::collections::HashMap; const ODICT_ITER_REVERSED: u8 = 1; @@ -977,6 +978,29 @@ pub(crate) mod ordered_dict { set_item_view_number_xor(a, b, is_item_view(a) && is_item_view(b), vm) } + fn ordered_view_repr( + view: &PyObject, + od: &PyRef, + kind: u8, + name: &str, + vm: &VirtualMachine, + ) -> PyResult { + let Some(_guard) = ReprGuard::enter(vm, view) else { + return Ok(vm.ctx.intern_str("...").to_owned()); + }; + // Snapshot the view before calling user-defined repr methods, which + // can mutate the underlying ordered dictionary. + let iterator = PyODictIter::new(od.clone(), kind).to_pyobject(vm); + let items = PyIter::try_from_object(vm, iterator)? + .iter::(vm)? + .collect::>>()?; + let list_repr = vm.ctx.new_list(items).as_object().repr(vm)?; + let mut result = Wtf8Buf::from(format!("{name}(")); + result.push_wtf8(list_repr.as_wtf8()); + result.push_str(")"); + Ok(vm.ctx.new_str(result)) + } + #[pyattr] #[pyclass(name = "odict_keys", module = "builtins", unhashable = true, traverse)] #[derive(Debug, PyPayload)] @@ -984,7 +1008,7 @@ pub(crate) mod ordered_dict { od: PyRef, } - #[pyclass(with(Iterable, Comparable, AsMapping, AsSequence, AsNumber))] + #[pyclass(with(Iterable, Comparable, AsMapping, AsSequence, AsNumber, Representable))] impl Py { #[pymethod] fn __reversed__(&self, vm: &VirtualMachine) -> PyObjectRef { @@ -1007,6 +1031,22 @@ pub(crate) mod ordered_dict { } } + impl Representable for PyODictKeys { + fn repr(zelf: &Py, vm: &VirtualMachine) -> PyResult { + ordered_view_repr( + zelf.as_object(), + &zelf.od, + ODICT_ITER_KEYS, + "odict_keys", + vm, + ) + } + + fn repr_str(_zelf: &Py, _vm: &VirtualMachine) -> PyResult { + unreachable!("use repr instead") + } + } + impl AsMapping for PyODictKeys { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: PyMappingMethods = PyMappingMethods { @@ -1077,7 +1117,7 @@ pub(crate) mod ordered_dict { od: PyRef, } - #[pyclass(with(Iterable, Comparable, AsMapping))] + #[pyclass(with(Iterable, Comparable, AsMapping, Representable))] impl Py { #[pymethod] fn __reversed__(&self, vm: &VirtualMachine) -> PyObjectRef { @@ -1091,6 +1131,22 @@ pub(crate) mod ordered_dict { } } + impl Representable for PyODictValues { + fn repr(zelf: &Py, vm: &VirtualMachine) -> PyResult { + ordered_view_repr( + zelf.as_object(), + &zelf.od, + ODICT_ITER_VALUES, + "odict_values", + vm, + ) + } + + fn repr_str(_zelf: &Py, _vm: &VirtualMachine) -> PyResult { + unreachable!("use repr instead") + } + } + impl AsMapping for PyODictValues { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: PyMappingMethods = PyMappingMethods { @@ -1128,7 +1184,7 @@ pub(crate) mod ordered_dict { od: PyRef, } - #[pyclass(with(Iterable, Comparable, AsMapping, AsSequence, AsNumber))] + #[pyclass(with(Iterable, Comparable, AsMapping, AsSequence, AsNumber, Representable))] impl Py { #[pymethod] fn __reversed__(&self, vm: &VirtualMachine) -> PyObjectRef { @@ -1163,6 +1219,22 @@ pub(crate) mod ordered_dict { } } + impl Representable for PyOrderedDictItems { + fn repr(zelf: &Py, vm: &VirtualMachine) -> PyResult { + ordered_view_repr( + zelf.as_object(), + &zelf.od, + ODICT_ITER_ITEMS, + "odict_items", + vm, + ) + } + + fn repr_str(_zelf: &Py, _vm: &VirtualMachine) -> PyResult { + unreachable!("use repr instead") + } + } + impl AsMapping for PyOrderedDictItems { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: PyMappingMethods = PyMappingMethods { diff --git a/extra_tests/snippets/stdlib_collections.py b/extra_tests/snippets/stdlib_collections.py index 2da8df87c56..be9e7a99cb2 100644 --- a/extra_tests/snippets/stdlib_collections.py +++ b/extra_tests/snippets/stdlib_collections.py @@ -91,6 +91,60 @@ def __repr__(self): ordered = OrderedDict() _count_elements(ordered, "bab") assert list(ordered.items()) == [("b", 2), ("a", 1)] + + +# OrderedDict views show their contents in insertion order, including recursion. +def check_ordered_dict_view_repr(): + ordered = OrderedDict([("b", 2), ("a", 1)]) + for method, contents in ( + ("keys", "['b', 'a']"), + ("values", "[2, 1]"), + ("items", "[('b', 2), ('a', 1)]"), + ): + assert repr(getattr(OrderedDict(), method)()) == "odict_" + method + "([])" + assert ( + repr(getattr(ordered, method)()) == "odict_" + method + "(" + contents + ")" + ) + + recursive = OrderedDict() + values = recursive.values() + items = recursive.items() + recursive["self"] = values + assert repr(values) == "odict_values([...])" + recursive["self"] = items + assert repr(items) == "odict_items([('self', ...)])" + + # Repr must snapshot the contents before arbitrary repr code mutates them. + for method in ("keys", "values", "items"): + + class MutatingRepr: + def __repr__(self): + ordered.clear() + return "mutated" + + item = MutatingRepr() + ordered = OrderedDict([(item, item), ("tail", "last")]) + expected = { + "keys": "odict_keys([mutated, 'tail'])", + "values": "odict_values([mutated, 'last'])", + "items": "odict_items([(mutated, mutated), ('tail', 'last')])", + } + assert repr(getattr(ordered, method)()) == expected[method] + assert not ordered + + class BadRepr: + def __repr__(self): + raise ValueError("view repr") + + ordered = OrderedDict([(BadRepr(), BadRepr())]) + for method in ("keys", "values", "items"): + with assert_raises(ValueError) as caught: + repr(getattr(ordered, method)()) + assert str(caught.exception) == "view repr" + + +check_ordered_dict_view_repr() + factory = defaultdict(lambda: 100) _count_elements(factory, "a") assert factory == {"a": 1} From e3374c8a24282968ff40598d7fc3ea8054c872af Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:32:52 -0700 Subject: [PATCH 069/229] Support pickling slot wrapper descriptors (#9037) Extract wrapper descriptor reduction from #8954 for Python 3.14. Reduce to getattr on the defining class and cover direct reduction plus one roundtrip in the existing pickle snippet. Preserve canonical expected failures for unrelated staticmethod descriptor behavior. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/descriptor.rs | 10 ++++++++++ extra_tests/snippets/stdlib_pickle.py | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index 55c2275f5bb..e8329fbb572 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -2062,6 +2062,16 @@ impl Callable for PyWrapper { flags(DISALLOW_INSTANTIATION) )] impl Py { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let getattr = vm.builtins.get_attr("getattr", vm)?; + let args = vm.ctx.new_tuple(vec![ + self.typ.to_owned().into(), + self.name.to_owned().into(), + ]); + Ok(vm.ctx.new_tuple(vec![getattr, args.into()]).into()) + } + #[pygetset] fn __qualname__(&self) -> String { format!("{}.{}", self.typ.name(), self.name) diff --git a/extra_tests/snippets/stdlib_pickle.py b/extra_tests/snippets/stdlib_pickle.py index 23d137df43b..855bd79691e 100644 --- a/extra_tests/snippets/stdlib_pickle.py +++ b/extra_tests/snippets/stdlib_pickle.py @@ -9,3 +9,7 @@ assert list(memo) == [1, 7] assert memo[1] is left assert memo[7] is left + +# Slot wrapper descriptors reduce to lookup on their defining class. +assert object.__str__.__reduce__() == (getattr, (object, "__str__")) +assert _pickle.loads(_pickle.dumps(object.__str__)) is object.__str__ From 5dba000d8be55469e7b4755e334eeb1ddb79ebae Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:33:32 -0700 Subject: [PATCH 070/229] Preserve raw mappingproxy rich-comparison results (#9039) Extract the Python 3.14 comparison contract from #8954 without its 3.15 ordering restrictions or defensive copies. Return the wrapped mapping comparison result without truth conversion. Cover all six direct and reflected operations in the existing mappingproxy snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/mappingproxy.rs | 43 ++++++++--------- extra_tests/snippets/builtin_mappingproxy.py | 49 ++++++++++++++++++++ 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs index 9ee9ff17228..7342fb3c0d9 100644 --- a/crates/vm/src/builtins/mappingproxy.rs +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -5,12 +5,12 @@ use crate::{ class::PyClassImpl, common::{hash, lock::LazyLock}, convert::ToPyObject, - function::{ArgMapping, OptionalArg, PyArithmeticValue, PyComparisonValue}, + function::{ArgMapping, Either, OptionalArg, PyComparisonValue}, object::{Traverse, TraverseFn}, protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods}, types::{ - AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, Iterable, - PyComparisonOp, Representable, + AsMapping, AsNumber, AsSequence, Constructor, Hashable, Iterable, PyComparisonOp, + Representable, }, }; use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat}; @@ -175,12 +175,30 @@ impl PyMappingProxy { Iterable, Constructor, AsSequence, - Comparable, Hashable, AsNumber, Representable ))] impl Py { + #[pyslot] + fn slot_richcompare( + zelf: &PyObject, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult> { + let zelf = zelf.downcast_ref::().ok_or_else(|| { + vm.new_type_error(format!( + "unexpected payload for {}", + op.method_name(&vm.ctx).as_str() + )) + })?; + let obj = zelf.to_object(vm)?; + // CPython parity (Objects/descrobject.c::mappingproxy_richcompare): + // delegate to PyObject_RichCompare on the underlying mapping. + obj.rich_compare(other.to_owned(), op, vm).map(Either::A) + } + #[pymethod] fn get( &self, @@ -238,23 +256,6 @@ impl Py { } } -impl Comparable for PyMappingProxy { - fn cmp( - zelf: &Py, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult { - let obj = zelf.to_object(vm)?; - // CPython parity (Objects/descrobject.c::mappingproxy_richcompare): - // delegate to PyObject_RichCompare on the underlying mapping. - let res = obj.rich_compare(other.to_owned(), op, vm)?; - PyArithmeticValue::from_object(vm, res) - .map(|o| o.try_to_bool(vm)) - .transpose() - } -} - impl Hashable for PyMappingProxy { #[inline] fn hash(zelf: &Py, vm: &VirtualMachine) -> PyResult { diff --git a/extra_tests/snippets/builtin_mappingproxy.py b/extra_tests/snippets/builtin_mappingproxy.py index fdd653c408d..c4d302136b8 100644 --- a/extra_tests/snippets/builtin_mappingproxy.py +++ b/extra_tests/snippets/builtin_mappingproxy.py @@ -23,3 +23,52 @@ def b(): assert A.__dict__.get("not here", "default") == "default" assert A.__dict__.get("a", "default") is A.a assert A.__dict__.get("not here") is None + + +# A mappingproxy must preserve arbitrary rich-comparison results unchanged. +def check_mappingproxy_comparison_results(): + import operator + from types import MappingProxyType + + class Result: + def __bool__(self): + raise AssertionError("comparison result must not be coerced to bool") + + result = Result() + calls = [] + + def compare(name): + def method(self, other): + calls.append(name) + return result + + return method + + class Compared: + __eq__ = compare("eq") + __ne__ = compare("ne") + __lt__ = compare("lt") + __le__ = compare("le") + __gt__ = compare("gt") + __ge__ = compare("ge") + + class CustomMapping(Compared, dict): + pass + + for operation, reflected in ( + (operator.eq, "eq"), + (operator.ne, "ne"), + (operator.lt, "gt"), + (operator.le, "ge"), + (operator.gt, "lt"), + (operator.ge, "le"), + ): + calls.clear() + assert operation(MappingProxyType(CustomMapping()), {}) is result + assert calls == [operation.__name__], calls + calls.clear() + assert operation(MappingProxyType({}), Compared()) is result + assert calls == [reflected], calls + + +check_mappingproxy_comparison_results() From 9c8b745bfef49c7a27fe81c543ef5dea83c5062a Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:34:12 -0700 Subject: [PATCH 071/229] Reject conflicting member layouts in multiple inheritance (#9041) Extract the Python 3.14-compatible solid-base check from #8954. Prefix member cells contribute to instance layout even though they do not change basicsize. Include member_count in shape comparison while preserving all native layout safeguards, and cover conflicting and compatible bases in the existing type snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/type.rs | 6 +++- extra_tests/snippets/builtin_type.py | 50 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 3eb9c94df52..a2449f136ef 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -3677,7 +3677,11 @@ fn mro_internal(typ: &Py, vm: &VirtualMachine) -> PyResult { /// Returns true if the two types have different instance layouts. fn shape_differs(t1: &Py, t2: &Py) -> bool { - t1.slots.basicsize != t2.slots.basicsize || t1.slots.itemsize != t2.slots.itemsize + // Prefix member cells do not change basicsize, but still make a base + // solid: unrelated slotted bases cannot share the same member indexes. + t1.slots.basicsize != t2.slots.basicsize + || t1.slots.itemsize != t2.slots.itemsize + || t1.slots.member_count != t2.slots.member_count } fn solid_base<'a>(typ: &'a Py, vm: &VirtualMachine) -> &'a Py { diff --git a/extra_tests/snippets/builtin_type.py b/extra_tests/snippets/builtin_type.py index f00baaa57c7..36e0d6330dd 100644 --- a/extra_tests/snippets/builtin_type.py +++ b/extra_tests/snippets/builtin_type.py @@ -748,3 +748,53 @@ class StrKey(str): warnings.simplefilter("error") with assert_raises(RuntimeWarning): type("NonStringKeyRaises", (), {6: 7}) + + +class Left: + __slots__ = ("left",) + + +class Right: + __slots__ = ("right",) + + +with assert_raises(TypeError): + + class Conflict(Left, Right): + pass + + +with assert_raises(TypeError): + + class ReverseConflict(Right, Left): + pass + + +class Empty: + __slots__ = () + + +class Compatible(Left, Empty): + pass + + +value = Compatible() +value.left = 42 +assert value.left == 42 + + +class FirstChild(Left): + __slots__ = () + + +class SecondChild(Left): + __slots__ = () + + +class Diamond(FirstChild, SecondChild): + pass + + +value = Diamond() +value.left = 43 +assert value.left == 43 From 7c9a0e32b3c52aa2fa7e2e3daa248be7ab3e13a7 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:34:41 -0700 Subject: [PATCH 072/229] Traverse references held by operator callables (#9044) Extract attrgetter, itemgetter and methodcaller GC traversal from #8954. Add four bounded reference-cycle cases to the existing operator snippet, including a non-dotted str subclass retained by attrgetter. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_operator.rs | 6 +++--- extra_tests/snippets/stdlib_operator.py | 28 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/stdlib/_operator.rs b/crates/vm/src/stdlib/_operator.rs index 975521af1fe..61548a859fc 100644 --- a/crates/vm/src/stdlib/_operator.rs +++ b/crates/vm/src/stdlib/_operator.rs @@ -351,7 +351,7 @@ mod _operator { } #[pyattr] - #[pyclass(name = "attrgetter", module = "operator")] + #[pyclass(name = "attrgetter", module = "operator", traverse)] #[derive(Debug, PyPayload)] struct PyAttrGetter { attrs: Vec, @@ -470,7 +470,7 @@ mod _operator { } #[pyattr] - #[pyclass(name = "itemgetter", module = "operator")] + #[pyclass(name = "itemgetter", module = "operator", traverse)] #[derive(Debug, PyPayload)] struct PyItemGetter { items: Vec, @@ -556,7 +556,7 @@ mod _operator { } #[pyattr] - #[pyclass(name = "methodcaller", module = "operator")] + #[pyclass(name = "methodcaller", module = "operator", traverse)] #[derive(Debug, PyPayload)] struct PyMethodCaller { name: PyUtf8StrRef, diff --git a/extra_tests/snippets/stdlib_operator.py b/extra_tests/snippets/stdlib_operator.py index d0954b12261..1fa1b177023 100644 --- a/extra_tests/snippets/stdlib_operator.py +++ b/extra_tests/snippets/stdlib_operator.py @@ -3,3 +3,31 @@ assert _operator._compare_digest("abcdef", "abcdef") assert not _operator._compare_digest("abcdef", "abc") assert not _operator._compare_digest("abc", "abcdef") + + +def check_operator_reference_cycles(): + import gc + import weakref + + class Holder: + pass + + class Name(str): + pass + + for make, make_payload in ( + (_operator.itemgetter, Holder), + (lambda value: _operator.methodcaller("method", value), Holder), + (lambda value: _operator.methodcaller("method", value=value), Holder), + (_operator.attrgetter, lambda: Name("attribute")), + ): + payload = make_payload() + wrapper = make(payload) + payload.wrapper = wrapper + reference = weakref.ref(payload) + del wrapper, payload + gc.collect() + assert reference() is None + + +check_operator_reference_cycles() From dcdd5d45c3d01c4d52fcf3e226d5078fc0ac2908 Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:35:48 +0300 Subject: [PATCH 073/229] Avoid copying strings when replace finds no match (#9029) Assisted-by: Codex:gpt-6 --- crates/vm/src/builtins/str.rs | 32 +++++++++++++++++++---------- extra_tests/snippets/builtin_str.py | 17 +++++++++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index 7d339ad1b42..5e3c10f77a0 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -1245,19 +1245,29 @@ impl Py { } let s = zelf.as_wtf8(); - let replaced = if count < 0 { - s.replace(old.as_wtf8(), new.as_wtf8()) + let limit = if count < 0 { + usize::MAX } else { - let s_is_empty = s.is_empty(); - let old_is_empty = old.is_empty(); - - if s_is_empty && !old_is_empty { - s.to_owned() - } else if s_is_empty && old_is_empty { - new.as_wtf8().to_owned() - } else { - s.replacen(old.as_wtf8(), new.as_wtf8(), count as usize) + count as usize + }; + let replaced = if old.is_empty() { + s.replacen(old.as_wtf8(), new.as_wtf8(), limit) + } else { + let mut matches = s.find_iter(old.as_wtf8()).take(limit); + let Some(first) = matches.next() else { + drop(matches); + return PyStr::result_unchanged(zelf, vm); + }; + let mut result = Vec::with_capacity(s.len()); + let mut last = 0; + for start in core::iter::once(first).chain(matches) { + result.extend_from_slice(&s.as_bytes()[last..start]); + result.extend_from_slice(new.as_bytes()); + last = start + old.byte_len(); } + result.extend_from_slice(&s.as_bytes()[last..]); + // SAFETY: A nonempty WTF-8 needle matches only at code point boundaries. + unsafe { Wtf8Buf::from_bytes_unchecked(result) } }; vm.ctx.new_str(replaced) } diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index a8da6ab96bb..9ab9538c585 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -1098,6 +1098,23 @@ def test_replace_unchanged(): test_replace_unchanged() +def test_replace_missing_pattern(): + for text in ("ordinary text", "\u00e1\u03b2\U0001f600", "a\ud800b\udfff"): + for count in (-1, 1, 10): + assert text.replace("\r", "replacement", count) is text + result = MyString(text).replace("\r", "replacement", count) + assert type(result) is str + assert result == text + + # Continue the same search after the first match, including lone surrogates. + assert "ababa".replace("aba", "x") == "xba" + assert "\ud800x\ud800".replace("\ud800", "\udfff", 1) == "\udfffx\ud800" + assert "\ud800x\ud800".replace("\ud800", "\udfff") == "\udfffx\udfff" + + +test_replace_missing_pattern() + + def test_expandtabs_zero_tabsize(): # With no width to advance to, the tabs come out and nothing else moves. # A tab that followed a character used to ask for a run of usize::MAX From 7079e70b7909e859dee0294bd14df5307829ff7b Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:36:20 -0700 Subject: [PATCH 074/229] Honor warning instance categories before validating category arguments (#9034) Extract the Python 3.14-compatible ordering fix from #8954. An existing Warning instance determines its own category, regardless of the category argument. Keep canonical invalid-category coverage and add only the missing instance case to the existing warnings snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_warnings.rs | 11 ++++------- extra_tests/snippets/stdlib_warnings.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/vm/src/stdlib/_warnings.rs b/crates/vm/src/stdlib/_warnings.rs index 8a8f92b77da..35aca82d176 100644 --- a/crates/vm/src/stdlib/_warnings.rs +++ b/crates/vm/src/stdlib/_warnings.rs @@ -100,15 +100,12 @@ mod _warnings { category: Option, vm: &VirtualMachine, ) -> PyResult> { + if message.fast_isinstance(vm.ctx.exceptions.warning) { + return Ok(Some(message.class().to_owned())); + } let cat_obj = match category { Some(c) if !vm.is_none(&c) => c, - _ => { - return Ok(if message.fast_isinstance(vm.ctx.exceptions.warning) { - Some(message.class().to_owned()) - } else { - None // will default to UserWarning in warn_explicit - }); - } + _ => return Ok(None), // will default to UserWarning in warn_explicit }; let cat = PyTypeRef::try_from_object(vm, cat_obj.clone()).map_err(|_| { diff --git a/extra_tests/snippets/stdlib_warnings.py b/extra_tests/snippets/stdlib_warnings.py index 95092e50668..73ccc512ea7 100644 --- a/extra_tests/snippets/stdlib_warnings.py +++ b/extra_tests/snippets/stdlib_warnings.py @@ -1,3 +1,20 @@ import _warnings _warnings.warn("Test") + + +import warnings + + +class CustomWarning(UserWarning): + pass + + +for category in (None, DeprecationWarning, int, 42): + message = CustomWarning("sentinel") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + warnings.warn(message, category) + assert len(caught) == 1 + assert caught[0].message is message + assert caught[0].category is CustomWarning From 1c7c801a3b344b7b59a0571bd67d557870156b23 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:36:57 -0700 Subject: [PATCH 075/229] Accept dict subclasses returned by module annotation functions (#9033) Extract the Python 3.14-compatible annotation result check from #8954 without its Python 3.15 diagnostics. Preserve the returned dict subclass and cached identity, and cover the behavior in the existing module snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/module.rs | 2 +- extra_tests/snippets/builtins_module.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index aae75da816a..ac01c901c17 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -366,7 +366,7 @@ impl PyModule { { // Call __annotate__(1) where 1 is FORMAT_VALUE let result = annotate.call((1i32,), vm)?; - if !result.class().is(vm.ctx.types.dict_type) { + if !result.downcastable::() { return Err(vm.new_type_error(format!( "__annotate__ returned non-dict of type '{}'", result.class().name() diff --git a/extra_tests/snippets/builtins_module.py b/extra_tests/snippets/builtins_module.py index bf762425c89..a3a35a29739 100644 --- a/extra_tests/snippets/builtins_module.py +++ b/extra_tests/snippets/builtins_module.py @@ -42,3 +42,26 @@ def test_func(): __builtins__ # noqa: F821 assert print + + +from types import ModuleType + + +class AnnotationDict(dict): + pass + + +module = ModuleType("annotated_module") +annotations = AnnotationDict(answer=int) +calls = [] + + +def annotate(format): + calls.append(format) + return annotations + + +module.__annotate__ = annotate +assert module.__annotations__ is annotations +assert module.__annotations__ is annotations +assert calls == [1] From 386cd86aef678a1af180bde89ea389e4f4de92a0 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:41:23 -0700 Subject: [PATCH 076/229] Preserve FunctionType qualified names when overriding names (#9032) Extract the independent Python 3.14-compatible constructor fix from #8954. An explicit name replaces __name__ while __qualname__ remains the code qualified name. Add a focused regression to the existing types snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/function.rs | 4 +--- extra_tests/snippets/stdlib_types.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index c6c4aaf2b79..b0bd5a010b5 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -1321,9 +1321,7 @@ impl Constructor for PyFunction { let mut func = Self::new(args.code.clone(), args.globals.clone(), vm)?; // Set function name if provided if let Some(name) = args.name.into_option() { - *func.name.lock() = name.clone(); - // Also update qualname to match the name - *func.qualname.lock() = name; + *func.name.lock() = name; } // Now set additional attributes directly if let Some(closure_tuple) = closure { diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index bffc9f82445..892daeb4ba5 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -66,3 +66,18 @@ def _run_missing_type_params_regression(): deep_alias = list[deep_alias] with assert_raises(RecursionError): hash(deep_alias) + + +# FunctionType preserves the code qualified name when a name is supplied. +def outer(): + def nested(): + pass + + return nested + + +original = outer() +for name in ("renamed", ""): + function = types.FunctionType(original.__code__, globals(), name) + assert function.__name__ == name + assert function.__qualname__ == "outer..nested" From c516ca95dfd77ea76ac4df2b152aca9e88d3a6bc Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:47:59 -0700 Subject: [PATCH 077/229] Preserve requested encoding names in codec lookup errors (#9036) Extract the Python 3.14-compatible lookup diagnostic fix from #8954 while retaining the existing registry normalization. Cover original error spelling and normalized search-hook names in the existing codec snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/codecs.rs | 3 +- extra_tests/snippets/stdlib_codecs_wide.py | 36 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index 587701d6528..f3b6b13342f 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -261,6 +261,7 @@ impl CodecsRegistry { } pub fn lookup(&self, encoding: &str, vm: &VirtualMachine) -> PyResult { + let original_encoding = encoding; let encoding = normalize_encoding_name(encoding); let search_path = { let inner = self.inner.read(); @@ -286,7 +287,7 @@ impl CodecsRegistry { } } - Err(vm.new_lookup_error(format!("unknown encoding: {encoding}"))) + Err(vm.new_lookup_error(format!("unknown encoding: {original_encoding}"))) } fn _lookup_text_encoding( diff --git a/extra_tests/snippets/stdlib_codecs_wide.py b/extra_tests/snippets/stdlib_codecs_wide.py index d5a7d68fe8c..4b950ba70ba 100644 --- a/extra_tests/snippets/stdlib_codecs_wide.py +++ b/extra_tests/snippets/stdlib_codecs_wide.py @@ -28,3 +28,39 @@ _codecs.utf_32_ex_decode(b"\xff\xfe\x00\x00h\x00\x00\x00", "strict", 0, True)[0] == "h" ) + + +# Lookup errors preserve the requested spelling, while search hooks receive +# the Python 3.14 normalized registry key. +import codecs + +seen_encoding_names = [] + + +def record_unknown_encoding(name): + seen_encoding_names.append(name) + return None + + +codecs.register(record_unknown_encoding) +try: + for requested, normalized in ( + ("NO Such Codec!", "no_such_codec"), + ("x_ weird--Ω .Ab", "x_weird_.ab"), + ): + for lookup in ( + codecs.lookup, + lambda name: codecs.encode("value", name), + lambda name: codecs.decode(b"value", name), + lambda name: "value".encode(name), + lambda name: b"value".decode(name), + ): + try: + lookup(requested) + except LookupError as error: + assert str(error) == "unknown encoding: " + requested + else: + assert False, "unknown encoding was accepted" + assert seen_encoding_names[-1] == normalized +finally: + codecs.unregister(record_unknown_encoding) From fd062caaf46e81057983d12aedcafc8a66d2af8f Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 03:54:20 -0700 Subject: [PATCH 078/229] Use current locale for byte regular-expression matching (#9042) * Use current locale for byte regular-expression matching Extract the Python 3.14-compatible LOCALE matching paths from #8954, keeping host callbacks outside the no_std engine and preserving the C-locale fallback. Existing canonical regex tests cover locale changes and compiled patterns. Keep expected-failure markers until patched native execution proves they are obsolete; omit duplicate snippet tests and Python 3.15 prefixmatch additions. Assisted-by: OpenAI Codex:model identifier unavailable * Enable the repaired locale caching regression The exact patched native interpreter passes the existing locale flag and compiled-pattern tests and unexpectedly passes locale caching. Remove only the obsolete expected-failure decorator; preserve all canonical assertions and platform skips. Assisted-by: OpenAI Codex:model identifier unavailable --- Lib/test/test_re.py | 1 - crates/host_env/src/locale.rs | 47 +++++++++++++++++++++ crates/sre_engine/src/engine.rs | 73 ++++++++++++++++++++------------- crates/sre_engine/src/lib.rs | 2 +- crates/sre_engine/src/string.rs | 55 ++++++++++++++++--------- crates/vm/src/stdlib/_sre.rs | 12 +++++- 6 files changed, 139 insertions(+), 51 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 80c21f6b198..7a480a62b6c 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2179,7 +2179,6 @@ def test_bug_20998(self): # with ignore case. self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3)) - @unittest.expectedFailure # TODO: RUSTPYTHON; self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))\n AssertionError: None is not true @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") @unittest.skipIf(sys.platform.startswith("sunos"), "test doesn't work on Solaris, gh-91214") diff --git a/crates/host_env/src/locale.rs b/crates/host_env/src/locale.rs index e9df052c208..07dfa512d65 100644 --- a/crates/host_env/src/locale.rs +++ b/crates/host_env/src/locale.rs @@ -3,6 +3,53 @@ use core::{ffi::CStr, ptr}; pub use libc::{LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, LC_NUMERIC, LC_TIME}; +// CPython uses the wide-character APIs after btowc() on Apple platforms. +// Darwin's wint_t is int; libc does not expose these declarations. +#[cfg(target_vendor = "apple")] +unsafe extern "C" { + fn btowc(ch: libc::c_int) -> libc::c_int; + fn iswalnum(ch: libc::c_int) -> libc::c_int; + fn towlower(ch: libc::c_int) -> libc::c_int; + fn towupper(ch: libc::c_int) -> libc::c_int; +} + +/// Classify a byte using the current LC_CTYPE locale. +pub fn byte_isalnum(ch: u8) -> bool { + // ctype functions require EOF or an unsigned-char value, never a signed byte. + #[cfg(not(target_vendor = "apple"))] + unsafe { + libc::isalnum(ch.into()) != 0 + } + #[cfg(target_vendor = "apple")] + unsafe { + iswalnum(btowc(ch.into())) != 0 + } +} + +/// Lowercase a byte using the current LC_CTYPE locale. +pub fn byte_tolower(ch: u8) -> u32 { + #[cfg(not(target_vendor = "apple"))] + unsafe { + libc::tolower(ch.into()) as u32 + } + #[cfg(target_vendor = "apple")] + unsafe { + towlower(btowc(ch.into())) as u32 + } +} + +/// Uppercase a byte using the current LC_CTYPE locale. +pub fn byte_toupper(ch: u8) -> u32 { + #[cfg(not(target_vendor = "apple"))] + unsafe { + libc::toupper(ch.into()) as u32 + } + #[cfg(target_vendor = "apple")] + unsafe { + towupper(btowc(ch.into())) as u32 + } +} + #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))] pub use libc::LC_MESSAGES; diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs index c2a6ac81975..7224b4e878f 100644 --- a/crates/sre_engine/src/engine.rs +++ b/crates/sre_engine/src/engine.rs @@ -1,11 +1,11 @@ // good luck to those that follow; here be dragons use crate::string::{ - is_digit, is_linebreak, is_loc_word, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, - is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode, + is_digit, is_linebreak, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, is_uni_word, + is_word, lower_ascii, lower_unicode, upper_unicode, }; -use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; +use super::{Locale, MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; use alloc::{vec, vec::Vec}; use core::{convert::TryFrom, ptr::null}; use optional::Optioned; @@ -18,6 +18,7 @@ pub struct Request<'a, S> { pub pattern_codes: &'a [u32], pub match_all: bool, pub must_advance: bool, + pub locale: Locale, } impl<'a, S: StrDrive> Request<'a, S> { @@ -38,6 +39,7 @@ impl<'a, S: StrDrive> Request<'a, S> { pattern_codes, match_all, must_advance: false, + locale: Locale::default(), } } } @@ -674,20 +676,24 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo } SreOpcode::CATEGORY => { let cat_code = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap(); - if ctx.at_end(req) || !category(cat_code, ctx.peek_char::()) { + if ctx.at_end(req) + || !category(cat_code, ctx.peek_char::(), req.locale) + { break 'result false; } ctx.skip_code(2); ctx.advance_char::(); } - SreOpcode::IN => general_op_in!(charset), + SreOpcode::IN => general_op_in!(|set, c| charset(set, c, req.locale)), SreOpcode::IN_IGNORE => { - general_op_in!(|set, c| charset(set, lower_ascii(c))) + general_op_in!(|set, c| charset(set, lower_ascii(c), req.locale)) } SreOpcode::IN_UNI_IGNORE => { - general_op_in!(|set, c| charset(set, lower_unicode(c))) + general_op_in!(|set, c| charset(set, lower_unicode(c), req.locale)) + } + SreOpcode::IN_LOC_IGNORE => { + general_op_in!(|set, c| charset_loc_ignore(set, c, req.locale)) } - SreOpcode::IN_LOC_IGNORE => general_op_in!(charset_loc_ignore), SreOpcode::MARK => { state .marks @@ -837,13 +843,17 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo SreOpcode::NOT_LITERAL_UNI_IGNORE => { general_op_literal!(|code, c| code != lower_unicode(c)) } - SreOpcode::LITERAL_LOC_IGNORE => general_op_literal!(char_loc_ignore), + SreOpcode::LITERAL_LOC_IGNORE => { + general_op_literal!(|code, c| char_loc_ignore(code, c, req.locale)) + } SreOpcode::NOT_LITERAL_LOC_IGNORE => { - general_op_literal!(|code, c| !char_loc_ignore(code, c)) + general_op_literal!(|code, c| !char_loc_ignore(code, c, req.locale)) } SreOpcode::GROUPREF => general_op_groupref!(|x| x), SreOpcode::GROUPREF_IGNORE => general_op_groupref!(lower_ascii), - SreOpcode::GROUPREF_LOC_IGNORE => general_op_groupref!(lower_locate), + SreOpcode::GROUPREF_LOC_IGNORE => { + general_op_groupref!(|c| req.locale.lower(c)) + } SreOpcode::GROUPREF_UNI_IGNORE => general_op_groupref!(lower_unicode), SreOpcode::GROUPREF_EXISTS => { let (group_start, group_end) = @@ -1042,7 +1052,7 @@ fn search_info_charset( req.must_advance = false; loop { - while !ctx.at_end(req) && !charset(set, ctx.peek_char::()) { + while !ctx.at_end(req) && !charset(set, ctx.peek_char::(), req.locale) { ctx.advance_char::(); } if ctx.at_end(req) { @@ -1226,27 +1236,27 @@ fn at(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode) } SreAtCode::END_LINE => ctx.at_linebreak(req) || ctx.at_end(req), SreAtCode::END_STRING => ctx.at_end(req), - SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, is_loc_word), - SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, is_loc_word), + SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, |ch| req.locale.is_word(ch)), + SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, |ch| req.locale.is_word(ch)), SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, is_uni_word), SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, is_uni_word), } } -fn char_loc_ignore(code: u32, c: u32) -> bool { - code == c || code == lower_locate(c) || code == upper_locate(c) +fn char_loc_ignore(code: u32, c: u32, locale: Locale) -> bool { + code == c || code == locale.lower(c) || code == locale.upper(c) } -fn charset_loc_ignore(set: &[u32], c: u32) -> bool { - let lo = lower_locate(c); - if charset(set, c) { +fn charset_loc_ignore(set: &[u32], c: u32, locale: Locale) -> bool { + let lo = locale.lower(c); + if charset(set, lo, locale) { return true; } - let up = upper_locate(c); - up != lo && charset(set, up) + let up = locale.upper(c); + up != lo && charset(set, up, locale) } -fn category(cat_code: SreCatCode, c: u32) -> bool { +fn category(cat_code: SreCatCode, c: u32, locale: Locale) -> bool { match cat_code { SreCatCode::DIGIT => is_digit(c), SreCatCode::NOT_DIGIT => !is_digit(c), @@ -1256,8 +1266,8 @@ fn category(cat_code: SreCatCode, c: u32) -> bool { SreCatCode::NOT_WORD => !is_word(c), SreCatCode::LINEBREAK => is_linebreak(c), SreCatCode::NOT_LINEBREAK => !is_linebreak(c), - SreCatCode::LOC_WORD => is_loc_word(c), - SreCatCode::LOC_NOT_WORD => !is_loc_word(c), + SreCatCode::LOC_WORD => locale.is_word(c), + SreCatCode::LOC_NOT_WORD => !locale.is_word(c), SreCatCode::UNI_DIGIT => is_uni_digit(c), SreCatCode::UNI_NOT_DIGIT => !is_uni_digit(c), SreCatCode::UNI_SPACE => is_uni_space(c), @@ -1269,7 +1279,7 @@ fn category(cat_code: SreCatCode, c: u32) -> bool { } } -fn charset(set: &[u32], ch: u32) -> bool { +fn charset(set: &[u32], ch: u32, locale: Locale) -> bool { /* check if character is a member of the given set */ let mut ok = true; let mut i = 0; @@ -1292,7 +1302,7 @@ fn charset(set: &[u32], ch: u32) -> bool { break; } }; - if category(cat_code, ch) { + if category(cat_code, ch, locale) { return ok; } i += 2; @@ -1382,7 +1392,8 @@ fn _count( ctx.skip_char::(max_count); } SreOpcode::IN => { - while ctx.cursor.position < end && charset(&ctx.pattern(req)[2..], ctx.peek_char::()) + while ctx.cursor.position < end + && charset(&ctx.pattern(req)[2..], ctx.peek_char::(), req.locale) { ctx.advance_char::(); } @@ -1400,10 +1411,14 @@ fn _count( general_count_literal(req, ctx, end, |code, c| code != lower_ascii(c)); } SreOpcode::LITERAL_LOC_IGNORE => { - general_count_literal(req, ctx, end, char_loc_ignore); + general_count_literal(req, ctx, end, |code, c| { + char_loc_ignore(code, c, req.locale) + }); } SreOpcode::NOT_LITERAL_LOC_IGNORE => { - general_count_literal(req, ctx, end, |code, c| !char_loc_ignore(code, c)); + general_count_literal(req, ctx, end, |code, c| { + !char_loc_ignore(code, c, req.locale) + }); } SreOpcode::LITERAL_UNI_IGNORE => { general_count_literal(req, ctx, end, |code, c| code == lower_unicode(c)); diff --git a/crates/sre_engine/src/lib.rs b/crates/sre_engine/src/lib.rs index df598974ba9..7c31a6bae9d 100644 --- a/crates/sre_engine/src/lib.rs +++ b/crates/sre_engine/src/lib.rs @@ -8,7 +8,7 @@ pub mod string; pub use constants::{SRE_MAGIC, SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode}; pub use engine::{Request, SearchIter, State}; -pub use string::{StrDrive, StringCursor}; +pub use string::{Locale, StrDrive, StringCursor}; pub const CODESIZE: usize = 4; diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index d16a3ee88af..52b427a9414 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -391,15 +391,44 @@ pub(crate) fn is_digit(ch: u32) -> bool { u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) } -#[inline] -pub(crate) fn is_loc_alnum(ch: u32) -> bool { - // FIXME: Ignore the locales - u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) +/// Byte classification and case conversion in the current locale. +/// +/// Callbacks keep host APIs outside this `no_std` engine. They are called while +/// matching, rather than cached in a compiled pattern, so a locale change also +/// affects existing patterns and scanners. Without host callbacks, matching uses +/// the ASCII rules of the C locale. +#[derive(Debug, Clone, Copy)] +pub struct Locale { + pub is_alnum: fn(u8) -> bool, + pub to_lower: fn(u8) -> u32, + pub to_upper: fn(u8) -> u32, } -#[inline] -pub(crate) fn is_loc_word(ch: u32) -> bool { - ch == UNDERSCORE || is_loc_alnum(ch) +impl Default for Locale { + fn default() -> Self { + Self { + is_alnum: |ch| ch.is_ascii_alphanumeric(), + to_lower: |ch| ch.to_ascii_lowercase() as u32, + to_upper: |ch| ch.to_ascii_uppercase() as u32, + } + } +} + +impl Locale { + #[inline] + pub(crate) fn is_word(self, ch: u32) -> bool { + ch == UNDERSCORE || u8::try_from(ch).is_ok_and(self.is_alnum) + } + + #[inline] + pub(crate) fn lower(self, ch: u32) -> u32 { + u8::try_from(ch).map_or(ch, self.to_lower) + } + + #[inline] + pub(crate) fn upper(self, ch: u32) -> u32 { + u8::try_from(ch).map_or(ch, self.to_upper) + } } #[inline] @@ -413,18 +442,6 @@ pub fn lower_ascii(ch: u32) -> u32 { u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32) } -#[inline] -pub(crate) fn lower_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - lower_ascii(ch) -} - -#[inline] -pub(crate) fn upper_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32) -} - #[inline] pub(crate) fn is_uni_digit(ch: u32) -> bool { // SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL), diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs index 9fe57a86955..2e24b1477cc 100644 --- a/crates/vm/src/stdlib/_sre.rs +++ b/crates/vm/src/stdlib/_sre.rs @@ -59,7 +59,17 @@ mod _sre { fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef; fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<'_, Self> { - Request::new(self, start, end, &pattern.code, false) + let request = Request::new(self, start, end, &pattern.code, false); + #[cfg(all(feature = "host_env", any(unix, windows)))] + let request = Request { + locale: rustpython_sre_engine::Locale { + is_alnum: rustpython_host_env::locale::byte_isalnum, + to_lower: rustpython_host_env::locale::byte_tolower, + to_upper: rustpython_host_env::locale::byte_toupper, + }, + ..request + }; + request } } From bc5b9abd756dbf241df79b49561a38a7f9addba3 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 04:09:45 -0700 Subject: [PATCH 079/229] Export the stable ABI PyCFunction_Call shim (#9055) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/capi/src/methodobject.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/capi/src/methodobject.rs b/crates/capi/src/methodobject.rs index 2e82c5e9daa..3b0795deeb3 100644 --- a/crates/capi/src/methodobject.rs +++ b/crates/capi/src/methodobject.rs @@ -11,6 +11,16 @@ use rustpython_vm::{AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine}; define_py_check!(fn PyCFunction_Check, types.builtin_function_or_method_type); define_py_check!(exact fn PyCFunction_CheckExact, types.builtin_function_or_method_type); +// Removed from the public API in 3.13, but retained in the stable ABI. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyCFunction_Call( + callable: *mut PyObject, + args: *mut PyObject, + kwargs: *mut PyObject, +) -> *mut PyObject { + unsafe { crate::abstract_::PyObject_Call(callable, args, kwargs) } +} + #[repr(C)] #[derive(Debug)] pub struct PyMethodDef { From 64061cd38a19b4ab197c555234d154ca6fd00e1d Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 04:12:12 -0700 Subject: [PATCH 080/229] Validate RLock acquisition arguments before reentrant success (#9035) Extract the Python 3.14-compatible timeout validation order from #8954. Share validation with Lock and run it before increasing the RLock recursion count. Keep canonical general-lock coverage and add only owned-lock validation regressions to the existing thread snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_thread.rs | 41 ++++++++++++++------------ extra_tests/snippets/builtin_thread.py | 18 +++++++++++ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 7ac20af9273..efe4ed30211 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -104,28 +104,29 @@ pub(crate) mod _thread { } } + fn validate_acquire_args(args: AcquireArgs, vm: &VirtualMachine) -> PyResult<(bool, f64)> { + let timeout = args.timeout.to_secs_f64(); + if !args.blocking && timeout != -1.0 { + return Err(vm.new_value_error("can't specify a timeout for a non-blocking call")); + } + if args.blocking && timeout < 0.0 && timeout != -1.0 { + return Err(vm.new_value_error("timeout value must be a non-negative number")); + } + if timeout > TIMEOUT_MAX { + return Err(vm.new_overflow_error("timeout value is too large")); + } + Ok((args.blocking, timeout)) + } + macro_rules! acquire_lock_impl { - ($mu:expr, $args:expr, $vm:expr) => {{ - let (mu, args, vm) = ($mu, $args, $vm); - let timeout = args.timeout.to_secs_f64(); - match args.blocking { + ($mu:expr, $blocking:expr, $timeout:expr, $vm:expr) => {{ + let (mu, blocking, timeout, vm) = ($mu, $blocking, $timeout, $vm); + match blocking { true if timeout == -1.0 => { vm.allow_threads(|| mu.lock()); Ok(true) } - true if timeout < 0.0 => { - Err(vm.new_value_error("timeout value must be a non-negative number")) - } - true => { - if timeout > TIMEOUT_MAX { - return Err(vm.new_overflow_error("timeout value is too large")); - } - - Ok(vm.allow_threads(|| mu.try_lock_for(Duration::from_secs_f64(timeout)))) - } - false if timeout != -1.0 => { - Err(vm.new_value_error("can't specify a timeout for a non-blocking call")) - } + true => Ok(vm.allow_threads(|| mu.try_lock_for(Duration::from_secs_f64(timeout)))), false => Ok(mu.try_lock()), } }}; @@ -166,7 +167,8 @@ pub(crate) mod _thread { #[pymethod] #[pymethod(name = "acquire_lock")] fn acquire(&self, args: AcquireArgs, vm: &VirtualMachine) -> PyResult { - acquire_lock_impl!(&self.mu, args, vm) + let (blocking, timeout) = validate_acquire_args(args, vm)?; + acquire_lock_impl!(&self.mu, blocking, timeout, vm) } #[pymethod] @@ -252,6 +254,7 @@ pub(crate) mod _thread { #[pymethod] #[pymethod(name = "acquire_lock")] fn acquire(&self, args: AcquireArgs, vm: &VirtualMachine) -> PyResult { + let (blocking, timeout) = validate_acquire_args(args, vm)?; if self.mu.is_owned_by_current_thread() { // Re-entrant acquisition: just increment our count. // parking_lot stays at 1 level; we track recursion ourselves. @@ -259,7 +262,7 @@ pub(crate) mod _thread { .fetch_add(1, core::sync::atomic::Ordering::Relaxed); return Ok(true); } - let result = acquire_lock_impl!(&self.mu, args, vm)?; + let result = acquire_lock_impl!(&self.mu, blocking, timeout, vm)?; if result { self.count.store(1, core::sync::atomic::Ordering::Relaxed); } diff --git a/extra_tests/snippets/builtin_thread.py b/extra_tests/snippets/builtin_thread.py index e9696cce6d4..3d12e76cc59 100644 --- a/extra_tests/snippets/builtin_thread.py +++ b/extra_tests/snippets/builtin_thread.py @@ -1,3 +1,21 @@ import _thread assert _thread.TIMEOUT_MAX in [9223372036.0, 4294967.0] + +from testutils import assert_raises + +# Reentrant calls must validate arguments before incrementing the count. +for error, kwargs in ( + (ValueError, {"blocking": False, "timeout": 1}), + (ValueError, {"timeout": -2}), + (OverflowError, {"timeout": _thread.TIMEOUT_MAX + 1}), +): + lock = _thread.RLock() + assert lock.acquire() + try: + with assert_raises(error): + lock.acquire(**kwargs) + assert lock._recursion_count() == 1 + finally: + while lock._is_owned(): + lock.release() From 082a48aec348a67fdf04a9347af4d16d7dea0a45 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 04:21:50 -0700 Subject: [PATCH 081/229] Preserve signed filesystem timestamps in os.utime (#9047) Extract the Python 3.14-compatible signed timestamp conversion from #8954, preserving floor rounding, float-or-index argument rules and signed divmod remainders. Reuse the existing Windows file-time helper rather than the mega patch range restriction, and keep the 3.14 argument diagnostics. Add three concise parser regressions to the existing os snippet without native filesystem range assumptions. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/host_env/src/os.rs | 14 +++++ crates/host_env/src/posix.rs | 20 +++--- crates/host_env/src/posix_wasi.rs | 12 ++-- crates/vm/src/stdlib/os.rs | 101 +++++++++++++++++++++++++----- extra_tests/snippets/stdlib_os.py | 5 ++ 5 files changed, 122 insertions(+), 30 deletions(-) diff --git a/crates/host_env/src/os.rs b/crates/host_env/src/os.rs index 7d6879984bd..f6bd04d21cb 100644 --- a/crates/host_env/src/os.rs +++ b/crates/host_env/src/os.rs @@ -605,6 +605,20 @@ pub fn seek_fd( unsafe { suppress_iph!(libc::lseek(fd.as_raw(), position, how)) }.check_libc_neg() } +#[cfg(windows)] +pub type FileTimeSeconds = libc::time64_t; +#[cfg(any(unix, target_os = "wasi"))] +#[cfg_attr(target_env = "musl", allow(deprecated))] +pub type FileTimeSeconds = libc::time_t; + +/// Seconds and nanoseconds passed to filesystem timestamp operations. +#[cfg(any(unix, windows, target_os = "wasi"))] +#[derive(Clone, Copy)] +pub struct FileTime { + pub seconds: FileTimeSeconds, + pub nanoseconds: libc::c_long, +} + #[cfg(windows)] fn filetime_from_duration(duration: Duration) -> FILETIME { let intervals = ((duration.as_secs() as i64 + 11644473600) * 10_000_000) diff --git a/crates/host_env/src/posix.rs b/crates/host_env/src/posix.rs index a6ae039fd24..ae7c0d3dedc 100644 --- a/crates/host_env/src/posix.rs +++ b/crates/host_env/src/posix.rs @@ -762,12 +762,12 @@ pub fn fchmod(fd: BorrowedFd<'_>, mode: u32) -> std::io::Result<()> { #[cfg(target_os = "redox")] pub fn utimes( path: &Path, - acc: core::time::Duration, - modif: core::time::Duration, + acc: crate::os::FileTime, + modif: crate::os::FileTime, ) -> std::io::Result<()> { - let tv = |d: core::time::Duration| libc::timeval { - tv_sec: d.as_secs() as _, - tv_usec: d.subsec_micros() as _, + let tv = |time: crate::os::FileTime| libc::timeval { + tv_sec: time.seconds, + tv_usec: (time.nanoseconds / 1_000) as _, }; nix::sys::stat::utimes(path, &tv(acc).into(), &tv(modif).into()).map_err(std::io::Error::from) } @@ -776,13 +776,13 @@ pub fn utimes( pub fn set_file_times_at( dir_fd: i32, path: &CStr, - access: core::time::Duration, - modified: core::time::Duration, + access: crate::os::FileTime, + modified: crate::os::FileTime, follow_symlinks: bool, ) -> std::io::Result<()> { - let ts = |d: core::time::Duration| libc::timespec { - tv_sec: d.as_secs() as _, - tv_nsec: d.subsec_nanos() as _, + let ts = |time: crate::os::FileTime| libc::timespec { + tv_sec: time.seconds, + tv_nsec: time.nanoseconds as _, }; let times = [ts(access), ts(modified)]; let ret = unsafe { diff --git a/crates/host_env/src/posix_wasi.rs b/crates/host_env/src/posix_wasi.rs index df09c08bc09..7c055f08426 100644 --- a/crates/host_env/src/posix_wasi.rs +++ b/crates/host_env/src/posix_wasi.rs @@ -1,5 +1,5 @@ use alloc::ffi::CString; -use core::{ffi::CStr, time::Duration}; +use core::ffi::CStr; use rustix::fd::AsFd; use std::{ffi::OsStr, io, path::Path}; @@ -14,13 +14,13 @@ pub fn stat_fd(fd: crate::crt_fd::Borrowed<'_>) -> io::Result io::Result<()> { - let ts = |d: Duration| libc::timespec { - tv_sec: d.as_secs() as _, - tv_nsec: d.subsec_nanos() as _, + let ts = |time: crate::os::FileTime| libc::timespec { + tv_sec: time.seconds, + tv_nsec: time.nanoseconds as _, }; let times = [ts(access), ts(modified)]; unsafe { diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 6f1dcca1556..b873cb4c251 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -206,9 +206,9 @@ pub(super) mod _os { #[cfg(any(unix, windows))] use crate::utils::ToCString; use crate::{ - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, builtins::{ - PyBytesRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, + PyBytesRef, PyFloat, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, }, class::PyClassDef, common::lock::{OnceCell, PyRwLock}, @@ -225,11 +225,12 @@ pub(super) mod _os { }, vm::VirtualMachine, }; + use core::hint::cold_path; #[cfg(not(windows))] use core::marker::PhantomData; - use core::{hint::cold_path, time::Duration}; use crossbeam_utils::atomic::AtomicCell; use rustpython_common::wtf8::Wtf8Buf; + use rustpython_host_env::os::{FileTime, FileTimeSeconds}; #[cfg(windows)] use rustpython_host_env::{nt as host_nt, windows::ToWideString}; @@ -1838,6 +1839,56 @@ pub(super) mod _os { follow_symlinks: FollowSymlinks, } + fn file_time_seconds(obj: &PyObject, vm: &VirtualMachine) -> PyResult { + obj.try_index(vm) + .and_then(|value| value.try_to_primitive(vm)) + .map_err(|err| { + if err.fast_isinstance(vm.ctx.exceptions.overflow_error) { + vm.new_overflow_error("timestamp out of range for platform time_t") + } else { + err + } + }) + } + + fn file_time(obj: &PyObject, vm: &VirtualMachine) -> PyResult { + if let Some(float) = obj.downcast_ref::() { + let value = float.to_f64(); + if value.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); + } + let mut seconds = value.trunc(); + let mut nanoseconds = (value.fract() * 1_000_000_000.0).floor(); + if nanoseconds < 0.0 { + nanoseconds += 1_000_000_000.0; + seconds -= 1.0; + } + let minimum = FileTimeSeconds::MIN as f64; + // The signed maximum rounds up when represented by f64. + if !(minimum..-minimum).contains(&seconds) { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(FileTime { + seconds: seconds as FileTimeSeconds, + nanoseconds: nanoseconds as libc::c_long, + }) + } else { + let seconds = match file_time_seconds(obj, vm) { + Err(err) if err.fast_isinstance(vm.ctx.exceptions.type_error) => { + return Err(vm.new_type_error(format!( + "argument must be int or float, not {}", + obj.class().fully_qualified_name(vm)? + ))); + } + result => result?, + }; + Ok(FileTime { + seconds, + nanoseconds: 0, + }) + } + } + #[pyfunction] fn utime(args: UtimeArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { let parse_tup = |tup: &Py| -> Option<(PyObjectRef, PyObjectRef)> { @@ -1852,13 +1903,13 @@ pub(super) mod _os { let (a, m) = parse_tup(&t).ok_or_else(|| { vm.new_type_error("utime: 'times' must be either a tuple of two ints or None") })?; - (a.try_into_value(vm)?, m.try_into_value(vm)?) + (file_time(&a, vm)?, file_time(&m, vm)?) } (None, Some(ns)) => { let (a, m) = parse_tup(&ns) .ok_or_else(|| vm.new_type_error("utime: 'ns' must be a tuple of two ints"))?; let ns_in_sec: PyObjectRef = vm.ctx.new_int(1_000_000_000).into(); - let ns_to_dur = |obj: PyObjectRef| { + let ns_to_time = |obj: PyObjectRef| { let divmod = vm._divmod(&obj, &ns_in_sec)?; let (div, rem) = divmod .downcast_ref::() @@ -1870,16 +1921,31 @@ pub(super) mod _os { divmod.class().name() )) })?; - let secs = div.try_index(vm)?.try_to_primitive(vm)?; - let ns = rem.try_index(vm)?.try_to_primitive(vm)?; - Ok(Duration::new(secs, ns)) + let seconds = file_time_seconds(&div, vm)?; + let rem = rem.try_index(vm)?; + let nanoseconds = rem.try_to_primitive::(vm).map_err(|err| { + if err.fast_isinstance(vm.ctx.exceptions.overflow_error) { + vm.new_overflow_error("Python int too large to convert to C long") + } else { + err + } + })?; + Ok(FileTime { + seconds, + nanoseconds, + }) }; - // TODO: do validation to make sure this doesn't.. underflow? - (ns_to_dur(a)?, ns_to_dur(m)?) + (ns_to_time(a)?, ns_to_time(m)?) } (None, None) => { let now = SystemTime::now(); let now = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + let now = FileTime { + seconds: now.as_secs().try_into().map_err(|_| { + vm.new_overflow_error("timestamp out of range for platform time_t") + })?, + nanoseconds: now.subsec_nanos() as libc::c_long, + }; (now, now) } (Some(_), Some(_)) => { @@ -1893,8 +1959,8 @@ pub(super) mod _os { fn utime_impl( path: OsPath, - acc: Duration, - modif: Duration, + acc: FileTime, + modif: FileTime, dir_fd: DirFd<'_, { UTIME_DIR_FD as usize }>, _follow_symlinks: FollowSymlinks, vm: &VirtualMachine, @@ -1934,8 +2000,15 @@ pub(super) mod _os { )); } - crate::host_env::os::set_file_times(&path, acc, modif) - .map_err(|err| OSErrorBuilder::with_filename(&err, path, vm)) + let wide_path = path.to_wide_cstring(vm)?; + host_nt::set_file_times( + &wide_path, + acc.seconds, + acc.nanoseconds.into(), + modif.seconds, + modif.nanoseconds.into(), + ) + .map_err(|err| OSErrorBuilder::with_filename(&err, path, vm)) } } diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index 2c673e3dee3..5cb1c03b46d 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -145,6 +145,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert os.write(fd, CONTENT2) == len(CONTENT2) os.close(fd) + # A valid negative access time must reach the invalid modification time. + for atime in (-1, -0.25): + assert_raises(TypeError, os.utime, fname, (atime, object())) + assert_raises(TypeError, os.utime, fname, ns=(-1, object())) + fd = os.open(fname, os.O_WRONLY | os.O_APPEND) assert os.write(fd, CONTENT3) == len(CONTENT3) os.close(fd) From c0f0680297087e7653c95f5f09ab8912a0ed207e Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 04:41:54 -0700 Subject: [PATCH 082/229] Reject invalid character lengths before index conversion (#9052) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/cformat.rs | 58 +++++++++++++++++------------ extra_tests/snippets/builtin_str.py | 25 +++++++++++++ 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs index c9533835f6c..1b281880c3f 100644 --- a/crates/vm/src/cformat.rs +++ b/crates/vm/src/cformat.rs @@ -142,33 +142,39 @@ fn spec_format_bytes( CFormatType::Character(CCharacterType::Character) => { // CPython parity: bytes `%c` accepts a single byte or any object // with `__index__` in range(256). + let type_error = || { + // A bytes-like argument that is not one byte long is named by + // its length rather than by its type. + let what = if let Some(b) = obj.downcast_ref::() { + format!("a bytes object of length {}", b.as_bytes().len()) + } else if let Some(ba) = obj.downcast_ref::() { + format!("a bytearray object of length {}", ba.borrow_buf().len()) + } else { + obj.class().name().to_string() + }; + vm.new_type_error(format!( + "%c requires an integer in range(256) or a single byte, not {what}" + )) + }; if let Some(b) = obj.downcast_ref::() { if b.as_bytes().len() == 1 { return Ok(spec.format_char(b.as_bytes()[0])); } + return Err(type_error()); } else if let Some(ba) = obj.downcast_ref::() { let buf = ba.borrow_buf(); if buf.len() == 1 { return Ok(spec.format_char(buf[0])); } + drop(buf); + return Err(type_error()); } let int = if let Some(i) = obj.downcast_ref::() { i.to_owned() } else if let Some(int_result) = obj.try_index_opt(vm) { int_result? } else { - // A bytes-like argument that is not one byte long is named by - // its length rather than by its type. - let what = if let Some(b) = obj.downcast_ref::() { - format!("a bytes object of length {}", b.as_bytes().len()) - } else if let Some(ba) = obj.downcast_ref::() { - format!("a bytearray object of length {}", ba.borrow_buf().len()) - } else { - obj.class().name().to_string() - }; - return Err(vm.new_type_error(format!( - "%c requires an integer in range(256) or a single byte, not {what}" - ))); + return Err(type_error()); }; let ch = int .try_to_primitive::(vm) @@ -238,25 +244,29 @@ fn spec_format_string( CFormatType::Character(CCharacterType::Character) => { // CPython parity: `%c` accepts a single-char str or any object with // `__index__` (the latter via PyNumber_Index dispatch). - if let Some(s) = obj.downcast_ref::() - && let Ok(ch) = s.as_wtf8().code_points().exactly_one() - { - return Ok(spec.format_char(ch)); - } - let int = if let Some(i) = obj.downcast_ref::() { - i.to_owned() - } else if let Some(int_result) = obj.try_index_opt(vm) { - int_result? - } else { + let type_error = || { // A string argument that is not one character long is named by // its length rather than by its type. let what = match obj.downcast_ref::() { Some(s) => format!("a string of length {}", s.char_len()), None => obj.class().name().to_string(), }; - return Err(vm.new_type_error(format!( + vm.new_type_error(format!( "%c requires an int or a unicode character, not {what}" - ))); + )) + }; + if let Some(s) = obj.downcast_ref::() { + if let Ok(ch) = s.as_wtf8().code_points().exactly_one() { + return Ok(spec.format_char(ch)); + } + return Err(type_error()); + } + let int = if let Some(i) = obj.downcast_ref::() { + i.to_owned() + } else if let Some(int_result) = obj.try_index_opt(vm) { + int_result? + } else { + return Err(type_error()); }; let ch = int .as_bigint() diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index 9ab9538c585..f54082325db 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -427,6 +427,31 @@ def __int__(self): assert "%x" % PercentInt(3) == "7" assert b"%o" % PercentInt(3) == b"7" + +def test_percent_character_subclasses(): + for base, value, templates in ( + (str, "ab", ("%c",)), + (bytes, b"ab", (b"%c", bytearray(b"%c"))), + (bytearray, b"ab", (b"%c", bytearray(b"%c"))), + ): + + class Indexable(base): + def __index__(self): + raise AssertionError("%c must check the character length first") + + for template in templates: + assert template % Indexable(value[:1]) == template % value[:1] + for text in (value[:0], value): + with assert_raises(TypeError) as expected: + template % base(text) + with assert_raises(TypeError) as actual: + template % Indexable(text) + assert str(actual.exception) == str(expected.exception) + assert template % PercentIndex() == template % 7 + + +test_percent_character_subclasses() + assert ( "%(first)s %(second)s" % {"second": "World!", "first": "Hello,"} == "Hello, World!" ) From 02917b6d9ca73257d081ca9e2af4ecf88d185e7e Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 04:49:08 -0700 Subject: [PATCH 083/229] Preserve reflected addition for sequence subclasses (#9049) * Preserve reflected addition for native sequence subclasses Extract the Python 3.14-compatible sequence-add slot distinction from #8954 without its unconditional numeric-slot clearing. Preserve stateful duplicate-slot behavior, exact descriptor identity, inherited reflected priority and a stable wrapper function address. Extend the existing addition snippet only for the uncovered inherited-native and slot-update cases. Assisted-by: OpenAI Codex:model identifier unavailable * Use stored MRO for sequence descriptor ownership Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/types/slot.rs | 39 ++++++++--- extra_tests/snippets/builtin_str_add.py | 91 +++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 8 deletions(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 31f8bba6bb3..f2b1db1dcbc 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -452,6 +452,11 @@ macro_rules! number_binary_op_wrapper { |a, b, vm| vm.call_special_method(a, identifier!(vm, $name), (b.to_owned(),)) }; } +// binary_op1 needs one function address across slot updates and deletions. +fn number_add_wrapper(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(a, identifier!(vm, __add__), (b.to_owned(),)) +} + macro_rules! number_binary_right_op_wrapper { ($name:ident) => { |a, b, vm| vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(),)) @@ -888,7 +893,7 @@ impl PyType { name: &'static PyStrInterned, ctx: &Context, ) { - use crate::builtins::descriptor::SlotFunc; + use crate::builtins::descriptor::{PyWrapper, SlotFunc}; // Helper macro for main slots macro_rules! update_main_slot { @@ -1250,13 +1255,31 @@ impl PyType { number_binary_right_op_wrapper!(__radd__), NumBinary ) - } else { - update_sub_slot!( - as_number, - add, - number_binary_op_wrapper!(__add__), - NumBinary - ) + } + let native_concat = self + .get_attr(identifier!(ctx, __add__)) + .is_some_and(|attr| { + attr.class().is(ctx.types.wrapper_descriptor_type) + && attr.downcast_ref::().is_some_and(|descr| { + descr.name.as_str() == "__add__" + && self.mro.read().iter().any(|cls| cls.is(descr.typ)) + && matches!(descr.wrapped, SlotFunc::SeqConcat(_)) + }) + }); + if native_concat { + // Native sequence addition stays in sq_concat unless nb_add + // already participates or __radd__ needs its shared dispatcher. + // Like CPython's resolve_slotdups, preserve an existing numeric + // wrapper when an override is replaced or deleted. + let needs_numeric = self.slots.as_number.add.load().is_some() + || self.slots.as_sequence.concat.load().is_none() + || self.get_attr(identifier!(ctx, __radd__)).is_some(); + self.slots + .as_number + .add + .store(needs_numeric.then_some(number_add_wrapper)); + } else if name.as_str() == "__add__" { + update_sub_slot!(as_number, add, number_add_wrapper, NumBinary) } } SlotAccessor::NbInplaceAdd => { diff --git a/extra_tests/snippets/builtin_str_add.py b/extra_tests/snippets/builtin_str_add.py index 075939ae8d2..cafedc6a5e4 100644 --- a/extra_tests/snippets/builtin_str_add.py +++ b/extra_tests/snippets/builtin_str_add.py @@ -86,3 +86,94 @@ class Child(Override): del Override.__add__ for cls in (Override, Child): assert operator.add(cls(left), right) == left + right + + +# Inheriting native concatenation must leave reflected numeric addition first. +for base, left in ( + (str, "a"), + (bytes, b"a"), + (tuple, (1,)), + (list, [1]), + (bytearray, b"a"), +): + + class Native(base): + pass + + class Child(Native): + pass + + class Explicit(base): + __add__ = base.__add__ + + for cls in (Native, Child, Explicit): + for result in ("reflected", NotImplemented): + other = Reflected(result) + if result is NotImplemented: + assert_raises(TypeError, operator.add, cls(left), other) + else: + assert operator.add(cls(left), other) == result + assert other.calls == 1 + + class ReflectedSequence(base): + calls = 0 + + def __radd__(self, other): + self.calls += 1 + return NotImplemented + + other = ReflectedSequence(left) + assert operator.add(Native(left), other) == base(left) + base(left) + assert other.calls == 1 + + class AddOverride(ReflectedSequence): + def __add__(self, other): + return "overridden" + + other = AddOverride(left) + assert operator.add(ReflectedSequence(left), other) == base(left) + base(left) + assert other.calls == 0 + del AddOverride.__add__ + other = AddOverride(left) + assert operator.add(ReflectedSequence(left), other) == base(left) + base(left) + assert other.calls == 0 + + # CPython keeps numeric dispatch after either addition method is overridden, + # including when deletion or replacement exposes the native __add__ again. + Native.__radd__ = lambda self, other: NotImplemented + for change in ( + lambda: None, + lambda: delattr(Native, "__radd__"), + lambda: setattr(Native, "__add__", base.__add__), + lambda: delattr(Native, "__add__"), + ): + change() + for cls in (Native, Child): + other = Reflected("reflected") + assert_raises(TypeError, operator.add, cls(left), other) + assert other.calls == 0 + + +# A differently named descriptor assigned to __add__ needs numeric dispatch. +class InplaceAlias(list): + __add__ = list.__iadd__ + + +other = Reflected("reflected") +assert_raises(TypeError, operator.add, InplaceAlias([1]), other) +assert other.calls == 0 + + +class NativeIntAdd(int): + __add__ = int.__add__ + + def __radd__(self, other): + raise AssertionError("inherited reflected addition took priority") + + +class IntAddOverride(NativeIntAdd): + def __add__(self, other): + return "overridden" + + +assert operator.add(NativeIntAdd(1), IntAddOverride(2)) == 3 From bc495a8b306754f55dd76963cf54b6a6d6c26715 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 04:56:48 -0700 Subject: [PATCH 084/229] Call descriptor __get__ attributes without pre-binding (#9043) Extract the Python 3.14-compatible descriptor slot dispatch from #8954. Call the raw type attribute with descriptor, instance and owner, preserving ownership and recursion handling. Extend the existing Python snippet for staticmethod, classmethod, callable-object and ordinary __get__ replacements. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/types/slot.rs | 20 +++++++++++----- extra_tests/snippets/builtin_object.py | 32 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index f2b1db1dcbc..8a7f750cdb6 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -779,13 +779,21 @@ fn descr_get_wrapper( cls: Option<&PyObject>, vm: &VirtualMachine, ) -> PyResult { - // A descriptor whose `__get__` is the descriptor itself resolves it by - // fetching `__get__` again, and none of that pushes a Python frame. + // slot_tp_descr_get calls the raw type attribute with all three arguments. + // Binding it first would omit self for builtin functions and incorrectly + // invoke the descriptor protocol of static/class/custom __get__ objects. + // Keep both the callable and self owned across user code that deletes them. vm.with_recursion(" while calling a Python object", || { - vm.call_special_method( - zelf, - identifier!(vm, __get__), - (obj.map(PyObject::to_owned), cls.map(PyObject::to_owned)), + let Some(get) = zelf.class().get_attr(identifier!(vm, __get__)) else { + return Ok(zelf.to_owned()); + }; + get.call( + ( + zelf.to_owned(), + obj.map(PyObject::to_owned), + cls.map(PyObject::to_owned), + ), + vm, ) }) } diff --git a/extra_tests/snippets/builtin_object.py b/extra_tests/snippets/builtin_object.py index 0b90c44a320..9d343eafecb 100644 --- a/extra_tests/snippets/builtin_object.py +++ b/extra_tests/snippets/builtin_object.py @@ -161,3 +161,35 @@ def __getattr__(self, name): with assert_raises(ValueError) as caught: RaisingAttributeHook().absent assert caught.exception.__context__ is outer + + +# The descriptor slot calls the raw __get__ attribute without binding it first. +class RawGetDescriptor: + __get__ = staticmethod(lambda *args: args) + + +class RawGetOwner: + value = RawGetDescriptor() + + +raw_descriptor = vars(RawGetOwner)["value"] +raw_instance = RawGetOwner() +assert raw_instance.value == (raw_descriptor, raw_instance, RawGetOwner) +assert RawGetOwner.value == (raw_descriptor, None, RawGetOwner) + + +class CallableGet: + def __get__(self, instance, owner): + raise AssertionError("__get__ must not be bound before the slot call") + + def __call__(self, *args): + return args + + +RawGetDescriptor.__get__ = CallableGet() +assert raw_instance.value == (raw_descriptor, raw_instance, RawGetOwner) +RawGetDescriptor.__get__ = classmethod(lambda *args: args) +with assert_raises(TypeError): + raw_instance.value +RawGetDescriptor.__get__ = lambda self, instance, owner: (self, instance, owner) +assert raw_instance.value == (raw_descriptor, raw_instance, RawGetOwner) From d674ea11e3c95d007b8d9328c4d64c9b77a63c29 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 04:57:07 -0700 Subject: [PATCH 085/229] Drop the legacy abi3-only PyCFunction_Call export (#9059) Follow up on #9055: RustPython targets abi3t, which does not require this legacy binary-compatibility entry point. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/capi/src/methodobject.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/crates/capi/src/methodobject.rs b/crates/capi/src/methodobject.rs index 3b0795deeb3..2e82c5e9daa 100644 --- a/crates/capi/src/methodobject.rs +++ b/crates/capi/src/methodobject.rs @@ -11,16 +11,6 @@ use rustpython_vm::{AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine}; define_py_check!(fn PyCFunction_Check, types.builtin_function_or_method_type); define_py_check!(exact fn PyCFunction_CheckExact, types.builtin_function_or_method_type); -// Removed from the public API in 3.13, but retained in the stable ABI. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn PyCFunction_Call( - callable: *mut PyObject, - args: *mut PyObject, - kwargs: *mut PyObject, -) -> *mut PyObject { - unsafe { crate::abstract_::PyObject_Call(callable, args, kwargs) } -} - #[repr(C)] #[derive(Debug)] pub struct PyMethodDef { From c770c41499baf8ffaa1f664529edb3c11484afea Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 05:26:17 -0700 Subject: [PATCH 086/229] Honor no-site mode before importing site (#9045) Extract the startup import_site guard from #8954. Preserve existing logging and comments, and cover the missing sys.modules behavior under -S in the existing sys snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- extra_tests/snippets/stdlib_sys.py | 11 +++++++++++ src/lib.rs | 3 +-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index 9dba301fb01..ba2fcef4c99 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -149,6 +149,17 @@ def recursive_call(n): assert proc.stdout.rstrip() == "True" assert proc.returncode == 0, proc +# -S must prevent the site import itself, not just its customizations. +subprocess.run( + [ + sys.executable, + "-S", + "-c", + "import sys; assert sys.flags.no_site; assert 'site' not in sys.modules", + ], + check=True, +) + assert sys._getframemodulename() == "__main__", sys._getframemodulename() diff --git a/src/lib.rs b/src/lib.rs index e3798512602..33b3498eda2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -311,8 +311,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { // Import site first, before setting sys.path[0] // This matches CPython's behavior where site.removeduppaths() runs // before sys.path[0] is set, preventing '' from being converted to cwd - let site_result = vm.import("site", 0); - if site_result.is_err() { + if vm.state.config.settings.import_site && vm.import("site", 0).is_err() { warn!( "Failed to import site, consider adding the Lib directory to your RUSTPYTHONPATH \ environment variable", From b63c3b0134cf8c75e2b55cee69dbe599e1314999 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 05:33:20 -0700 Subject: [PATCH 087/229] Reuse exact dictionary hashes when merging call keywords (#9053) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/dict.rs | 4 ++ crates/vm/src/dict_inner.rs | 11 ++++- crates/vm/src/frame.rs | 12 ++--- extra_tests/snippets/syntax_function_args.py | 47 ++++++++++++++++++++ 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index a0de012fc86..a0f58236a32 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -136,6 +136,10 @@ impl PyDict { self.entries.items() } + pub(crate) fn items_with_hashes_vec(&self) -> Vec<(PyObjectRef, PyObjectRef, PyHash)> { + self.entries.items_with_hashes() + } + fn merge_object_with_override( &self, other: PyObjectRef, diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 513b2ca7089..aaa3b9276d1 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -581,7 +581,8 @@ impl Dict { /// Store a key. /// /// `hash` must be computed for `key` or read from [`Self::keys_with_hashes`] - /// or [`Self::next_entry_with_hash`] on a container holding this same key. + /// or [`Self::items_with_hashes`] or [`Self::next_entry_with_hash`] on a + /// container holding this same key. /// A wrong hash lands the entry in a bucket no lookup probes, silently /// losing the key. pub(crate) fn insert( @@ -1182,6 +1183,14 @@ impl Dict { .collect() } + pub(crate) fn items_with_hashes(&self) -> Vec<(PyObjectRef, T, HashValue)> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| (v.key.clone(), v.value.clone(), v.hash))) + .collect() + } + /// Fold stored hashes under the read lock; the callback must not run Python. pub(crate) fn fold_hashes(&self, init: Acc, f: Fold) -> Acc where diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 90ea0d2db61..b56dbc9e96e 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4623,11 +4623,11 @@ impl ExecutingFrame<'_> { let src_dict = source .downcast_ref::() .expect("exact dict must have a PyDict payload"); - // Snapshot under a single read lock so a mutation of `source` - // triggered by `dict.set_item` (e.g. via a target subclass, or - // aliasing) can't be observed mid-iteration. - for (key, value) in src_dict.items_vec() { - if dict.contains_key(&*key, vm) { + // Preserve stored hashes: copying an exact dict must not call + // a key's __hash__ before the callable receives its arguments. + // Snapshot under one lock before any equality callbacks run. + for (key, value, hash) in src_dict.items_with_hashes_vec() { + if dict.contains_known_hash(&key, hash, vm)? { let key_str = key.str(vm)?; return Err(vm.new_type_error(format!( "{} got multiple values for keyword argument '{}'", @@ -4635,7 +4635,7 @@ impl ExecutingFrame<'_> { key_str.as_wtf8() ))); } - dict.set_item(&*key, value, vm)?; + dict.set_item_known_hash(&key, hash, value, vm)?; } return Ok(None); } else { diff --git a/extra_tests/snippets/syntax_function_args.py b/extra_tests/snippets/syntax_function_args.py index 96cbaf37272..6fd39c951e7 100644 --- a/extra_tests/snippets/syntax_function_args.py +++ b/extra_tests/snippets/syntax_function_args.py @@ -213,3 +213,50 @@ def f(a, b, /, c, d, *, e, f): f(1,2,3,4,5,f=6) with assert_raises(TypeError): f(1,2,3,4,5,6) + + +def test_keyword_dict_stored_hashes(): + class Keyword(str): + def __hash__(self): + assert not getattr(self, "hashed", False), "keyword was rehashed" + self.hashed = True + return str.__hash__(self) + + def accept(*, value): + return value + + source = {Keyword("value"): 42} + assert accept(**source) == 42 + with assert_raises(TypeError) as raised: + accept(value=1, **source) + assert str(raised.exception).endswith( + "got multiple values for keyword argument 'value'" + ) + + class NonString: + def __hash__(self): + assert not getattr(self, "hashed", False), "non-string key was rehashed" + self.hashed = True + return 0 + + source = {NonString(): 42} + with assert_raises(TypeError) as raised: + accept(**source) + assert str(raised.exception) == "keywords must be strings" + + failure = RuntimeError("keyword comparison failed") + + class BadEquality(str): + __hash__ = str.__hash__ + + def __eq__(self, other): + raise failure + + left = {BadEquality("value"): 1} + right = {BadEquality("value"): 2} + with assert_raises(RuntimeError) as raised: + accept(**left, **right) + assert raised.exception is failure + + +test_keyword_dict_stored_hashes() From 1889ee627fb6d17ac88a68e3e10b850419b60d89 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 05:44:03 -0700 Subject: [PATCH 088/229] Track live entries in reverse dictionary iterators (#9046) * Track live entries in reverse dictionary iterators Extract the Python 3.14-compatible remaining-count and same-size storage handling from #8954. Preserve the older deferred exhaustion and extra-yield contract, using explicit wrapping arithmetic rather than the Python 3.15 extra-yield error and cursor clamp. Cover deleted-slot hints and mutation after the last yield in the existing Python snippet. Assisted-by: OpenAI Codex:model identifier unavailable * Enable the repaired reverse-dict restoration regression The patched native interpreter unexpectedly passes the existing reverse-dict clear/restore test. Remove only its obsolete expected-failure decorator, preserving the canonical assertions and all other markers. Assisted-by: OpenAI Codex:model identifier unavailable * Preserve reverse iterator invalidation on atomic relocation Assisted-by: OpenAI Codex:model identifier unavailable --- Lib/test/test_dict.py | 1 - crates/vm/src/builtins/dict.rs | 25 ++++++++++++++---------- crates/vm/src/dict_inner.rs | 29 +++++++++++++++++++++------- extra_tests/snippets/builtin_dict.py | 19 ++++++++++++++++++ 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index b0a999c54ef..195466e6658 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1405,7 +1405,6 @@ def __init__(self, x, y): self.assertEqual(list(reversed(A(1, 0).__dict__)), ['x']) self.assertEqual(list(reversed(A(0, 1).__dict__)), ['y']) - @unittest.expectedFailure # TODO: RUSTPYTHON; RuntimeError: dictionary changed size during iteration def test_reversed_dict_after_clear_and_restore(self): d = {} for i in range(1000): diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index a0f58236a32..a98ba4a4356 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -1464,6 +1464,7 @@ macro_rules! dict_view { pub(crate) size: dict_inner::DictSize, /// As in `$iter_name`. changed: PyAtomic, + remaining: PyAtomic, internal: PyMutex>, } @@ -1477,8 +1478,10 @@ macro_rules! dict_view { impl $reverse_iter_name { fn new(dict: PyDictRef) -> Self { let size = dict.size(); - let position = size.entries_size.saturating_sub(1); + // usize::MAX represents the position before the first entry. + let position = size.entries_size.wrapping_sub(1); $reverse_iter_name { + remaining: Radium::new(size.used), size, changed: Radium::new(false), internal: PyMutex::new(PositionIterInternal::new(dict, position)), @@ -1520,8 +1523,10 @@ macro_rules! dict_view { } let internal = self.internal.lock(); match &internal.status { - IterStatus::Active(dict) if dict.size() == self.size => { - internal.rev_length_hint(|_| self.size.entries_size) + IterStatus::Active(dict) + if dict.size().has_same_used_and_relocations(&self.size) => + { + self.remaining.load(Ordering::Relaxed) } _ => 0, } @@ -1553,13 +1558,13 @@ macro_rules! dict_view { (Err(mutated()), None) } Ok(Some((found_index, item))) => { - let released = if found_index == 0 { - internal.exhaust() - } else { - internal.position = found_index - 1; - None - }; - (Ok(PyIterReturn::Return(($result_fn)(vm, item))), released) + // CPython 3.14 exposes the remaining count as size_t, + // even after same-size key replacement yields extra entries. + let remaining = zelf.remaining.load(Ordering::Relaxed); + zelf.remaining + .store(remaining.wrapping_sub(1), Ordering::Relaxed); + internal.position = found_index.wrapping_sub(1); + (Ok(PyIterReturn::Return(($result_fn)(vm, item))), None) } Ok(None) => (Ok(PyIterReturn::StopIteration(None)), internal.exhaust()), } diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index aaa3b9276d1..9bbc2550a75 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -184,6 +184,8 @@ impl IndexEntry { struct DictInner { used: usize, filled: usize, + // Only private atomic relocations invalidate same-size reverse iteration. + relocation_epoch: u64, indices: Vec, entries: Vec>>, } @@ -210,7 +212,7 @@ impl Clone for Dict { } // Keep dense copies cheap, but do not carry mostly empty storage into // a new dictionary. pop_back can leave oversized indices without holes. - let inner = if inner.used >= inner.entries.len() - inner.entries.len() / 3 + let mut inner = if inner.used >= inner.entries.len() - inner.entries.len() / 3 && (inner.indices.len() <= 8 || inner.used >= inner.indices.len() / 4) { inner.clone() @@ -218,6 +220,7 @@ impl Clone for Dict { let mut copy = DictInner { used: inner.used, filled: inner.used, + relocation_epoch: 0, indices: Vec::new(), entries: Vec::with_capacity(inner.used), }; @@ -226,6 +229,7 @@ impl Clone for Dict { copy.resize(inner.used * 2); copy }; + inner.relocation_epoch = 0; Self { inner: PyRwLock::new(inner), keys_version: AtomicU32::new(0), @@ -239,6 +243,7 @@ impl Default for Dict { inner: PyRwLock::new(DictInner { used: 0, filled: 0, + relocation_epoch: 0, indices: vec![IndexEntry::FREE; 8], entries: Vec::new(), }), @@ -262,9 +267,16 @@ pub struct DictSize { pub entries_size: usize, pub used: usize, filled: usize, + relocation_epoch: u64, } -/// The dict was resized under an iterator holding an older [`DictSize`]. +impl DictSize { + pub(crate) fn has_same_used_and_relocations(&self, other: &Self) -> bool { + self.used == other.used && self.relocation_epoch == other.relocation_epoch + } +} + +/// The dict changed under an iterator holding an older [`DictSize`]. #[derive(Debug)] pub(crate) struct DictChanged; @@ -387,6 +399,7 @@ impl DictInner { entries_size: self.entries.len(), used: self.used, filled: self.filled, + relocation_epoch: self.relocation_epoch, } } @@ -856,10 +869,12 @@ impl Dict { /// Install an owned table, releasing the old entries after unlocking. pub(crate) fn replace_contents(&self, other: Self) { - let replacement = other.inner.into_inner(); + let mut replacement = other.inner.into_inner(); let _removed = { let mut inner = self.write(); self.invalidate_keys_version(); + // Relocation history belongs to this dict, not the incoming table. + replacement.relocation_epoch = inner.relocation_epoch; core::mem::replace(&mut *inner, replacement) }; } @@ -1082,7 +1097,8 @@ impl Dict { } } - /// [`Self::next_entry_checked`] in reverse. + /// Step backward, validating the live size and private relocation history + /// under the same read guard. Ordinary same-size mutations may change layout. pub(crate) fn prev_entry_checked( &self, mut position: EntryIndex, @@ -1090,7 +1106,7 @@ impl Dict { project: impl FnOnce(&PyObject, &T) -> R, ) -> Result, DictChanged> { let inner = self.read(); - if inner.size() != *old { + if !inner.size().has_same_used_and_relocations(old) { return Err(DictChanged); } loop { @@ -1471,6 +1487,7 @@ impl Dict { // this critical section. inner.entries.reserve(1); self.invalidate_keys_version(); + inner.relocation_epoch = inner.relocation_epoch.wrapping_add(1); let entry = inner.entries[entry_index].take().unwrap(); let new_index = inner.entries.len(); inner.indices[entry.index] = unsafe { @@ -1482,8 +1499,6 @@ impl Dict { if holes >= 2 && holes >= inner.used { // Moves do not increase `filled`, so ordinary insertion's // resize threshold cannot bound their accumulated holes. - // Two holes also ensure this operation changes DictSize, - // allowing existing iterators to detect the relocation. let indices_size = inner.indices.len(); inner.resize(indices_size); } diff --git a/extra_tests/snippets/builtin_dict.py b/extra_tests/snippets/builtin_dict.py index fb84bf48a90..e593b92fd87 100644 --- a/extra_tests/snippets/builtin_dict.py +++ b/extra_tests/snippets/builtin_dict.py @@ -378,6 +378,25 @@ def __eq__(self, other): assert_raises(StopIteration, next, it) assert it.__length_hint__() == 0 +# Reverse hints count live entries, including when storage has deleted slots. +for reverse_iter in ( + reversed, + lambda d: reversed(d.keys()), + lambda d: reversed(d.values()), + lambda d: reversed(d.items()), +): + d = {0: 0, 1: 1, 2: 2, 3: 3} + del d[1] + it = reverse_iter(d) + for remaining in (3, 2, 1): + assert it.__length_hint__() == remaining + next(it) + assert it.__length_hint__() == 0 + # Exhaustion is observed by next(), rather than the final successful yield. + d[4] = 4 + assert_raises(RuntimeError, next, it) + assert it.__length_hint__() == 0 + # Test dictionary unpacking with non-mapping objects # This should raise TypeError for non-mapping objects with assert_raises(TypeError) as cm: From 146bb809bb4c418074f9ac20b0e17a2c8cd1ba88 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 05:52:23 -0700 Subject: [PATCH 089/229] Validate sleep against signed nanosecond time bounds (#9051) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/time.rs | 87 ++++++++++++++++++++++------- extra_tests/snippets/stdlib_time.py | 32 +++++++++++ 2 files changed, 100 insertions(+), 19 deletions(-) diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index 7760b7ed8d2..9ea375e13c0 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -14,10 +14,13 @@ mod decl { #[cfg(any(unix, windows))] use crate::builtins::PyBaseExceptionRef; + #[cfg(any(unix, windows))] + use crate::convert::{ToPyException, ToPyObject}; use crate::{ AsObject, Py, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyStr, PyStrRef, PyTypeRef}, + builtins::{PyFloat, PyStr, PyStrRef, PyTypeRef}, class::PyClassDef, + common::wtf8::{Wtf8Buf, wtf8_concat}, function::{Either, FuncArgs, OptionalArg, OptionalOption}, types::{PyStructSequence, PyStructSequenceData, struct_sequence_new}, }; @@ -26,14 +29,10 @@ mod decl { PyRef, builtins::{PyNamespace, PyUtf8StrRef}, }; - #[cfg(any(unix, windows))] - use crate::{ - common::wtf8::Wtf8Buf, - convert::{ToPyException, ToPyObject}, - }; use core::time::Duration; #[cfg(not(any(unix, windows)))] use jiff::{Timestamp, Zoned, civil::DateTime, tz::TimeZone}; + use num_traits::ToPrimitive; #[cfg(target_os = "wasi")] use rustpython_host_env::time::ClockId; #[cfg(any(unix, windows))] @@ -128,21 +127,71 @@ mod decl { fn sleep(object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { vm.audit("time.sleep", || (object.clone(),))?; - let seconds_type_name = object.class().name().to_owned(); - let dur = object.try_into_value::(vm).map_err(|e| { - if e.class().is(vm.ctx.exceptions.value_error) - && let Some(s) = e.args().as_slice().first().and_then(|arg| arg.str(vm).ok()) - && s.as_bytes() == b"negative duration" - { - return vm.new_value_error("sleep length must be non-negative"); + let overflow = || vm.new_overflow_error("timestamp out of range for C PyTime_t"); + let nanoseconds = if let Some(float) = object.downcast_ref::() { + let seconds = float.to_f64(); + if seconds.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); } - if e.class().is(vm.ctx.exceptions.type_error) { - return vm.new_type_error(format!( - "'{seconds_type_name}' object cannot be interpreted as an integer or float" - )); + // _PyTime_ROUND_TIMEOUT rounds away from zero. + let nanoseconds = seconds * SEC_TO_NS as f64; + let nanoseconds = if nanoseconds >= 0.0 { + nanoseconds.ceil() + } else { + nanoseconds.floor() + }; + let minimum = i64::MIN as f64; + // The signed maximum rounds up to 2**63 as a double. + if !(minimum..-minimum).contains(&nanoseconds) { + return Err(overflow()); } - e - })?; + nanoseconds as i64 + } else { + let seconds = object.try_index(vm).or_else(|e| { + if e.fast_isinstance(vm.ctx.exceptions.overflow_error) { + Err(overflow()) + } else if e.fast_isinstance(vm.ctx.exceptions.type_error) { + // CPython's %T reads stored type metadata, preserving surrogates. + let class = object.class(); + let name = if let Some(heap_type) = &class.heaptype_ext { + let qualname = heap_type.qualname.read().clone(); + let module = if let Some(dict) = class.attributes.as_dict() { + dict.get_item_opt(identifier!(vm, __module__), vm)? + .ok_or_else(|| vm.new_attribute_error("__module__"))? + } else { + class.__module__(vm)? + }; + match module.downcast_ref::() { + Some(module) + if module.as_wtf8() != "builtins" + && module.as_wtf8() != "__main__" => + { + wtf8_concat!(module.as_wtf8(), ".", qualname.as_wtf8()) + } + _ => qualname.as_wtf8().to_owned(), + } + } else { + Wtf8Buf::from(class.slot_name().to_owned()) + }; + Err(vm.new_type_error(wtf8_concat!( + "'", + name, + "' object cannot be interpreted as an integer or float" + ))) + } else { + Err(e) + } + })?; + seconds + .as_bigint() + .to_i64() + .and_then(|seconds| seconds.checked_mul(SEC_TO_NS)) + .ok_or_else(overflow)? + }; + if nanoseconds < 0 { + return Err(vm.new_value_error("sleep length must be non-negative")); + } + let dur = Duration::from_nanos(nanoseconds as u64); #[cfg(unix)] { diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index dd43d3f0b5a..4cdb78631c6 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -1,6 +1,8 @@ import sys import time +from testutils import assert_raises + if sys.platform == "win32": # Record the native runtime before evaluating the range regression below. print("Windows time runtime:", sys.version, sys.implementation, sys.flags) @@ -165,6 +167,36 @@ assert monotonic_elapsed >= 0.01 assert perf_elapsed >= 0.01 +# Sleep validates signed nanoseconds before rejecting negative durations. +for duration in ( + 2**63, + -(2**63) - 1, + 2**34, + -(2**34), + 2**63 / 1_000_000_000, + -1e10, + float("inf"), + -float("inf"), +): + with assert_raises(OverflowError) as error: + time.sleep(duration) + assert str(error.exception) == "timestamp out of range for C PyTime_t" +assert_raises(ValueError, time.sleep, float("nan")) + + +class SleepTypeNamespace: + class Invalid: + pass + + +for module, prefix in (("__main__", ""), ("sleep_test", "sleep_test.")): + SleepTypeNamespace.Invalid.__module__ = module + with assert_raises(TypeError) as error: + time.sleep(SleepTypeNamespace.Invalid()) + assert str(error.exception) == ( + f"'{prefix}SleepTypeNamespace.Invalid' object cannot be interpreted as an integer or float" + ) + # The optional second argument fills the fields that are not part of the # sequence. fields = (2024, 1, 2, 3, 4, 5, 6, 7, 0) From eb162b4b5b388609f7f44b88ecf67a426c62de94 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 06:10:07 -0700 Subject: [PATCH 090/229] Decode strftime locale output through wide characters (#9050) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/host_env/src/time.rs | 39 +++++++++++++++++++++++++++-- extra_tests/snippets/stdlib_time.py | 23 +++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/crates/host_env/src/time.rs b/crates/host_env/src/time.rs index 5cac2ff4887..85427bb9ab0 100644 --- a/crates/host_env/src/time.rs +++ b/crates/host_env/src/time.rs @@ -682,7 +682,7 @@ pub fn mktime_tm_from_parts(parts: MktimeTmParts) -> Result Result { let fmt_c = CString::new(fmt).map_err(|_| CheckedTmError::EmbeddedNul)?; let mut size = 1024usize; @@ -704,7 +704,42 @@ pub fn strftime_ascii(fmt: &str, tm: &libc::tm) -> Result Result { + if fmt.contains('\0') { + return Err(CheckedTmError::EmbeddedNul); + } + let fmt_wide: Vec = fmt + .chars() + .map(|ch| ch as libc::wchar_t) + .chain([0]) + .collect(); + let mut size = 1024usize; + let max_scale = 256usize.saturating_mul(fmt.len().max(1)); + loop { + let mut out = vec![0 as libc::wchar_t; size]; + let written = unsafe { + wcsftime( + out.as_mut_ptr(), + out.len(), + fmt_wide.as_ptr(), + tm as *const libc::tm, + ) + }; + if written > 0 || size >= max_scale { + return Ok(out[..written] + .iter() + .map(|ch| char::from_u32(*ch as u32).unwrap_or(char::REPLACEMENT_CHARACTER)) + .collect()); + } + size = size.saturating_mul(2); + } +} + +#[cfg(any(windows, target_os = "linux", target_os = "macos"))] unsafe extern "C" { fn wcsftime( s: *mut libc::wchar_t, diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index 4cdb78631c6..33b356c23f5 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -96,6 +96,29 @@ # print(s) assert s == "1970-01-01-00-16-40" +if sys.platform in ("linux", "darwin"): + import locale + + old_locale = locale.setlocale(locale.LC_ALL) + try: + # Locale output must be Unicode even when the locale is not UTF-8. + # Keep LC_CTYPE and LC_TIME consistent for wcsftime on macOS. + # These locales are optional on the host running the snippet. + for time_locale in ("fr_FR.ISO8859-1", "fr_FR.UTF-8"): + try: + locale.setlocale(locale.LC_ALL, time_locale) + except locale.Error: + continue + february = (2024, 2, 1, 12, 0, 0, 3, 32, 0) + assert time.strftime("%B", february) == "février" + assert ( + time.strftime("[%B] 🐍 \ud800\0%Y", february) + == "[février] 🐍 \ud800\0" + "2024" + ) + assert time.strftime("%B" * 300, february) == "février" * 300 + finally: + locale.setlocale(locale.LC_ALL, old_locale) + if sys.platform != "wasi": # _strptime depends on time.tzname, which is not available on WASI yet. x2 = time.strptime(s, "%Y-%m-%d-%H-%M-%S") From 9d6bf4d22420709f28882d3de2fc5c679f4baa7c Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 06:17:16 -0700 Subject: [PATCH 091/229] Support directory descriptors in chmod (#9054) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/posix.rs | 75 ++++++++++++++++++++++--------- extra_tests/snippets/stdlib_os.py | 63 ++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 21 deletions(-) diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index a5fe3f4e88a..96bb0066702 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -36,7 +36,7 @@ pub mod module { use crate::{builtins::PyUtf8StrRef, utils::ToCString}; use alloc::ffi::CString; use core::ffi::CStr; - use rustpython_host_env::os::ffi::OsStringExt; + use rustpython_host_env::os::{AT_FDCWD, ffi::OsStringExt}; use rustpython_host_env::posix as host_posix; use std::{ fs, io, @@ -1184,33 +1184,39 @@ pub mod module { fn _chmod( path: OsPath, - dir_fd: DirFd<'_, 0>, + dir_fd: i32, mode: u32, follow_symlinks: bool, vm: &VirtualMachine, ) -> PyResult<()> { - let [] = dir_fd.0; - #[cfg(all( - unix, - not(target_os = "redox"), - not(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd")) - ))] - if !follow_symlinks { + #[cfg(target_os = "redox")] + let _ = dir_fd; + #[cfg(not(target_os = "redox"))] + if dir_fd != AT_FDCWD || !follow_symlinks { let err_path = path.clone(); let c_path = path.into_cstring(vm)?; + let flags = if follow_symlinks { + 0 + } else { + libc::AT_SYMLINK_NOFOLLOW + }; return rustpython_host_env::posix::fchmodat( - libc::AT_FDCWD, + dir_fd, &c_path, mode as libc::mode_t, - libc::AT_SYMLINK_NOFOLLOW, + flags, ) .map_err(|err| { let enotsup = err.raw_os_error() == Some(libc::EOPNOTSUPP) || err.raw_os_error() == Some(libc::ENOTSUP); - if enotsup { - vm.new_not_implemented_error( - "chmod: follow_symlinks unavailable on this platform".to_owned(), - ) + if !follow_symlinks && enotsup { + if dir_fd != AT_FDCWD { + vm.new_value_error("chmod: cannot use dir_fd and follow_symlinks together") + } else { + vm.new_not_implemented_error( + "chmod: follow_symlinks unavailable on this platform".to_owned(), + ) + } } else { OSErrorBuilder::with_filename(&err, err_path, vm) } @@ -1232,6 +1238,31 @@ pub mod module { rustpython_host_env::posix::fchmod(fd, mode).map_err(|err| err.into_pyexception(vm)) } + // Absolute paths ignore dir_fd, so even -1 must reach fchmodat unchanged. + #[cfg(not(target_os = "redox"))] + struct ChmodDirFd(i32); + + #[cfg(not(target_os = "redox"))] + impl TryFromObject for ChmodDirFd { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + warn_if_bool_fd(&obj, vm)?; + let fd = obj.try_index_opt(vm).unwrap_or_else(|| { + Err(vm.new_type_error(format!( + "argument should be integer or None, not {}", + obj.class().name() + ))) + })?; + i32::try_from(fd.as_bigint()).map(Self).map_err(|_| { + let message = if fd.as_bigint().sign() == malachite_bigint::Sign::Minus { + "fd is less than minimum" + } else { + "fd is greater than maximum" + }; + vm.new_overflow_error(message) + }) + } + } + #[cfg(not(target_os = "redox"))] #[derive(FromArgs)] struct ChmodArgs<'fd> { @@ -1239,8 +1270,8 @@ pub mod module { path: OsPathOrFd<'fd>, #[pyarg(any)] mode: u32, - #[pyarg(flatten)] - dir_fd: DirFd<'fd, 0>, + #[pyarg(named, default, py_default = "None")] + dir_fd: Option, // CPython writes the platform expression; on posix it is always true. #[pyarg(named, default = true, py_default = "(os.name != 'nt')")] follow_symlinks: bool, @@ -1255,10 +1286,11 @@ pub mod module { dir_fd, follow_symlinks, } = args; + let dir_fd = dir_fd.map_or(AT_FDCWD, |fd| fd.0); match path { OsPathOrFd::Path(path) => { #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd",))] - if !follow_symlinks && dir_fd == Default::default() { + if !follow_symlinks && dir_fd == AT_FDCWD { return lchmod(LchmodArgs { path, mode }, vm); } _chmod(path, dir_fd, mode, follow_symlinks, vm) @@ -1290,7 +1322,8 @@ pub mod module { dir_fd, follow_symlinks, } = args; - _chmod(path, dir_fd, mode, follow_symlinks, vm) + let [] = dir_fd.0; + _chmod(path, AT_FDCWD, mode, follow_symlinks, vm) } #[cfg(not(target_os = "redox"))] @@ -2194,8 +2227,8 @@ pub mod module { vec![ SupportFunc::new( "chmod", - Some(false), - Some(false), + Some(cfg!(not(target_os = "redox"))), + Some(cfg!(not(target_os = "redox"))), Some(cfg!(any( target_os = "macos", target_os = "freebsd", diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index 5cb1c03b46d..df7c723c967 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -28,6 +28,69 @@ def stat_rejects_negative_file_descriptors(): stat_rejects_negative_file_descriptors() +def chmod_supports_directory_descriptors(): + import errno + import tempfile + + assert os.chmod in os.supports_dir_fd + assert os.chmod in os.supports_fd + with tempfile.TemporaryDirectory() as directory: + filename = os.path.join(directory, "file") + with open(filename, "wb"): + pass + dir_fd = os.open(directory, os.O_RDONLY) + try: + os.chmod("file", 0o640, dir_fd=dir_fd) + assert stat.S_IMODE(os.stat(filename).st_mode) == 0o640 + + # Absolute paths ignore even an invalid directory descriptor. + os.chmod(path=filename, mode=0o600, dir_fd=-1) + assert stat.S_IMODE(os.stat(filename).st_mode) == 0o600 + + class DirectoryDescriptor: + def __index__(self): + return dir_fd + + os.chmod(b"file", 0o640, dir_fd=DirectoryDescriptor()) + assert stat.S_IMODE(os.stat(filename).st_mode) == 0o640 + with assert_raises(OSError) as error: + os.chmod("file", 0o600, dir_fd=-1) + assert error.exception.errno == errno.EBADF + assert error.exception.filename == "file" + for invalid, message in ( + (-(2**31) - 1, "fd is less than minimum"), + (2**31, "fd is greater than maximum"), + ): + with assert_raises(OverflowError) as error: + os.chmod(filename, 0o600, dir_fd=invalid) + assert str(error.exception) == message + + link = os.path.join(directory, "link") + os.symlink("file", link) + os.chmod("link", 0o600, dir_fd=dir_fd) + assert stat.S_IMODE(os.stat(filename).st_mode) == 0o600 + for path, options, error_type, message in ( + (link, {}, NotImplementedError, "follow_symlinks unavailable"), + ( + "link", + {"dir_fd": dir_fd}, + ValueError, + "cannot use dir_fd and follow_symlinks together", + ), + ): + try: + os.chmod(path, 0o640, follow_symlinks=False, **options) + except error_type as error: + assert message in str(error) + assert stat.S_IMODE(os.stat(filename).st_mode) == 0o600 + finally: + os.close(dir_fd) + + +if os.name == "posix" and hasattr(os, "fchmod"): + chmod_supports_directory_descriptors() + + fd = os.open("README.md", os.O_RDONLY) assert fd > 0 From be52c97463c371584fc11aee2facea1f33f6171d Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 08:51:40 -0700 Subject: [PATCH 092/229] Reject oversized pickle lengths with OverflowError (#9040) Preserve the wrapper-pickle regressions merged into main while refreshing the independent length checks. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/pickle.rs | 17 +++++++++++------ extra_tests/snippets/stdlib_pickle.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/stdlib/src/pickle.rs b/crates/stdlib/src/pickle.rs index 031c700fdf9..51427c0c9b0 100644 --- a/crates/stdlib/src/pickle.rs +++ b/crates/stdlib/src/pickle.rs @@ -818,13 +818,18 @@ mod _pickle { .ok() .filter(|n| isize::try_from(*n).is_ok()) .ok_or_else(|| { - new_unpickling_error( - vm, + let message = if what == "FRAME" { + format!( + "FRAME length exceeds system's maximum of {} bytes", + isize::MAX + ) + } else { format!( "{what} exceeds system's maximum size of {} bytes", isize::MAX - ), - ) + ) + }; + vm.new_overflow_error(message) }) } @@ -1090,7 +1095,7 @@ mod _pickle { } Some(BINBYTES8) => { let arr: [u8; 8] = st.read_n(8, vm)?.try_into().unwrap(); - let n = read_size(&arr, vm, "BINBYTES8")?; + let n = read_size(&arr, vm, "BINBYTES")?; let data = st.read_n(n, vm)?.to_vec(); stack.push(vm.ctx.new_bytes(data).into()); } @@ -1129,7 +1134,7 @@ mod _pickle { } Some(BINUNICODE8) => { let arr: [u8; 8] = st.read_n(8, vm)?.try_into().unwrap(); - let n = read_size(&arr, vm, "BINUNICODE8")?; + let n = read_size(&arr, vm, "BINUNICODE")?; let data = st.read_n(n, vm)?.to_vec(); stack.push(utf8_surrogatepass(&data, vm)?); } diff --git a/extra_tests/snippets/stdlib_pickle.py b/extra_tests/snippets/stdlib_pickle.py index 855bd79691e..e1872a24bfa 100644 --- a/extra_tests/snippets/stdlib_pickle.py +++ b/extra_tests/snippets/stdlib_pickle.py @@ -1,5 +1,8 @@ import _pickle import io +import sys + +from testutils import assert_raises # BINPUT leaves holes; MEMOIZE uses the number of present entries. unpickler = _pickle.Unpickler(io.BytesIO(b"\x80\x04]q\x07\x94h\x01\x86.")) @@ -13,3 +16,22 @@ # Slot wrapper descriptors reduce to lookup on their defining class. assert object.__str__.__reduce__() == (getattr, (object, "__str__")) assert _pickle.loads(_pickle.dumps(object.__str__)) is object.__str__ + +# Oversized advertised lengths are rejected before trying to read or allocate. +for opcode, name in ( + (b"\x8e", "BINBYTES"), + (b"\x8d", "BINUNICODE"), + (b"\x95", "FRAME"), +): + data = b"\x80\x04" + opcode + (sys.maxsize + 1).to_bytes(8, "little") + for load in ( + _pickle.loads, + lambda data: _pickle.Unpickler(io.BytesIO(data)).load(), + ): + with assert_raises(OverflowError) as caught: + load(data) + if name == "FRAME": + expected = f"FRAME length exceeds system's maximum of {sys.maxsize} bytes" + else: + expected = f"{name} exceeds system's maximum size of {sys.maxsize} bytes" + assert str(caught.exception) == expected From 29f819ec404d0f85d0a0d0ca29ed53ad876f5342 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 10:01:08 -0700 Subject: [PATCH 093/229] Implement the stable ABI bytes formatting functions (#9060) Assisted-by: OpenAI Codex:model identifier unavailable --- crates/capi/src/bytesobject.rs | 132 ++++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/crates/capi/src/bytesobject.rs b/crates/capi/src/bytesobject.rs index f9aa9eb8f2e..f1ffb7a903a 100644 --- a/crates/capi/src/bytesobject.rs +++ b/crates/capi/src/bytesobject.rs @@ -1,7 +1,7 @@ use crate::object::define_py_check; use crate::util::FfiPtrExt; use crate::{PyObject, pystate::with_vm}; -use core::ffi::{CStr, c_char, c_int}; +use core::ffi::{CStr, VaList, c_char, c_int, c_long, c_uint, c_ulong, c_void}; use rustpython_vm::builtins::PyBytes; define_py_check!(fn PyBytes_Check, types.bytes_type); @@ -38,6 +38,136 @@ pub unsafe extern "C" fn PyBytes_FromString(s: *const c_char) -> *mut PyObject { }) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyBytes_FromFormat(format: *const c_char, args: ...) -> *mut PyObject { + unsafe { PyBytes_FromFormatV(format, args) } +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn PyBytes_FromFormatV( + format: *const c_char, + mut args: VaList<'_>, +) -> *mut PyObject { + with_vm(|vm| { + let format = unsafe { CStr::from_ptr(format) }.to_bytes(); + let mut result = Vec::with_capacity(format.len()); + let mut pos = 0; + while pos < format.len() { + if format[pos] != b'%' { + result.push(format[pos]); + pos += 1; + continue; + } + let start = pos; + pos += 1; + // The C API ignores field width and applies precision only to %s. + while format.get(pos).is_some_and(u8::is_ascii_digit) { + pos += 1; + } + let mut precision = 0usize; + if format.get(pos) == Some(&b'.') { + pos += 1; + while let Some(&digit) = format.get(pos).filter(|b| b.is_ascii_digit()) { + precision = precision + .saturating_mul(10) + .saturating_add((digit - b'0').into()); + pos += 1; + } + } + while format + .get(pos) + .is_some_and(|b| *b != b'%' && !b.is_ascii_alphabetic()) + { + pos += 1; + } + let modifier = if matches!(format.get(pos), Some(b'l' | b'z')) + && matches!(format.get(pos + 1), Some(b'd' | b'u')) + { + let modifier = format[pos]; + pos += 1; + Some(modifier) + } else { + None + }; + match format.get(pos) { + Some(b'c') => { + let value = unsafe { args.next_arg::() }; + let value = u8::try_from(value).map_err(|_| { + vm.new_overflow_error( + "PyBytes_FromFormatV(): %c format expects an integer in range [0; 255]", + ) + })?; + result.push(value); + } + Some(b'd') => { + let value = match modifier { + Some(b'l') => unsafe { args.next_arg::() }.to_string(), + Some(b'z') => unsafe { args.next_arg::() }.to_string(), + _ => unsafe { args.next_arg::() }.to_string(), + }; + result.extend_from_slice(value.as_bytes()); + } + Some(b'u') => { + let value = match modifier { + Some(b'l') => unsafe { args.next_arg::() }.to_string(), + Some(b'z') => unsafe { args.next_arg::() }.to_string(), + _ => unsafe { args.next_arg::() }.to_string(), + }; + result.extend_from_slice(value.as_bytes()); + } + Some(b'i') => result + .extend_from_slice(unsafe { args.next_arg::() }.to_string().as_bytes()), + Some(b'x') => result.extend_from_slice( + format!("{:x}", unsafe { args.next_arg::() } as c_uint).as_bytes(), + ), + Some(b's') => { + let string = unsafe { args.next_arg::<*const c_char>() }; + if precision == 0 { + result.extend_from_slice(unsafe { CStr::from_ptr(string) }.to_bytes()); + } else { + // A precision-limited string need not have a terminator + // beyond the specified readable region. + for offset in 0..precision { + let byte = unsafe { *string.add(offset) } as u8; + if byte == 0 { + break; + } + result.push(byte); + } + } + } + Some(b'p') => { + let pointer = unsafe { args.next_arg::<*const c_void>() }; + // Preserve the platform's %p spelling, including Windows + // uppercase digits, but always supply the C API's 0x prefix. + let mut buffer = [0 as c_char; 2 * size_of::() + 16]; + unsafe { + libc::snprintf(buffer.as_mut_ptr(), buffer.len(), c"%p".as_ptr(), pointer); + } + let pointer = unsafe { CStr::from_ptr(buffer.as_ptr()) }.to_bytes(); + result.extend_from_slice(b"0x"); + result.extend_from_slice( + if pointer.starts_with(b"0x") || pointer.starts_with(b"0X") { + &pointer[2..] + } else { + pointer + }, + ); + } + Some(b'%') => result.push(b'%'), + _ => { + // The first unrecognized format copies the remaining + // format verbatim, without consuming any more arguments. + result.extend_from_slice(&format[start..]); + break; + } + } + pos += 1; + } + Ok(vm.ctx.new_bytes(result)) + }) +} + #[unsafe(no_mangle)] pub unsafe extern "C" fn PyBytes_FromObject(obj: *mut PyObject) -> *mut PyObject { with_vm(|vm| { From d23dc631eee68e90444dfabba89a8ee86304ebe8 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 10:55:52 -0700 Subject: [PATCH 094/229] Validate lock timeout range before acquire restrictions (#9061) Preserve CPython signed nanosecond conversion precedence for reentrant and nonblocking acquisition, without changing the platform timeout maximum. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_thread.rs | 5 +++++ extra_tests/snippets/builtin_thread.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index efe4ed30211..a2bab628a71 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -106,6 +106,11 @@ pub(crate) mod _thread { fn validate_acquire_args(args: AcquireArgs, vm: &VirtualMachine) -> PyResult<(bool, f64)> { let timeout = args.timeout.to_secs_f64(); + // CPython checks the signed nanosecond range before acquire-specific restrictions. + let timeout_ns = timeout * 1_000_000_000.0; + if !(i64::MIN as f64..-(i64::MIN as f64)).contains(&timeout_ns) { + return Err(vm.new_overflow_error("timestamp out of range for C PyTime_t")); + } if !args.blocking && timeout != -1.0 { return Err(vm.new_value_error("can't specify a timeout for a non-blocking call")); } diff --git a/extra_tests/snippets/builtin_thread.py b/extra_tests/snippets/builtin_thread.py index 3d12e76cc59..b98a2a42e34 100644 --- a/extra_tests/snippets/builtin_thread.py +++ b/extra_tests/snippets/builtin_thread.py @@ -9,6 +9,16 @@ (ValueError, {"blocking": False, "timeout": 1}), (ValueError, {"timeout": -2}), (OverflowError, {"timeout": _thread.TIMEOUT_MAX + 1}), + (OverflowError, {"blocking": False, "timeout": 2**63 / 1e9}), + (OverflowError, {"blocking": False, "timeout": -9_223_372_037}), + (OverflowError, {"timeout": -9_223_372_037}), + (OverflowError, {"blocking": False, "timeout": float("inf")}), + (OverflowError, {"timeout": float("inf")}), + (OverflowError, {"blocking": False, "timeout": float("-inf")}), + (OverflowError, {"timeout": float("-inf")}), + (ValueError, {"timeout": -(2**63) / 1e9}), + (ValueError, {"blocking": False, "timeout": 4_294_968}), + (ValueError, {"timeout": -4_294_968}), ): lock = _thread.RLock() assert lock.acquire() From 92204385cd69525c9a8d51a68cd777b3c2bee8b9 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 12:26:08 -0700 Subject: [PATCH 095/229] Count the complete Windows locale encoding suffix (#9065) Include modifiers in the 15-byte bound before calling the CRT, matching Python 3.14. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/locale.rs | 4 +--- extra_tests/snippets/stdlib_locale.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 extra_tests/snippets/stdlib_locale.py diff --git a/crates/stdlib/src/locale.rs b/crates/stdlib/src/locale.rs index aba3b22433f..8195302d356 100644 --- a/crates/stdlib/src/locale.rs +++ b/crates/stdlib/src/locale.rs @@ -194,9 +194,7 @@ mod _locale { fn check_locale_name(locale: &str) -> bool { if let Some(dot_pos) = locale.find('.') { let encoding_part = &locale[dot_pos + 1..]; - // Find the end of encoding (could be followed by '@' modifier) - let encoding_len = encoding_part.find('@').unwrap_or(encoding_part.len()); - encoding_len <= MAX_CP_LEN + encoding_part.len() <= MAX_CP_LEN } else { true } diff --git a/extra_tests/snippets/stdlib_locale.py b/extra_tests/snippets/stdlib_locale.py new file mode 100644 index 00000000000..3c6edbbbf7c --- /dev/null +++ b/extra_tests/snippets/stdlib_locale.py @@ -0,0 +1,14 @@ +import locale +import sys + +from testutils import assert_raises + +# Windows counts the modifier toward the code-page length before calling the CRT. +if sys.platform == "win32": + for category, name in ( + (locale.LC_CTYPE, "English.1252@" + "x" * 16), + (locale.LC_ALL, "LC_CTYPE=English.1252@" + "x" * 16), + ): + with assert_raises(locale.Error) as caught: + locale.setlocale(category, name) + assert caught.exception.args == ("unsupported locale setting",) From 1ff5bbd920bf0caa84981d140998efe109a4f254 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 12:31:54 -0700 Subject: [PATCH 096/229] Accept the format keyword in Struct initialization (#9064) Keep the existing conversion and initialization path while binding format as positional-or-keyword, matching Python 3.14. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/pystruct.rs | 11 +++++++++-- extra_tests/snippets/stdlib_struct.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs index 2a3c0dfe341..f586c14483e 100644 --- a/crates/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -289,10 +289,17 @@ pub(crate) mod _struct { } } + #[derive(FromArgs)] + struct StructArgs { + #[pyarg(any)] + format: IntoStructFormatBytes, + } + impl Initializer for PyStruct { - type Args = IntoStructFormatBytes; + type Args = StructArgs; - fn init(zelf: &Py, fmt: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + fn init(zelf: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let fmt = args.format; // The format is read before anything is replaced, so a format that // cannot be read leaves the object as it was. let spec = fmt.format_spec(vm)?; diff --git a/extra_tests/snippets/stdlib_struct.py b/extra_tests/snippets/stdlib_struct.py index c80f5511713..0f379df343b 100644 --- a/extra_tests/snippets/stdlib_struct.py +++ b/extra_tests/snippets/stdlib_struct.py @@ -184,3 +184,21 @@ def __index__(self): buffer = bytearray(2) struct.pack_into("h", b">h"): + s = cls(format=fmt) + assert s.format == ">h" + assert s.pack(7) == b"\x00\x07" + s.__init__(format=">hh") + assert s.unpack(b"\x00\x01\x00\x02") == (1, 2) + with assert_raises(struct.error): + s.__init__(format="$") + assert s.format == ">hh" + assert s.pack(1, 2) == b"\x00\x01\x00\x02" From 44858012f430c203f9fbedb96e288268a3543fcf Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 14:05:21 -0700 Subject: [PATCH 097/229] Preserve missing caller modules in eager type aliases (#9073) Only lazy aliases may obtain their module from the evaluation function. Eager aliases without a caller __name__ must not inspect their stored value. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_typing.rs | 4 +++- extra_tests/snippets/stdlib_typing.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/_typing.rs b/crates/vm/src/stdlib/_typing.rs index 863dd8367b2..fad43d8eb4b 100644 --- a/crates/vm/src/stdlib/_typing.rs +++ b/crates/vm/src/stdlib/_typing.rs @@ -321,7 +321,9 @@ pub(crate) mod decl { return module.clone(); } // Fall back to compute_value's __module__ (like PyFunction_GetModule) - if let Ok(module) = self.compute_value.get_attr("__module__", vm) { + if self.is_lazy + && let Ok(module) = self.compute_value.get_attr("__module__", vm) + { return module; } vm.ctx.none() diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py index 4082d683f8d..8ba64844c52 100644 --- a/extra_tests/snippets/stdlib_typing.py +++ b/extra_tests/snippets/stdlib_typing.py @@ -63,3 +63,24 @@ def method(self, value: Union[int, float]) -> Union[str, bytes]: repr(nested) except RecursionError: pass + + +# Eager aliases without a caller module never inspect their aliased value. +alias_module_lookups = [] + + +class AliasModuleValue: + @property + def __module__(self): + alias_module_lookups.append(True) + return "unexpected_value_module" + + +for alias_value in (int, AliasModuleValue()): + alias_namespace = {"value": alias_value} + exec( + "from typing import TypeAliasType; alias = TypeAliasType('Alias', value)", + alias_namespace, + ) + assert alias_namespace["alias"].__module__ is None +assert alias_module_lookups == [] From eb6cefd76509b6cd3103ff2e90deca5072e23050 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 14:17:08 -0700 Subject: [PATCH 098/229] Mark LRU cache wrappers as method descriptors (#9072) Match the Python 3.14 wrapper type flag and enable the existing guarded method-call fast path. Keep the partial vectorcall changes separate. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_functools.rs | 2 +- extra_tests/snippets/stdlib_functools.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/_functools.rs b/crates/vm/src/stdlib/_functools.rs index 4c9725289f7..49292707ad2 100644 --- a/crates/vm/src/stdlib/_functools.rs +++ b/crates/vm/src/stdlib/_functools.rs @@ -842,7 +842,7 @@ mod _functools { #[pyclass( with(Constructor, Callable, GetDescriptor), - flags(HAS_DICT, HAS_WEAKREF) + flags(HAS_DICT, HAS_WEAKREF, METHOD_DESCRIPTOR) )] impl Py { #[pymethod] diff --git a/extra_tests/snippets/stdlib_functools.py b/extra_tests/snippets/stdlib_functools.py index 355c3187616..2d5c8389d3e 100644 --- a/extra_tests/snippets/stdlib_functools.py +++ b/extra_tests/snippets/stdlib_functools.py @@ -1,9 +1,13 @@ import gc import weakref +from _functools import _lru_cache_wrapper from functools import cache, cmp_to_key, lru_cache, partial, reduce from testutils import assert_raises +# Cached wrappers support method-descriptor lookup and binding. +assert _lru_cache_wrapper.__flags__ & (1 << 17) + class Squares: def __init__(self, max): From 1cbae85e2019327f342ff1f36caf9d5c71c2d824 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 14:25:50 -0700 Subject: [PATCH 099/229] Keep coroutine generator expressions on the ordinary call path (#9075) Match Python 3.14 by excluding coroutine-marked generator expressions from builtin call optimization. Reuse the existing cursor-preserving comprehension lookup so nested outer iterable scopes do not select the wrong symbol table; propagate lookup errors. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/codegen/src/compile.rs | 71 ++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 3aea9089de0..95c1ceb8098 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -9395,36 +9395,38 @@ impl<'warnings> Compiler<'warnings> { Ok(()) } - /// The called name of a `name(genexpr)` call, the shape - /// `maybe_optimize_function_call()` reserves a `skip_optimization` label - /// for. An `await` or an `async for` inside the generator does not - /// disqualify it: the inlined loop raises the same `TypeError` that - /// calling the builtin on an async generator would. + /// The called name of a synchronous `name(genexpr)` call, the shape + /// `maybe_optimize_function_call()` reserves a `skip_optimization` label for. fn cpython_genexpr_call_name<'a>( - &self, + &mut self, func: &'a ast::Expr, args: &ast::Arguments, - ) -> Option<&'a str> { + ) -> CompileResult> { let ast::Expr::Name(ast::ExprName { id, .. }) = func else { - return None; + return Ok(None); }; - let [ast::Expr::Generator(ast::ExprGenerator { .. })] = &args.args[..] else { - return None; + let [ast::Expr::Generator(ast::ExprGenerator { generators, .. })] = &args.args[..] else { + return Ok(None); }; if !args.keywords.is_empty() { - return None; + return Ok(None); } - let table = self.current_symbol_table(); - table.sub_tables.get(table.next_sub_table)?; - Some(id.as_str()) + let Some(outermost) = generators.first() else { + return Ok(None); + }; + let (comp_table, _) = self + .lookup_comprehension_symbol_table_after_skipped_nested_scopes_in_expr( + &outermost.iter, + ComprehensionType::Generator, + )?; + if comp_table.is_coroutine { + return Ok(None); + } + Ok(Some(id.as_str())) } - fn detect_builtin_generator_call( - &self, - func: &ast::Expr, - args: &ast::Arguments, - ) -> Option { - match self.cpython_genexpr_call_name(func, args)? { + fn detect_builtin_generator_call(&self, name: &str) -> Option { + match name { "tuple" => Some(BuiltinGeneratorCallKind::Tuple), "all" => Some(BuiltinGeneratorCallKind::All), "any" => Some(BuiltinGeneratorCallKind::Any), @@ -9615,6 +9617,11 @@ impl<'warnings> Compiler<'warnings> { let call_range = self.current_source_range; self.validate_keywords(&args.keywords)?; let uses_ex_call = self.call_uses_ex_call(args); + let genexpr_call_name = if uses_ex_call { + None + } else { + self.cpython_genexpr_call_name(func, args)? + }; // Method call: obj → LOAD_ATTR_METHOD → [method, self_or_null] → args → CALL // Regular call: func → PUSH_NULL → args → CALL @@ -9681,9 +9688,8 @@ impl<'warnings> Compiler<'warnings> { self.emit_load_attr_method(idx); self.compile_method_call_arguments(args, method_call_range, attr_access_range)?; } - } else if let Some(kind) = (!uses_ex_call) - .then(|| self.detect_builtin_generator_call(func, args)) - .flatten() + } else if let Some(kind) = + genexpr_call_name.and_then(|name| self.detect_builtin_generator_call(name)) { let skip_normal_call = self.new_block(); self.check_caller(func)?; @@ -9705,13 +9711,9 @@ impl<'warnings> Compiler<'warnings> { // `skip_normal_call`, even when `maybe_optimize_function_call()` // leaves it untargeted. let skip_normal_call = self.current_code_info().new_instr_sequence_label(); - let genexpr_call_name = (!uses_ex_call) - .then(|| self.cpython_genexpr_call_name(func, args)) - .flatten() - .is_some(); self.check_caller(func)?; self.compile_expression(func)?; - if genexpr_call_name { + if genexpr_call_name.is_some() { // CPython `maybe_optimize_function_call()` creates and uses // `skip_optimization` for every name(genexpr) shape after // loading the function, even when the name is not all/any/tuple. @@ -20608,24 +20610,25 @@ def f(xs): } #[test] - fn builtin_any_async_genexpr_call_is_optimized_like_cpython() { + fn builtin_any_async_genexpr_call_is_not_optimized_like_cpython() { for source in [ "async def f(xs):\n return any(x async for x in xs)\n", "async def f(xs):\n return any(await x for x in xs)\n", + "async def f(xs):\n return any(x async for x in (lambda: xs)())\n", + "async def f(xs):\n return any(await x for x in (y for y in xs))\n", ] { let code = compile_exec(source); let f = find_code(&code, "f").expect("missing function code"); assert!( - has_common_constant(f, bytecode::CommonConstant::BuiltinAny), - "maybe_optimize_function_call() guards any(genexpr) whether or not the \ - generator is a coroutine: {source}" + !has_common_constant(f, bytecode::CommonConstant::BuiltinAny), + "coroutine generator expressions must use the ordinary call path: {source}" ); assert!( - f.instructions + !f.instructions .iter() .any(|unit| matches!(unit.op, Instruction::ForIter { .. })), - "the guarded path inlines the loop: {source}" + "the ordinary call path must not inline an async generator loop: {source}" ); assert!( f.instructions From 8d27253fa2e3af1f6713864536f6fdb2ffd43909 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 17:17:01 -0700 Subject: [PATCH 100/229] Canonicalize nested template format specs in annotations (#9076) Use the existing f-string body unparser for template interpolation format specs, while preserving the outer interpolation source text. Add compile-only Python annotation-constant coverage. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/codegen/src/unparse.rs | 2 +- extra_tests/snippets/syntax_annotations.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 3ada6644cbb..868f1b1bab1 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -814,7 +814,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { if let Some(format_spec) = &interpolation.format_spec { self.p(":")?; - self.unparse_tstring_body(&format_spec.elements)?; + self.unparse_fstring_body(&format_spec.elements)?; } self.p("}") diff --git a/extra_tests/snippets/syntax_annotations.py b/extra_tests/snippets/syntax_annotations.py index 0f78bdc2be1..9b7d0c0ce89 100644 --- a/extra_tests/snippets/syntax_annotations.py +++ b/extra_tests/snippets/syntax_annotations.py @@ -8,3 +8,15 @@ def func(s: str) -> int: # The order of type hints matters for certain functions # e.g. functools.singledispatch assert list(hints.items()) == [('s', str), ('return', int)] + + +# Template format specs use canonical expression text inside future annotations. +for template, expected in ( + ("t'{value:04}'", "t'{value:04}'"), + ("t'{value:{(width)}}'", "t'{value:{width}}'"), + ("t'{(value):{(width)!r}}'", "t'{(value):{width!r}}'"), +): + annotation_code = compile( + "from __future__ import annotations\nx: " + template, "", "exec" + ) + assert expected in annotation_code.co_consts From 4dc50cc4729e029ce374a5375dde20adb369ff69 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 20:35:15 -0700 Subject: [PATCH 101/229] Validate mmap assignments against current storage (#9071) * Preserve ParamSpec bounds as in Python 3.14.8 Extract the backported bound-validation removal and align the CPython reference pin to 3.14.8 so both snippet interpreters exercise the same contract. Assisted-by: OpenAI Codex:model identifier unavailable * Align TextIOWrapper chunk-size deletion with Python 3.14.8 The 3.14.8 reference pin exposes the Argument Clinic deletion guard moved before initialization and attachment checks by CPython 90a1f02b7da0c356d10a23f1faee7dc063f887ed. Preserve assignment checks and align the exact deletion error, including subclass type names. Assisted-by: OpenAI Codex:model identifier unavailable * Validate mmap assignments against current storage Use the existing mapping guard for bounds and writes after argument conversion, including empty assignments. Match the Python 3.14.8 ordering while preserving export guards and ownership APIs. Assisted-by: OpenAI Codex:model identifier unavailable * Align mmap boundary regression with Python 3.14.8 Include the matching upstream GH-158027 test-data correction: use an integer RHS so the unchanged IndexError assertion checks index bounds after conversion. Preserve the runtime assignment and storage guards. Assisted-by: OpenAI Codex:model identifier unavailable --------- Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- .python-version | 2 +- Lib/test/test_mmap.py | 2 +- crates/stdlib/src/mmap.rs | 58 ++++++++++++--------------- crates/vm/src/stdlib/_io.rs | 42 ++++++++++--------- crates/vm/src/stdlib/typevar.rs | 5 +-- extra_tests/snippets/stdlib_io.py | 22 ++++++++-- extra_tests/snippets/stdlib_mmap.py | 38 ++++++++++++++++++ extra_tests/snippets/stdlib_typing.py | 5 +++ 8 files changed, 112 insertions(+), 62 deletions(-) diff --git a/.python-version b/.python-version index a128d5c0d97..9b8fe2fbb85 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.14.7 +3.14.8 diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index bd8744a0ca8..309c0d8c933 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -73,7 +73,7 @@ def test_basic(self): # Shouldn't crash on boundary (Issue #5292) self.assertRaises(IndexError, m.__getitem__, len(m)) - self.assertRaises(IndexError, m.__setitem__, len(m), b'\0') + self.assertRaises(IndexError, m.__setitem__, len(m), 0) # Modify the file's content m[0] = b'3'[0] diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index a242f8f1d29..371a800c477 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -1455,17 +1455,15 @@ mod mmap { value: &PyObject, vm: &VirtualMachine, ) -> PyResult<()> { - let i: usize = i - .wrapped_at(self.__len__()) - .ok_or_else(|| vm.new_index_error("mmap index out of range"))?; - let b = value_from_object(vm, value)?; self.try_writable(vm, |mmap| { + let i = i + .wrapped_at(mmap.len()) + .ok_or_else(|| vm.new_index_error("mmap index out of range"))?; mmap[i] = b; - })?; - - Ok(()) + Ok(()) + })? } fn setitem_by_slice( @@ -1474,43 +1472,37 @@ mod mmap { value: &PyObject, vm: &VirtualMachine, ) -> PyResult<()> { - let (range, step, slice_len) = slice.adjust_indices(self.__len__()); - let bytes = bytes_from_object(vm, value)?; - if bytes.len() != slice_len { - return Err(vm.new_index_error("mmap slice assignment is wrong size")); - } + self.try_writable(vm, |mmap| { + let (range, step, slice_len) = slice.adjust_indices(mmap.len()); - if slice_len == 0 { - // do nothing - Ok(()) - } else if step == 1 { - self.try_writable(vm, |mmap| { + if bytes.len() != slice_len { + return Err(vm.new_index_error("mmap slice assignment is wrong size")); + } + + if slice_len == 0 { + // do nothing + } else if step == 1 { (&mut mmap[range]) .write(&bytes) .map_err(|err| err.to_pyexception(vm))?; - Ok(()) - })? - } else { - let mut bi = 0; // bytes index - if step.is_negative() { - for i in range.rev().step_by(step.unsigned_abs()) { - self.try_writable(vm, |mmap| { - mmap[i] = bytes[bi]; - })?; - bi += 1; - } } else { - for i in range.step_by(step.unsigned_abs()) { - self.try_writable(vm, |mmap| { + let mut bi = 0; // bytes index + if step.is_negative() { + for i in range.rev().step_by(step.unsigned_abs()) { + mmap[i] = bytes[bi]; + bi += 1; + } + } else { + for i in range.step_by(step.unsigned_abs()) { mmap[i] = bytes[bi]; - })?; - bi += 1; + bi += 1; + } } } Ok(()) - } + })? } } diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index d4cd66e1491..14ca013a831 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -3568,6 +3568,17 @@ mod _io { #[pygetset(setter, name = "_CHUNK_SIZE")] fn set_chunksize(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let PySetterValue::Assign(object_value) = value else { + let type_name = self.class().slot_name(); + let mut end = type_name.len().min(100); + while !type_name.is_char_boundary(end) { + end -= 1; + } + return Err(vm.new_attribute_error(format!( + "attribute '_CHUNK_SIZE' of '{}' objects cannot be deleted", + &type_name[..end] + ))); + }; { let textio = self.lock(vm)?; if self.buffer.deref().is_none() { @@ -3576,27 +3587,20 @@ mod _io { drop(textio); } - let chunk_size: isize = match value { - PySetterValue::Assign(object_value) => { - let integer = object_value.try_index(vm)?; + let integer = object_value.try_index(vm)?; - integer.try_to_primitive::(vm).map_err(|_| { - let class = object_value.class(); - let type_name = class.name(); - let mut end = type_name.len().min(200); - while !type_name.is_char_boundary(end) { - end -= 1; - } - vm.new_value_error(format!( - "cannot fit '{}' into an index-sized integer", - &type_name[..end] - )) - })? + let chunk_size = integer.try_to_primitive::(vm).map_err(|_| { + let class = object_value.class(); + let type_name = class.name(); + let mut end = type_name.len().min(200); + while !type_name.is_char_boundary(end) { + end -= 1; } - PySetterValue::Delete => { - return Err(vm.new_attribute_error("cannot delete attribute")); - } - }; + vm.new_value_error(format!( + "cannot fit '{}' into an index-sized integer", + &type_name[..end] + )) + })?; if chunk_size <= 0 { return Err(vm.new_value_error("a strictly positive integer is required")); diff --git a/crates/vm/src/stdlib/typevar.rs b/crates/vm/src/stdlib/typevar.rs index 19ad3562540..3cc4b9b4b66 100644 --- a/crates/vm/src/stdlib/typevar.rs +++ b/crates/vm/src/stdlib/typevar.rs @@ -583,10 +583,7 @@ pub(crate) mod typevar { return Err(vm.new_type_error("ParamSpec() takes at most 1 positional argument")); }; - let bound = kwargs - .swap_remove("bound") - .map(|b| type_check(b, "Bound must be a type.", vm)) - .transpose()?; + let bound = kwargs.swap_remove("bound"); let covariant = kwargs .swap_remove("covariant") .map(|v| v.try_to_bool(vm)) diff --git a/extra_tests/snippets/stdlib_io.py b/extra_tests/snippets/stdlib_io.py index 8346ddbb62d..07b127999ea 100644 --- a/extra_tests/snippets/stdlib_io.py +++ b/extra_tests/snippets/stdlib_io.py @@ -166,10 +166,24 @@ def __index__(self): "underlying buffer has been detached", lambda: setattr(detached_textio, "_CHUNK_SIZE", 16), ) -expect_value_error( - "underlying buffer has been detached", - lambda: delattr(detached_textio, "_CHUNK_SIZE"), -) +for wrapper in (textio, uninitialized_textio, detached_textio): + with assert_raises(AttributeError) as error: + del wrapper._CHUNK_SIZE + assert str(error.exception) == ( + "attribute '_CHUNK_SIZE' of '_io.TextIOWrapper' objects cannot be deleted" + ) + + +for wrapper_name in ("ChunkWrapper", "€" * 34): + wrapper_type = type(wrapper_name, (TextIOWrapper,), {}) + wrapper = wrapper_type(BytesIO()) + with assert_raises(AttributeError) as error: + del wrapper._CHUNK_SIZE + displayed_name = wrapper_name.encode("utf-8")[:100].decode("utf-8", "ignore") + assert str(error.exception) == ( + f"attribute '_CHUNK_SIZE' of '{displayed_name}' objects cannot be deleted" + ) + wrapper.close() long_type_name = "X" * 250 diff --git a/extra_tests/snippets/stdlib_mmap.py b/extra_tests/snippets/stdlib_mmap.py index 2c4efdf58bf..1dc89e29d80 100644 --- a/extra_tests/snippets/stdlib_mmap.py +++ b/extra_tests/snippets/stdlib_mmap.py @@ -77,3 +77,41 @@ with assert_raises(ValueError): mapped.resize(4) assert mapped.closed + + +# Assignment validates current storage after argument conversion, even if empty. +with mmap.mmap(-1, 4) as mapped: + with assert_raises(ValueError): + mapped[4] = 256 + mapped[0:0] = b"" +with assert_raises(ValueError): + mapped[0:0] = b"" + +with mmap.mmap(-1, 4, access=mmap.ACCESS_READ) as mapped: + with assert_raises(TypeError): + mapped[0:0] = b"" + + +class CloseOnIndex: + def __init__(self, mapped): + self.mapped = mapped + + def __index__(self): + self.mapped.close() + return 0 + + +for close_from in ("key", "value", "slice"): + mapped = mmap.mmap(-1, 4) + closer = CloseOnIndex(mapped) + try: + with assert_raises(ValueError): + if close_from == "key": + mapped[closer] = 0 + elif close_from == "value": + mapped[0] = closer + else: + mapped[closer:0] = b"" + assert mapped.closed + finally: + mapped.close() diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py index 8ba64844c52..4bebfae4540 100644 --- a/extra_tests/snippets/stdlib_typing.py +++ b/extra_tests/snippets/stdlib_typing.py @@ -65,6 +65,11 @@ def method(self, value: Union[int, float]) -> Union[str, bytes]: pass +# Python 3.14.8 retains ParamSpec bounds without converting or validating them. +assert ParamSpec("Unbounded").__bound__ is None +for bound in (None, "int", "not valid Python!", (int, str), object()): + assert ParamSpec("BoundSpec", bound=bound).__bound__ is bound + # Eager aliases without a caller module never inspect their aliased value. alias_module_lookups = [] From 06007c81fa99e813cbf4583dab326764ba0ef017 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 10 Oct 2026 20:57:49 -0700 Subject: [PATCH 102/229] Begin the Python 3.15 target and essential-library migration (#8950) * Target Python 3.15 using the final upstream reference Update active version and comparison configuration to Python 3.15.0. Preserve the alpha implementation status, actual bytecode format, and existing libraries pending separate migration. Align generated library-upgrade workflow pins without changing permissions or actions. Assisted-by: OpenAI Codex:model identifier unavailable * Adapt essential snippet contracts to Python 3.15 final Preserve existing coverage while following verified final behavior for script metadata, builtin signatures and loading, AST fixture construction, memoryview diagnostics, mmap search bounds, and struct format errors. The unchanged final comparator exposes the earlier expectations; focused native baseline failures establish the required implementation changes. Assisted-by: OpenAI Codex:model identifier unavailable * Keep final-reference regression paths and remove expired sysconfig argument Preserve reverse multiplication syntax and native exact overflow diagnostics. Accept the two source-proven CPython 3.15 specialization messages only in affected reverse str/bytes/tuple cases. Use an actually out-of-range Windows datetime errno fixture. Remove the sysconfig check_home argument at its existing canonical 3.15 removal test boundary. Assisted-by: OpenAI Codex:model identifier unavailable * Keep README wording unchanged in Python 3.15 transition Assisted-by: OpenAI Codex:model identifier unavailable * Match Python 3.15 repetition specialization and keep universal tests Use exact sequence-type guards and preserve generic fallback diagnostics. Validate both cold and warmed expressions without implementation-specific assertions. Reject stale CPython source checkouts and cross-version library caches. Assisted-by: OpenAI Codex:model identifier unavailable * Limit in-place specialization additions to exact sequence repetition Assisted-by: OpenAI Codex:model identifier unavailable * Keep subclass regression scoped to repetition dispatch Assisted-by: OpenAI Codex:model identifier unavailable * Provision final CPython 3.15 with pinned uv builds Assisted-by: OpenAI Codex:model identifier unavailable * Enforce Python 3.15 AST constructor requirements Reject retired missing-field and arbitrary-keyword forms while preserving defaults and explicit placeholders. Sync only matching final CPython constructor test hunks. Assisted-by: OpenAI Codex:model identifier unavailable * Keep seeded Python ahead of its Windows DLL directory Assisted-by: OpenAI Codex:model identifier unavailable * Keep opcode metadata tied to the implemented bytecode format The interpreter target and reference tests use Python 3.15.0, but the current Rust opcode enum retains the 3.14 bytecode format. Generate metadata from its matching source rather than importing unsupported instruction families. Assisted-by: OpenAI Codex:model identifier unavailable * Align CSV, logarithm and decimal formatting with Python 3.15 Propagate dialect lookup errors except AttributeError, retry integer logarithms after float overflow, and update the decimal conversion diagnostic. Exercise the final reference contracts without implementation-specific branches. Assisted-by: OpenAI Codex:model identifier unavailable * Bind the logarithm index result before borrowing its integer Assisted-by: OpenAI Codex:model identifier unavailable * Align required canonical test expectations with Python 3.15 Take only demonstrated contract changes from CPython v3.15.0, d220409a13b77de7a6c30abe16ea90db01528edc: removed module __cached__, Struct format exception classes, and decimal-format diagnostic. Preserve unrelated tests and expectations. Assisted-by: OpenAI Codex:model identifier unavailable * Track pending frozendict pickle support during the 3.15 transition Keep the two existing frozendict pickle tests running as expected failures until #8951 supplies the builtin. Unexpected success will signal removal of these temporary markers. Assisted-by: OpenAI Codex:model identifier unavailable * Support final Python 3.15 Windows negative timestamps Use FILETIME for pre-epoch conversions and preserve precise Windows timestamp errors. Assisted-by: OpenAI Codex:model identifier unavailable * Derive historical Windows timezone offsets from wall time FILETIME reports historical DST as unknown, so calculate the actual offset rather than selecting standard-time bias. Assisted-by: OpenAI Codex:model identifier unavailable * Identify decimal formatting arguments in Python 3.15 errors Track consumed positional arguments, including star width and precision, and preserve mapping key representations in decimal conversion diagnostics. Assisted-by: OpenAI Codex:model identifier unavailable * Reuse mapping format keys for lookup and error context Assisted-by: OpenAI Codex:model identifier unavailable * Borrow mapping diagnostic keys as Python objects Assisted-by: OpenAI Codex:model identifier unavailable * Avoid empty components in the reference Python library path Assisted-by: OpenAI Codex:model identifier unavailable * Isolate RustPython bytecode caches during the 3.15 transition Assisted-by: OpenAI Codex:model identifier unavailable * Sync final Windows timestamp test expectations Import only the final CPython platform-limit guards and mktime roundtrip expectation required by the existing Windows CI suite. Assisted-by: OpenAI Codex:model identifier unavailable * Restore official Python setup for 3.15 final Use the existing pinned setup-python action now that its official catalog supplies CPython 3.15.0. Remove the temporary standalone provisioning action and its sparse-checkout entry while retaining interpreter outputs and requested versions. Assisted-by: OpenAI Codex:model identifier unavailable * Align merged runtime contracts with Python 3.15 final Restrict mappingproxy ordering, use final codec registry normalization, and prefer index conversion before float conversion for sleep. Retain universal Python assertions and import the matching final codec/time canonical test hunks plus the non-ASCII codec-name deprecation warning. Upstream source: CPython d220409a13b77de7a6c30abe16ea90db01528edc. The existing codec normalization implementation and bytecode contract remain unchanged. Assisted-by: OpenAI Codex:model identifier unavailable * Limit Windows launcher version check to its interpreter Exclude only the CPython py.exe version-identity case from RustPython CI. The installed launcher reports its own final version, while RustPython correctly remains alpha. Preserve the other 45 launcher cases and all RustPython version assertions without modifying Lib. Assisted-by: OpenAI Codex:model identifier unavailable --------- Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- .claude/scripts/setup-env.sh | 18 ++- .github/workflows/ci.yaml | 15 +-- .github/workflows/pyperformance.yaml | 2 +- .github/workflows/update-doc-db.yml | 2 +- .github/workflows/upgrade-pylib.lock.yml | 8 +- .github/workflows/upgrade-pylib.md | 17 ++- .python-version | 2 +- AGENTS.md | 2 +- CONTRIBUTING.md | 2 +- Lib/encodings/__init__.py | 6 + Lib/sysconfig/__init__.py | 13 +- Lib/test/datetimetester.py | 2 + Lib/test/pickletester.py | 4 + Lib/test/test_ast/test_ast.py | 79 +++++------- Lib/test/test_cmd_line_script.py | 3 - Lib/test/test_codecs.py | 35 ++--- Lib/test/test_peepholer.py | 3 +- Lib/test/test_struct.py | 4 +- Lib/test/test_time.py | 7 +- README.md | 2 +- crates/host_env/src/time.rs | 73 ++++++++++- crates/stdlib/src/csv.rs | 5 +- crates/stdlib/src/math.rs | 89 +++++++------ crates/stdlib/src/mmap.rs | 13 +- crates/stdlib/src/pystruct.rs | 31 ++--- crates/vm/build.rs | 4 +- crates/vm/src/builtins/mappingproxy.rs | 6 +- crates/vm/src/builtins/memory.rs | 2 +- crates/vm/src/cformat.rs | 102 ++++++++++----- crates/vm/src/codecs.rs | 26 +++- crates/vm/src/exceptions.rs | 12 ++ crates/vm/src/frame.rs | 129 +++++++++++++++++-- crates/vm/src/stdlib/_ast.rs | 6 +- crates/vm/src/stdlib/_ast/python.rs | 87 ++++++------- crates/vm/src/stdlib/_imp.rs | 46 ++++++- crates/vm/src/stdlib/builtins.rs | 8 +- crates/vm/src/stdlib/sys.rs | 4 +- crates/vm/src/stdlib/time.rs | 82 ++++++------ crates/vm/src/vm/mod.rs | 6 +- extra_tests/snippets/builtin___main__.py | 2 +- extra_tests/snippets/builtin_mappingproxy.py | 18 ++- extra_tests/snippets/builtin_memoryview.py | 2 +- extra_tests/snippets/builtin_signature.py | 4 +- extra_tests/snippets/builtin_str.py | 29 ++++- extra_tests/snippets/operator_arithmetic.py | 85 +++++++++++- extra_tests/snippets/stdlib_ast.py | 77 ++++++++++- extra_tests/snippets/stdlib_codecs_wide.py | 6 +- extra_tests/snippets/stdlib_csv.py | 12 +- extra_tests/snippets/stdlib_datetime.py | 27 ++-- extra_tests/snippets/stdlib_imp.py | 3 +- extra_tests/snippets/stdlib_math.py | 65 +++++++++- extra_tests/snippets/stdlib_mmap.py | 6 +- extra_tests/snippets/stdlib_struct.py | 18 +-- extra_tests/snippets/stdlib_sys.py | 7 + extra_tests/snippets/stdlib_time.py | 62 ++++++++- scripts/libc_posix.py | 2 +- scripts/whats_left.py | 6 +- 57 files changed, 975 insertions(+), 413 deletions(-) diff --git a/.claude/scripts/setup-env.sh b/.claude/scripts/setup-env.sh index f5d28a3f14c..97e38f22976 100755 --- a/.claude/scripts/setup-env.sh +++ b/.claude/scripts/setup-env.sh @@ -16,7 +16,7 @@ if [ "$(printf '%s\n' "3.13" "$CURRENT_PY" | sort -V | head -1)" != "3.13" ]; th echo "Upgrading python3 default to 3.13..." # Find best available Python >= 3.13 TARGET="" - for ver in python3.14 python3.13; do + for ver in python3.15 python3.14 python3.13; do if command -v "$ver" &>/dev/null; then TARGET=$(command -v "$ver") break @@ -38,13 +38,17 @@ else echo "python3 already >= 3.13: $(python3 --version)" fi -# 2. Clone CPython source if not present (needed for scripts/update_lib) -if [ ! -d "cpython" ]; then - echo "Cloning CPython v3.14.3 (shallow)..." - git clone --depth 1 --branch v3.14.3 https://github.com/python/cpython.git cpython - echo "CPython source ready." +# 2. Ensure the CPython source matches the target (needed for scripts/update_lib) +PYTHON_VERSION="v3.15.0" +if [ ! -e "cpython" ]; then + git clone --depth 1 --branch "$PYTHON_VERSION" https://github.com/python/cpython.git cpython +elif [ ! -d "cpython/.git" ] || [ ! -d "cpython/Lib" ] || + [ "$(git -C cpython rev-parse HEAD)" != "$(git -C cpython rev-parse --verify "$PYTHON_VERSION^{commit}" 2>/dev/null)" ] || + [ -n "$(git -C cpython status --porcelain)" ]; then + echo "ERROR: cpython must be a clean checkout of $PYTHON_VERSION. Move the existing directory aside and rerun." >&2 + exit 1 else - echo "CPython source already present." + echo "CPython $PYTHON_VERSION source already present." fi echo "=== Setup complete ===" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2f62d91990f..c48bd99a105 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -502,6 +502,9 @@ jobs: - '-u all' - '--timeout 600' - '--dont-add-python-opts' + # CPython's installed py.exe reports its own version, not RustPython's. + # Keep the remaining launcher integration tests and runtime version checks. + - '--ignore test.test_launcher.TestLauncher.test_version' env_polluting_tests: [] skips: [] timeout: 50 @@ -781,18 +784,14 @@ jobs: - name: prek run run: prek run --show-diff-on-failure --color=always --all-files - - name: Get target CPython version - id: cpython-version - run: | - version=$(cat .python-version) - echo "version=${version}" >> "$GITHUB_OUTPUT" - - - name: Clone CPython + - name: Clone CPython opcode metadata source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: python/cpython path: cpython - ref: "v${{ steps.cpython-version.outputs.version }}" + # RustPython still uses the 3.14 bytecode format; this source is only + # for opcode metadata, independent of the 3.15 runtime target. + ref: "v3.14.7" persist-credentials: false - name: prek run (manual stage) diff --git a/.github/workflows/pyperformance.yaml b/.github/workflows/pyperformance.yaml index 1d304582ce2..05f76266e26 100644 --- a/.github/workflows/pyperformance.yaml +++ b/.github/workflows/pyperformance.yaml @@ -63,7 +63,7 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 id: cpython with: - python-version: "3.14" + python-version: "3.15.0" - name: Restore cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml index 8874c3aeac8..3abdf27de72 100644 --- a/.github/workflows/update-doc-db.yml +++ b/.github/workflows/update-doc-db.yml @@ -8,7 +8,7 @@ on: python-version: description: Target python version to generate doc db for type: string - default: "3.14.7" + default: "3.15.0" base-ref: description: Base branch to create the update branch from type: string diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml index c08d19058e3..6fa651d5035 100644 --- a/.github/workflows/upgrade-pylib.lock.yml +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -24,7 +24,7 @@ # Pick an out-of-sync Python library from the todo list and upgrade it # by running `scripts/update_lib quick`, then open a pull request. # -# frontmatter-hash: 3129480d6628afe028911bb8b31b6bb3b5eb251e395f00ed0677922cd21727cb +# frontmatter-hash: bf5b6c1bf4e49141df567a8ff69cce1fc96faa991e1cc353ed6a77aa7c2ba719 name: "Upgrade Python Library" "on": @@ -44,7 +44,7 @@ run-name: "Upgrade Python Library" env: ISSUE_ID: "6839" - PYTHON_VERSION: v3.14.3 + PYTHON_VERSION: v3.15.0 # Cache configuration from frontmatter was processed and added to the main job steps @@ -109,7 +109,7 @@ jobs: - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: '3.14' + python-version: '3.15' - name: Create gh-aw temp directory run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh # Cache configuration from frontmatter processed below @@ -118,8 +118,6 @@ jobs: with: key: cpython-lib-${{ env.PYTHON_VERSION }} path: cpython - restore-keys: | - cpython-lib- - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} diff --git a/.github/workflows/upgrade-pylib.md b/.github/workflows/upgrade-pylib.md index cd05eb77734..238c0d43bef 100644 --- a/.github/workflows/upgrade-pylib.md +++ b/.github/workflows/upgrade-pylib.md @@ -28,7 +28,7 @@ engine: copilot runtimes: python: - version: "3.14" + version: "3.15" tools: bash: @@ -48,11 +48,9 @@ safe-outputs: cache: key: cpython-lib-${{ env.PYTHON_VERSION }} path: cpython - restore-keys: - - cpython-lib- env: - PYTHON_VERSION: "v3.14.7" + PYTHON_VERSION: "v3.15.0" ISSUE_ID: "6839" --- @@ -65,10 +63,15 @@ You are an automated maintenance agent for RustPython, a Python 3 interpreter wr The CPython source may already be cached. Check if the `cpython` directory exists and has the correct version: ```bash -if [ -d "cpython/Lib" ]; then - echo "CPython cache hit, skipping clone" -else +if [ ! -e "cpython" ]; then git clone --depth 1 --branch "$PYTHON_VERSION" https://github.com/python/cpython.git cpython +elif [ ! -d "cpython/.git" ] || [ ! -d "cpython/Lib" ] || + [ "$(git -C cpython rev-parse HEAD)" != "$(git -C cpython rev-parse --verify "$PYTHON_VERSION^{commit}" 2>/dev/null)" ] || + [ -n "$(git -C cpython status --porcelain)" ]; then + echo "ERROR: cpython must be a clean checkout of $PYTHON_VERSION. Move the existing directory aside and rerun." >&2 + exit 1 +else + echo "CPython $PYTHON_VERSION source already present." fi ``` diff --git a/.python-version b/.python-version index 9b8fe2fbb85..f02113fe87c 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.14.8 +3.15.0 diff --git a/AGENTS.md b/AGENTS.md index ae24d41671d..9aa4cc6777a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This document provides guidelines for AI coding agents (GitHub Copilot, Claude C ## Project Overview -RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0+ compatibility. The project aims to provide: +RustPython is a Python 3 interpreter written in Rust, targeting Python 3.15 with compatibility work in progress. The project aims to provide: - A complete Python-3 environment entirely in Rust (not CPython bindings) - A clean implementation without compatibility hacks diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 637ecf0993a..710bb2cafd3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ RustPython requires the following: stable version: `rustup update stable` - If you do not have Rust installed, use [rustup](https://rustup.rs/) to do so. -- CPython version 3.14 or higher +- CPython version 3.15 or higher - CPython can be installed by your operating system's package manager, from the [Python website](https://www.python.org/downloads/), or using a third-party distribution, such as diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index 8548bbe04ad..18f5da70caf 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -56,6 +56,12 @@ def normalize_encoding(encoding): if isinstance(encoding, bytes): encoding = str(encoding, "ascii") + if not encoding.isascii(): + import warnings + warnings.warn( + "Support for non-ascii encoding names will be removed in 3.17", + DeprecationWarning, stacklevel=2) + chars = [] punct = False for c in encoding: diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 029d5bcfcbe..45f74bc2dad 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -219,18 +219,7 @@ def _safe_realpath(path): if "_PYTHON_PROJECT_BASE" in os.environ: _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) -def is_python_build(check_home=None): - if check_home is not None: - import warnings - warnings.warn( - ( - 'The check_home argument of sysconfig.is_python_build is ' - 'deprecated and its value is ignored. ' - 'It will be removed in Python 3.15.' - ), - DeprecationWarning, - stacklevel=2, - ) +def is_python_build(): for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): return True diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 1b71b71c544..3d3ac2dbc72 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2745,6 +2745,7 @@ def test_timestamp_limits(self): # If that assumption changes, this value can change as well self.assertEqual(max_ts, 253402300799.0) + @unittest.skipIf(sys.platform == "win32", "Windows doesn't support min timestamp") def test_fromtimestamp_limits(self): try: self.theclass.fromtimestamp(-2**32 - 1) @@ -2784,6 +2785,7 @@ def test_fromtimestamp_limits(self): # OverflowError, especially on 32-bit platforms. self.theclass.fromtimestamp(ts) + @unittest.skipIf(sys.platform == "win32", "Windows doesn't support min timestamp") def test_utcfromtimestamp_limits(self): with self.assertWarns(DeprecationWarning): try: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 75e2afa7ae6..41e4087bacb 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -4394,6 +4394,8 @@ def test_fast_save_enter_frozenset(self): def test_fast_save_enter_set(self): self.fast_save_enter(lambda i: set([i])) + # TODO: RUSTPYTHON; frozendict support: https://github.com/RustPython/RustPython/pull/8951 + @unittest.expectedFailure def test_fast_save_enter_frozendict(self): if self.py_version < (3, 15): self.skipTest('need frozendict') @@ -4441,6 +4443,8 @@ def test_deep_nested_struct_set(self): depth=FAST_NESTING_LIMIT+1, compare_equal=False) + # TODO: RUSTPYTHON; frozendict support: https://github.com/RustPython/RustPython/pull/8951 + @unittest.expectedFailure def test_deep_nested_struct_frozendict(self): if self.py_version < (3, 15): self.skipTest('need frozendict') diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index aeaa4872e28..b006e513fe4 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -460,14 +460,9 @@ def test_field_attr_writable(self): self.assertEqual(x._fields, 666) def test_classattrs(self): - with self.assertWarns(DeprecationWarning): - x = ast.Constant() + x = ast.Constant(42) self.assertEqual(x._fields, ('value', 'kind')) - with self.assertRaises(AttributeError): - x.value - - x = ast.Constant(42) self.assertEqual(x.value, 42) with self.assertRaises(AttributeError): @@ -487,11 +482,8 @@ def test_classattrs(self): self.assertRaises(TypeError, ast.Constant, 1, None, 2) self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - # Arbitrary keyword arguments are supported (but deprecated) - with self.assertWarns(DeprecationWarning): - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): + msg = "ast.Constant got multiple values for argument 'value'" + with self.assertRaisesRegex(TypeError, re.escape(msg)): ast.Constant(1, value=2) self.assertEqual(ast.Constant(42).value, 42) @@ -530,23 +522,24 @@ def test_module(self): self.assertEqual(x.body, body) def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed (but deprecated) - with self.assertWarns(DeprecationWarning): - x = ast.BinOp() - self.assertEqual(x._fields, ('left', 'op', 'right')) - - # Random attribute allowed too - x.foobarbaz = 5 - self.assertEqual(x.foobarbaz, 5) + # Zero arguments constructor is not allowed + msg = "ast.BinOp.__init__ missing 3 required positional arguments: 'left', 'op', and 'right'" + self.assertRaisesRegex(TypeError, re.escape(msg), ast.BinOp) n1 = ast.Constant(1) n3 = ast.Constant(3) addop = ast.Add() x = ast.BinOp(n1, addop, n3) + self.assertEqual(x._fields, ('left', 'op', 'right')) self.assertEqual(x.left, n1) self.assertEqual(x.op, addop) self.assertEqual(x.right, n3) + # Arbitrary attributes are allowed + x.foobarbaz = 5 + self.assertEqual(x.foobarbaz, 5) + self.assertEqual(x._fields, ('left', 'op', 'right')) + x = ast.BinOp(1, 2, 3) self.assertEqual(x.left, 1) self.assertEqual(x.op, 2) @@ -570,10 +563,10 @@ def test_nodeclasses(self): self.assertEqual(x.right, 3) self.assertEqual(x.lineno, 0) - # Random kwargs also allowed (but deprecated) - with self.assertWarns(DeprecationWarning): - x = ast.BinOp(1, 2, 3, foobarbaz=42) - self.assertEqual(x.foobarbaz, 42) + # Arbitrary keyword arguments are not allowed + msg = "ast.BinOp.__init__ got an unexpected keyword argument 'foobarbaz'" + with self.assertRaisesRegex(TypeError, re.escape(msg)): + ast.BinOp(1, 2, 3, foobarbaz=42) def test_no_fields(self): # this used to fail because Sub._fields was None @@ -3183,11 +3176,10 @@ def test_FunctionDef(self): args = ast.arguments() self.assertEqual(args.args, []) self.assertEqual(args.posonlyargs, []) - with self.assertWarnsRegex(DeprecationWarning, - r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): - node = ast.FunctionDef(args=args) - self.assertNotHasAttr(node, "name") - self.assertEqual(node.decorator_list, []) + msg = "ast.FunctionDef.__init__ missing 1 required positional argument: 'name'" + with self.assertRaisesRegex(TypeError, re.escape(msg)): + ast.FunctionDef(args=args) + node = ast.FunctionDef(name='foo', args=args) self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) @@ -3205,9 +3197,8 @@ def test_expr_context(self): self.assertEqual(name3.id, "x") self.assertIsInstance(name3.ctx, ast.Del) - with self.assertWarnsRegex(DeprecationWarning, - r"Name\.__init__ missing 1 required positional argument: 'id'"): - name3 = ast.Name() + msg = "ast.Name.__init__ missing 1 required positional argument: 'id'" + self.assertRaisesRegex(TypeError, re.escape(msg), ast.Name) def test_custom_subclass_with_no_fields(self): class NoInit(ast.AST): @@ -3246,20 +3237,18 @@ class MyAttrs(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) - with self.assertWarnsRegex(DeprecationWarning, - r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): - obj = MyAttrs(c=3) + msg = "MyAttrs.__init__ got an unexpected keyword argument 'c'" + with self.assertRaisesRegex(TypeError, re.escape(msg)): + MyAttrs(c=3) def test_fields_and_types_no_default(self): class FieldsAndTypesNoDefault(ast.AST): _fields = ('a',) _field_types = {'a': int} - with self.assertWarnsRegex(DeprecationWarning, - r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\."): - obj = FieldsAndTypesNoDefault() - with self.assertRaises(AttributeError): - obj.a + msg = "FieldsAndTypesNoDefault.__init__ missing 1 required positional argument: 'a'" + self.assertRaisesRegex(TypeError, re.escape(msg), FieldsAndTypesNoDefault) + obj = FieldsAndTypesNoDefault(a=1) self.assertEqual(obj.a, 1) @@ -3270,13 +3259,8 @@ class MoreFieldsThanTypes(ast.AST): a: int | None = None b: int | None = None - with self.assertWarnsRegex( - DeprecationWarning, - r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" - ): - obj = MoreFieldsThanTypes() - self.assertIs(obj.a, None) - self.assertIs(obj.b, None) + msg = r"Field 'b' is missing from .*\.MoreFieldsThanTypes\._field_types" + self.assertRaisesRegex(TypeError, msg, MoreFieldsThanTypes) obj = MoreFieldsThanTypes(a=1, b=2) self.assertEqual(obj.a, 1) @@ -3288,8 +3272,7 @@ class BadFields(ast.AST): _field_types = {'a': int} # This should not crash - with self.assertWarnsRegex(DeprecationWarning, r"Field b'\\xff\\xff.*' .*"): - obj = BadFields() + self.assertRaisesRegex(TypeError, r"Field b'\\xff\\xff.*' .*", BadFields) def test_complete_field_types(self): class _AllFieldTypes(ast.AST): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 8dd64afcc00..a2ea0535389 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -44,7 +44,6 @@ def f(): _loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__) print('__loader__==%a' % _loader) print('__file__==%a' % __file__) -print('__cached__==%a' % __cached__) print('__package__==%r' % __package__) # Check PEP 451 details import os.path @@ -58,8 +57,6 @@ def f(): assertEqual(__spec__.parent, __package__) assertIdentical(__spec__.submodule_search_locations, None) assertEqual(__spec__.origin, __file__) - if __spec__.cached is not None: - assertEqual(__spec__.cached, __cached__) # Check the sys module import sys assertIdentical(globals(), sys.modules[__name__].__dict__) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index fd21c7b05b0..bcaffb79c48 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -13,6 +13,7 @@ from test import support from test.support import os_helper +from test.support import warnings_helper try: import _testlimitedcapi @@ -3875,32 +3876,27 @@ def test_rot13_func(self): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" def test_codecs_lookup(self): - FOUND = (1, 2, 3, 4) - NOT_FOUND = (None, None, None, None) def search_function(encoding): - if encoding == "aaa_8": - return FOUND + if encoding.startswith("test."): + return (encoding, 2, 3, 4) else: - return NOT_FOUND + return None codecs.register(search_function) self.addCleanup(codecs.unregister, search_function) - self.assertEqual(FOUND, codecs.lookup('aaa_8')) - self.assertEqual(FOUND, codecs.lookup('AAA-8')) - self.assertEqual(FOUND, codecs.lookup('AAA---8')) - self.assertEqual(FOUND, codecs.lookup('AAA 8')) - self.assertEqual(FOUND, codecs.lookup('aaa\xe9\u20ac-8')) - self.assertEqual(NOT_FOUND, codecs.lookup('AAA.8')) - self.assertEqual(NOT_FOUND, codecs.lookup('AAA...8')) - self.assertEqual(NOT_FOUND, codecs.lookup('BBB-8')) - self.assertEqual(NOT_FOUND, codecs.lookup('BBB.8')) - self.assertEqual(NOT_FOUND, codecs.lookup('a\xe9\u20ac-8')) + self.assertEqual(codecs.lookup('test.aaa_8'), ('test.aaa_8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA-8'), ('test.aaa-8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa-8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA---8'), ('test.aaa---8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa---8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA.8'), ('test.aaa.8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA...8'), ('test.aaa...8', 2, 3, 4)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4)) def test_encodings_normalize_encoding(self): - # encodings.normalize_encoding() ignores non-ASCII characters. normalize = encodings.normalize_encoding self.assertEqual(normalize('utf_8'), 'utf_8') - self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') self.assertEqual(normalize('utf 8'), 'utf_8') # encodings.normalize_encoding() doesn't convert # characters to lower case. @@ -3908,6 +3904,11 @@ def test_encodings_normalize_encoding(self): self.assertEqual(normalize('utf.8'), 'utf.8') self.assertEqual(normalize('utf...8'), 'utf...8') + # Non-ASCII *encoding* is deprecated. + msg = "Support for non-ascii encoding names will be removed in 3.17" + with warnings_helper.check_warnings((msg, DeprecationWarning)): + self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') + class CodecCacheTest(unittest.TestCase): def test_cache_bounded(self): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index d06b9a4b6ef..66af5ae7cf9 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -743,7 +743,8 @@ def test_format_errors(self): eval("'%s%z' % (x,)", {'x': 1234}) with self.assertRaisesRegex(ValueError, 'unsupported format character'): eval("'%s%z' % (x, 5)", {'x': 1234}) - with self.assertRaisesRegex(TypeError, 'a real number is required, not str'): + with self.assertRaisesRegex(TypeError, + 'format argument 1: %d requires a real number, not str'): eval("'%d' % (x,)", {'x': '1234'}) with self.assertRaisesRegex(TypeError, 'an integer is required, not float'): eval("'%x' % (x,)", {'x': 1234.56}) diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index e18bad9c990..13e323de994 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -622,7 +622,7 @@ def test_Struct_reinitialization(self): self.assertEqual(s.pack(1, 2), packed) self.assertEqual(s.unpack(packed), (1, 2)) - with self.assertRaises(UnicodeEncodeError): + with self.assertRaises(ValueError): s.__init__('\udc00') self.assertEqual(s.format, '>hh') self.assertEqual(s.pack(1, 2), packed) @@ -882,7 +882,7 @@ def test_operations_on_half_initialized_Struct(self): self.assertRaises(RuntimeError, S.pack_into, spam, 1) self.assertRaises(RuntimeError, S.unpack, spam) self.assertRaises(RuntimeError, S.unpack_from, spam) - self.assertRaises(RuntimeError, getattr, S, 'format') + self.assertRaises(AttributeError, getattr, S, 'format') self.assertRaises(RuntimeError, S.__sizeof__) self.assertRaises(RuntimeError, repr, S) self.assertEqual(S.size, -1) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index f66a37edb4c..298c45faf27 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -170,7 +170,7 @@ def test_sleep_exceptions(self): # Improved exception #81267 with self.assertRaises(TypeError) as errmsg: time.sleep([]) - self.assertIn("integer or float", str(errmsg.exception)) + self.assertIn("real number", str(errmsg.exception)) def test_sleep(self): for value in [-0.0, 0, 0.0, 1e-100, 1e-9, 1e-6, 1, 1.2]: @@ -498,12 +498,13 @@ def test_localtime_without_arg(self): def test_mktime(self): # Issue #1726687 for t in (-2, -1, 0, 1): + t_struct = time.localtime(t) try: - tt = time.localtime(t) + t1 = time.mktime(t_struct) except (OverflowError, OSError): pass else: - self.assertEqual(time.mktime(tt), t) + self.assertEqual(t1, t) # Issue #13309: passing extreme values to mktime() or localtime() # borks the glibc's internal timezone data. diff --git a/README.md b/README.md index 1fc891b7901..c3aaa00c9af 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # [RustPython](https://rustpython.github.io/) -A Python-3 (CPython >= 3.14.0) Interpreter written in Rust :snake: :scream: +A Python-3 (CPython >= 3.15.0) Interpreter written in Rust :snake: :scream: :metal:. [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) diff --git a/crates/host_env/src/time.rs b/crates/host_env/src/time.rs index 85427bb9ab0..94448a8acbb 100644 --- a/crates/host_env/src/time.rs +++ b/crates/host_env/src/time.rs @@ -145,23 +145,84 @@ pub fn current_time_t() -> TimeT { } #[cfg(windows)] -#[cfg_attr(target_env = "musl", allow(deprecated))] -pub fn gmtime_from_timestamp(when: TimeT) -> std::io::Result { +#[derive(Debug)] +pub enum TimestampError { + FiletimeOutOfRange, + Os(std::io::Error), +} + +#[cfg(windows)] +fn windows_filetime(when: TimeT, local: bool) -> Result { + use windows_sys::Win32::{ + Foundation::{FILETIME, SYSTEMTIME}, + System::Time::{FileTimeToSystemTime, SystemTimeToTzSpecificLocalTime}, + }; + + const SECS_BETWEEN_EPOCHS: i64 = 11_644_473_600; + if when < -SECS_BETWEEN_EPOCHS { + return Err(TimestampError::FiletimeOutOfRange); + } + // Only negative timestamps reach this fallback, so the FILETIME cannot overflow. + debug_assert!(when < 0); + let ticks = ((when + SECS_BETWEEN_EPOCHS) as u64) * 10_000_000; + let filetime = FILETIME { + dwLowDateTime: ticks as u32, + dwHighDateTime: (ticks >> 32) as u32, + }; + let mut utc = SYSTEMTIME::default(); + if unsafe { FileTimeToSystemTime(&filetime, &mut utc) } == 0 { + return Err(TimestampError::Os(std::io::Error::last_os_error())); + } + let time = if local { + let mut result = SYSTEMTIME::default(); + // Unlike FileTimeToLocalFileTime, this applies DST rules for the timestamp. + if unsafe { SystemTimeToTzSpecificLocalTime(core::ptr::null(), &utc, &mut result) } == 0 { + return Err(TimestampError::Os(std::io::Error::last_os_error())); + } + result + } else { + utc + }; + const DAYS_BEFORE_MONTH: [i32; 12] = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]; + let year = i32::from(time.wYear); + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let yday = DAYS_BEFORE_MONTH[usize::from(time.wMonth - 1)] + i32::from(time.wDay) - 1 + + i32::from(time.wMonth > 2 && leap); + Ok(libc::tm { + tm_year: year - 1900, + tm_mon: i32::from(time.wMonth) - 1, + tm_mday: i32::from(time.wDay), + tm_hour: i32::from(time.wHour), + tm_min: i32::from(time.wMinute), + tm_sec: i32::from(time.wSecond), + tm_wday: i32::from(time.wDayOfWeek), + tm_yday: yday, + tm_isdst: if local { -1 } else { 0 }, + }) +} + +#[cfg(windows)] +pub fn gmtime_from_timestamp(when: TimeT) -> Result { + if when < 0 { + return windows_filetime(when, false); + } let mut out = core::mem::MaybeUninit::::uninit(); let err = unsafe { _gmtime64_s(out.as_mut_ptr(), &when) }; if err != 0 { - return Err(crate::os::io_error_from_errno(err)); + return Err(TimestampError::Os(crate::os::io_error_from_errno(err))); } Ok(unsafe { out.assume_init() }) } #[cfg(windows)] -#[cfg_attr(target_env = "musl", allow(deprecated))] -pub fn localtime_from_timestamp(when: TimeT) -> std::io::Result { +pub fn localtime_from_timestamp(when: TimeT) -> Result { + if when < 0 { + return windows_filetime(when, true); + } let mut out = core::mem::MaybeUninit::::uninit(); let err = unsafe { _localtime64_s(out.as_mut_ptr(), &when) }; if err != 0 { - return Err(crate::os::io_error_from_errno(err)); + return Err(TimestampError::Os(crate::os::io_error_from_errno(err))); } Ok(unsafe { out.assume_init() }) } diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 51bba58c91e..792546a6dfa 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -473,12 +473,11 @@ mod _csv { } else { obj }; - // CPython ignores attribute lookup errors, but not errors converting - // the retrieved values. Read all non-overridden attributes first. + // Read all non-overridden attributes before converting their values. macro_rules! fill_from_dialect { ($($name:ident),* $(,)?) => { $(if self.$name.is_none() { - self.$name = obj.get_attr(stringify!($name), vm).ok(); + self.$name = vm.get_attribute_opt(&obj, stringify!($name))?; })* }; } diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index 3f5dab6b9b0..97e34efcce0 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -93,24 +93,51 @@ mod math { pymath::math::expm1(x.into_float()).map_err(|err| pymath_exception(err, vm)) } + fn logarithm( + x: &PyObject, + float_log: impl Fn(f64) -> Result, + int_log: impl Fn(&BigInt) -> Result, + vm: &VirtualMachine, + ) -> PyResult { + let integer_log = |i: &PyInt| { + int_log(i.as_bigint()).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input"), + _ => pymath_exception(err, vm), + }) + }; + if let Some(i) = x.downcast_ref::() { + return integer_log(i); + } + let val = match x.try_float(vm) { + Ok(value) => value.to_f64(), + Err(error) => { + if error.fast_isinstance(vm.ctx.exceptions.overflow_error) + && let Some(index) = x.try_index_opt(vm) + { + let index = index?; + return integer_log(&index); + } + return Err(error); + } + }; + float_log(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + #[pyfunction] fn log(x: PyObjectRef, base: OptionalArg, vm: &VirtualMachine) -> PyResult { let natural_log = |x: &PyObject| { - // Handle BigInt specially for large values (only for actual int type, not float) - if let Some(i) = x.downcast_ref::() { - return pymath::math::log_bigint(i.as_bigint(), None).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error("expected a positive input"), - _ => pymath_exception(err, vm), - }); - } - let val = x.try_float(vm)?.to_f64(); - pymath::math::log(val, None).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error(format!( - "expected a positive input, got {}", - super::float_repr(val) - )), - _ => pymath_exception(err, vm), - }) + logarithm( + x, + |value| pymath::math::log(value, None), + |value| pymath::math::log_bigint(value, None), + vm, + ) }; let numerator = natural_log(&x)?; match base { @@ -133,40 +160,12 @@ mod math { #[pyfunction] fn log2(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // Handle BigInt specially for large values (only for actual int type, not float) - if let Some(i) = x.downcast_ref::() { - return pymath::math::log2_bigint(i.as_bigint()).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error("expected a positive input"), - _ => pymath_exception(err, vm), - }); - } - let val = x.try_float(vm)?.to_f64(); - pymath::math::log2(val).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error(format!( - "expected a positive input, got {}", - super::float_repr(val) - )), - _ => pymath_exception(err, vm), - }) + logarithm(&x, pymath::math::log2, pymath::math::log2_bigint, vm) } #[pyfunction] fn log10(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // Handle BigInt specially for large values (only for actual int type, not float) - if let Some(i) = x.downcast_ref::() { - return pymath::math::log10_bigint(i.as_bigint()).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error("expected a positive input"), - _ => pymath_exception(err, vm), - }); - } - let val = x.try_float(vm)?.to_f64(); - pymath::math::log10(val).map_err(|err| match err { - pymath::Error::EDOM => vm.new_value_error(format!( - "expected a positive input, got {}", - super::float_repr(val) - )), - _ => pymath_exception(err, vm), - }) + logarithm(&x, pymath::math::log10, pymath::math::log10_bigint, vm) } #[pyfunction] diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index 371a800c477..790a7bc0fef 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -854,9 +854,16 @@ mod mmap { fn get_find_range(&self, options: &FindOptions) -> (usize, usize) { let size = self.__len__(); - let start = options - .start - .map_or_else(|| self.pos(), |start| start.saturated_at(size)); + let start = options.start.map_or_else( + || self.pos(), + |start| { + if start < 0 { + start.saturated_at(size) + } else { + start as usize + } + }, + ); let end = options.end.map_or(size, |end| end.saturated_at(size)); (start, end) } diff --git a/crates/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs index f586c14483e..d71452f6b4e 100644 --- a/crates/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -30,33 +30,18 @@ pub(crate) mod _struct { impl TryFromObject for IntoStructFormatBytes { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - // CPython turns str to bytes (via str.encode('ascii')) but we keep str. - // The error reporting for non-ASCII input still matches CPython: - // - str input with non-ASCII char: UnicodeEncodeError, the same exception - // str.encode('ascii') would produce. - // - bytes input with non-ASCII byte: struct.error("bad char in struct format"), - // matching CPython where bytes are passed through to the format parser. + // Keep ASCII formats as str and reject non-ASCII str or bytes with + // the same ValueError as CPython's format parser. let fmt = match_class!(match obj { s @ PyStr => { if !s.isascii() { - let start = s - .as_wtf8() - .code_points() - .position(|cp| !cp.to_char().is_some_and(|c| c.is_ascii())) - .unwrap_or(0); - return Err(vm.new_unicode_encode_error( - vm.ctx.new_str("ascii"), - s, - start, - start + 1, - vm.ctx.new_str("ordinal not in range(128)"), - )); + return Err(vm.new_value_error("non-ASCII character in struct format")); } s } b @ PyBytes => { let ascii_str = ascii::AsciiStr::from_ascii(&b) - .map_err(|_| new_struct_error(vm, "bad char in struct format"))?; + .map_err(|_| vm.new_value_error("non-ASCII character in struct format"))?; vm.ctx.new_str(ascii_str) } other => @@ -322,7 +307,13 @@ pub(crate) mod _struct { #[pygetset] fn format(zelf: &Py, vm: &VirtualMachine) -> PyResult { - Ok(zelf.ready(vm)?.format.clone()) + zelf.inner + .read() + .as_ref() + .map(|inner| inner.format.clone()) + .ok_or_else(|| { + vm.new_no_attribute_error(zelf.as_object().to_owned(), vm.ctx.new_str("format")) + }) } // The size an uninitialized `Struct` reports, which no format has diff --git a/crates/vm/build.rs b/crates/vm/build.rs index 867cf577502..098cf05aa12 100644 --- a/crates/vm/build.rs +++ b/crates/vm/build.rs @@ -22,8 +22,8 @@ fn main() { } println!("cargo:rerun-if-changed=../../Lib/importlib/_bootstrap.py"); - // = 3.14.0alpha - python_version(3, 14, 0, "alpha", 0); + // = 3.15.0alpha + python_version(3, 15, 0, "alpha", 0); println!("cargo:rustc-env=RUSTPYTHON_GIT_HASH={}", git_hash()); println!( diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs index 7342fb3c0d9..3e53f1729a5 100644 --- a/crates/vm/src/builtins/mappingproxy.rs +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -193,9 +193,11 @@ impl Py { op.method_name(&vm.ctx).as_str() )) })?; + if !matches!(op, PyComparisonOp::Eq | PyComparisonOp::Ne) { + return Ok(Either::B(PyComparisonValue::NotImplemented)); + } let obj = zelf.to_object(vm)?; - // CPython parity (Objects/descrobject.c::mappingproxy_richcompare): - // delegate to PyObject_RichCompare on the underlying mapping. + // Equality comparisons preserve the underlying mapping's result. obj.rich_compare(other.to_owned(), op, vm).map(Either::A) } diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index 33481ce22a5..6dd4d71fa3a 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -747,7 +747,7 @@ impl PyMemoryView { let format_str = format.as_str(); let Some(dest_char) = Self::native_fmtchar(format_str) else { return Err(vm.new_value_error( - "memoryview: destination format must be a native single character format prefixed with an optional '@'", + "memoryview: destination format must be a native format prefixed with an optional '@'", )); }; // One side has to be bytes. Casting between two item types would diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs index 1b281880c3f..abb845f9d2c 100644 --- a/crates/vm/src/cformat.rs +++ b/crates/vm/src/cformat.rs @@ -26,31 +26,44 @@ use crate::{ stdlib::builtins, }; +enum FormatArgument<'a> { + Scalar, + Positional(usize), + Mapping(&'a PyObject), +} + +impl FormatArgument<'_> { + fn description(&self, vm: &VirtualMachine) -> PyResult { + Ok(match self { + Self::Scalar => "format argument".to_owned(), + Self::Positional(index) => format!("format argument {index}"), + Self::Mapping(key) => format!("format argument {}", key.repr(vm)?), + }) + } +} + fn format_decimal_object( vm: &VirtualMachine, spec: &CFormatSpec, obj: &PyObject, + argument: FormatArgument<'_>, ) -> PyResult { - let type_error = || { - vm.new_type_error(format!( - "%{} format: a real number is required, not {}", - spec.format_type.to_char(), - obj.class().slot_name() - )) - }; // Decimal conversions prefer __int__; __index__ is only a fallback when absent. - let i = obj - .number() - .int(vm) - .or_else(|| obj.try_index_opt(vm)) - .ok_or_else(type_error)? - .map_err(|error| { - if error.fast_isinstance(vm.ctx.exceptions.type_error) { - type_error() - } else { - error - } - })?; + let conversion = obj.number().int(vm).or_else(|| obj.try_index_opt(vm)); + let i = match conversion { + Some(Ok(value)) => value, + Some(Err(error)) if !error.fast_isinstance(vm.ctx.exceptions.type_error) => { + return Err(error); + } + _ => { + return Err(vm.new_type_error(format!( + "{}: %{} requires a real number, not {}", + argument.description(vm)?, + spec.format_type.to_char(), + obj.class().slot_name() + ))); + } + }; check_int_to_str_digits(i.as_bigint(), vm)?; Ok(spec.format_number(i.as_bigint())) } @@ -59,6 +72,7 @@ fn spec_format_bytes( vm: &VirtualMachine, spec: &CFormatSpec, obj: PyObjectRef, + argument: FormatArgument<'_>, ) -> PyResult> { match &spec.format_type { CFormatType::Unsupported { ch, index } => Err(vm.new_value_error(format!( @@ -106,7 +120,7 @@ fn spec_format_bytes( check_int_to_str_digits(&bigint, vm)?; Ok(spec.format_number(&bigint).into_bytes()) } else { - format_decimal_object(vm, spec, &obj).map(String::into_bytes) + format_decimal_object(vm, spec, &obj, argument).map(String::into_bytes) } } _ => { @@ -188,6 +202,7 @@ fn spec_format_string( vm: &VirtualMachine, spec: &CFormatSpec, obj: PyObjectRef, + argument: FormatArgument<'_>, ) -> PyResult { match &spec.format_type { CFormatType::Unsupported { ch, index } => Err(vm.new_value_error(format!( @@ -217,7 +232,7 @@ fn spec_format_string( check_int_to_str_digits(&bigint, vm)?; Ok(spec.format_number(&bigint).into()) } else { - format_decimal_object(vm, spec, &obj).map(Into::into) + format_decimal_object(vm, spec, &obj, argument).map(Into::into) } } _ => { @@ -379,8 +394,14 @@ pub(crate) fn cformat_bytes( CFormatPart::Literal(literal) => result.extend(literal), CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => { let key = mapping_key.unwrap(); - let value = values_obj.get_item(&key, vm)?; - let part_result = spec_format_bytes(vm, &spec, value)?; + let key = vm.ctx.new_bytes(key); + let value = values_obj.get_item(key.as_object(), vm)?; + let part_result = spec_format_bytes( + vm, + &spec, + value, + FormatArgument::Mapping(key.as_object()), + )?; result.extend(part_result); } } @@ -392,7 +413,7 @@ pub(crate) fn cformat_bytes( // tuple let mut slice_iter; let mut once_iter; - let mut value_iter: &mut dyn Iterator = + let value_iter: &mut dyn Iterator = if let Some(tup) = values_obj.downcast_ref::() { slice_iter = tup.as_slice().iter().map(|v| &**v); &mut slice_iter @@ -400,6 +421,9 @@ pub(crate) fn cformat_bytes( once_iter = core::iter::once(values_obj); &mut once_iter }; + let argument_index = core::cell::Cell::new(0); + let mut value_iter = value_iter.inspect(|_| argument_index.set(argument_index.get() + 1)); + let is_tuple = values_obj.downcast_ref::().is_some(); for (_, part) in format { match part { @@ -417,7 +441,12 @@ pub(crate) fn cformat_bytes( return Err(vm.new_type_error("not enough arguments for format string")); }; - let part_result = spec_format_bytes(vm, &spec, value.to_owned())?; + let argument = if is_tuple { + FormatArgument::Positional(argument_index.get()) + } else { + FormatArgument::Scalar + }; + let part_result = spec_format_bytes(vm, &spec, value.to_owned(), argument)?; result.extend(part_result); } } @@ -479,8 +508,15 @@ pub(crate) fn cformat_string( match part { CFormatPart::Literal(literal) => result.push_wtf8(&literal), CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => { - let value = values_obj.get_item(&mapping_key.unwrap(), vm)?; - let part_result = spec_format_string(vm, &spec, value)?; + let key = mapping_key.unwrap(); + let key = vm.ctx.new_str(key); + let value = values_obj.get_item(key.as_object(), vm)?; + let part_result = spec_format_string( + vm, + &spec, + value, + FormatArgument::Mapping(key.as_object()), + )?; result.push_wtf8(&part_result); } } @@ -492,7 +528,7 @@ pub(crate) fn cformat_string( // tuple let mut slice_iter; let mut once_iter; - let mut value_iter: &mut dyn Iterator = + let value_iter: &mut dyn Iterator = if let Some(tup) = values_obj.downcast_ref::() { slice_iter = tup.as_slice().iter().map(|v| &**v); &mut slice_iter @@ -500,6 +536,9 @@ pub(crate) fn cformat_string( once_iter = core::iter::once(values_obj); &mut once_iter }; + let argument_index = core::cell::Cell::new(0); + let mut value_iter = value_iter.inspect(|_| argument_index.set(argument_index.get() + 1)); + let is_tuple = values_obj.downcast_ref::().is_some(); for (_, part) in format { match part { @@ -517,7 +556,12 @@ pub(crate) fn cformat_string( return Err(vm.new_type_error("not enough arguments for format string")); }; - let part_result = spec_format_string(vm, &spec, value.to_owned())?; + let argument = if is_tuple { + FormatArgument::Positional(argument_index.get()) + } else { + FormatArgument::Scalar + }; + let part_result = spec_format_string(vm, &spec, value.to_owned(), argument)?; result.push_wtf8(&part_result); } } diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index f3b6b13342f..32724984e7a 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -253,7 +253,7 @@ impl CodecsRegistry { } pub(crate) fn register_manual(&self, name: &str, codec: PyCodec) { - let name = normalize_encoding_name(name); + let name = normalize_registry_name(name); self.inner .write() .search_cache @@ -262,7 +262,7 @@ impl CodecsRegistry { pub fn lookup(&self, encoding: &str, vm: &VirtualMachine) -> PyResult { let original_encoding = encoding; - let encoding = normalize_encoding_name(encoding); + let encoding = normalize_registry_name(encoding); let search_path = { let inner = self.inner.read(); if let Some(codec) = inner.search_cache.get(encoding.as_ref()) { @@ -308,7 +308,7 @@ impl CodecsRegistry { } pub fn forget(&self, encoding: &str) -> Option { - let encoding = normalize_encoding_name(encoding); + let encoding = normalize_registry_name(encoding); self.inner.write().search_cache.remove(encoding.as_ref()) } @@ -623,6 +623,26 @@ impl FastCodec { } } +fn normalize_registry_name(encoding: &str) -> Cow<'_, str> { + if !encoding + .bytes() + .any(|b| b.is_ascii_uppercase() || b == b' ') + { + return encoding.into(); + } + encoding + .chars() + .map(|c| { + if c == ' ' { + '-' + } else { + c.to_ascii_lowercase() + } + }) + .collect::() + .into() +} + fn normalize_encoding_name(encoding: &str) -> Cow<'_, str> { // _Py_normalize_encoding: collapse non-alphanumeric/non-dot chars into // single underscore, strip non-ASCII, lowercase ASCII letters. diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index dbbdb920422..f7869fccd48 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -1576,6 +1576,18 @@ impl ToPyException for rustpython_host_env::socket::AncillaryPackError { } } +#[cfg(windows)] +impl ToPyException for rustpython_host_env::time::TimestampError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + match self { + Self::FiletimeOutOfRange => { + vm.new_overflow_error("timestamp out of range for Windows FILETIME") + } + Self::Os(error) => error.to_pyexception(vm), + } + } +} + #[cfg(any(unix, windows))] impl ToPyException for rustpython_host_env::time::CheckedTmError { fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index b56dbc9e96e..2c9d6075106 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -6,9 +6,10 @@ use crate::{ AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, PyStackRef, TryFromObject, VirtualMachine, builtins::{ - PyBaseException, PyBaseExceptionRef, PyBaseObject, PyCode, PyCoroutine, PyDict, PyDictRef, - PyFloat, PyFrozenSet, PyGenerator, PyInt, PyInterpolation, PyList, PyModule, PyProperty, - PySet, PySlice, PyStr, PyStrInterned, PyTemplate, PyTraceback, PyType, PyUtf8Str, + PyBaseException, PyBaseExceptionRef, PyBaseObject, PyBytes, PyCode, PyCoroutine, PyDict, + PyDictRef, PyFloat, PyFrozenSet, PyGenerator, PyInt, PyInterpolation, PyList, PyModule, + PyProperty, PySet, PySlice, PyStr, PyStrInterned, PyTemplate, PyTraceback, PyType, + PyUtf8Str, builtin_func::PyNativeFunction, descriptor::{PyMemberDescriptor, PyMethodDescriptor}, frame::stack_analysis, @@ -3098,7 +3099,7 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachi } type BinaryOpExtendGuard = fn(&PyObject, &PyObject, &VirtualMachine) -> bool; -type BinaryOpExtendAction = fn(&PyObject, &PyObject, &VirtualMachine) -> Option; +type BinaryOpExtendAction = fn(&PyObject, &PyObject, &VirtualMachine) -> Option; struct BinaryOpExtendSpecializationDescr { oparg: bytecode::BinaryOperator, @@ -3159,10 +3160,10 @@ struct FusedBoolJump { macro_rules! bitwise_longs_action { ($name:ident, $op:tt) => { #[inline] - fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { + fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { let lhs_val = compact_int_from_obj(lhs, vm)?; let rhs_val = compact_int_from_obj(rhs, vm)?; - Some(vm.ctx.new_int(lhs_val $op rhs_val).into()) + Some(Ok(vm.ctx.new_int(lhs_val $op rhs_val).into())) } }; } @@ -3184,10 +3185,10 @@ fn nonzero_float_compactlong_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualM macro_rules! float_long_action { ($name:ident, $op:tt) => { #[inline] - fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { + fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { let lhs_val = exact_float_from_obj(lhs, vm)?; let rhs_val = compact_int_from_obj(rhs, vm)?; - Some(vm.ctx.new_float(lhs_val $op rhs_val as f64).into()) + Some(Ok(vm.ctx.new_float(lhs_val $op rhs_val as f64).into())) } }; } @@ -3210,10 +3211,10 @@ fn nonzero_compactlong_float_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualM macro_rules! long_float_action { ($name:ident, $op:tt) => { #[inline] - fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { + fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { let lhs_val = compact_int_from_obj(lhs, vm)?; let rhs_val = exact_float_from_obj(rhs, vm)?; - Some(vm.ctx.new_float(lhs_val as f64 $op rhs_val).into()) + Some(Ok(vm.ctx.new_float(lhs_val as f64 $op rhs_val).into())) } }; } @@ -3222,7 +3223,108 @@ long_float_action!(compactlong_float_subtract, -); long_float_action!(compactlong_float_multiply, *); long_float_action!(compactlong_float_true_div, /); +fn sequence_int_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { + (lhs.downcast_ref_if_exact::(vm).is_some() + || lhs.downcast_ref_if_exact::(vm).is_some() + || lhs.downcast_ref_if_exact::(vm).is_some()) + && rhs.downcast_ref_if_exact::(vm).is_some() +} + +fn int_sequence_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { + sequence_int_guard(rhs, lhs, vm) +} + +fn exact_sequence_int_guard( + lhs: &PyObject, + rhs: &PyObject, + vm: &VirtualMachine, +) -> bool { + lhs.downcast_ref_if_exact::(vm).is_some() && rhs.downcast_ref_if_exact::(vm).is_some() +} + +fn exact_int_sequence_guard( + lhs: &PyObject, + rhs: &PyObject, + vm: &VirtualMachine, +) -> bool { + exact_sequence_int_guard::(rhs, lhs, vm) +} + +fn sequence_int_multiply(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { + let count = rhs.downcast_ref_if_exact::(vm)?; + // CPython 3.15's specialized repetition uses PyLong_AsSsize_t, unlike + // the generic sequence dispatch's PyNumber_AsSsize_t conversion. + Some(match count.as_bigint().to_isize() { + Some(count) => lhs.sequence_unchecked().repeat(count, vm), + None => Err(vm.new_overflow_error("Python int too large to convert to C ssize_t")), + }) +} + +fn int_sequence_multiply(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option { + sequence_int_multiply(rhs, lhs, vm) +} + static BINARY_OP_EXTEND_DESCRIPTORS: &[BinaryOpExtendSpecializationDescr] = &[ + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: exact_sequence_int_guard::, + action: sequence_int_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: exact_int_sequence_guard::, + action: int_sequence_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceMultiply, + guard: exact_sequence_int_guard::, + action: sequence_int_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceMultiply, + guard: exact_int_sequence_guard::, + action: int_sequence_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: exact_sequence_int_guard::, + action: sequence_int_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: exact_int_sequence_guard::, + action: int_sequence_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceMultiply, + guard: exact_sequence_int_guard::, + action: sequence_int_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceMultiply, + guard: exact_int_sequence_guard::, + action: int_sequence_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: exact_sequence_int_guard::, + action: sequence_int_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: exact_int_sequence_guard::, + action: int_sequence_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceMultiply, + guard: exact_sequence_int_guard::, + action: sequence_int_multiply, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceMultiply, + guard: exact_int_sequence_guard::, + action: int_sequence_multiply, + }, // long-long arithmetic BinaryOpExtendSpecializationDescr { oparg: bytecode::BinaryOperator::Or, @@ -6522,7 +6624,7 @@ impl ExecutingFrame<'_> { { self.pop_stackref(); self.pop_stackref(); - self.push_value(result); + self.push_value(result?); Ok(None) } else { self.execute_bin_op(vm, op) @@ -10661,6 +10763,11 @@ impl ExecutingFrame<'_> { && b.downcast_ref_if_exact::(vm).is_some() { Some(Instruction::BinaryOpMultiplyFloat) + } else if (sequence_int_guard(a, b, vm) || int_sequence_guard(a, b, vm)) + && let Some(descr) = self.binary_op_extended_specialization(op, a, b, vm) + { + cached_extend_descr = Some(descr); + Some(Instruction::BinaryOpExtend) } else { None } diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index 497dec48b49..6560d7ee1dc 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -1804,7 +1804,7 @@ pub(crate) fn parse>( ) -> Result { let source_file = SourceFileBuilder::new(filename.to_owned(), source.to_owned()).finish(); let mut options = parser::ParseOptions::from(mode); - let target_version = target_version.unwrap_or(ast::PythonVersion::PY314); + let target_version = target_version.unwrap_or(ast::PythonVersion::PY315); if let Some(error) = feature_version_syntax_error(source, &source_file, target_version) { return Err(error.into()); } @@ -2045,7 +2045,7 @@ pub(crate) fn parse_func_type>( let parse_expr = |expr_src: &str| -> Result { let source_file = SourceFileBuilder::new(filename.to_owned(), expr_src.to_owned()).finish(); let options = parser::ParseOptions::from(parser::Mode::Expression) - .with_target_version(target_version.unwrap_or(ast::PythonVersion::PY314)); + .with_target_version(target_version.unwrap_or(ast::PythonVersion::PY315)); let parsed = parser::parse(expr_src, options).map_err(|parse_error| { let range = text_range_to_source_range(&source_file, parse_error.location); CompileError::from(ParseError { @@ -2080,7 +2080,7 @@ pub(crate) fn parse_func_type>( let call_source = format!("{ARG_PREFIX}{inner})"); let source_file = SourceFileBuilder::new(filename.to_owned(), call_source.clone()).finish(); let options = parser::ParseOptions::from(parser::Mode::Expression) - .with_target_version(target_version.unwrap_or(ast::PythonVersion::PY314)); + .with_target_version(target_version.unwrap_or(ast::PythonVersion::PY315)); let parsed = parser::parse(&call_source, options).map_err(|parse_error| { let range = text_range_to_source_range(&source_file, parse_error.location); CompileError::from(ParseError { diff --git a/crates/vm/src/stdlib/_ast/python.rs b/crates/vm/src/stdlib/_ast/python.rs index cb9027aeb99..297ffbc6787 100644 --- a/crates/vm/src/stdlib/_ast/python.rs +++ b/crates/vm/src/stdlib/_ast/python.rs @@ -13,7 +13,6 @@ pub(crate) mod _ast { function::{ArgIterable, FuncArgs, KwArgs, PyMethodDef, PyMethodFlags}, stdlib::_ast::repr, types::{Constructor, Initializer}, - warn, }; #[pyattr] #[pyclass(module = "ast", name = "AST")] @@ -388,9 +387,9 @@ pub(crate) mod _ast { if contains { if !ast_replace_set_discard(&remaining_fields, &key_obj, vm)? { return Err(vm.new_type_error(format!( - "{} got multiple values for argument '{}'", - zelf.class().name(), - key + "{} got multiple values for argument {}", + zelf.class().fully_qualified_name(vm)?, + key_obj.repr(vm)? ))); } } else { @@ -410,19 +409,11 @@ pub(crate) mod _ast { attributes.as_deref().unwrap() }; if !attrs.sequence_unchecked().contains(&key_obj, vm)? { - let message = vm.ctx.new_str(format!( - "{}.__init__ got an unexpected keyword argument '{}'. \ -Support for arbitrary keyword arguments is deprecated and will be removed in Python 3.15.", - zelf.class().name(), - key - )); - warn::warn( - message.into(), - Some(vm.ctx.exceptions.deprecation_warning.to_owned()), - 1, - None, - vm, - )?; + return Err(vm.new_type_error(format!( + "{}.__init__ got an unexpected keyword argument {}", + zelf.class().fully_qualified_name(vm)?, + key_obj.repr(vm)? + ))); } } @@ -438,7 +429,13 @@ Support for arbitrary keyword arguments is deprecated and will be removed in Pyt let expr_ctx_type: PyObjectRef = super::super::pyast::NodeExprContext::make_static_type().into(); - for field in remaining_fields.elements() { + let mut missing = Vec::new(); + // Render missing arguments in their declared positional order. + for i in 0..numfields { + let field = fields_seq.get_item(i as isize, vm)?; + if !ast_replace_set_discard(&remaining_fields, &field, vm)? { + continue; + } if let Some(ftype) = ft_dict.get_item_opt(&*field, vm)? { if ftype.fast_isinstance(vm.ctx.types.union_type) { // Optional field (T | None) — no default @@ -457,39 +454,35 @@ Support for arbitrary keyword arguments is deprecated and will be removed in Pyt }); ast_set_attr(zelf, &field, load_instance, vm)?; } else { - // Required field missing: emit DeprecationWarning. - let field_repr = field.repr(vm)?; - let message = vm.ctx.new_str(format!( - "{}.__init__ missing 1 required positional argument: {}. \ -This will become an error in Python 3.15.", - zelf.class().name(), - field_repr - )); - warn::warn( - message.into(), - Some(vm.ctx.exceptions.deprecation_warning.to_owned()), - 1, - None, - vm, - )?; + missing.push(format!("'{}'", field.str(vm)?)); } } else { - let field_repr = field.repr(vm)?; - let message = vm.ctx.new_str(format!( - "Field {} is missing from {}._field_types. \ -This will become an error in Python 3.15.", - field_repr, - zelf.class().name() - )); - warn::warn( - message.into(), - Some(vm.ctx.exceptions.deprecation_warning.to_owned()), - 1, - None, - vm, - )?; + return Err(vm.new_type_error(format!( + "Field {} is missing from {}._field_types", + field.repr(vm)?, + zelf.class().fully_qualified_name(vm)? + ))); } } + if !missing.is_empty() { + let count = missing.len(); + let names = match missing.as_slice() { + [name] => name.clone(), + [first, last] => format!("{first} and {last}"), + names => format!( + "{}, and {}", + names[..count - 1].join(", "), + names[count - 1] + ), + }; + return Err(vm.new_type_error(format!( + "{}.__init__ missing {} required positional argument{}: {}", + zelf.class().fully_qualified_name(vm)?, + count, + if count == 1 { "" } else { "s" }, + names + ))); + } } Ok(()) diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index 07503ea91d8..d9ee8d1bba7 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -194,7 +194,10 @@ fn find_frozen(name: &str, vm: &VirtualMachine) -> Result PyResult { - let sys_modules = vm.sys_module.get_attr("modules", vm).unwrap(); - let name: PyUtf8StrRef = spec.get_attr("name", vm)?.try_into_value(vm)?; + let name_obj = spec.get_attr("name", vm)?; + let name: PyStrRef = name_obj.downcast().map_err(|obj| { + vm.new_type_error(format!( + "name must be string, not {:.200}", + obj.class().name() + )) + })?; + if name.is_empty() { + return Err(vm.new_value_error("name must not be empty")); + } + // Builtin module names use ASCII even when an unknown name is supplied. + if !name.as_bytes().is_ascii() { + vm.state + .codec_registry + .encode_text(name.clone(), "ascii", None, vm)?; + } + let name_str = name.as_utf8().expect("ASCII module name").as_str(); + if name_str.contains('\0') { + return Err(vm.new_value_error("embedded null character")); + } + if !matches!(name_str, "sys" | "builtins") && !vm.state.module_defs.contains_key(name_str) { + let exc = vm.new_payload_exception::( + vm.ctx.exceptions.module_not_found_error.to_owned(), + vec![ + vm.ctx + .new_str(format!("{name_str} module not found")) + .into(), + ] + .into(), + )?; + exc.as_object().set_attr("name", name.clone(), vm)?; + return Err(exc.upcast()); + } - // Check sys.modules first + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + // Only registered builtins (or the two core modules) may use this cache. if let Ok(module) = sys_modules.get_item(&*name, vm) { return Ok(module); } - let name_str = name.as_str(); if let Some(&def) = vm.state.module_defs.get(name_str) { // Phase 1: Create module (use create slot if provided, else default creation) let module = if let Some(create) = def.slots.create { @@ -274,7 +308,7 @@ mod _imp { module.__init_methods(vm)?; // Add to sys.modules BEFORE exec (critical for circular import handling) - sys_modules.set_item(name.as_pystr(), module.clone().into(), vm)?; + sys_modules.set_item(&*name, module.clone().into(), vm)?; // Phase 2: Call exec slot (can safely import other modules now) if let Some(exec) = def.slots.exec { diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 23e4c7ea7d7..655f30f87d8 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -40,8 +40,8 @@ mod builtins { "can't compile() to bytecode when the `codegen` feature of rustpython is disabled"; #[pyfunction] - fn abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._abs(&x) + fn abs(number: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._abs(&number) } #[pyfunction] @@ -70,8 +70,8 @@ mod builtins { } #[pyfunction] - fn bin(number: ArgIndex) -> String { - let number = number.into_int_ref(); + fn bin(integer: ArgIndex) -> String { + let number = integer.into_int_ref(); let x = number.as_bigint(); if x.is_negative() { format!("-0b{:b}", x.abs()) diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index a77fbb10fd3..28f1078625a 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -667,8 +667,8 @@ pub mod sys { fn implementation(vm: &VirtualMachine) -> PyRef { const NAME: &str = "rustpython"; - // cache tag uses 'cpython' because our compiler is cpython compatible - let cache_tag = format!("cpython-{}{}", version::MAJOR, version::MINOR); + // Keep implementation-specific bytecode caches separate. + let cache_tag = format!("{NAME}-{}{}", version::MAJOR, version::MINOR); let ctx = &vm.ctx; py_namespace!(vm, { "name" => ctx.new_str(NAME), diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index 9ea375e13c0..867891effc0 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -18,9 +18,9 @@ mod decl { use crate::convert::{ToPyException, ToPyObject}; use crate::{ AsObject, Py, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyFloat, PyStr, PyStrRef, PyTypeRef}, + builtins::{PyStr, PyStrRef, PyTypeRef}, class::PyClassDef, - common::wtf8::{Wtf8Buf, wtf8_concat}, + common::wtf8::Wtf8Buf, function::{Either, FuncArgs, OptionalArg, OptionalOption}, types::{PyStructSequence, PyStructSequenceData, struct_sequence_new}, }; @@ -128,8 +128,21 @@ mod decl { vm.audit("time.sleep", || (object.clone(),))?; let overflow = || vm.new_overflow_error("timestamp out of range for C PyTime_t"); - let nanoseconds = if let Some(float) = object.downcast_ref::() { - let seconds = float.to_f64(); + let nanoseconds = if object.number().is_index() { + let seconds = object.try_index(vm).map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.overflow_error) { + overflow() + } else { + e + } + })?; + seconds + .as_bigint() + .to_i64() + .and_then(|seconds| seconds.checked_mul(SEC_TO_NS)) + .ok_or_else(overflow)? + } else { + let seconds = object.try_float(vm)?.to_f64(); if seconds.is_nan() { return Err(vm.new_value_error("Invalid value NaN (not a number)")); } @@ -146,47 +159,6 @@ mod decl { return Err(overflow()); } nanoseconds as i64 - } else { - let seconds = object.try_index(vm).or_else(|e| { - if e.fast_isinstance(vm.ctx.exceptions.overflow_error) { - Err(overflow()) - } else if e.fast_isinstance(vm.ctx.exceptions.type_error) { - // CPython's %T reads stored type metadata, preserving surrogates. - let class = object.class(); - let name = if let Some(heap_type) = &class.heaptype_ext { - let qualname = heap_type.qualname.read().clone(); - let module = if let Some(dict) = class.attributes.as_dict() { - dict.get_item_opt(identifier!(vm, __module__), vm)? - .ok_or_else(|| vm.new_attribute_error("__module__"))? - } else { - class.__module__(vm)? - }; - match module.downcast_ref::() { - Some(module) - if module.as_wtf8() != "builtins" - && module.as_wtf8() != "__main__" => - { - wtf8_concat!(module.as_wtf8(), ".", qualname.as_wtf8()) - } - _ => qualname.as_wtf8().to_owned(), - } - } else { - Wtf8Buf::from(class.slot_name().to_owned()) - }; - Err(vm.new_type_error(wtf8_concat!( - "'", - name, - "' object cannot be interpreted as an integer or float" - ))) - } else { - Err(e) - } - })?; - seconds - .as_bigint() - .to_i64() - .and_then(|seconds| seconds.checked_mul(SEC_TO_NS)) - .ok_or_else(overflow)? }; if nanoseconds < 0 { return Err(vm.new_value_error("sleep length must be non-negative")); @@ -1353,7 +1325,25 @@ mod platform { (info.standard_bias, &info.standard_name) }; - let gmtoff = -(info.bias + bias) * 60; + let gmtoff = if when < 0 { + // FILETIME marks historical DST as unknown. Derive the offset from + // the converted wall time rather than treating unknown as standard. + let wall_time = jiff::civil::DateTime::new( + (tm.tm_year + 1900) as i16, + (tm.tm_mon + 1) as i8, + tm.tm_mday as i8, + tm.tm_hour as i8, + tm.tm_min as i8, + tm.tm_sec as i8, + 0, + ) + .and_then(|dt| dt.to_zoned(jiff::tz::TimeZone::UTC)) + .map_err(|_| vm.new_overflow_error("timestamp out of range for Windows FILETIME"))?; + i32::try_from(wall_time.timestamp().as_second() - when) + .map_err(|_| vm.new_overflow_error("timezone offset out of range"))? + } else { + -(info.bias + bias) * 60 + }; Ok(struct_time_from_tm(vm, tm, name, gmtoff)) } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 238b19a2b50..304991f9504 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1607,7 +1607,7 @@ impl VirtualMachine { /// ```no_run /// use rustpython_vm::Interpreter; /// Interpreter::without_stdlib(Default::default()).enter(|vm| { - /// let bytes = std::fs::read("__pycache__/.rustpython-314.pyc").unwrap(); + /// let bytes = std::fs::read("__pycache__/.rustpython-315.pyc").unwrap(); /// let main_scope = vm.new_scope_with_main().unwrap(); /// vm.run_pyc_bytes(&bytes, main_scope); /// }); @@ -2258,17 +2258,15 @@ impl VirtualMachine { self.ctx.new_str(path).into(), self, )?; - module_dict.set_item(identifier!(self, __cached__), self.ctx.none(), self)?; } let result = run(&module_dict); self.flush_io(); - // Cleanup __file__ and __cached__ after execution + // Cleanup __file__ after execution if set_file_name { let _ = module_dict.del_item(identifier!(self, __file__), self); - let _ = module_dict.del_item(identifier!(self, __cached__), self); } result diff --git a/extra_tests/snippets/builtin___main__.py b/extra_tests/snippets/builtin___main__.py index 97e76ce324a..14cd3f3b984 100644 --- a/extra_tests/snippets/builtin___main__.py +++ b/extra_tests/snippets/builtin___main__.py @@ -2,4 +2,4 @@ main_module = sys.modules["__main__"] assert main_module.__file__.endswith("builtin___main__.py") -assert main_module.__cached__ is None +assert not hasattr(main_module, "__cached__") diff --git a/extra_tests/snippets/builtin_mappingproxy.py b/extra_tests/snippets/builtin_mappingproxy.py index c4d302136b8..8adf4d7e9bb 100644 --- a/extra_tests/snippets/builtin_mappingproxy.py +++ b/extra_tests/snippets/builtin_mappingproxy.py @@ -25,11 +25,13 @@ def b(): assert A.__dict__.get("not here") is None -# A mappingproxy must preserve arbitrary rich-comparison results unchanged. +# A mappingproxy preserves arbitrary equality-comparison results unchanged. def check_mappingproxy_comparison_results(): import operator from types import MappingProxyType + from testutils import assert_raises + class Result: def __bool__(self): raise AssertionError("comparison result must not be coerced to bool") @@ -58,14 +60,24 @@ class CustomMapping(Compared, dict): for operation, reflected in ( (operator.eq, "eq"), (operator.ne, "ne"), + ): + calls.clear() + assert operation(MappingProxyType(CustomMapping()), {}) is result + assert calls == [operation.__name__], calls + calls.clear() + assert operation(MappingProxyType({}), Compared()) is result + assert calls == [reflected], calls + + for operation, reflected in ( (operator.lt, "gt"), (operator.le, "ge"), (operator.gt, "lt"), (operator.ge, "le"), ): calls.clear() - assert operation(MappingProxyType(CustomMapping()), {}) is result - assert calls == [operation.__name__], calls + with assert_raises(TypeError): + operation(MappingProxyType(CustomMapping()), {}) + assert calls == [], calls calls.clear() assert operation(MappingProxyType({}), Compared()) is result assert calls == [reflected], calls diff --git a/extra_tests/snippets/builtin_memoryview.py b/extra_tests/snippets/builtin_memoryview.py index ef28ffc3d79..ea0475d7c63 100644 --- a/extra_tests/snippets/builtin_memoryview.py +++ b/extra_tests/snippets/builtin_memoryview.py @@ -399,7 +399,7 @@ def test_cast_rejects_non_native_format(): memoryview(b"").cast(fmt) raise AssertionError(f"cast accepted {fmt!r}") except ValueError as e: - assert "native single character format" in str(e), e + assert "native format" in str(e), e assert memoryview(bytes(8)).cast("@i").itemsize == 4 diff --git a/extra_tests/snippets/builtin_signature.py b/extra_tests/snippets/builtin_signature.py index 68998566a7e..06d5cf92bea 100644 --- a/extra_tests/snippets/builtin_signature.py +++ b/extra_tests/snippets/builtin_signature.py @@ -19,7 +19,7 @@ raise AssertionError("len() should not accept keyword arguments") assert str(inspect.signature(len)) == "(obj, /)" -assert str(inspect.signature(abs)) == "(x, /)" +assert str(inspect.signature(abs)) == "(number, /)" assert str(inspect.signature(hash)) == "(obj, /)" assert str(inspect.signature(chr)) == "(i, /)" assert str(inspect.signature(callable)) == "(obj, /)" @@ -37,7 +37,7 @@ ], breakpoint_kinds # Parameter names follow CPython, so signatures are directly comparable. -assert str(inspect.signature(bin)) == "(number, /)" +assert str(inspect.signature(bin)) == "(integer, /)" assert str(inspect.signature(ord)) == "(character, /)" assert str(inspect.signature(divmod)) == "(x, y, /)" assert str(inspect.signature(hasattr)) == "(obj, name, /)" diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index f54082325db..579976fd495 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -411,7 +411,10 @@ def __int__(self): for value in (None, TypeError("conversion failed")): with assert_raises(TypeError) as cm: template % PercentInt(value) - assert str(cm.exception).endswith("a real number is required, not PercentInt") + conversion = chr(template[-1]) if isinstance(template, bytes) else template[-1] + assert str(cm.exception) == ( + f"format argument: %{conversion} requires a real number, not PercentInt" + ) assert_raises( RuntimeError, template.__mod__, PercentInt(RuntimeError("conversion failed")) ) @@ -1224,3 +1227,27 @@ def message(index): test_subscript_typeerror_message() + + +# Decimal conversion errors identify the consumed argument, including star arguments. +for is_bytes in (False, True): + for conversion in "diu": + for template, values, context in ( + ("%" + conversion, "bad", ""), + ("%" + conversion, ("bad",), " 1"), + ("%s %" + conversion, ("ok", "bad"), " 2"), + ("%*.*" + conversion, (5, 2, "bad"), " 3"), + ("%(value)" + conversion, {"value": "bad"}, " 'value'"), + ): + if is_bytes: + template = template.encode() + if isinstance(values, dict): + values = {b"value": "bad"} + context = " b'value'" + elif isinstance(values, tuple) and len(values) == 2: + values = (b"ok", "bad") + with assert_raises(TypeError) as caught: + template % values + assert str(caught.exception) == ( + f"format argument{context}: %{conversion} requires a real number, not str" + ) diff --git a/extra_tests/snippets/operator_arithmetic.py b/extra_tests/snippets/operator_arithmetic.py index 14763107b5c..559ad7a4f73 100644 --- a/extra_tests/snippets/operator_arithmetic.py +++ b/extra_tests/snippets/operator_arithmetic.py @@ -89,11 +89,21 @@ def __getitem__(self, index): # Native sequence slots use the same count conversion in both directions. for value in ([1], (1,), "a", b"a", bytearray(b"a")): - for operation in (mul, lambda seq, count: count * seq, imul): + namespace = {} + exec("def reverse(seq, count):\n return count * seq", namespace) + for operation_index, operation in enumerate((mul, namespace["reverse"], imul)): + for _ in range(64): + operation(value, 1) for count, error, message in ( (1.5, TypeError, "can't multiply sequence by non-int of type 'float'"), (2**100, OverflowError, "cannot fit 'int' into an index-sized integer"), ): + if ( + type(value) in (str, bytes, tuple) + and operation_index == 1 + and error is OverflowError + ): + message = "Python int too large to convert to C ssize_t" check_repeat_error(error, message, operation, value, count) @@ -123,6 +133,79 @@ def __index__(self): ) +# Each expression has fresh bytecode so cold and warmed paths are both checked. +for value in ("a", b"a", (1,), "", b"", ()): + for expression in ( + "return seq * count", + "return count * seq", + "seq *= count; return seq", + "count *= seq; return count", + ): + namespace = {} + exec("def repeat(seq, count):\n " + expression, namespace) + repeat = namespace["repeat"] + check_repeat_error( + OverflowError, + "cannot fit 'int' into an index-sized integer", + repeat, + value, + 2**100, + ) + for _ in range(64): + assert repeat(value, 2) == mul(value, 2) + assert repeat(value, -2) == value[:0] + assert repeat(value, 0) == value[:0] + assert repeat(value, 1) == value + for count in (2**100, -(2**100)): + check_repeat_error( + OverflowError, + "Python int too large to convert to C ssize_t", + repeat, + value, + count, + ) + for other in ("a", b"a", (1,)): + if type(other) is not type(value): + check_repeat_error( + OverflowError, + "cannot fit 'int' into an index-sized integer", + repeat, + other, + 2**100, + ) + if expression == "count *= seq; return count": + check_repeat_error( + TypeError, + f"unsupported operand type(s) for *=: 'RepeatIndex' and '{type(value).__name__}'", + repeat, + value, + RepeatIndex(2), + ) + else: + assert repeat(value, RepeatIndex(2)) == mul(value, 2) + check_repeat_error( + OverflowError, + "cannot fit 'RepeatIndex' into an index-sized integer", + repeat, + value, + RepeatIndex(2**100), + ) + subclass = type("RepeatSubclass", (type(value),), {})(value) + assert repeat(subclass, 2) == mul(value, 2) + overridden = type( + "OverrideRepeat", + (type(value),), + { + "__mul__": lambda self, count: "forward", + "__rmul__": lambda self, count: "reverse", + }, + )(value) + expected = ( + "forward" if expression.startswith(("return seq", "seq *=")) else "reverse" + ) + assert repeat(overridden, 2) == expected + + class DeclineRepeat(GetItemOnly): calls = [] diff --git a/extra_tests/snippets/stdlib_ast.py b/extra_tests/snippets/stdlib_ast.py index 2dd2276723c..c9a3ae13db4 100644 --- a/extra_tests/snippets/stdlib_ast.py +++ b/extra_tests/snippets/stdlib_ast.py @@ -69,12 +69,13 @@ def foo(): def _cyclic_cases(): # Box descents - u = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + u = ast.UnaryOp(op=ast.Not(), operand=None, lineno=0, col_offset=0) u.operand = u yield "UnaryOp", u b = ast.BinOp( op=ast.Add(), + left=None, right=ast.Constant(value=0, lineno=0, col_offset=0), lineno=0, col_offset=0, @@ -82,11 +83,11 @@ def _cyclic_cases(): b.left = b yield "BinOp", b - c = ast.Call(args=[], keywords=[], lineno=0, col_offset=0) + c = ast.Call(func=None, args=[], keywords=[], lineno=0, col_offset=0) c.func = c yield "Call", c - a = ast.Attribute(attr="x", ctx=ast.Load(), lineno=0, col_offset=0) + a = ast.Attribute(value=None, attr="x", ctx=ast.Load(), lineno=0, col_offset=0) a.value = a yield "Attribute", a @@ -108,3 +109,73 @@ def _cyclic_cases(): raise AssertionError(f"cyclic {name} should raise RecursionError") except RecursionError: pass # expected; matches CPython + + +# Python 3.15 rejects omitted required fields and unknown constructor keywords. +from testutils import assert_raises + +with assert_raises(TypeError) as exc: + ast.Name() +assert ( + str(exc.exception) + == "ast.Name.__init__ missing 1 required positional argument: 'id'" +) + +with assert_raises(TypeError) as exc: + ast.BinOp() +assert str(exc.exception) == ( + "ast.BinOp.__init__ missing 3 required positional arguments: 'left', 'op', and 'right'" +) + +with assert_raises(TypeError) as exc: + ast.BinOp(op=ast.Add()) +assert str(exc.exception) == ( + "ast.BinOp.__init__ missing 2 required positional arguments: 'left' and 'right'" +) + +with assert_raises(TypeError) as exc: + ast.Name(id="x", extra=True) +assert ( + str(exc.exception) == "ast.Name.__init__ got an unexpected keyword argument 'extra'" +) + +with assert_raises(TypeError) as exc: + ast.Name("x", id="y") +assert str(exc.exception) == "ast.Name got multiple values for argument 'id'" + +# Constructor defaults and explicit placeholders still work. +assert ast.Name("x").ctx is ast.Name("y").ctx +assert isinstance(ast.Name("x").ctx, ast.Load) +assert ast.FunctionDef(name="f", args=ast.arguments()).returns is None +assert ast.Module().body == [] +assert ast.Module().body is not ast.Module().body +assert ast.BinOp(left=None, op=None, right=None).left is None + + +class FieldsWithoutTypes(ast.AST): + _fields = ("value",) + + +assert not hasattr(FieldsWithoutTypes(), "value") +assert FieldsWithoutTypes(value=1).value == 1 + + +class IncompleteFieldTypes(ast.AST): + _fields = ("value",) + _field_types = {} + + +with assert_raises(TypeError) as exc: + IncompleteFieldTypes() +assert ( + str(exc.exception) + == "Field 'value' is missing from IncompleteFieldTypes._field_types" +) + + +for key in ("a'b", "line\nbreak"): + with assert_raises(TypeError) as exc: + ast.Name(id="x", **{key: 1}) + assert str(exc.exception) == ( + f"ast.Name.__init__ got an unexpected keyword argument {key!r}" + ) diff --git a/extra_tests/snippets/stdlib_codecs_wide.py b/extra_tests/snippets/stdlib_codecs_wide.py index 4b950ba70ba..0c45cc91cca 100644 --- a/extra_tests/snippets/stdlib_codecs_wide.py +++ b/extra_tests/snippets/stdlib_codecs_wide.py @@ -31,7 +31,7 @@ # Lookup errors preserve the requested spelling, while search hooks receive -# the Python 3.14 normalized registry key. +# the Python 3.15 normalized registry key. import codecs seen_encoding_names = [] @@ -45,8 +45,8 @@ def record_unknown_encoding(name): codecs.register(record_unknown_encoding) try: for requested, normalized in ( - ("NO Such Codec!", "no_such_codec"), - ("x_ weird--Ω .Ab", "x_weird_.ab"), + ("NO Such Codec!", "no-such--codec!"), + ("x_ weird--Ω .Ab", "x_-weird--Ω-.ab"), ): for lookup in ( codecs.lookup, diff --git a/extra_tests/snippets/stdlib_csv.py b/extra_tests/snippets/stdlib_csv.py index 08adeaf5294..5911b0a1c09 100644 --- a/extra_tests/snippets/stdlib_csv.py +++ b/extra_tests/snippets/stdlib_csv.py @@ -493,9 +493,15 @@ def __getattribute__(self, name): events.append(name) raise error_type(name) - result = construct(Missing()) - assert tuple(getattr(result, field) for field in fields) == defaults - assert events == list(fields) + if error_type is AttributeError: + result = construct(Missing()) + assert tuple(getattr(result, field) for field in fields) == defaults + assert events == list(fields) + else: + with assert_raises(error_type) as caught: + construct(Missing()) + assert caught.exception.args == ("delimiter",) + assert events == ["delimiter"] for dialect in (None, object(), _csv.Dialect(csv.excel)): result = construct(dialect) assert tuple(getattr(result, field) for field in fields) == defaults diff --git a/extra_tests/snippets/stdlib_datetime.py b/extra_tests/snippets/stdlib_datetime.py index 41efd03a837..817f9237114 100644 --- a/extra_tests/snippets/stdlib_datetime.py +++ b/extra_tests/snippets/stdlib_datetime.py @@ -3944,17 +3944,28 @@ class Name(str): def test_timestamp_errno(): if sys.platform == "win32": try: - datetime_module.datetime.fromtimestamp(-1) + datetime_module.datetime.fromtimestamp(2**40) except OSError as error: assert error.errno == errno.EINVAL else: - raise AssertionError("Windows localtime accepted a negative timestamp") - try: - datetime_module.datetime.fromtimestamp(-86400, datetime_module.timezone.utc) - except OSError as error: - assert error.errno == errno.EINVAL - else: - raise AssertionError("Windows gmtime accepted an out-of-range timestamp") + raise AssertionError( + "Windows localtime accepted a timestamp beyond the CRT range" + ) + assert datetime_module.datetime.fromtimestamp( + -86400, datetime_module.timezone.utc + ) == datetime_module.datetime(1969, 12, 31, tzinfo=datetime_module.timezone.utc) + assert datetime_module.datetime.fromtimestamp( + -11_644_473_600, datetime_module.timezone.utc + ) == datetime_module.datetime(1601, 1, 1, tzinfo=datetime_module.timezone.utc) + for zone in (None, datetime_module.timezone.utc): + try: + datetime_module.datetime.fromtimestamp(-11_644_473_601, zone) + except OverflowError as error: + assert str(error) == "timestamp out of range for Windows FILETIME" + else: + raise AssertionError( + "pre-FILETIME datetime did not raise OverflowError" + ) test_astimezone_subclass_state() diff --git a/extra_tests/snippets/stdlib_imp.py b/extra_tests/snippets/stdlib_imp.py index 8153790c327..b5184e89a01 100644 --- a/extra_tests/snippets/stdlib_imp.py +++ b/extra_tests/snippets/stdlib_imp.py @@ -23,7 +23,8 @@ def __init__(self, name): # assert imp_time.sleep == import_time.sleep B = FakeSpec("not existing module") -assert _imp.create_builtin(B) == None +with assert_raises(ModuleNotFoundError): + _imp.create_builtin(B) _imp.exec_builtin(imp_time) == 0 diff --git a/extra_tests/snippets/stdlib_math.py b/extra_tests/snippets/stdlib_math.py index 29824043aa2..6e4e74b04bd 100644 --- a/extra_tests/snippets/stdlib_math.py +++ b/extra_tests/snippets/stdlib_math.py @@ -48,14 +48,12 @@ def __float__(self): class LogFloatOverflow: def __float__(self): + log_conversions.append("float") raise OverflowError("float conversion failed") def __index__(self): - raise AssertionError("must not retry __index__ after __float__ fails") - - -assert_raises(OverflowError, math.log, LogFloatOverflow()) -assert_raises(OverflowError, math.log, 8, LogFloatOverflow()) + log_conversions.append("index") + return 2**2000 class LogIndexOnly: @@ -63,8 +61,61 @@ def __index__(self): return 2**2000 -assert_raises(OverflowError, math.log, LogIndexOnly()) -assert_raises(OverflowError, math.log, 8, LogIndexOnly()) +for logarithm in (math.log, math.log2, math.log10): + log_conversions.clear() + assert math.isclose(logarithm(LogFloatOverflow()), logarithm(2**2000)) + assert log_conversions == ["float", "index"] + assert math.isclose(logarithm(LogIndexOnly()), logarithm(2**2000)) + + for error_type in (TypeError, ValueError, RuntimeError): + error = error_type("float conversion failed") + + class FailedFloat(LogFloatOverflow): + def __float__(self): + raise error + + with assert_raises(error_type) as caught: + logarithm(FailedFloat()) + assert caught.exception is error + + error = RuntimeError("index conversion failed") + + class FailedIndex(LogFloatOverflow): + def __index__(self): + raise error + + with assert_raises(RuntimeError) as caught: + logarithm(FailedIndex()) + assert caught.exception is error + + class NegativeIndex(LogFloatOverflow): + def __index__(self): + return -1 + + assert_raises(ValueError, logarithm, NegativeIndex()) + + class InvalidIndex(LogFloatOverflow): + def __index__(self): + return 1.0 + + assert_raises(TypeError, logarithm, InvalidIndex()) + + class NoIndex: + __float__ = LogFloatOverflow.__float__ + + assert_raises(OverflowError, logarithm, NoIndex()) + + class SuccessfulFloat: + def __float__(self): + return 8.0 + + def __index__(self): + raise AssertionError("successful float conversion must not retry index") + + assert logarithm(SuccessfulFloat()) == logarithm(8.0) + +assert math.isclose(math.log(8, LogFloatOverflow()), 3 / 2000) +assert math.isclose(math.log(8, LogIndexOnly()), 3 / 2000) class LogInt(int): diff --git a/extra_tests/snippets/stdlib_mmap.py b/extra_tests/snippets/stdlib_mmap.py index 1dc89e29d80..094c1aa9d74 100644 --- a/extra_tests/snippets/stdlib_mmap.py +++ b/extra_tests/snippets/stdlib_mmap.py @@ -25,9 +25,9 @@ assert mapped.rfind(b"") == 3 assert mapped.find(b"", 1, 2) == 1 assert mapped.rfind(b"", 1, 2) == 2 -# offsets past the end are clamped and negative ones count from the end -assert mapped.find(b"", 5) == 3 -assert mapped.rfind(b"", 5) == 3 +# starts past the end find nothing and negative offsets count from the end +assert mapped.find(b"", 5) == -1 +assert mapped.rfind(b"", 5) == -1 assert mapped.find(b"", -1) == 2 assert mapped.rfind(b"", 0, -1) == 2 # an inverted range holds nothing, not even the empty subsequence diff --git a/extra_tests/snippets/stdlib_struct.py b/extra_tests/snippets/stdlib_struct.py index 0f379df343b..5184493b14c 100644 --- a/extra_tests/snippets/stdlib_struct.py +++ b/extra_tests/snippets/stdlib_struct.py @@ -78,19 +78,18 @@ def __index__(self): assert struct.error.__name__ == "error" # Non-ASCII format string: error type matches CPython. -# str → UnicodeEncodeError (encoding='ascii') -# bytes → struct.error +# str and bytes → ValueError try: struct.Struct("\udc00") -except UnicodeEncodeError as e: - assert e.encoding == "ascii" +except ValueError as e: + assert str(e) == "non-ASCII character in struct format" else: - raise AssertionError("expected UnicodeEncodeError") + raise AssertionError("expected ValueError") -with assert_raises(UnicodeEncodeError): +with assert_raises(ValueError): struct.Struct("한") -with assert_raises(struct.error): +with assert_raises(ValueError): struct.Struct(b"\xff") @@ -128,7 +127,7 @@ def __index__(self): # A format that cannot be read leaves the Struct as it was. for bad in ("\udc00", "$"): - with assert_raises((UnicodeEncodeError, struct.error)): + with assert_raises((ValueError, struct.error)): s.__init__(bad) assert s.format == ">hh" assert s.pack(1, 2) == b"\x00\x01\x00\x02" @@ -145,8 +144,9 @@ def __init__(self): # Until __init__ runs there is no format to answer with. blank = struct.Struct.__new__(struct.Struct) assert blank.size == -1 +with assert_raises(AttributeError): + blank.format for call in ( - lambda: blank.format, lambda: blank.pack(1), lambda: blank.unpack(b"aa"), lambda: blank.unpack_from(b"aaaa"), diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index ba2fcef4c99..185e89f8b19 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -25,6 +25,13 @@ assert isinstance(sys.implementation.name, str) assert isinstance(sys.implementation.cache_tag, str) +assert sys.version_info[:3] == (3, 15, 0) +assert sys.hexversion >> 8 == 0x030F00 +assert sys.version.startswith("3.15.0") +assert sys.implementation.cache_tag == ( + f"{sys.implementation.name}-{sys.version_info.major}{sys.version_info.minor}" +) + assert sys.getfilesystemencoding() == "utf-8" assert sys.getfilesystemencodeerrors().startswith("surrogate") diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index 33b356c23f5..614d9610576 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -75,9 +75,8 @@ import errno for converter in (time.gmtime, time.localtime, time.ctime): - # Python 3.14 preserves UCRT conversion failures as OSError. Use a - # negative timestamp below gmtime's near-epoch allowance in UCRT. - for timestamp in (-86400, 2**40, 2**63 - 1, -(2**63), -float(2**63)): + # Non-negative timestamps still use UCRT and preserve errno. + for timestamp in (2**40, 2**63 - 1): try: converter(timestamp) except OSError as error: @@ -85,6 +84,29 @@ else: raise AssertionError("UCRT timestamp failure did not raise OSError") + # Negative timestamps use FILETIME, whose epoch is 1601-01-01. + for timestamp in (-11_644_473_601, -(2**63), -float(2**63)): + try: + converter(timestamp) + except OverflowError as error: + assert str(error) == "timestamp out of range for Windows FILETIME" + else: + raise AssertionError( + "pre-FILETIME timestamp did not raise OverflowError" + ) + + assert time.gmtime(-11_644_473_600) == (1601, 1, 1, 0, 0, 0, 0, 1, 0) + assert time.gmtime(-86400) == (1969, 12, 31, 0, 0, 0, 2, 365, 0) + assert time.gmtime(-1) == (1969, 12, 31, 23, 59, 59, 2, 365, 0) + # Leap-year accounting uses the historical date, including century rules. + assert time.gmtime(-2_203_891_200).tm_yday == 60 # 1900-03-01 + assert time.gmtime(-2_330_035_200).tm_yday == 61 # 1896-03-01 + for timestamp in (-15_897_600, -86400, -43201, -1): + local = time.localtime(timestamp) + assert local.tm_isdst == -1 + assert local[:6] == time.gmtime(timestamp + local.tm_gmtoff)[:6] + assert time.ctime(timestamp) == time.asctime(local) + x = time.gmtime(1000) assert x.tm_year == 1970 @@ -212,13 +234,39 @@ class Invalid: pass -for module, prefix in (("__main__", ""), ("sleep_test", "sleep_test.")): +for module in ("__main__", "sleep_test"): SleepTypeNamespace.Invalid.__module__ = module with assert_raises(TypeError) as error: time.sleep(SleepTypeNamespace.Invalid()) - assert str(error.exception) == ( - f"'{prefix}SleepTypeNamespace.Invalid' object cannot be interpreted as an integer or float" - ) + assert str(error.exception) == "must be real number, not Invalid" + + +class FloatSleepDuration: + def __float__(self): + return -0.5 + + +class IndexedSleepDuration(FloatSleepDuration): + def __index__(self): + return -1 + + def __float__(self): + raise AssertionError("sleep must prefer __index__") + + +class InvalidIndexedSleepDuration(FloatSleepDuration): + def __index__(self): + raise TypeError("index conversion failed") + + +for duration in (FloatSleepDuration(), IndexedSleepDuration()): + with assert_raises(ValueError) as error: + time.sleep(duration) + assert str(error.exception) == "sleep length must be non-negative" + +with assert_raises(TypeError) as error: + time.sleep(InvalidIndexedSleepDuration()) +assert str(error.exception) == "index conversion failed" # The optional second argument fills the fields that are not part of the # sequence. diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py index 82f3eb96d83..1392352442f 100644 --- a/scripts/libc_posix.py +++ b/scripts/libc_posix.py @@ -8,7 +8,7 @@ import tomllib -CPYTHON_VERSION = "3.14" +CPYTHON_VERSION = "3.15" CONSTS_PATTERN = re.compile(r"\b_*[A-Z]+(?:_+[A-Z]+)*_*\b") diff --git a/scripts/whats_left.py b/scripts/whats_left.py index 40b7b5be9d8..9adae4c9c86 100755 --- a/scripts/whats_left.py +++ b/scripts/whats_left.py @@ -1,6 +1,6 @@ #!/usr/bin/env -S python3 -I # /// script -# requires-python = ">=3.14" +# requires-python = ">=3.15.0" # /// # This script generates Lib/snippets/whats_left_data.py with these variables defined: @@ -39,9 +39,9 @@ implementation = platform.python_implementation() if implementation != "CPython": sys.exit(f"whats_left.py must be run under CPython, got {implementation} instead") -if sys.version_info[:2] < (3, 14): +if sys.version_info[:2] < (3, 15): sys.exit( - f"whats_left.py must be run under CPython 3.14 or newer, got {implementation} {sys.version} instead. If you have uv, try `uv run python -I scripts/whats_left.py` to select a proper Python interpreter easier." + f"whats_left.py must be run under CPython 3.15 or newer, got {implementation} {sys.version} instead. If you have uv, try `uv run python -I scripts/whats_left.py` to select a proper Python interpreter easier." ) From d88c9350fb5de38b8c3be74ba8b53c8dd68a9b8f Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 05:36:02 -0700 Subject: [PATCH 103/229] Restore Python 3.15 TextIOWrapper chunk-size deletion semantics (#9082) Check initialization and attachment before rejecting deletion, and use the final reference error message. Preserve initialized, uninitialized, detached, ASCII-subclass, and non-ASCII-subclass regression cases with their exact Python 3.15 outcomes. No bundled Lib files change. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_io.rs | 14 +++----------- extra_tests/snippets/stdlib_io.py | 17 ++++++++--------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 14ca013a831..1d86717a211 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -3568,17 +3568,6 @@ mod _io { #[pygetset(setter, name = "_CHUNK_SIZE")] fn set_chunksize(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - let PySetterValue::Assign(object_value) = value else { - let type_name = self.class().slot_name(); - let mut end = type_name.len().min(100); - while !type_name.is_char_boundary(end) { - end -= 1; - } - return Err(vm.new_attribute_error(format!( - "attribute '_CHUNK_SIZE' of '{}' objects cannot be deleted", - &type_name[..end] - ))); - }; { let textio = self.lock(vm)?; if self.buffer.deref().is_none() { @@ -3587,6 +3576,9 @@ mod _io { drop(textio); } + let PySetterValue::Assign(object_value) = value else { + return Err(vm.new_attribute_error("cannot delete attribute")); + }; let integer = object_value.try_index(vm)?; let chunk_size = integer.try_to_primitive::(vm).map_err(|_| { diff --git a/extra_tests/snippets/stdlib_io.py b/extra_tests/snippets/stdlib_io.py index 07b127999ea..b0f497359b7 100644 --- a/extra_tests/snippets/stdlib_io.py +++ b/extra_tests/snippets/stdlib_io.py @@ -166,12 +166,14 @@ def __index__(self): "underlying buffer has been detached", lambda: setattr(detached_textio, "_CHUNK_SIZE", 16), ) -for wrapper in (textio, uninitialized_textio, detached_textio): - with assert_raises(AttributeError) as error: +for wrapper, error_type, message in ( + (textio, AttributeError, "cannot delete attribute"), + (uninitialized_textio, ValueError, "I/O operation on uninitialized object"), + (detached_textio, ValueError, "underlying buffer has been detached"), +): + with assert_raises(error_type) as error: del wrapper._CHUNK_SIZE - assert str(error.exception) == ( - "attribute '_CHUNK_SIZE' of '_io.TextIOWrapper' objects cannot be deleted" - ) + assert str(error.exception) == message for wrapper_name in ("ChunkWrapper", "€" * 34): @@ -179,10 +181,7 @@ def __index__(self): wrapper = wrapper_type(BytesIO()) with assert_raises(AttributeError) as error: del wrapper._CHUNK_SIZE - displayed_name = wrapper_name.encode("utf-8")[:100].decode("utf-8", "ignore") - assert str(error.exception) == ( - f"attribute '_CHUNK_SIZE' of '{displayed_name}' objects cannot be deleted" - ) + assert str(error.exception) == "cannot delete attribute" wrapper.close() From 4e3b2f40395a4235841aaca811843d97ed68af0d Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 06:52:19 -0700 Subject: [PATCH 104/229] Keep REPL history-save errors from replacing exit status (#9081) Restore the two-line fix from #9078 onto main after it was merged into the Python 3.15 staging branch. Assisted-by: OpenAI Codex:model identifier unavailable --- src/shell.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shell.rs b/src/shell.rs index a899370ac47..73496981521 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -217,14 +217,14 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { if let Err(exc) = result { if exc.fast_isinstance(vm.ctx.exceptions.system_exit) { - repl.save_history(&repl_history_path).unwrap(); + let _ = repl.save_history(&repl_history_path); return Err(exc); } vm.print_exception(&exc); } flush_stdio(vm); } - repl.save_history(&repl_history_path).unwrap(); + let _ = repl.save_history(&repl_history_path); Ok(()) } From 1fa883b9b5e53cb1de535a26469e2b467c1e952a Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 07:47:18 -0700 Subject: [PATCH 105/229] Match Element.remove argument type diagnostics (#9068) Match native type names, None spelling, and UTF-8 byte limits while preserving removal behavior. Keep all six regression cases in the existing XML snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/elementtree.rs | 16 +++++++++++++++- extra_tests/snippets/stdlib_xml.py | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/crates/stdlib/src/elementtree.rs b/crates/stdlib/src/elementtree.rs index 1e49c9ec1bb..ed8dc17c612 100644 --- a/crates/stdlib/src/elementtree.rs +++ b/crates/stdlib/src/elementtree.rs @@ -578,7 +578,21 @@ pub(crate) mod _elementtree { #[pymethod] fn remove(zelf: &Py, subelement: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - check_element(&subelement, vm)?; + if !subelement.downcastable::() { + let name = if vm.is_none(&subelement) { + "None".into() + } else { + subelement.class().slot_name() + }; + let mut end = name.len().min(50); + while !name.is_char_boundary(end) { + end -= 1; + } + return Err(vm.new_type_error(format!( + "remove() argument must be xml.etree.ElementTree.Element, not {}", + &name[..end] + ))); + } // Identity first, then equality, the way `element_remove` does; // the equality test can run arbitrary code, so the lock is // dropped for it and the list re-read afterwards. diff --git a/extra_tests/snippets/stdlib_xml.py b/extra_tests/snippets/stdlib_xml.py index 34f1a726e4c..18e73004513 100644 --- a/extra_tests/snippets/stdlib_xml.py +++ b/extra_tests/snippets/stdlib_xml.py @@ -1,4 +1,6 @@ +import array import xml.sax +from xml.etree.ElementTree import Element from xml.parsers import expat from testutils import assert_raises @@ -66,3 +68,18 @@ def endElement(self, name): ("characters", "text"), ("end-cdata",), ] + +root = Element("root") +for value, type_name in ( + (None, "None"), + (1, "int"), + ("child", "str"), + (array.array("b"), "array.array"), + (type("A" * 60, (), {})(), "A" * 50), + (type("€" * 20, (), {})(), "€" * 16), +): + with assert_raises(TypeError) as caught: + root.remove(value) + assert str(caught.exception) == ( + "remove() argument must be xml.etree.ElementTree.Element, not " + type_name + ) From 2ab8af332f4d102643393450ef07d994bd808f30 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 07:48:08 -0700 Subject: [PATCH 106/229] Preserve Expat qualified names without namespace expansion (#9070) Retain raw element and attribute prefixes when namespace_separator is None. Keep expanded names and parser configuration unchanged, with regression coverage in the pyexpat snippet. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/pyexpat.rs | 4 ++++ extra_tests/snippets/stdlib_pyexpat.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 extra_tests/snippets/stdlib_pyexpat.py diff --git a/crates/stdlib/src/pyexpat.rs b/crates/stdlib/src/pyexpat.rs index 5a516dd7821..724fdb75926 100644 --- a/crates/stdlib/src/pyexpat.rs +++ b/crates/stdlib/src/pyexpat.rs @@ -1051,6 +1051,10 @@ mod _pyexpat { fn make_name(&self, name: &xml::name::OwnedName) -> String { match (&self.namespace_separator, &name.namespace) { (Some(sep), Some(ns)) => format!("{}{}{}", ns, sep, name.local_name), + (None, _) => match &name.prefix { + Some(prefix) => format!("{prefix}:{}", name.local_name), + None => name.local_name.clone(), + }, _ => name.local_name.clone(), } } diff --git a/extra_tests/snippets/stdlib_pyexpat.py b/extra_tests/snippets/stdlib_pyexpat.py new file mode 100644 index 00000000000..e8e2652d3db --- /dev/null +++ b/extra_tests/snippets/stdlib_pyexpat.py @@ -0,0 +1,23 @@ +from xml.parsers import expat + +# Without namespace expansion, element and attribute names retain prefixes. +for separator, expected_name, expected_attribute in ( + (None, "p:x", "p:a"), + ("!", "urn:p!x", "urn:p!a"), +): + for ordered in (False, True): + parser = expat.ParserCreate(namespace_separator=separator) + parser.ordered_attributes = ordered + starts = [] + ends = [] + parser.StartElementHandler = lambda name, attrs: starts.append((name, attrs)) + parser.EndElementHandler = ends.append + parser.Parse('', True) + assert [name for name, attrs in starts] == ["r", expected_name] + assert ends == [expected_name, "r"] + expected_attributes = ( + [expected_attribute, "v", "a", "u"] + if ordered + else {expected_attribute: "v", "a": "u"} + ) + assert starts[1][1] == expected_attributes From a10744303f6fb113a6082dde133958ad882d25a4 Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:50:12 +0300 Subject: [PATCH 107/229] Release SQLite aggregate instances outside locks (#9014) Assisted-by: Codex:gpt-6 --- crates/stdlib/src/_sqlite3.rs | 234 ++++++++++++++++++++----- extra_tests/snippets/stdlib_sqlite.py | 236 ++++++++++++++++++++++++++ 2 files changed, 424 insertions(+), 46 deletions(-) diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs index e5953aef6ae..3eaa9533f71 100644 --- a/crates/stdlib/src/_sqlite3.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -49,6 +49,7 @@ mod _sqlite3 { atomic::{Ordering, PyAtomic, Radium}, hash::PyHash, lock::{PyMappedMutexGuard, PyMutex, PyMutexGuard}, + refcount::{try_defer_drop, with_deferred_drops}, static_cell, }; use rustpython_vm::{ @@ -515,7 +516,11 @@ mod _sqlite3 { let context = SqliteContext::from(context); let (cls, vm) = unsafe { (*context.user_data::()).retrieve() }; let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; - let instance = context.aggregate_context::<*const PyObject>(); + let instance = context.aggregate_context(true); + if instance.is_null() { + unsafe { sqlite3_result_error_nomem(context.ctx) }; + return; + } if unsafe { (*instance).is_null() } { match cls.call((), vm) { Ok(obj) => unsafe { *instance = obj.into_raw().as_ptr() }, @@ -536,12 +541,22 @@ mod _sqlite3 { unsafe extern "C" fn finalize_callback(context: *mut sqlite3_context) { let context = SqliteContext::from(context); let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; - let instance = context.aggregate_context::<*const PyObject>(); - let Some(instance) = (unsafe { (*instance).as_ref() }) else { + let instance = context.aggregate_context(false); + if instance.is_null() { + return; + } + // xFinal releases the reference acquired by xStep, even if the + // query or finalize() fails. Clear the slot before calling Python. + let object = unsafe { instance.replace(core::ptr::null()) }; + let Some(object) = NonNull::new(object.cast_mut()) else { return; }; + let instance = unsafe { PyObjectRef::from_raw(object) }; - Self::callback_result_from_method(context, instance, "finalize", vm); + Self::callback_result_from_method(context, &instance, "finalize", vm); + // __del__ and weakref callbacks must run after the caller releases + // its cursor, statement and connection locks. + try_defer_drop(move || drop(instance)); } unsafe extern "C" fn collation_callback( @@ -579,8 +594,10 @@ mod _sqlite3 { unsafe extern "C" fn value_callback(context: *mut sqlite3_context) { let context = SqliteContext::from(context); let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; - let instance = context.aggregate_context::<*const PyObject>(); - let instance = unsafe { &**instance }; + let instance = context.aggregate_context(false); + let Some(instance) = (unsafe { instance.as_ref().and_then(|obj| obj.as_ref()) }) else { + return; + }; Self::callback_result_from_method(context, instance, "value", vm); } @@ -593,8 +610,10 @@ mod _sqlite3 { let context = SqliteContext::from(context); let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; - let instance = context.aggregate_context::<*const PyObject>(); - let instance = unsafe { &**instance }; + let instance = context.aggregate_context(false); + let Some(instance) = (unsafe { instance.as_ref().and_then(|obj| obj.as_ref()) }) else { + return; + }; Self::call_method_with_args(context, instance, "inverse", args, vm); } @@ -995,44 +1014,50 @@ mod _sqlite3 { type Args = ConnectArgs; fn init(zelf: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - let was_initialized = Radium::swap(&zelf.initialized, false, Ordering::AcqRel); + with_deferred_drops(|| { + let was_initialized = Radium::swap(&zelf.initialized, false, Ordering::AcqRel); - // Reset factories to their defaults, matching CPython's behavior. - zelf.reset_factories(vm); + // Reset factories to their defaults, matching CPython's behavior. + zelf.reset_factories(vm); - if was_initialized { - zelf.drop_db(vm)?; - } - - // Attempt to open the new database before mutating other state so failures leave - // the connection uninitialized (and subsequent operations raise ProgrammingError). - let db = Self::initialize_db(&args, vm)?; - - let ConnectArgs { - detect_types, - isolation_level, - check_same_thread, - autocommit, - .. - } = args; - - zelf.detect_types.store(detect_types, Ordering::Relaxed); - zelf.check_same_thread - .store(check_same_thread, Ordering::Relaxed); - *zelf.autocommit.lock() = autocommit; - *zelf.thread_ident.lock() = std::thread::current().id(); - let _ = unsafe { zelf.isolation_level.swap(isolation_level.0) }; + if was_initialized { + zelf.drop_db(vm)?; + } - let mut guard = zelf.db.lock(); - *guard = Some(db); - Radium::store(&zelf.initialized, true, Ordering::Release); - Ok(()) + // Attempt to open the new database before mutating other state so failures leave + // the connection uninitialized (and subsequent operations raise ProgrammingError). + let db = Self::initialize_db(&args, vm)?; + + let ConnectArgs { + detect_types, + isolation_level, + check_same_thread, + autocommit, + .. + } = args; + + zelf.detect_types.store(detect_types, Ordering::Relaxed); + zelf.check_same_thread + .store(check_same_thread, Ordering::Relaxed); + *zelf.autocommit.lock() = autocommit; + *zelf.thread_ident.lock() = std::thread::current().id(); + let _ = unsafe { zelf.isolation_level.swap(isolation_level.0) }; + + let mut guard = zelf.db.lock(); + *guard = Some(db); + Radium::store(&zelf.initialized, true, Ordering::Release); + Ok(()) + }) } } #[pyclass(with(Constructor, Callable, Initializer), flags(BASETYPE, HAS_WEAKREF))] impl Connection { fn drop_db(&self, vm: &VirtualMachine) -> PyResult<()> { + with_deferred_drops(|| Self::drop_db_inner(self, vm)) + } + + fn drop_db_inner(&self, vm: &VirtualMachine) -> PyResult<()> { let mut guard = self.db.lock(); let rollback_result = if let Some(db) = guard.as_ref() && *self.autocommit.lock() == AutocommitMode::Disabled @@ -1128,6 +1153,14 @@ mod _sqlite3 { zelf: PyRef, args: BlobOpenArgs, vm: &VirtualMachine, + ) -> PyResult> { + with_deferred_drops(|| Self::blobopen_inner(zelf, args, vm)) + } + + fn blobopen_inner( + zelf: PyRef, + args: BlobOpenArgs, + vm: &VirtualMachine, ) -> PyResult> { let table = args.table.to_cstring(vm)?; let column = args.column.to_cstring(vm)?; @@ -1170,6 +1203,10 @@ mod _sqlite3 { #[pymethod] fn commit(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { + with_deferred_drops(|| Self::commit_inner(zelf, vm)) + } + + fn commit_inner(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { let db = zelf.db_lock(vm)?; let mode = *zelf.autocommit.lock(); match mode { @@ -1184,6 +1221,10 @@ mod _sqlite3 { #[pymethod] fn rollback(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { + with_deferred_drops(|| Self::rollback_inner(zelf, vm)) + } + + fn rollback_inner(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { let db = zelf.db_lock(vm)?; let mode = *zelf.autocommit.lock(); match mode { @@ -1316,6 +1357,14 @@ mod _sqlite3 { zelf: &Py, args: CreateFunctionArgs, vm: &VirtualMachine, + ) -> PyResult<()> { + with_deferred_drops(|| Self::create_function_inner(zelf, args, vm)) + } + + fn create_function_inner( + zelf: &Py, + args: CreateFunctionArgs, + vm: &VirtualMachine, ) -> PyResult<()> { let name = args.name.to_cstring(vm)?; let flags = if args.deterministic { @@ -1357,6 +1406,14 @@ mod _sqlite3 { zelf: &Py, args: CreateAggregateArgs, vm: &VirtualMachine, + ) -> PyResult<()> { + with_deferred_drops(|| Self::create_aggregate_inner(zelf, args, vm)) + } + + fn create_aggregate_inner( + zelf: &Py, + args: CreateAggregateArgs, + vm: &VirtualMachine, ) -> PyResult<()> { let name = args.name.to_cstring(vm)?; let db = zelf.db_lock(vm)?; @@ -1394,6 +1451,15 @@ mod _sqlite3 { name: PyUtf8StrRef, callable: PyObjectRef, vm: &VirtualMachine, + ) -> PyResult<()> { + with_deferred_drops(|| Self::create_collation_inner(zelf, name, callable, vm)) + } + + fn create_collation_inner( + zelf: &Py, + name: PyUtf8StrRef, + callable: PyObjectRef, + vm: &VirtualMachine, ) -> PyResult<()> { let name = name.to_cstring(vm)?; let db = zelf.db_lock(vm)?; @@ -1440,6 +1506,18 @@ mod _sqlite3 { narg: c_int, aggregate_class: PyObjectRef, vm: &VirtualMachine, + ) -> PyResult<()> { + with_deferred_drops(|| { + Self::create_window_function_inner(zelf, name, narg, aggregate_class, vm) + }) + } + + fn create_window_function_inner( + zelf: &Py, + name: PyStrRef, + narg: c_int, + aggregate_class: PyObjectRef, + vm: &VirtualMachine, ) -> PyResult<()> { let name = name.to_cstring(vm)?; let db = zelf.db_lock(vm)?; @@ -1679,6 +1757,14 @@ mod _sqlite3 { } #[pygetset(setter)] fn set_autocommit(zelf: &Py, val: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + with_deferred_drops(|| Self::set_autocommit_inner(zelf, val, vm)) + } + + fn set_autocommit_inner( + zelf: &Py, + val: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { let mode = AutocommitMode::try_from_borrowed_object(vm, &val)?; let db = zelf.db_lock(vm)?; *zelf.autocommit.lock() = mode; @@ -1912,6 +1998,15 @@ mod _sqlite3 { sql: PyUtf8StrRef, parameters: OptionalArg, vm: &VirtualMachine, + ) -> PyResult> { + with_deferred_drops(|| Self::execute_inner(zelf, sql, parameters, vm)) + } + + fn execute_inner( + zelf: PyRef, + sql: PyUtf8StrRef, + parameters: OptionalArg, + vm: &VirtualMachine, ) -> PyResult> { let mut inner = zelf.inner(vm)?; @@ -1994,6 +2089,15 @@ mod _sqlite3 { sql: PyUtf8StrRef, seq_of_params: ArgIterable, vm: &VirtualMachine, + ) -> PyResult> { + with_deferred_drops(|| Self::executemany_inner(zelf, sql, seq_of_params, vm)) + } + + fn executemany_inner( + zelf: PyRef, + sql: PyUtf8StrRef, + seq_of_params: ArgIterable, + vm: &VirtualMachine, ) -> PyResult> { let mut inner = zelf.inner(vm)?; @@ -2070,6 +2174,14 @@ mod _sqlite3 { zelf: PyRef, script: PyUtf8StrRef, vm: &VirtualMachine, + ) -> PyResult> { + with_deferred_drops(|| Self::executescript_inner(zelf, script, vm)) + } + + fn executescript_inner( + zelf: PyRef, + script: PyUtf8StrRef, + vm: &VirtualMachine, ) -> PyResult> { zelf.check_cursor_valid(vm)?; @@ -2146,6 +2258,10 @@ mod _sqlite3 { #[pymethod] fn close(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { + with_deferred_drops(|| Self::close_inner(zelf, vm)) + } + + fn close_inner(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { // Check if __init__ was called let mut guard = zelf.inner.lock(); @@ -2299,6 +2415,12 @@ mod _sqlite3 { impl SelfIter for Cursor {} impl IterNext for Cursor { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { + with_deferred_drops(|| Self::next_inner(zelf, vm)) + } + } + + impl Cursor { + fn next_inner(zelf: &Py, vm: &VirtualMachine) -> PyResult { // Check if connection is closed first, and if so, clear statement to release file lock if zelf.connection.is_closed() { let mut guard = zelf.inner.lock(); @@ -2576,7 +2698,7 @@ mod _sqlite3 { impl Drop for BlobInner { fn drop(&mut self) { - unsafe { sqlite3_blob_close(self.blob.blob) }; + with_deferred_drops(|| unsafe { sqlite3_blob_close(self.blob.blob) }); } } @@ -2584,7 +2706,9 @@ mod _sqlite3 { impl Blob { #[pymethod] fn close(zelf: &Py) { - zelf.inner.lock().take(); + with_deferred_drops(|| { + zelf.inner.lock().take(); + }); } fn ensure_connection_open(&self, vm: &VirtualMachine) -> PyResult<()> { @@ -2742,6 +2866,10 @@ mod _sqlite3 { } fn subscript(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + with_deferred_drops(|| Self::subscript_inner(self, needle, vm)) + } + + fn subscript_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { self.ensure_connection_open(vm)?; let inner = self.inner(vm)?; if let Some(index) = needle.try_index_opt(vm) { @@ -2784,6 +2912,15 @@ mod _sqlite3 { needle: &PyObject, value: Option, vm: &VirtualMachine, + ) -> PyResult<()> { + with_deferred_drops(|| Self::ass_subscript_inner(self, needle, value, vm)) + } + + fn ass_subscript_inner( + &self, + needle: &PyObject, + value: Option, + vm: &VirtualMachine, ) -> PyResult<()> { self.ensure_connection_open(vm)?; let inner = self.inner(vm)?; @@ -3042,7 +3179,7 @@ mod _sqlite3 { impl Drop for Sqlite { fn drop(&mut self) { // Use sqlite3_close_v2 for safe closing even with active statements - unsafe { sqlite3_close_v2(self.raw.db) }; + with_deferred_drops(|| unsafe { sqlite3_close_v2(self.raw.db) }); } } @@ -3261,9 +3398,11 @@ mod _sqlite3 { impl Drop for SqliteStatement { fn drop(&mut self) { - unsafe { + // A cursor may be dropped without an explicit close(). Nesting + // preserves any outer scope that still owns SQLite locks. + with_deferred_drops(|| unsafe { sqlite3_finalize(self.raw.st); - } + }); } } @@ -3591,10 +3730,13 @@ mod _sqlite3 { unsafe { sqlite3_user_data(self.ctx).cast() } } - fn aggregate_context(self) -> *mut T { - unsafe { - sqlite3_aggregate_context(self.ctx, core::mem::size_of::() as c_int).cast() - } + fn aggregate_context(self, create: bool) -> *mut *const PyObject { + let size = if create { + core::mem::size_of::<*const PyObject>() as c_int + } else { + 0 + }; + unsafe { sqlite3_aggregate_context(self.ctx, size).cast() } } fn result_exception(self, vm: &VirtualMachine, exc: PyBaseExceptionRef, msg: &CStr) { diff --git a/extra_tests/snippets/stdlib_sqlite.py b/extra_tests/snippets/stdlib_sqlite.py index 83fc7db94f0..76c8b7adde1 100644 --- a/extra_tests/snippets/stdlib_sqlite.py +++ b/extra_tests/snippets/stdlib_sqlite.py @@ -70,3 +70,239 @@ def finalize(self): expected = b"t5i4 3l2b1" + data[10:] assert actual == expected, f"got {actual!r}, expected {expected!r}" blob.close() + + +# Aggregate instances belong to a query, including its error and cancellation paths. +import gc +import weakref + +from testutils import assert_raises + + +def check_aggregate_lifetime(failure): + refs = [] + + class TrackedAggregate: + def __init__(self): + refs.append(weakref.ref(self)) + + def step(self, value): + if failure == "step": + raise ValueError("step") + + def finalize(self): + if failure == "finalize": + raise ValueError("finalize") + return object() if failure == "result" else 1 + + cx.create_aggregate("tracked", 1, TrackedAggregate) + query = "SELECT tracked(key) FROM foo GROUP BY key % 2" + if failure: + with assert_raises(sqlite.OperationalError): + cx.execute(query).fetchall() + else: + assert cx.execute(query).fetchall() == [(1,), (1,)] + assert refs + gc.collect() + assert all(ref() is None for ref in refs), failure + + # xFinal may run without any preceding xStep. + refs.clear() + assert cx.execute("SELECT tracked(key) FROM foo WHERE 0").fetchall() == [(None,)] + assert not refs + + +for failure in (None, "step", "finalize", "result"): + check_aggregate_lifetime(failure) + + +# Window functions share aggregate finalization, including an unfinished cursor. +window_refs = [] + + +class TrackedWindow: + def __init__(self): + window_refs.append(weakref.ref(self)) + + def step(self, value): + pass + + def inverse(self, value): + pass + + def value(self): + return 1 + + def finalize(self): + return 1 + + +cx.create_window_function("tracked_window", 1, TrackedWindow) +window_query = """ + SELECT tracked_window(key) OVER ( + ORDER BY key ROWS BETWEEN 1 PRECEDING AND CURRENT ROW + ) FROM foo +""" +assert cx.execute(window_query).fetchall() == [(1,)] * 4 +assert window_refs +assert all(ref() is None for ref in window_refs) +window_refs.clear() +window_cursor = cx.execute(window_query) +assert window_cursor.fetchone() == (1,) +window_cursor.close() +gc.collect() +assert window_refs and all(ref() is None for ref in window_refs) + + +# Releasing the last reference can run Python that reads the cursor/connection. +def check_aggregate_destructor(operation): + connection = sqlite.connect(":memory:") + cursor = connection.cursor() + cursor_ref = weakref.ref(cursor) + released = [] + + class Aggregate: + def step(self, value): + pass + + def value(self): + return 1 + + finalize = value + + def __del__(self): + current = cursor_ref() + if current is not None: + try: + current.description + except sqlite.ProgrammingError: + pass # The cursor may already have been closed. + released.append(connection.total_changes) + + connection.create_aggregate("tracked", 1, Aggregate) + expected = 1 + if operation == "execute": + cursor.execute("SELECT tracked(1)") + elif operation == "script": + cursor.executescript("SELECT tracked(1);") + elif operation == "many": + connection.execute("CREATE TABLE results(value)") + cursor.executemany("INSERT INTO results SELECT tracked(?)", [(1,), (2,)]) + expected = 2 + else: + connection.create_window_function("tracked_window", 1, Aggregate) + cursor.execute(""" + SELECT tracked_window(x) OVER (ROWS UNBOUNDED PRECEDING) + FROM (SELECT 1 AS x UNION ALL SELECT 2 UNION ALL SELECT 3) + """) + assert not released + if operation == "fetch": + assert cursor.fetchall() == [(1,), (1,), (1,)] + elif operation == "close": + cursor.close() + elif operation == "replace": + cursor.execute("SELECT 2") + else: + del cursor + gc.collect() + assert len(released) == expected, (operation, released) + connection.close() + + +for operation in ("execute", "script", "many", "fetch", "close", "replace", "drop"): + check_aggregate_destructor(operation) + + +# A nested query must not release its aggregate while an outer callback holds locks. +def check_nested_aggregate_destructor(operation): + outer = sqlite.connect(":memory:", autocommit=False) + nested = sqlite.connect(":memory:") + released = [] + inspect = lambda: outer.total_changes + + class Aggregate: + def step(self, value): + pass + + def finalize(self): + return 1 + + def __del__(self): + try: + inspect() + except sqlite.ProgrammingError: + pass # Closing the outer connection may finish before cleanup. + released.append(True) + + nested.create_aggregate("tracked", 1, Aggregate) + outer.set_trace_callback(lambda sql: nested.execute("SELECT tracked(1)")) + if operation == "commit": + outer.commit() + elif operation == "rollback": + outer.rollback() + elif operation == "autocommit": + outer.autocommit = True + elif operation in ("blobopen", "blob_index"): + outer.execute("CREATE TABLE blobs(data BLOB)") + outer.execute("INSERT INTO blobs VALUES (zeroblob(1))") + released.clear() + + def progress(): + nested.execute("SELECT tracked(1)") + return 0 + + outer.set_progress_handler(progress, 1) + blob = outer.blobopen("blobs", "data", 1) + outer.set_progress_handler(None, 0) + if operation != "blobopen": + released.clear() + inspect = lambda: len(blob) + + class Index: + def __index__(self): + nested.execute("SELECT tracked(1)") + return 0 + + index = Index() + assert blob[index] == 0 + blob[index] = 1 + assert len(released) == 2 + blob.close() + elif operation == "replace_callback": + + class Callback: + def __call__(self): + return 1 + + def __del__(self): + nested.execute("SELECT tracked(1)") + + for register in ( + lambda cb: outer.create_function("hook", 0, cb), + lambda cb: outer.create_aggregate("hook", 0, cb), + lambda cb: outer.create_window_function("hook", 0, cb), + lambda cb: outer.create_collation("hook", cb), + ): + released.clear() + register(Callback()) + register(None) + assert released + else: + outer.close() + assert released, operation + if operation != "close": + outer.set_trace_callback(None) + outer.close() + nested.close() + + +for operation in ( + "commit", + "rollback", + "autocommit", + "close", + "blobopen", + "blob_index", + "replace_callback", +): + check_nested_aggregate_destructor(operation) From 37fdf4d2d99d705338d3b4db0065f95e7e0024a3 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 09:08:16 -0700 Subject: [PATCH 108/229] Fix native ABC checks when __class__ is missing (#9067) Retain the native fallback and Python regression without modifying bundled Lib sources. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_abc.rs | 4 +++- extra_tests/snippets/stdlib_abc.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/_abc.rs b/crates/vm/src/stdlib/_abc.rs index 19cf47415a9..4d2971949e5 100644 --- a/crates/vm/src/stdlib/_abc.rs +++ b/crates/vm/src/stdlib/_abc.rs @@ -272,7 +272,9 @@ mod _abc { let impl_data = get_impl(&cls, vm)?; // Get instance.__class__ - let subclass = instance.get_attr("__class__", vm)?; + let subclass = vm + .get_attribute_opt(&instance, "__class__")? + .unwrap_or_else(|| instance.class().to_owned().into()); // Check cache if in_weak_set(&impl_data.cache, &subclass, vm)? { diff --git a/extra_tests/snippets/stdlib_abc.py b/extra_tests/snippets/stdlib_abc.py index 30b4ef37068..df1833a419f 100644 --- a/extra_tests/snippets/stdlib_abc.py +++ b/extra_tests/snippets/stdlib_abc.py @@ -36,3 +36,28 @@ class SubConcrete(Concrete): assert isinstance(Concrete(), CustomInterface) assert isinstance(SubConcrete(), CustomInterface) assert not isinstance((), CustomInterface) + + +class NoClassAttribute: + @property + def __class__(self): + raise AttributeError("no class attribute") + + +instance = NoClassAttribute() +assert not isinstance(instance, CustomInterface) +CustomInterface.register(NoClassAttribute) +assert isinstance(instance, CustomInterface) + +class_error = RuntimeError("class lookup failed") + + +class BrokenClassAttribute: + @property + def __class__(self): + raise class_error + + +with assert_raises(RuntimeError) as caught: + isinstance(BrokenClassAttribute(), CustomInterface) +assert caught.exception is class_error From 4638b6a5f3596331b51f418a3dca5340a9b6f799 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 09:10:08 -0700 Subject: [PATCH 109/229] Raise IndexError for oversized SQLite blob indices (#9048) Use index-sized conversion, preserve __index__ exceptions, and bound native type names to the UTF-8-safe diagnostic limit. Include in-memory blob read and write regressions without changing callback ownership. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/_sqlite3.rs | 30 ++++++++++---- extra_tests/snippets/stdlib_sqlite.py | 56 +++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs index 3eaa9533f71..d0bb95a6c9f 100644 --- a/crates/stdlib/src/_sqlite3.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -2839,15 +2839,31 @@ mod _sqlite3 { } } - fn wrapped_index(index: PyIntRef, length: c_int, vm: &VirtualMachine) -> PyResult { - let mut index = index.try_to_primitive::(vm)?; + fn wrapped_index( + index: PyIntRef, + object: &PyObject, + length: c_int, + vm: &VirtualMachine, + ) -> PyResult { + let mut index = index.try_to_primitive::(vm).map_err(|_| { + let class = object.class(); + let type_name = class.slot_name(); + let mut end = type_name.len().min(200); + while !type_name.is_char_boundary(end) { + end -= 1; + } + vm.new_index_error(format!( + "cannot fit '{}' into an index-sized integer", + &type_name[..end] + )) + })?; if index < 0 { - index += length; + index += length as isize; } - if index < 0 || index >= length { + if index < 0 || index >= length as isize { Err(vm.new_index_error("Blob index out of range")) } else { - Ok(index) + Ok(index as c_int) } } @@ -2874,7 +2890,7 @@ mod _sqlite3 { let inner = self.inner(vm)?; if let Some(index) = needle.try_index_opt(vm) { let blob_len = inner.blob.bytes(); - let index = Self::wrapped_index(index?, blob_len, vm)?; + let index = Self::wrapped_index(index?, needle, blob_len, vm)?; let mut byte: u8 = 0; let ret = inner.blob.read_single(&mut byte, index); self.check(ret, vm).map(|_| vm.ctx.new_int(byte).into()) @@ -2937,7 +2953,7 @@ mod _sqlite3 { ))); }; let blob_len = inner.blob.bytes(); - let index = Self::wrapped_index(index?, blob_len, vm)?; + let index = Self::wrapped_index(index?, needle, blob_len, vm)?; // Mirror CPython ass_subscript_index: use PyLong_AsLong, treat any // overflow (e.g. 2**65) as -1, then validate the [0, 255] range. let val = int_val.as_bigint().to_i64().unwrap_or(-1); diff --git a/extra_tests/snippets/stdlib_sqlite.py b/extra_tests/snippets/stdlib_sqlite.py index 76c8b7adde1..36babea6144 100644 --- a/extra_tests/snippets/stdlib_sqlite.py +++ b/extra_tests/snippets/stdlib_sqlite.py @@ -306,3 +306,59 @@ def __del__(self): "replace_callback", ): check_nested_aggregate_destructor(operation) + + +class BlobIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + +class BlobInt(int): + pass + + +class BrokenBlobIndex: + def __index__(self): + raise OverflowError("index conversion failed") + + +cx.execute("CREATE TABLE blobindices(b BLOB)") +cx.execute("INSERT INTO blobindices(b) VALUES (?)", (b"abc",)) +with cx.blobopen("blobindices", "b", 1) as blob: + assert blob[BlobIndex(-1)] == ord("c") + blob[BlobIndex(0)] = ord("A") + assert blob[0] == ord("A") + + test = unittest.TestCase() + for index in (sys.maxsize, -sys.maxsize - 1): + with test.assertRaisesRegex(IndexError, "^Blob index out of range$"): + blob[index] + with test.assertRaisesRegex(IndexError, "^Blob index out of range$"): + blob[index] = 0 + + for value in (sys.maxsize + 1, -sys.maxsize - 2): + for index in (value, BlobInt(value), BlobIndex(value)): + message = ( + f"^cannot fit '{type(index).__name__}' into an index-sized integer$" + ) + with test.assertRaisesRegex(IndexError, message): + blob[index] + with test.assertRaisesRegex(IndexError, message): + blob[index] = 0 + + for name, bounded_name in (("X" * 250, "X" * 200), ("€" * 67, "€" * 66)): + long_index = type(name, (), {"__index__": lambda self: sys.maxsize + 1})() + message = f"^cannot fit '{bounded_name}' into an index-sized integer$" + with test.assertRaisesRegex(IndexError, message): + blob[long_index] + with test.assertRaisesRegex(IndexError, message): + blob[long_index] = 0 + + with test.assertRaisesRegex(OverflowError, "^index conversion failed$"): + blob[BrokenBlobIndex()] + with test.assertRaisesRegex(OverflowError, "^index conversion failed$"): + blob[BrokenBlobIndex()] = 0 + assert blob[:] == b"Abc" From 924ef2f434758e602460e18e2ca86bfb489ba935 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 10:06:41 -0700 Subject: [PATCH 110/229] Match hashlib constructor name type diagnostics (#9066) Use native type names, None spelling, and UTF-8 byte precision for invalid name arguments without changing accepted strings or hashing behavior. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/stdlib/src/hashlib.rs | 30 +++++++++++++++++++++++--- extra_tests/snippets/stdlib_hashlib.py | 16 ++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs index f28f27b1055..ceca00d0bd9 100644 --- a/crates/stdlib/src/hashlib.rs +++ b/crates/stdlib/src/hashlib.rs @@ -8,7 +8,7 @@ pub(crate) use _hashlib::module_def; #[pymodule] pub(crate) mod _hashlib { use crate::vm::{ - Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, builtins::{ PyBaseExceptionRef, PyBytes, PyFrozenSet, PyStr, PyType, PyTypeRef, PyUtf8StrRef, PyValueError, @@ -67,11 +67,35 @@ pub(crate) mod _hashlib { dict.into() } + #[derive(Debug)] + struct HashName(PyUtf8StrRef); + + impl TryFromObject for HashName { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if !obj.downcastable::() { + let name = if vm.is_none(&obj) { + "None".into() + } else { + obj.class().slot_name() + }; + let mut end = name.len().min(50); + while !name.is_char_boundary(end) { + end -= 1; + } + return Err(vm.new_type_error(format!( + "new() argument 'name' must be str, not {}", + &name[..end] + ))); + } + PyUtf8StrRef::try_from_object(vm, obj).map(Self) + } + } + #[derive(FromArgs, Debug)] #[allow(unused)] struct NewHashArgs { #[pyarg(any)] - name: PyUtf8StrRef, + name: HashName, // Missing still allows the string keyword; b'' does not. #[pyarg(any, optional, py_default = "b''")] data: OptionalArg, @@ -785,7 +809,7 @@ pub(crate) mod _hashlib { #[pyfunction(name = "new")] fn hashlib_new(args: NewHashArgs, vm: &VirtualMachine) -> PyResult { let data = resolve_data(args.data, args.string, vm)?; - match args.name.as_str().to_lowercase().as_str() { + match args.name.0.as_str().to_lowercase().as_str() { "md5" => Ok(new_fixed_hasher("md5", data).into_pyobject(vm)), "sha1" => Ok(new_fixed_hasher("sha1", data).into_pyobject(vm)), "sha224" => Ok(new_fixed_hasher("sha224", data).into_pyobject(vm)), diff --git a/extra_tests/snippets/stdlib_hashlib.py b/extra_tests/snippets/stdlib_hashlib.py index f3400aed57d..9cbc6bffc7e 100644 --- a/extra_tests/snippets/stdlib_hashlib.py +++ b/extra_tests/snippets/stdlib_hashlib.py @@ -1,5 +1,7 @@ +import _hashlib import _md5 import _sha1 +import array import hashlib from testutils import assert_raises @@ -64,3 +66,17 @@ # narrower than the length asked for, converting the argument fails first. with assert_raises(OverflowError): hashlib.pbkdf2_hmac("sha256", b"password", b"salt", 1, 2**62) + + +# Invalid names use the native constructor's bounded type diagnostic. +for value, name in ( + (None, "None"), + (42, "int"), + (b"sha256", "bytes"), + (array.array("b"), "array.array"), + (type("x" * 60, (), {})(), "x" * 50), + (type("€" * 17, (), {})(), "€" * 16), +): + with assert_raises(TypeError) as caught: + _hashlib.new(value) + assert str(caught.exception) == f"new() argument 'name' must be str, not {name}" From f39da82d9d38cd5bdceb9e645205e830f65126fb Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 10:07:08 -0700 Subject: [PATCH 111/229] Prefer resolved executable symlinks in POSIX virtual environments (#9058) Resolve executable symlinks for virtual-environment paths while retaining macOS framework launcher behavior. Include directory-canonicalization and launcher regressions, using core for UTF-8 checks. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/getpath.rs | 59 ++++++++++++- extra_tests/snippets/stdlib_sys.py | 132 +++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs index eef1417ab14..f869922f984 100644 --- a/crates/vm/src/getpath.rs +++ b/crates/vm/src/getpath.rs @@ -299,12 +299,69 @@ fn calculate_exec_prefix(exe_dir: Option<&PathBuf>, prefix: &str) -> String { } } +/// Resolve final-component links without resolving directory aliases (ref: getpath.c). +#[cfg(unix)] +fn resolve_executable_symlinks(executable: &Path) -> Option { + use std::os::unix::ffi::OsStrExt; + + let mut path = executable.to_path_buf(); + // CPython rejects 40 successful hops, matching the Linux 4.2 kernel limit. + for _ in 0..40 { + let Ok(target) = std::fs::read_link(&path) else { + return Some(path); + }; + if target.is_absolute() { + path = target; + continue; + } + + // Match getpath.c's raw dirname join, then normalize lexically without + // resolving directory symlinks or losing a double-leading-slash root. + let bytes = path.as_os_str().as_bytes(); + let end = bytes.iter().rposition(|&b| b == b'/'); + let dirname = &bytes[..end.unwrap_or(bytes.len())]; + let mut joined = PathBuf::from(std::ffi::OsStr::from_bytes(dirname)); + // Filesystem text uses UTF-8/surrogateescape; count characters, not bytes. + let multiple_chars = + core::str::from_utf8(dirname).map_or(dirname.len() > 1, |s| s.chars().nth(1).is_some()); + if multiple_chars && !dirname.ends_with(b"/") { + joined.as_mut_os_string().push("/"); + } + joined.as_mut_os_string().push(target.as_os_str()); + let bytes = joined.as_os_str().as_bytes(); + let double_root = bytes.starts_with(b"//") && !bytes.starts_with(b"///"); + path.clear(); + for component in joined.components() { + if component == std::path::Component::ParentDir + && !path.as_os_str().is_empty() + && !path.ends_with("..") + { + path.pop(); + } else if component != std::path::Component::CurDir { + path.push(component.as_os_str()); + } + } + if double_root { + path = Path::new("//").join(path.strip_prefix("/").ok()?); + } + } + None +} + /// Calculate base_executable fn calculate_base_executable(executable: Option<&PathBuf>, home_dir: Option<&PathBuf>) -> String { - // If in venv and we have home, construct base_executable from home + // If in venv and we have home, prefer the actual target of a symlink. if let (Some(exe), Some(home)) = (executable, home_dir) && let Some(exe_name) = exe.file_name() { + #[cfg(unix)] + if let Some(resolved) = resolve_executable_symlinks(exe) + && !resolved.as_os_str().is_empty() + && resolved.as_os_str() != exe.as_os_str() + { + return resolved.to_string_lossy().into_owned(); + } + let base = home.join(exe_name); return base.to_string_lossy().into_owned(); } diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index 185e89f8b19..c785a092efc 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -1,7 +1,10 @@ import os +import shutil import subprocess import sys +import tempfile import warnings +from pathlib import Path from testutils import assert_raises @@ -192,3 +195,132 @@ def test_getframemodulename(): del os.environ["PYTHONBREAKPOINT"] else: os.environ["PYTHONBREAKPOINT"] = saved_breakpoint_env + + +# A venv's symlink target takes precedence over home/. +if os.name == "posix" and sys.platform not in ("wasi", "emscripten"): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + interpreter = Path(sys.executable).resolve() + # A framework build can also run its embedded app directly. Match the + # installed launcher's file identity before applying launcher semantics. + framework_base = ( + Path(sys.base_prefix) + / "bin" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + ) + framework_launcher = ( + sys.platform == "darwin" + and bool(getattr(sys, "_framework", "")) + and framework_base.is_file() + and interpreter.samefile(framework_base) + ) + home = root / "home" + home.mkdir() + venv_env = dict(os.environ) + for name in ("PYTHONHOME", "PYTHONEXECUTABLE", "__PYVENV_LAUNCHER__"): + venv_env.pop(name, None) + # Keep startup imports available even with the synthetic home directory. + venv_env["PYTHONPATH"] = os.pathsep.join(sys.path) + path_code = ( + "import sys; print(sys.executable); " + "print(sys._base_executable); print(sys.prefix)" + ) + + def executable_paths(executable, *, cwd=None, env=venv_env): + result = subprocess.run( + [str(executable), "-B", "-S", "-c", path_code], + cwd=cwd, + env=env, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + return result.stdout.splitlines() + + for kind in ("absolute", "relative", "chain"): + bin_dir = root / kind / "bin" + bin_dir.mkdir(parents=True) + # Exercise both parent and adjacent pyvenv.cfg discovery. + prefix = bin_dir if kind == "chain" else bin_dir.parent + (prefix / "pyvenv.cfg").write_text(f"home = {home}\n") + executable = bin_dir / kind + # This regular file must never be preferred to a symlink's target. + (home / kind).touch() + if kind == "absolute": + executable.symlink_to(interpreter) + elif kind == "relative": + executable.symlink_to(os.path.relpath(interpreter, bin_dir)) + else: + (bin_dir / "target").symlink_to(interpreter) + executable.symlink_to("target") + invocation = f"./bin/{kind}" if kind == "relative" else executable + assert executable_paths(invocation, cwd=bin_dir.parent) == [ + str(executable), + str(framework_base if framework_launcher else interpreter), + str(prefix), + ] + + # Framework launchers preset their base; ordinary binaries keep argv[0]. + plain = root / "plain" / "python" + plain.parent.mkdir() + plain.symlink_to(interpreter) + assert executable_paths(plain)[:2] == [ + str(plain), + str(framework_base if framework_launcher else plain), + ] + + # Without a launcher preset, copies fall back to home/. + copied = root / "copied" / "python" + copied.parent.mkdir() + shutil.copy2(interpreter, copied) + (home / copied.name).touch() + (copied.parent / "pyvenv.cfg").write_text(f"home = {home}\n") + assert executable_paths(copied) == [ + str(copied), + str(framework_base if framework_launcher else home / copied.name), + str(copied.parent), + ] + + # A directory alias does not turn a copied executable into a symlink. + directory_alias = root / "directory-alias" + directory_alias.symlink_to(copied.parent, target_is_directory=True) + aliased_copy = directory_alias / copied.name + # The framework launcher canonicalizes its directory before dispatch. + assert executable_paths(aliased_copy) == [ + str(copied if framework_launcher else aliased_copy), + str(framework_base if framework_launcher else home / copied.name), + str(copied.parent if framework_launcher else directory_alias), + ] + + # Without a launcher preset, final-link resolution preserves target aliases. + target_link = root / "linked" / "python" + target_link.parent.mkdir() + (target_link.parent / "pyvenv.cfg").write_text(f"home = {home}\n") + target_link.symlink_to("../directory-alias/python") + assert executable_paths(target_link) == [ + str(target_link), + str(framework_base if framework_launcher else aliased_copy), + str(target_link.parent), + ] + + # macOS framework builds use a framework-derived base for overrides. + if sys.platform == "linux": + # Relative targets also preserve a double-leading-slash directory. + double_target = root / "double-target" + double_target.symlink_to("directory-alias/python") + target_link.unlink() + target_link.symlink_to("/" + str(double_target)) + assert executable_paths(target_link) == [ + str(target_link), + "/" + str(aliased_copy), + str(target_link.parent), + ] + + override_env = dict(venv_env, __PYVENV_LAUNCHER__=str(executable)) + assert executable_paths(plain, env=override_env) == [ + str(executable), + str(plain), + str(prefix), + ] From 896e07d9fa95dc0283ef185c61ab67595a004e65 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 21:36:37 -0700 Subject: [PATCH 112/229] Implement mkfifo with directory descriptor support (#9056) Support directory-relative FIFO creation. Retry interrupted path opens while releasing the interpreter and honoring signal-handler exceptions; preserve raw directory descriptors across callbacks. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/host_env/src/crt_fd.rs | 8 +- crates/host_env/src/posix.rs | 61 +++++++++++++++ crates/vm/src/stdlib/_io.rs | 19 +++-- crates/vm/src/stdlib/os.rs | 118 +++++++++++++++++++++------- crates/vm/src/stdlib/posix.rs | 92 ++++++++++++++++++++++ extra_tests/snippets/stdlib_os.py | 125 ++++++++++++++++++++++++++++++ 6 files changed, 385 insertions(+), 38 deletions(-) diff --git a/crates/host_env/src/crt_fd.rs b/crates/host_env/src/crt_fd.rs index cdedf177b04..c9af646122f 100644 --- a/crates/host_env/src/crt_fd.rs +++ b/crates/host_env/src/crt_fd.rs @@ -331,7 +331,13 @@ pub fn wopen(path: &widestring::WideCStr, flags: i32, mode: i32) -> io::Result, path: &ffi::CStr, flags: i32, mode: i32) -> io::Result { - cvt_fd(unsafe { c::openat(dir.as_raw(), path.as_ptr(), flags, mode) }) + openat_raw(dir.as_raw(), path, flags, mode) +} + +#[cfg(all(any(unix, target_os = "wasi"), not(target_os = "redox")))] +// Accept a descriptor number without asserting it stays open across calls. +pub fn openat_raw(dir: Raw, path: &ffi::CStr, flags: i32, mode: i32) -> io::Result { + cvt_fd(unsafe { c::openat(dir, path.as_ptr(), flags, mode) }) } pub fn fsync(fd: Borrowed<'_>) -> io::Result<()> { diff --git a/crates/host_env/src/posix.rs b/crates/host_env/src/posix.rs index ae7c0d3dedc..053148ca4a4 100644 --- a/crates/host_env/src/posix.rs +++ b/crates/host_env/src/posix.rs @@ -569,6 +569,67 @@ pub fn statvfs_fd(fd: i32) -> std::io::Result { } } +#[cfg(not(target_os = "redox"))] +pub fn mkfifo(path: &CStr, mode: libc::mode_t) -> std::io::Result<()> { + let ret = unsafe { libc::mkfifo(path.as_ptr(), mode) }; + if ret == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(target_vendor = "apple")] +type MkfifoAt = unsafe extern "C" fn(libc::c_int, *const libc::c_char, libc::mode_t) -> libc::c_int; + +#[cfg(target_vendor = "apple")] +fn mkfifoat_symbol() -> Option { + static MKFIFOAT: std::sync::OnceLock> = std::sync::OnceLock::new(); + *MKFIFOAT.get_or_init(|| { + // mkfifoat was added in macOS 13 / iOS 16. Resolve it at runtime so + // ordinary mkfifo remains usable on older Apple systems. + let symbol = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"mkfifoat".as_ptr()) }; + if symbol.is_null() { + None + } else { + // SAFETY: the system symbol has the MkfifoAt signature and stays + // loaded for the lifetime of the process. + Some(unsafe { core::mem::transmute::<*mut libc::c_void, MkfifoAt>(symbol) }) + } + }) +} + +#[cfg(not(target_os = "redox"))] +pub fn has_mkfifoat() -> bool { + #[cfg(target_vendor = "apple")] + { + mkfifoat_symbol().is_some() + } + #[cfg(not(target_vendor = "apple"))] + { + !cfg!(target_os = "android") + } +} + +#[cfg(not(any(target_os = "redox", target_os = "android")))] +pub fn mkfifoat(dir_fd: i32, path: &CStr, mode: libc::mode_t) -> std::io::Result<()> { + #[cfg(target_vendor = "apple")] + let mkfifoat = mkfifoat_symbol().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::Unsupported, + "mkfifoat unavailable on this platform", + ) + })?; + #[cfg(not(target_vendor = "apple"))] + let mkfifoat = libc::mkfifoat; + let ret = unsafe { mkfifoat(dir_fd, path.as_ptr(), mode) }; + if ret == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + #[cfg(not(target_os = "redox"))] pub fn mknod(path: &CStr, mode: libc::mode_t, device: libc::dev_t) -> std::io::Result<()> { let ret = unsafe { libc::mknod(path.as_ptr(), mode, device) }; diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 1d86717a211..5bcd8f64d29 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -5930,7 +5930,7 @@ mod _io { #[pymodule] mod fileio { use super::{_io::*, Offset, iobase_finalize}; - use crate::host_env::crt_fd; + use crate::host_env::{crt_fd, os::ErrorExt}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyUtf8Str, PyUtf8StrRef}, @@ -6044,14 +6044,19 @@ mod fileio { } else { let path = OsPath::try_from_fspath(name.clone(), vm)?; #[cfg(any(unix, target_os = "wasi"))] - let fd = host_io::open_path(&path.clone().into_cstring(vm)?, flags, 0o666); + let raw_path = path.clone().into_cstring(vm)?; #[cfg(windows)] - let fd = host_io::open_path(&path.to_wide_cstring(vm)?, flags, 0o666); + let raw_path = path.to_wide_cstring(vm)?; let filename = OsPathOrFd::Path(path); - match fd { - Ok(fd) => (fd.into_raw(), Some(filename)), - Err(e) => { - return Err(OSErrorBuilder::with_filename_from_errno(&e, filename, vm)); + loop { + match vm.allow_threads(|| host_io::open_path(&raw_path, flags, 0o666)) { + Ok(fd) => break (fd.into_raw(), Some(filename)), + Err(e) if e.posix_errno() == libc::EINTR => vm.check_signals()?, + Err(e) => { + return Err(OSErrorBuilder::with_filename_from_errno( + &e, filename, vm, + )); + } } } } diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index b873cb4c251..7419c8f9260 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -199,7 +199,10 @@ impl ToPyObject for crt_fd::Borrowed<'_> { #[pymodule(sub, name = "posix")] pub(super) mod _os { - use super::{DirFd, DstDirFd, FollowSymlinks, RawMode, SrcDirFd, SupportFunc}; + use super::{ + AT_FDCWD, DefaultDirFd, DirFd, DirFdKeyword, DstDirFd, FollowSymlinks, RawMode, SrcDirFd, + SupportFunc, warn_if_bool_fd, + }; #[cfg(not(windows))] use crate::exceptions; use crate::host_env::fileutils::StatStruct; @@ -212,9 +215,11 @@ pub(super) mod _os { }, class::PyClassDef, common::lock::{OnceCell, PyRwLock}, - convert::{IntoPyException, ToPyObject}, + convert::{IntoPyException, ToPyException, ToPyObject}, exceptions::{OSErrorBuilder, ToOSErrorBuilder}, - function::{ArgBytesLike, ArgMemoryBuffer, FsPath, FuncArgs}, + function::{ + ArgBytesLike, ArgMemoryBuffer, ArgumentError, FromArgs, FsPath, FuncArgs, Param, + }, host_env::crt_fd, ospath::{OsPath, OsPathOrFd, OutputMode, PathConverter}, protocol::PyIterReturn, @@ -276,20 +281,57 @@ pub(super) mod _os { crt_fd::closerange(fd_low, fd_high); } + // Signal callbacks can close this descriptor between open attempts. + #[cfg(any(unix, windows, target_os = "wasi"))] + struct OpenDirFd(crt_fd::Raw); + + #[cfg(any(unix, windows, target_os = "wasi"))] + impl FromArgs for OpenDirFd { + const PARAMS: Option<&'static [Param]> = Some(DefaultDirFd::PARAMS); + + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result { + let fd = match args.take_keyword("dir_fd") { + Some(o) if vm.is_none(&o) => AT_FDCWD, + None => AT_FDCWD, + Some(o) => { + warn_if_bool_fd(&o, vm).map_err(Into::::into)?; + let fd = o.try_index_opt(vm).unwrap_or_else(|| { + Err(vm.new_type_error(format!( + "argument should be integer or None, not {}", + o.class().name() + ))) + })?; + fd.try_to_primitive(vm)? + } + }; + if fd == -1 { + return Err(io::Error::from_raw_os_error(libc::EBADF) + .to_pyexception(vm) + .into()); + } + if !OPEN_DIR_FD && fd != AT_FDCWD { + return Err(vm + .new_not_implemented_error("dir_fd unavailable on this platform") + .into()); + } + Ok(Self(fd)) + } + } + #[cfg(any(unix, windows, target_os = "wasi"))] #[derive(FromArgs)] - struct OpenArgs<'fd> { + struct OpenArgs { path: OsPath, flags: i32, #[pyarg(any, default = 0o777)] mode: i32, #[pyarg(flatten)] - dir_fd: DirFd<'fd, { OPEN_DIR_FD as usize }>, + dir_fd: OpenDirFd, } #[pyfunction] - fn open(args: OpenArgs<'_>, vm: &VirtualMachine) -> PyResult { - os_open(args.path, args.flags, args.mode, args.dir_fd, vm) + fn open(args: OpenArgs, vm: &VirtualMachine) -> PyResult { + os_open(args.path, args.flags, args.mode, args.dir_fd.0, vm) } #[cfg(any(unix, windows, target_os = "wasi"))] @@ -297,34 +339,50 @@ pub(super) mod _os { name: OsPath, flags: i32, mode: i32, - dir_fd: DirFd<'_, { OPEN_DIR_FD as usize }>, + dir_fd: crt_fd::Raw, vm: &VirtualMachine, ) -> PyResult { + use crate::host_env::os::ErrorExt; + #[cfg(windows)] - let fd = { - let [] = dir_fd.0; - let name = name.to_wide_cstring(vm)?; - let flags = flags | crate::host_env::os::O_NOINHERIT; - crt_fd::wopen(&name, flags, mode) - }; + let raw_name = name.to_wide_cstring(vm)?; #[cfg(not(windows))] - let fd = { - let name = name.clone().into_cstring(vm)?; - #[cfg(not(target_os = "wasi"))] - let flags = flags | crate::host_env::os::O_CLOEXEC; - #[cfg(not(target_os = "redox"))] - if let Some(dir_fd) = dir_fd.get_opt() { - crt_fd::openat(dir_fd, &name, flags, mode) - } else { - crt_fd::open(&name, flags, mode) - } - #[cfg(target_os = "redox")] - { - let [] = dir_fd.0; - crt_fd::open(&name, flags, mode) + let raw_name = name.clone().into_cstring(vm)?; + #[cfg(windows)] + let flags = flags | crate::host_env::os::O_NOINHERIT; + #[cfg(all(not(windows), not(target_os = "wasi")))] + let flags = flags | crate::host_env::os::O_CLOEXEC; + + loop { + let fd = vm.allow_threads(|| { + #[cfg(windows)] + { + let _ = dir_fd; + crt_fd::wopen(&raw_name, flags, mode) + } + #[cfg(not(windows))] + { + #[cfg(not(target_os = "redox"))] + if dir_fd != AT_FDCWD { + crt_fd::openat_raw(dir_fd, &raw_name, flags, mode) + } else { + crt_fd::open(&raw_name, flags, mode) + } + #[cfg(target_os = "redox")] + { + let _ = dir_fd; + crt_fd::open(&raw_name, flags, mode) + } + } + }); + match fd { + Ok(fd) => return Ok(fd), + Err(err) if err.posix_errno() == libc::EINTR => vm.check_signals()?, + Err(err) => { + return Err(OSErrorBuilder::with_filename_from_errno(&err, name, vm)); + } } - }; - fd.map_err(|err| OSErrorBuilder::with_filename_from_errno(&err, name, vm)) + } } #[derive(FromArgs)] diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 96bb0066702..388771acbc6 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1012,6 +1012,91 @@ pub mod module { } } + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + struct MkfifoPath(OsPath); + + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + impl TryFromObject for MkfifoPath { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + crate::ospath::PathConverter::new() + .function("mkfifo") + .try_path(obj, vm) + .map(Self) + } + } + + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + struct MkfifoMode(i32); + + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + impl TryFromObject for MkfifoMode { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let mode = obj.try_index(vm)?; + i32::try_from(mode.as_bigint()) + .map(Self) + .map_err(|_| vm.new_overflow_error("Python int too large to convert to C int")) + } + } + + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + #[derive(FromArgs)] + struct MkfifoArgs { + #[pyarg(any)] + path: MkfifoPath, + #[pyarg(any, default = MkfifoMode(0o666), py_default = "438")] + mode: MkfifoMode, + #[pyarg(named, optional, py_default = "None")] + dir_fd: OptionalArg, + } + + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + #[pyfunction] + fn mkfifo(args: MkfifoArgs, vm: &VirtualMachine) -> PyResult<()> { + let c_path = args.path.0.into_cstring(vm)?; + let mode = args.mode.0 as libc::mode_t; + // Keep a raw C int: even -1 is valid with an absolute path, for which + // mkfifoat ignores dir_fd. BorrowedFd cannot represent that value. + let dir_fd = match args.dir_fd { + OptionalArg::Present(fd) if !vm.is_none(&fd) => { + warn_if_bool_fd(&fd, vm)?; + let fd = fd.try_index_opt(vm).unwrap_or_else(|| { + Err(vm.new_type_error(format!( + "argument should be integer or None, not {}", + fd.class().name() + ))) + })?; + let fd = i32::try_from(fd.as_bigint()).map_err(|_| { + let message = if fd.as_bigint().sign() == malachite_bigint::Sign::Minus { + "fd is less than minimum" + } else { + "fd is greater than maximum" + }; + vm.new_overflow_error(message) + })?; + (fd != rustpython_host_env::os::AT_FDCWD).then_some(fd) + } + _ => None, + }; + if dir_fd.is_some() && !host_posix::has_mkfifoat() { + return Err(vm.new_not_implemented_error("dir_fd unavailable on this platform")); + } + + loop { + let result = vm.allow_threads(|| { + #[cfg(not(target_os = "android"))] + if let Some(dir_fd) = dir_fd { + return host_posix::mkfifoat(dir_fd, &c_path, mode); + } + host_posix::mkfifo(&c_path, mode) + }); + match result { + Ok(()) => return Ok(()), + Err(err) if err.raw_os_error() == Some(libc::EINTR) => vm.check_signals()?, + Err(err) => return Err(err.into_pyexception(vm)), + } + } + } + #[cfg(not(target_os = "redox"))] const MKNOD_DIR_FD: bool = cfg!(not(target_vendor = "apple")); @@ -2243,6 +2328,13 @@ pub mod module { SupportFunc::new("lchown", None, None, None), #[cfg(not(target_os = "redox"))] SupportFunc::new("fchown", Some(true), None, Some(true)), + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + SupportFunc::new( + "mkfifo", + Some(false), + Some(host_posix::has_mkfifoat()), + Some(false), + ), #[cfg(not(target_os = "redox"))] SupportFunc::new("mknod", Some(true), Some(MKNOD_DIR_FD), Some(false)), SupportFunc::new("umask", Some(false), Some(false), Some(false)), diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index df7c723c967..50ce16be191 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -633,3 +633,128 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert os.stat_result(tuple(range(10)), {"st_atime": 1.5}).st_atime == 1.5 with assert_raises(TypeError): os.stat_result(tuple(range(10)), ["st_atime"]) + + +def mkfifo_paths_and_directory_descriptors(): + import errno + import inspect + import tempfile + import warnings + + assert str(inspect.signature(os.mkfifo)) == "(path, mode=438, *, dir_fd=None)" + assert os.mkfifo not in os.supports_fd + assert os.mkfifo not in os.supports_follow_symlinks + if sys.platform.startswith("linux"): + assert os.mkfifo in os.supports_dir_fd + + class Path: + def __init__(self, value): + self.value = value + + def __fspath__(self): + return self.value + + class Index: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + class BadIndex: + def __index__(self): + raise RuntimeError("mkfifo index marker") + + # Never open a FIFO: creating, inspecting and unlinking it cannot wait for + # another process to connect. All paths belong to this temporary directory. + with tempfile.TemporaryDirectory(prefix="rustpython-mkfifo-") as directory: + path = os.path.join(directory, "fifo") + for value in (path, os.fsencode(path), Path(path), Path(os.fsencode(path))): + assert os.mkfifo(path=value, mode=0) is None + info = os.stat(path) + assert stat.S_ISFIFO(info.st_mode) + assert stat.S_IMODE(info.st_mode) == 0 + with assert_raises(FileExistsError) as error: + os.mkfifo(value) + assert error.exception.errno == errno.EEXIST + assert error.exception.filename is None + os.unlink(path) + + assert os.mkfifo(path, dir_fd=None) is None + assert stat.S_ISFIFO(os.stat(path).st_mode) + assert stat.S_IMODE(os.stat(path).st_mode) & ~0o666 == 0 + os.unlink(path) + + for value in (None, 1.5): + assert_raises(TypeError, os.mkfifo, path, value) + for value in (2**31, -(2**31) - 1, 2**100, -(2**100)): + with assert_raises(OverflowError) as error: + os.mkfifo(path, value) + assert str(error.exception) == "Python int too large to convert to C int" + with assert_raises(OverflowError) as error: + os.mkfifo(path, dir_fd=value) + expected = ( + "fd is greater than maximum" if value > 0 else "fd is less than minimum" + ) + assert str(error.exception) == expected + for value in (1.5, "fd"): + assert_raises(TypeError, os.mkfifo, path, dir_fd=value) + for value in (None, 1): + with assert_raises(TypeError) as error: + os.mkfifo(value) + expected = "NoneType" if value is None else "int" + assert str(error.exception) == ( + "mkfifo: path should be string, bytes or os.PathLike, not " + expected + ) + assert_raises(TypeError, os.mkfifo, Path(None)) + for value in (path + "\0", os.fsencode(path) + b"\0"): + with assert_raises(ValueError) as error: + os.mkfifo(value) + assert str(error.exception) == "mkfifo: embedded null character in path" + assert_raises(TypeError, os.mkfifo, path, 0o600, None) + for options in ({"mode": BadIndex()}, {"dir_fd": BadIndex()}): + with assert_raises(RuntimeError) as error: + os.mkfifo(path, **options) + assert str(error.exception) == "mkfifo index marker" + with assert_raises(FileNotFoundError) as error: + os.mkfifo(os.path.join(directory, "missing", "fifo")) + assert error.exception.errno == errno.ENOENT + assert error.exception.filename is None + + directory_fd = os.open(directory, os.O_RDONLY) + try: + if os.mkfifo in os.supports_dir_fd: + assert ( + os.mkfifo(Path(b"fifo"), Index(0o600), dir_fd=Index(directory_fd)) + is None + ) + assert stat.S_ISFIFO(os.stat(path).st_mode) + assert stat.S_IMODE(os.stat(path).st_mode) & ~0o600 == 0 + os.unlink(path) + for value in (-1, -(2**31)): + assert os.mkfifo(path, dir_fd=value) is None + assert stat.S_ISFIFO(os.stat(path).st_mode) + os.unlink(path) + with assert_raises(OSError) as error: + os.mkfifo("fifo", dir_fd=-1) + assert error.exception.errno == errno.EBADF + assert error.exception.filename is None + with warnings.catch_warnings(record=True) as records: + warnings.simplefilter("always") + assert os.mkfifo(path, dir_fd=True) is None + assert len(records) == 1 + assert records[0].category is RuntimeWarning + assert str(records[0].message) == "bool is used as a file descriptor" + os.unlink(path) + else: + with assert_raises(NotImplementedError) as error: + os.mkfifo(path, dir_fd=directory_fd) + assert str(error.exception) == "dir_fd unavailable on this platform" + finally: + os.close(directory_fd) + + +if sys.platform.startswith("linux"): + assert hasattr(os, "mkfifo") +if hasattr(os, "mkfifo"): + mkfifo_paths_and_directory_descriptors() From 9ea138c8891ebf0bdd7865c629d11466d768d2a9 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 21:38:23 -0700 Subject: [PATCH 113/229] Canonicalize f-string field spacing in annotations (#9077) Ignore incidental formatted-value whitespace while retaining brace-leading safeguards and literal/debug text. Include compile-only annotation-constant coverage. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/codegen/src/unparse.rs | 14 +------------- extra_tests/snippets/syntax_annotations.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 868f1b1bab1..7c139e77a8e 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -628,19 +628,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { // put a space to avoid escaping the bracket "{ " } else { - // Preserve leading whitespace between '{' and the expression - let source_text = self.source.source_text(); - let start = val.range().start().to_usize(); - if start > 0 - && source_text - .as_bytes() - .get(start - 1) - .is_some_and(|b| b.is_ascii_whitespace()) - { - "{ " - } else { - "{" - } + "{" }; self.p(brace)?; self.p(&buffered)?; diff --git a/extra_tests/snippets/syntax_annotations.py b/extra_tests/snippets/syntax_annotations.py index 9b7d0c0ce89..b6859a89b3f 100644 --- a/extra_tests/snippets/syntax_annotations.py +++ b/extra_tests/snippets/syntax_annotations.py @@ -1,5 +1,18 @@ from typing import get_type_hints +# Future f-string annotations canonicalize fields, not their literal padding. +for formatted, expected in ( + ("f'{ value }'", "f'{value}'"), + ("f'{ value = }'", "f' value = {value!r}'"), + ("f'{ value : >8}'", "f'{value: >8}'"), + ("f'{ {1, 2} }'", "f'{ {1, 2}}'"), +): + formatted_code = compile( + "from __future__ import annotations\nx: " + formatted, "", "exec" + ) + assert expected in formatted_code.co_consts + + def func(s: str) -> int: return int(s) From 1a1605dee91e24609fd71f9b80d8d7d471b04a87 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sun, 11 Oct 2026 22:49:33 -0700 Subject: [PATCH 114/229] Use active builtins in descriptor and method reductions (#9057) Use frame-aware getattr lookup consistently across descriptor and method reductions. Preserve defining types and exception order, and perform generic builtin-mapping subclass lookup only once. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/builtin_func.rs | 2 +- crates/vm/src/builtins/descriptor.rs | 16 ++--- crates/vm/src/builtins/function.rs | 4 +- crates/vm/src/vm/mod.rs | 2 +- extra_tests/snippets/stdlib_pickle.py | 81 ++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 11 deletions(-) diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index e0399987049..ac89021b9f6 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -215,7 +215,7 @@ impl PyNativeFunction { .as_ref() .filter(|bound| !bound.class().is_subtype(vm.ctx.types.module_type)) { - let getattr = vm.builtins.get_attr("getattr", vm)?; + let getattr = vm.eval_get_builtin(vm.ctx.intern_str("getattr"))?; Ok(vm .new_tuple((getattr, (bound.clone(), zelf.value.name))) .into()) diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index e8329fbb572..79afc8cb5a8 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -155,10 +155,12 @@ impl Py { fn __reduce__( &self, vm: &VirtualMachine, - ) -> (Option, (Option, &'static str)) { - let builtins_getattr = vm.builtins.get_attr("getattr", vm).ok(); - let classname = vm.builtins.get_attr(&self.common.typ.__name__(vm), vm).ok(); - (builtins_getattr, (classname, self.method.name)) + ) -> PyResult<(PyObjectRef, (PyTypeRef, &'static str))> { + let builtins_getattr = vm.eval_get_builtin(vm.ctx.intern_str("getattr"))?; + Ok(( + builtins_getattr, + (self.common.typ.to_owned(), self.method.name), + )) } } @@ -851,7 +853,7 @@ impl Py { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { - let builtins_getattr = vm.builtins.get_attr("getattr", vm)?; + let builtins_getattr = vm.eval_get_builtin(vm.ctx.intern_str("getattr"))?; Ok(vm .ctx .new_tuple(vec![ @@ -2064,7 +2066,7 @@ impl Callable for PyWrapper { impl Py { #[pymethod] fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { - let getattr = vm.builtins.get_attr("getattr", vm)?; + let getattr = vm.eval_get_builtin(vm.ctx.intern_str("getattr"))?; let args = vm.ctx.new_tuple(vec![ self.typ.to_owned().into(), self.name.to_owned().into(), @@ -2181,7 +2183,7 @@ impl Py { #[pymethod] fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - let builtins_getattr = vm.builtins.get_attr("getattr", vm)?; + let builtins_getattr = vm.eval_get_builtin(vm.ctx.intern_str("getattr"))?; Ok(vm .ctx .new_tuple(vec![ diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index b0bd5a010b5..be4994a6204 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -1470,9 +1470,9 @@ impl Py { &self, vm: &VirtualMachine, ) -> PyResult<(PyObjectRef, (PyObjectRef, PyObjectRef))> { - let builtins_getattr = vm.builtins.get_attr("getattr", vm)?; - let func_self = self.object.clone(); let func_name = self.function.get_attr("__name__", vm)?; + let builtins_getattr = vm.eval_get_builtin(vm.ctx.intern_str("getattr"))?; + let func_self = self.object.clone(); Ok((builtins_getattr, (func_self, func_name))) } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 304991f9504..0c5f8e2b088 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -3170,7 +3170,7 @@ impl VirtualMachine { pub fn eval_get_builtin(&self, name: &'static PyStrInterned) -> PyResult { let builtins = crate::frame::current_builtins().unwrap_or_else(|| self.builtins.dict().into()); - if let Some(dict) = builtins.downcast_ref::() { + if let Some(dict) = builtins.downcast_ref_if_exact::(self) { match dict.get_item_opt(name, self)? { Some(value) => Ok(value), None => Err(self.new_attribute_error(name.to_string())), diff --git a/extra_tests/snippets/stdlib_pickle.py b/extra_tests/snippets/stdlib_pickle.py index e1872a24bfa..999106d4235 100644 --- a/extra_tests/snippets/stdlib_pickle.py +++ b/extra_tests/snippets/stdlib_pickle.py @@ -1,6 +1,7 @@ import _pickle import io import sys +from types import MappingProxyType, MethodType from testutils import assert_raises @@ -35,3 +36,83 @@ else: expected = f"{name} exceeds system's maximum size of {sys.maxsize} bytes" assert str(caught.exception) == expected + + +# Reduction returns getattr from the active frame's builtins without calling it. +class ReductionOwner: + __slots__ = ("slot",) + + def method(self): + pass + + +owner = ReductionOwner() +items = [] +replacement_getattr = object() +for descriptor, receiver, name in ( + (object.__str__, object, "__str__"), + (str.join, str, "join"), + (io.BytesIO.getvalue, io.BytesIO, "getvalue"), + (ReductionOwner.slot, ReductionOwner, "slot"), + (owner.__str__, owner, "__str__"), + (items.append, items, "append"), + (dict.fromkeys, dict, "fromkeys"), + (owner.method, owner, "method"), +): + for builtins_type in (dict, MappingProxyType): + namespace = { + "__builtins__": builtins_type({"getattr": replacement_getattr}), + "descriptor": descriptor, + } + exec("reduced = descriptor.__reduce__()", namespace) + assert namespace["reduced"] == (replacement_getattr, (receiver, name)) + + namespace["__builtins__"] = builtins_type({}) + with assert_raises(AttributeError) as caught: + exec("descriptor.__reduce__()", namespace) + assert caught.exception.args == ("getattr",) + +# Module-bound functions reduce to a name without looking up getattr. +namespace = {"__builtins__": {}, "function": len} +exec("reduced = function.__reduce__()", namespace) +assert namespace["reduced"] == "len" + + +class MissingBuiltin(dict): + def __getitem__(self, key): + raise KeyError(key) + + def __missing__(self, key): + raise AssertionError("must not retry __missing__ after custom __getitem__") + + +for descriptor, name in ( + (object.__str__, "getattr"), + (iter([1]), "iter"), + (reversed([1]), "reversed"), +): + namespace = {"__builtins__": MissingBuiltin(), "descriptor": descriptor} + with assert_raises(AttributeError) as caught: + exec("descriptor.__reduce__()", namespace) + assert caught.exception.args == (name,) + + +name_error = RuntimeError("function name first") + + +class NamedCallable: + def __call__(self, *args): + pass + + @property + def __name__(self): + raise name_error + + +namespace = { + "__builtins__": {}, + "method": MethodType(NamedCallable(), owner), +} +with assert_raises(RuntimeError) as caught: + exec("method.__reduce__()", namespace) +assert caught.exception is name_error From 597a4221d2b11a1c6414fa320b6d16dd94721904 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 00:10:29 -0700 Subject: [PATCH 115/229] Serialize bytearray exports with resizing (#9087) * Serialize bytearray exports with resizing Keep descriptor acquisition and export retention under the bytearray read lock, and recheck exports after acquiring a resize lock. Guard initializer and sequence deletion writes, preserve zero-index initialization callbacks, and match RawIOBase.read buffer ownership without bypassing export checks. Add bounded Python export-lifetime and concurrent decode/memoryview regression coverage. Assisted-by: OpenAI Codex:model identifier unavailable * Release bytearray initializer storage and check deletion indices Reset initializer storage before zero-count callbacks and returns. Validate integer deletion indices with a nonblocking snapshot before export rejection where possible, then revalidate under the resize guard. Preserve fast export rejection when contended to avoid codec callback deadlocks. Add Python regressions for exported-empty rejection, initializer capacity release, callback ordering, and ordinary deletion error precedence. Assisted-by: OpenAI Codex:model identifier unavailable * Fix bytearray concatenation buffer callback reentrancy Acquire both operand exports before borrowing their bytes and release them in CPython order. Cover reentrant initialization, mutable callbacks and export cleanup through public concatenation APIs. Assisted-by: OpenAI Codex:model identifier unavailable * Stage bytearray buffer callbacks outside receiver locks Keep operation-specific export and release ordering for comparisons, searches, affix tuples and translation. Resolve padding arguments before borrowing the receiver, and retain buffer owners through computation without holding bytearray locks across Python callbacks. Add bounded Python regressions for callback mutation, reentrant initialization, lazy affixes, subclass protocols and ordered cleanup. Assisted-by: OpenAI Codex:model identifier unavailable * Consolidate bytearray regressions in the builtin snippet Preserve all existing regression bodies and coverage while following the requested snippet organization. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/anystr.rs | 13 + crates/vm/src/builtins/bytearray.rs | 327 ++++++++++-- crates/vm/src/bytes_inner.rs | 51 +- crates/vm/src/stdlib/_io.rs | 11 +- extra_tests/snippets/builtin_bytearray.py | 591 ++++++++++++++++++++++ 5 files changed, 925 insertions(+), 68 deletions(-) diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs index b9ce612500b..1b7b20641e1 100644 --- a/crates/vm/src/anystr.rs +++ b/crates/vm/src/anystr.rs @@ -47,6 +47,19 @@ pub(crate) struct StartsEndsWithArgs { } impl StartsEndsWithArgs { + pub(crate) fn parts(&self, len: usize) -> (&PyObject, Option>) { + let range = if self.start.is_some() || self.end.is_some() { + Some(adjust_indices( + self.start.as_ref().map(AsRef::as_ref), + self.end.as_ref().map(AsRef::as_ref), + len, + )) + } else { + None + }; + (&self.affix, range) + } + pub(crate) fn get_value(self, len: usize) -> (PyObjectRef, Option>) { let range = if self.start.is_some() || self.end.is_some() { Some(adjust_indices( diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index 5afeec62833..c463489df61 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -6,7 +6,7 @@ use super::{ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - anystr::{self, AnyStr}, + anystr::{self, AnyStr, StringRange}, atomic_func, byte::{bytearray_extend_from_object, bytearray_from_object, value_from_object}, bytes_inner::{ @@ -65,6 +65,21 @@ impl PyPayload for PyByteArray { } } +// Acquisitions can call Python, and releases must follow acquisition order. +// No bytearray lock may be held when this owner is dropped. +#[derive(Default)] +struct ByteArrayBuffers { + first: Option, + second: Option, +} + +impl Drop for ByteArrayBuffers { + fn drop(&mut self) { + drop(self.first.take()); + drop(self.second.take()); + } +} + /// Fill bytearray class methods dictionary. pub(crate) fn init(context: &'static Context) { PyByteArray::extend_class(context, context.types.bytearray_type); @@ -92,6 +107,105 @@ impl PyByteArray { PyDetachingRwLockWriteGuard::map(self.inner.write(), |inner| &mut inner.elements) } + fn with_export(&self, f: impl FnOnce(usize) -> R) -> R { + self.exports.fetch_add(1, Ordering::Release); + scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } + // Reading the length also waits for any earlier resize to finish. + f(self.__len__()) + } + + fn buffer_snapshot(buffer: &PyBuffer, vm: &VirtualMachine) -> PyResult> { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(buffer.desc.len) + .map_err(|_| vm.no_memory_error())?; + buffer.append_to(&mut bytes); + Ok(bytes) + } + + fn search_exported( + &self, + options: ByteInnerFindOptions, + search: impl FnOnce(&[u8], &[u8], core::ops::Range) -> R, + vm: &VirtualMachine, + ) -> PyResult { + self.with_export(|len| { + let (sub, range) = options.parts(len); + let buffer = if sub.check_buffer() { + Some(PyBuffer::from_object(vm, sub, BufferFlags::SIMPLE)?) + } else { + None + }; + let needle = match &buffer { + Some(buffer) => Self::buffer_snapshot(buffer, vm)?, + None => ByteInnerSub::try_from_object(vm, sub.to_owned())?.into_vec(vm)?, + }; + let result = search(&self.borrow_buf(), &needle, range); + drop(buffer); + Ok(result) + }) + } + + fn starts_ends_with( + &self, + options: anystr::StartsEndsWithArgs, + ends: bool, + vm: &VirtualMachine, + ) -> PyResult { + self.with_export(|len| { + let (affix, range) = options.parts(len); + let tailmatch = |item: &PyObject| -> PyResult { + // bytes subclasses use their bytes directly, as in tailmatch. + let buffer = if item.downcast_ref::().is_some() { + None + } else { + Some(PyBuffer::from_object(vm, item, BufferFlags::SIMPLE)?) + }; + let needle = match &buffer { + Some(buffer) => Self::buffer_snapshot(buffer, vm)?, + None => item.downcast_ref::().unwrap().as_bytes().to_vec(), + }; + let result = { + let bytes = self.borrow_buf(); + let slice = match &range { + Some(range) if !range.is_normal() => None, + Some(range) => bytes.get(range.clone()), + None => Some(&*bytes), + }; + slice.is_some_and(|slice| { + if ends { + slice.ends_with(&needle) + } else { + slice.starts_with(&needle) + } + }) + }; + drop(buffer); + Ok(result) + }; + if let Some(tuple) = affix.downcast_ref::() { + for item in tuple { + if tailmatch(item)? { + return Ok(true); + } + } + Ok(false) + } else { + tailmatch(affix).map_err(|error| { + if error.fast_isinstance(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "{} first arg must be bytes or a tuple of bytes, not {}", + if ends { "endswith" } else { "startswith" }, + affix.class().name() + )) + } else { + error + } + }) + } + }) + } + fn repeat(&self, value: isize, vm: &VirtualMachine) -> PyResult { self.inner().mul(value, vm).map(|x| x.into()) } @@ -140,7 +254,7 @@ impl PyByteArray { pub fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { match SequenceIndex::try_from_borrowed_object::(vm, needle)? { - SequenceIndex::Int(i) => self.try_resizable(vm)?.elements.delitem_by_index(vm, i), + SequenceIndex::Int(i) => self.delitem_by_index(i, vm), SequenceIndex::Slice(slice) => { // TODO: delete 0 elements don't need resizable self.try_resizable(vm)?.elements.delitem_by_slice(vm, slice) @@ -148,6 +262,27 @@ impl PyByteArray { } } + fn delitem_by_index(&self, index: isize, vm: &VirtualMachine) -> PyResult<()> { + // Validate before export rejection when the bytes are available without + // waiting. A codec callback may already hold a read guard with a writer + // queued, so a blocking read here could deadlock before that rejection. + let valid = self + .inner + .try_read() + .map(|inner| inner.elements.wrap_index(index).is_some()); + if valid == Some(false) { + return Err(vm.new_index_error("bytearray index out of range")); + } + let mut inner = self.try_resizable(vm)?; + // Another thread may have resized the bytearray between the locks. + let index = inner + .elements + .wrap_index(index) + .ok_or_else(|| vm.new_index_error("bytearray index out of range"))?; + inner.elements.remove(index); + Ok(()) + } + fn irepeat(zelf: &Py, n: isize, vm: &VirtualMachine) -> PyResult<()> { if n == 1 { return Ok(()); @@ -173,11 +308,6 @@ impl PyByteArray { fn inner(&self) -> PyDetachingRwLockReadGuard<'_, PyBytesInner> { self.inner.read() } - #[inline] - fn inner_mut(&self) -> PyDetachingRwLockWriteGuard<'_, PyBytesInner> { - self.inner.write() - } - fn __len__(&self) -> usize { self.borrow_buf().len() } @@ -496,7 +626,11 @@ impl Py { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult { - Ok(self.inner().center(options, vm)?.into()) + let (width, fillchar) = options.get_value("center", vm)?; + Ok(self + .inner() + .pad_with(width, fillchar, AnyStr::py_center, vm)? + .into()) } #[pymethod] @@ -505,7 +639,11 @@ impl Py { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult { - Ok(self.inner().ljust(options, vm)?.into()) + let (width, fillchar) = options.get_value("ljust", vm)?; + Ok(self + .inner() + .pad_with(width, fillchar, AnyStr::py_ljust, vm)? + .into()) } #[pymethod] @@ -514,14 +652,20 @@ impl Py { options: ByteInnerPaddingOptions, vm: &VirtualMachine, ) -> PyResult { - Ok(self.inner().rjust(options, vm)?.into()) + let (width, fillchar) = options.get_value("rjust", vm)?; + Ok(self + .inner() + .pad_with(width, fillchar, AnyStr::py_rjust, vm)? + .into()) } #[pymethod] fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { - self.exports.fetch_add(1, Ordering::Release); - scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } - self.inner().count(options, vm) + self.search_exported( + options, + |bytes, needle, range| bytes.py_count(needle, range, |h, n| h.find_iter(n).count()), + vm, + ) } #[pymethod] @@ -539,19 +683,7 @@ impl Py { #[pymethod] fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult { - let borrowed = self.borrow_buf(); - let (affix, substr) = - match options.prepare(&*borrowed, borrowed.len(), |s, r| s.get_bytes(r)) { - Some(x) => x, - None => return Ok(false), - }; - substr.py_starts_ends_with( - &affix, - "endswith", - "bytes", - |s, x: PyBytesInner| s.ends_with(x.as_bytes()), - vm, - ) + self.starts_ends_with(options, true, vm) } #[pymethod] @@ -560,19 +692,7 @@ impl Py { options: anystr::StartsEndsWithArgs, vm: &VirtualMachine, ) -> PyResult { - let borrowed = self.borrow_buf(); - let (affix, substr) = - match options.prepare(&*borrowed, borrowed.len(), |s, r| s.get_bytes(r)) { - Some(x) => x, - None => return Ok(false), - }; - substr.py_starts_ends_with( - &affix, - "startswith", - "bytes", - |s, x: PyBytesInner| s.starts_with(x.as_bytes()), - vm, - ) + self.starts_ends_with(options, false, vm) } /// Converting the needle runs Python, which must not resize this @@ -586,9 +706,11 @@ impl Py { where F: Fn(&[u8], &[u8]) -> Option, { - self.exports.fetch_add(1, Ordering::Release); - scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } - self.inner().find(options, find, vm) + self.search_exported( + options, + |bytes, needle, range| bytes.py_find(needle, range, find), + vm, + ) } #[pymethod] @@ -621,7 +743,28 @@ impl Py { options: ByteInnerTranslateOptions, vm: &VirtualMachine, ) -> PyResult { - Ok(self.inner().translate(options, vm)?.into()) + let mut buffers = ByteArrayBuffers::default(); + if let Some(table) = &options.table { + let buffer = PyBuffer::from_object(vm, table, BufferFlags::SIMPLE)?; + if buffer.desc.len != 256 { + return Err(vm.new_value_error("translation table must be 256 characters long")); + } + buffers.first = Some(buffer); + } + buffers.second = Some(PyBuffer::from_object( + vm, + &options.delete, + BufferFlags::SIMPLE, + )?); + // Both acquisition callbacks finish before any input bytes are read. + let table = match &buffers.first { + Some(buffer) => PyByteArray::buffer_snapshot(buffer, vm)?, + None => (0..=u8::MAX).collect(), + }; + let delete = PyByteArray::buffer_snapshot(buffers.second.as_ref().unwrap(), vm)?; + let result = self.inner().translate_with(&table, &delete).into(); + drop(buffers); + Ok(result) } #[pymethod] @@ -781,9 +924,18 @@ impl Initializer for PyByteArray { type Args = ByteInnerNewOptions; fn init(zelf: &Py, options: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - // First unpack bytearray and *then* get a lock to set it. - let mut inner = options.get_inner(bytearray_from_object, vm)?; - core::mem::swap(&mut *zelf.inner_mut(), &mut inner); + // Reject exports and clear existing contents before converting source. + zelf.try_resizable(vm)?.elements = Vec::new(); + // Conversion can call Python, so do not hold the lock across it and + // check again for exports acquired by a callback or another thread. + let (mut inner, zero_count) = + options.get_inner_with_zero_count(bytearray_from_object, vm)?; + // A zero integer count performs no further writes, including when its + // __index__ callback changed this bytearray or acquired an export. + if zero_count { + return Ok(()); + } + core::mem::swap(&mut *zelf.try_resizable(vm)?, &mut inner); Ok(()) } } @@ -795,10 +947,33 @@ impl Comparable for PyByteArray { op: PyComparisonOp, vm: &VirtualMachine, ) -> PyResult { - if let Some(res) = op.identical_optimization(zelf, other) { - return Ok(res.into()); + if !zelf.as_object().check_buffer() || !other.check_buffer() { + return Ok(PyComparisonValue::NotImplemented); } - Ok(zelf.inner().cmp(other, op, vm)) + let first = match PyBuffer::from_object(vm, zelf.as_object(), BufferFlags::SIMPLE) { + Ok(buffer) => buffer, + Err(_) => return Ok(PyComparisonValue::NotImplemented), + }; + let mut buffers = ByteArrayBuffers { + first: Some(first), + second: None, + }; + buffers.second = match PyBuffer::from_object(vm, other, BufferFlags::SIMPLE) { + Ok(buffer) => Some(buffer), + Err(_) => return Ok(PyComparisonValue::NotImplemented), + }; + let left = buffers.first.as_ref().unwrap(); + let right = buffers.second.as_ref().unwrap(); + let result = if left.desc.len != right.desc.len && op.eval_eq(|| false).is_some() { + op.eval_eq(|| false).unwrap() + } else { + let left = Self::buffer_snapshot(left, vm)?; + let right = Self::buffer_snapshot(right, vm)?; + op.eval_eq(|| left == right) + .unwrap_or_else(|| op.eval_ord(left.cmp(&right))) + }; + drop(buffers); + Ok(result.into()) } } @@ -841,9 +1016,11 @@ impl AsBuffer for PyByteArray { } fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + // Keep the length stable until PyBuffer::new retains the export. + let inner = zelf.inner(); Ok(PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor::simple(zelf.__len__(), false), + BufferDescriptor::simple(inner.elements.len(), false), &BUFFER_METHODS, )) } @@ -855,7 +1032,12 @@ impl BufferResizeGuard for PyByteArray { fn try_resizable_opt(&self) -> Option> { // An export is a borrow someone else still holds, so it is answered // before the lock rather than by waiting on it. - (self.exports.load(Ordering::SeqCst) == 0).then(|| self.inner.write()) + if self.exports.load(Ordering::SeqCst) != 0 { + return None; + } + let inner = self.inner.write(); + // An export may have been acquired while we waited for the lock. + (self.exports.load(Ordering::SeqCst) == 0).then_some(inner) } } @@ -886,10 +1068,43 @@ impl AsSequence for PyByteArray { static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyByteArray::sequence_downcast(seq).__len__())), concat: atomic_func!(|seq, other, vm| { - PyByteArray::sequence_downcast(seq) - .inner() - .concat(other, vm) - .map(|x| PyByteArray::from(x).into_pyobject(vm)) + let concat_error = || { + vm.new_type_error(format!( + "can't concat {} to {}", + other.class().name(), + seq.obj.class().name() + )) + }; + // Protect the left operand before the right buffer callback, + // without holding either operand's lock across Python calls. + let left = PyBuffer::from_object(vm, seq.obj, BufferFlags::SIMPLE) + .map_err(|_| concat_error())?; + let right = match PyBuffer::from_object(vm, other, BufferFlags::SIMPLE) { + Ok(buffer) => buffer, + Err(_) => { + let error = concat_error(); + drop(left); + return Err(error); + } + }; + let result = (|| { + let len = left + .desc + .len + .checked_add(right.desc.len) + .ok_or_else(|| vm.no_memory_error())?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(len) + .map_err(|_| vm.no_memory_error())?; + left.append_to(&mut bytes); + right.append_to(&mut bytes); + Ok(PyByteArray::from(bytes).into_pyobject(vm)) + })(); + // Buffer release can run Python too; all slice guards are gone. + drop(left); + drop(right); + result }), repeat: atomic_func!(|seq, n, vm| { PyByteArray::sequence_downcast(seq) @@ -907,7 +1122,7 @@ impl AsSequence for PyByteArray { if let Some(value) = value { zelf._setitem_by_index(i, &value, vm) } else { - zelf.borrow_buf_mut().delitem_by_index(vm, i) + zelf.delitem_by_index(i, vm) } }), contains: atomic_func!(|seq, other, vm| { diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index f577e302085..8c52948c831 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -225,12 +225,24 @@ impl ByteInnerNewOptions { /// read: `bytes()` and `bytearray()` differ in whether they ask it how long /// it is. pub fn get_inner(self, from_object: FromObject, vm: &VirtualMachine) -> PyResult { - match (self.source, self.encoding, self.errors) { + self.get_inner_with_zero_count(from_object, vm) + .map(|(inner, _)| inner) + } + + pub(crate) fn get_inner_with_zero_count( + self, + from_object: FromObject, + vm: &VirtualMachine, + ) -> PyResult<(PyBytesInner, bool)> { + let mut zero_count = false; + let inner = match (self.source, self.encoding, self.errors) { (OptionalArg::Present(obj), OptionalArg::Missing, OptionalArg::Missing) => { // Try __index__ first to handle int-like objects that might raise custom exceptions if let Some(index_result) = obj.try_index_opt(vm) { match index_result { - Ok(index) => Self::get_value_from_size(&index, vm), + Ok(index) => Self::get_value_from_size(&index, vm).inspect(|inner| { + zero_count = inner.elements.is_empty(); + }), Err(e) => { // Only propagate non-TypeError exceptions // TypeError means the object doesn't support __index__, so fall back @@ -266,7 +278,8 @@ impl ByteInnerNewOptions { (OptionalArg::Present(_), OptionalArg::Missing, OptionalArg::Present(_)) => { Err(vm.new_type_error(STRING_WITHOUT_ENCODING.to_owned())) } - } + }?; + Ok((inner, zero_count)) } } @@ -302,7 +315,7 @@ impl ByteInnerSub { } } - fn into_vec(self, vm: &VirtualMachine) -> PyResult> { + pub(crate) fn into_vec(self, vm: &VirtualMachine) -> PyResult> { Ok(match self { Self::Buffer(buffer) => buffer.elements, Self::Byte(int) => vec![int.as_bigint().byte_or(vm)?], @@ -321,6 +334,15 @@ pub struct ByteInnerFindOptions { } impl ByteInnerFindOptions { + pub(crate) fn parts(&self, len: usize) -> (&PyObject, core::ops::Range) { + let range = anystr::adjust_indices( + self.start.as_ref().map(AsRef::as_ref), + self.end.as_ref().map(AsRef::as_ref), + len, + ); + (&self.sub, range) + } + pub fn get_value( self, len: usize, @@ -347,7 +369,7 @@ pub struct ByteInnerPaddingOptions { } impl ByteInnerPaddingOptions { - fn get_value(self, fn_name: &str, vm: &VirtualMachine) -> PyResult<(isize, u8)> { + pub(crate) fn get_value(self, fn_name: &str, vm: &VirtualMachine) -> PyResult<(isize, u8)> { let fillchar = { let v = self.fillchar; try_as_bytes(v.clone(), |bytes| bytes.iter().copied().exactly_one().ok()) @@ -368,9 +390,9 @@ impl ByteInnerPaddingOptions { #[derive(FromArgs)] pub struct ByteInnerTranslateOptions { #[pyarg(positional)] - table: Option, + pub(crate) table: Option, #[pyarg(any, default = b"")] - delete: PyObjectRef, + pub(crate) delete: PyObjectRef, } impl ByteInnerTranslateOptions { @@ -708,6 +730,16 @@ impl PyBytesInner { vm: &VirtualMachine, ) -> PyResult> { let (width, fillchar) = options.get_value("center", vm)?; + self.pad_with(width, fillchar, pad, vm) + } + + pub(crate) fn pad_with( + &self, + width: isize, + fillchar: u8, + pad: PadFn, + vm: &VirtualMachine, + ) -> PyResult> { let len = self.len(); if len as isize >= width { return Ok(Vec::from(&self.elements[..])); @@ -783,7 +815,10 @@ impl PyBytesInner { vm: &VirtualMachine, ) -> PyResult> { let (table, delete) = options.get_value(vm)?; + Ok(self.translate_with(&table, &delete)) + } + pub(crate) fn translate_with(&self, table: &[u8], delete: &[u8]) -> Vec { let mut res = if delete.is_empty() { Vec::with_capacity(self.elements.len()) } else { @@ -796,7 +831,7 @@ impl PyBytesInner { } } - Ok(res) + res } pub fn strip(&self, chars: Option) -> Vec { diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 5bcd8f64d29..d20cb367999 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -762,10 +762,13 @@ mod _io { ))); } let n = n as usize; - let mut bytes = b.borrow_buf_mut(); - bytes.truncate(n); - // FIXME: try to use Arc::unwrap on the bytearray to get at the inner buffer - bytes.clone().to_pyobject(vm) + let mut inner = b.try_resizable(vm)?; + let bytes = &mut inner.elements; + bytes + .try_reserve_exact(n.saturating_sub(bytes.len())) + .map_err(|_| vm.no_memory_error())?; + bytes.resize(n, 0); + core::mem::take(bytes).to_pyobject(vm) } }) } else { diff --git a/extra_tests/snippets/builtin_bytearray.py b/extra_tests/snippets/builtin_bytearray.py index d897ec94782..2f2793a76cf 100644 --- a/extra_tests/snippets/builtin_bytearray.py +++ b/extra_tests/snippets/builtin_bytearray.py @@ -913,3 +913,594 @@ class B(bytearray): empty = bytearray() empty *= sys.maxsize assert empty == bytearray() + + +def test_bytearray_exports(): + """Resizing paths must respect outstanding bytearray exports.""" + + import io + + from testutils import assert_raises + + for args in ((), (b"abc",), (b"x",), (b"",), (object(),)): + value = bytearray(b"abc") + with memoryview(value) as view: + with assert_raises(BufferError): + value.__init__(*args) + assert value == b"abc" + assert view.tobytes() == b"abc" + value.__init__(b"released") + assert value == b"released" + + value = bytearray(b"abc") + with assert_raises(TypeError): + value.__init__(object()) + assert value == b"" + value = bytearray(b"abc") + value.__init__(value) + assert value == b"" + + class ExportDuringInit: + def __init__(self, count, payload=b""): + self.count = count + self.payload = payload + self.calls = 0 + + def __index__(self): + self.calls += 1 + value.extend(self.payload) + self.view = memoryview(value) + return self.count + + value = bytearray(b"abc") + source = ExportDuringInit(2) + with assert_raises(BufferError): + value.__init__(source) + assert value == b"" + assert source.view.tobytes() == b"" + assert source.calls == 1 + source.view.release() + value.extend(b"released") + assert value == b"released" + + value = bytearray(b"abc") + source = ExportDuringInit(0) + value.__init__(source) + assert value == b"" + assert source.view.tobytes() == b"" + source.view.release() + + value = bytearray(b"abc") + source = ExportDuringInit(0, b"xy") + value.__init__(source) + assert value == b"xy" + assert source.view.tobytes() == b"xy" + assert source.calls == 1 + source.view.release() + + class FailingIndex: + def __index__(self): + assert value == b"" + raise ValueError("conversion failed") + + value = bytearray(b"abc") + with assert_raises(ValueError): + value.__init__(FailingIndex()) + assert value == b"" + + for key in (0, slice(0, 1)): + value = bytearray(b"abc") + with memoryview(value) as view: + with assert_raises(BufferError): + del value[key] + assert view.tobytes() == b"abc" + del value[key] + assert value == b"bc" + + class Reader(io.RawIOBase): + def __init__(self, count, retain, payload=b"abcd"): + self.count = count + self.retain = retain + self.payload = payload + + def readinto(self, buffer): + buffer[:] = self.payload + self.buffer = buffer + if self.retain: + self.view = memoryview(buffer) + return self.count + + for count in (0, 2, 4): + reader = Reader(count, False) + assert reader.read(4) == b"abcd"[:count] + assert reader.buffer == b"" + + reader = Reader(count, True) + with assert_raises(BufferError): + reader.read(4) + assert reader.view.tobytes() == b"abcd" + reader.view.release() + reader.buffer.clear() + + for count in (-1, 5): + reader = Reader(count, True) + with assert_raises(ValueError): + reader.read(4) + assert reader.view.tobytes() == b"abcd" + reader.view.release() + + reader = Reader(None, True) + assert reader.read(4) is None + assert reader.view.tobytes() == b"abcd" + reader.view.release() + + # The callback can shrink the temporary buffer, so slicing by the original + # requested size without checking the current buffer is not safe. + reader = Reader(2, False, b"xy") + assert reader.read(4) == b"xy" + assert reader.buffer == b"" + + reader = Reader(4, False, b"xy") + result = reader.read(4) + assert len(result) == 4 + assert result[:2] == b"xy" + # The contents of the newly grown region are unspecified by CPython. + assert reader.buffer == b"" + + +test_bytearray_exports() + + +def test_bytearray_threading(): + """Acquiring a bytearray export must be atomic with respect to resizing.""" + + import threading + + def exercise_exports(): + payload = b"x" * 65536 + value = bytearray(payload) + barrier = threading.Barrier(3, timeout=5) + errors = [] + + def worker(operation): + try: + for _ in range(256): + barrier.wait() + operation() + barrier.wait() + except BaseException as exc: + errors.append(exc) + barrier.abort() + + def resize(): + try: + value.clear() + value.extend(payload) + except BufferError: + pass + + def decode(): + result = value.decode("ascii") + assert result in ("", "x" * len(payload)) + + def export(): + with memoryview(value) as view: + size = len(view) + assert size in (0, len(payload)) + # The acquired export keeps its original length and contents even + # if the resizing thread runs between these two reads. + assert view.tobytes() == payload[:size] + assert len(view) == size + + threads = [ + threading.Thread(target=worker, args=(operation,), daemon=True) + for operation in (resize, decode, export) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + assert not any(thread.is_alive() for thread in threads), "worker did not finish" + if errors: + raise errors[0] + + exercise_exports() + + +test_bytearray_threading() + + +def test_bytearray_init_exports(): + """Initializer storage and integer-deletion ordering with buffer exports.""" + + from testutils import assert_raises + + class Zero: + def __init__(self, value): + self.value = value + self.observations = [] + + def __index__(self): + self.observations.append((len(self.value), self.value.__alloc__())) + return 0 + + # Final CPython rejects even an empty exported initializer before conversion. + for kind in ("missing", "integer", "callback"): + value = bytearray() + source = Zero(value) + with memoryview(value) as view: + with assert_raises(BufferError): + if kind == "missing": + value.__init__() + elif kind == "integer": + value.__init__(0) + else: + value.__init__(source) + assert value == b"" + assert view.tobytes() == b"" + assert source.observations == [] + + # Clearing an initializer must release its old storage before callbacks and + # before the successful zero-count early return. + for kind in ("missing", "integer", "callback"): + value = bytearray(b"x" * 64) + source = Zero(value) + assert value.__alloc__() > 0 + if kind == "missing": + value.__init__() + elif kind == "integer": + value.__init__(0) + else: + value.__init__(source) + assert source.observations == [(0, 0)] + assert value == b"" + assert value.__alloc__() == 0 + + # An invalid integer index is rejected before checking whether resizing is + # blocked, and neither error path may alter the exported bytes. + for initial, indices in ((b"abc", (3, -4)), (b"", (0, -1))): + for index in indices: + value = bytearray(initial) + with memoryview(value) as view: + try: + del value[index] + except IndexError as exc: + assert str(exc) == "bytearray index out of range" + else: + raise AssertionError("invalid index was accepted") + assert value == initial + assert view.tobytes() == initial + + for index in (0, -1): + value = bytearray(b"abc") + with memoryview(value) as view: + with assert_raises(BufferError): + del value[index] + assert view.tobytes() == b"abc" + del value[index] + assert value == (b"bc" if index == 0 else b"ab") + + +test_bytearray_init_exports() + + +def test_bytearray_concat_exports(): + import operator + + from testutils import assert_raises + + for concat in (operator.add, operator.concat, bytearray.__add__): + target = bytearray(b"a") + events = [] + + class ReentrantBuffer: + def __buffer__(self, flags): + events.append("acquire") + with assert_raises(BufferError): + target.__init__(object()) + with assert_raises(BufferError): + target.append(120) + target[0] = ord("A") + return memoryview(b"b") + + def __release_buffer__(self, view): + events.append("release") + # The left operand's export must already have been released. + target.append(99) + + assert concat(target, ReentrantBuffer()) == bytearray(b"Ab") + assert events == ["acquire", "release"] + assert target == bytearray(b"Ac") + target.append(100) + + class FailingBuffer: + def __buffer__(self, flags): + raise ValueError("buffer acquisition failed") + + with assert_raises(TypeError) as error: + concat(target, FailingBuffer()) + assert str(error.exception) == "can't concat FailingBuffer to bytearray" + target.append(101) + + events = [] + + class Left(bytearray): + def __buffer__(self, flags): + events.append("left acquire") + return memoryview(b"L") + + def __release_buffer__(self, view): + events.append("left release") + + class Right: + def __buffer__(self, flags): + events.append("right acquire") + return memoryview(b"R") + + def __release_buffer__(self, view): + events.append("right release") + + assert concat(Left(b"ignored"), Right()) == bytearray(b"LR") + assert events == [ + "left acquire", + "right acquire", + "left release", + "right release", + ] + + events = [] + with assert_raises(TypeError) as error: + concat(Left(), FailingBuffer()) + assert str(error.exception) == "can't concat FailingBuffer to Left" + assert events == ["left acquire", "left release"] + + class FailingLeft(bytearray): + def __buffer__(self, flags): + events.append("left acquire") + raise ValueError("left acquisition failed") + + events = [] + with assert_raises(TypeError) as error: + concat(FailingLeft(), Right()) + assert str(error.exception) == "can't concat Right to FailingLeft" + assert events == ["left acquire"] + + +test_bytearray_concat_exports() + + +def test_bytearray_callback_exports(): + import operator + + from testutils import assert_raises + + def check_comparison(): + for compare, expected in ( + (operator.eq, False), + (operator.ne, True), + (operator.lt, True), + (operator.le, True), + (operator.gt, False), + (operator.ge, False), + ): + target = bytearray(b"a") + events = [] + + class Other: + def __buffer__(self, flags): + events.append("acquire") + with assert_raises(BufferError): + target.__init__(object()) + target[0] = ord("A") + return memoryview(b"B") + + def __release_buffer__(self, view): + events.append("release") + target[0] = ord("Z") + target.append(99) + + assert compare(target, Other()) is expected + assert target == b"Zc" + assert events == ["acquire", "release"] + + events = [] + + class Left(bytearray): + def __buffer__(self, flags): + events.append("acquire") + return memoryview(b"L") + + def __release_buffer__(self, view): + events.append("release") + + left = Left(b"ignored") + assert left.__eq__(left) is True + assert events == ["acquire", "acquire", "release", "release"] + events.clear() + assert left.__eq__(object()) is NotImplemented + assert events == [] + + class Failing: + def __buffer__(self, flags): + raise ValueError("no buffer") + + assert left.__eq__(Failing()) is NotImplemented + assert events == ["acquire", "release"] + left.append(0) + + def check_search(): + for method, expected in ( + ("find", 0), + ("index", 0), + ("rfind", 2), + ("rindex", 2), + ("count", 2), + ): + target = bytearray(b"xxx") + events = [] + + class Needle: + def __buffer__(self, flags): + events.append("acquire") + with assert_raises(BufferError): + target.__init__(object()) + target[:] = b"aba" + return memoryview(b"a") + + def __release_buffer__(self, view): + events.append("release") + with assert_raises(BufferError): + target.append(0) + target[:] = b"zzz" + + assert getattr(target, method)(Needle()) == expected + assert target == b"zzz" + assert events == ["acquire", "release"] + target.append(0) + + target = bytearray(b"xxx") + + class IndexNeedle: + def __index__(self): + with assert_raises(BufferError): + target.__init__(object()) + target[:] = b"aba" + return ord("a") + + assert getattr(target, method)(IndexNeedle()) == expected + + target = bytearray(b"ab") + + class Start: + def __index__(self): + target.extend(b"ab") + return 1 + + assert target.count(ord("a"), Start()) == 1 + assert target == b"abab" + + def check_affixes(): + for method in ("startswith", "endswith"): + target = bytearray(b"aaa") + events = [] + + class Affix: + def __init__(self, value, after): + self.value = value + self.after = after + + def __buffer__(self, flags): + events.append(self.value) + with assert_raises(BufferError): + target.__init__(object()) + target[0] = target[0] + return memoryview(self.value) + + def __release_buffer__(self, view): + events.append("release") + with assert_raises(BufferError): + target.append(0) + target[:] = self.after + + class Unvisited: + def __buffer__(self, flags): + raise AssertionError("tuple must short-circuit") + + assert getattr(target, method)( + (Affix(b"x", b"yyy"), Affix(b"y", b"zzz"), Unvisited()) + ) + assert target == b"zzz" + assert events == [b"x", "release", b"y", "release"] + target.append(0) + + target = bytearray(b"aaa") + events.clear() + assert getattr(target, method)(Affix(b"a", b"zzz")) + assert target == b"zzz" + assert events == [b"a", "release"] + + events.clear() + assert not getattr(target, method)(Affix(b"z", b"aaa"), 100) + assert events == [b"z", "release"] + with assert_raises(TypeError): + getattr(target, method)((Affix(b"x", b"aaa"), object())) + target.append(0) + + class BytesAffix(bytes): + def __buffer__(self, flags): + raise AssertionError("bytes subclasses use their stored bytes") + + assert getattr(bytearray(b"a"), method)(BytesAffix(b"a")) + + def check_translate(): + target = bytearray(b"x") + table = bytearray(range(256)) + events = [] + + class Table: + def __buffer__(self, flags): + events.append("table acquire") + target.__init__(b"a") + return memoryview(table) + + def __release_buffer__(self, view): + events.append("table release") + target.__init__(b"t") + + class Delete: + def __buffer__(self, flags): + events.append("delete acquire") + with assert_raises(BufferError): + table.append(0) + table[ord("a")] = ord("b") + target.__init__(b"aa") + return memoryview(b"") + + def __release_buffer__(self, view): + events.append("delete release") + target.__init__(b"d") + + assert target.translate(Table(), delete=Delete()) == b"bb" + assert target == b"d" + assert events == [ + "table acquire", + "delete acquire", + "table release", + "delete release", + ] + table.append(0) + table.pop() + events.clear() + + class BadDelete: + def __buffer__(self, flags): + events.append("delete failure") + with assert_raises(BufferError): + table.append(0) + raise ValueError("bad delete") + + with assert_raises(ValueError): + target.translate(Table(), delete=BadDelete()) + assert events == ["table acquire", "delete failure", "table release"] + table.append(0) + + def check_padding(): + for method in ("center", "ljust", "rjust"): + target = bytearray(b"x") + assert getattr(target, method)(3, target) == b"xxx" + + class Width: + def __index__(self): + target.__init__(b"y") + return 3 + + assert getattr(target, method)(Width(), target) == b"yyy" + + check_comparison() + check_search() + check_affixes() + check_translate() + check_padding() + + +test_bytearray_callback_exports() From 5f3daa5386dcfa225fc75c8807c590f2784cfe28 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 00:12:24 -0700 Subject: [PATCH 116/229] Implement Python 3.15 base64 APIs (#9085) * Implement Python 3.15 base64 encoding and decoding Follow CPython 3.15.0 final d220409a13b77de7a6c30abe16ea90db01528edc for base64 options, alphabets, padding and canonical decoding. Keep reverse-table caches in traced module-native state and preserve module-bound callable semantics. Add bounded Python regressions without editing Lib. Assisted-by: OpenAI Codex:model identifier unavailable * Complete binascii codecs for canonical Python 3.15 tests Migrate Lib/test/test_binascii.py unchanged from CPython 3.15.0 final using scripts/update_lib and replace the superseded custom base64 snippet. Implement the base32, Base85, Ascii85 and hex ignorechars APIs required by the canonical suite. Share the existing module-bound original-key alphabet cache across base32/base64/base85 and preserve argument and buffer cleanup semantics. Keep the existing base64 kernels and VM state integration unchanged. Assisted-by: OpenAI Codex:model identifier unavailable --- Cargo.lock | 7 - Cargo.toml | 1 - Lib/test/test_binascii.py | 1181 +++++++++++++++++++++++- crates/common/Cargo.toml | 3 +- crates/common/src/binascii.rs | 187 ++-- crates/stdlib/src/binascii.rs | 570 +++++++++++- crates/stdlib/src/binascii/base_n.rs | 431 +++++++++ crates/vm/src/builtins/builtin_func.rs | 4 +- crates/vm/src/builtins/module.rs | 31 +- 9 files changed, 2281 insertions(+), 134 deletions(-) create mode 100644 crates/stdlib/src/binascii/base_n.rs diff --git a/Cargo.lock b/Cargo.lock index 9a0b3b46c9b..cf5e0f9c0cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -320,12 +320,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "base64ct" version = "1.8.3" @@ -3761,7 +3755,6 @@ name = "rustpython-common" version = "0.6.1" dependencies = [ "ascii", - "base64", "bitflags 2.13.2", "blake2b_simd", "blake2s_simd", diff --git a/Cargo.toml b/Cargo.toml index 792f2b981af..0b55f2e8277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -198,7 +198,6 @@ phf = { version = "0.14.0", default-features = false, features = ["macros"]} adler32 = "1.2.0" approx = "0.5.1" ascii = { version = "1.1", default-features = false } -base64 = "0.23" blake2b_simd = "1.0.4" blake2s_simd = "1.0.4" bitflags = "2.11.0" diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 82eabf4bb06..cedbdc61f18 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -4,14 +4,17 @@ import binascii import array import re -from test.support import bigmemtest, _1G, _4G +import sys +from test.support import bigmemtest, _1G, _4G, check_impl_detail from test.support.hypothesis_helper import hypothesis # Note: "*_hex" functions are aliases for "(un)hexlify" -b2a_functions = ['b2a_base64', 'b2a_hex', 'b2a_qp', 'b2a_uu', +b2a_functions = ['b2a_ascii85', 'b2a_base32', 'b2a_base64', 'b2a_base85', + 'b2a_hex', 'b2a_qp', 'b2a_uu', 'hexlify'] -a2b_functions = ['a2b_base64', 'a2b_hex', 'a2b_qp', 'a2b_uu', +a2b_functions = ['a2b_ascii85', 'a2b_base32', 'a2b_base64', 'a2b_base85', + 'a2b_hex', 'a2b_qp', 'a2b_uu', 'unhexlify'] all_functions = a2b_functions + b2a_functions + ['crc32', 'crc_hqx'] @@ -23,6 +26,8 @@ class BinASCIITest(unittest.TestCase): rawdata = b"The quick brown fox jumps over the lazy dog.\r\n" # Be slow so we don't depend on other modules rawdata += bytes(range(256)) + rawdata += b'\0'*32 + rawdata += b' '*32 rawdata += b"\r\nHello world.\n" def setUp(self): @@ -41,6 +46,40 @@ def test_exceptions(self): self.assertIsSubclass(binascii.Error, Exception) self.assertIsSubclass(binascii.Incomplete, Exception) + def test_constants(self): + self.assertEqual(binascii.BASE64_ALPHABET, + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + b'abcdefghijklmnopqrstuvwxyz' + b'0123456789+/') + self.assertEqual(binascii.URLSAFE_BASE64_ALPHABET, + binascii.BASE64_ALPHABET[:-2] + b'-_') + self.assertEqual(binascii.UU_ALPHABET, bytes(range(32, 32+64))) + self.assertEqual(binascii.CRYPT_ALPHABET, + b'./0123456789' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + b'abcdefghijklmnopqrstuvwxyz') + self.assertEqual(binascii.BINHEX_ALPHABET, + b'!"#$%&\'()*+,-012345689' + b'@ABCDEFGHIJKLMNPQRSTUVXYZ[' + b'`abcdefhijklmpqr') + + self.assertEqual(binascii.BASE85_ALPHABET, + b'0123456789' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + b'abcdefghijklmnopqrstuvwxyz' + b'!#$%&()*+-;<=>?@^_`{|}~') + self.assertEqual(binascii.ASCII85_ALPHABET, bytes(range(33, 33+85))) + self.assertEqual(binascii.Z85_ALPHABET, + b'0123456789' + b'abcdefghijklmnopqrstuvwxyz' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + b'.-:+=^!/*?&<>()[]{}@%$#') + + self.assertEqual(binascii.BASE32_ALPHABET, + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567') + self.assertEqual(binascii.BASE32HEX_ALPHABET, + b'0123456789ABCDEFGHIJKLMNOPQRSTUV') + def test_functions(self): # Check presence of all functions for name in all_functions: @@ -117,62 +156,170 @@ def addnoise(line): # empty strings. TBD: shouldn't it raise an exception instead ? self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'') - def test_base64_strict_mode(self): - # Test base64 with strict mode on - def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes): + def test_base64_bad_padding(self): + # Test malformed padding + def _assertRegexTemplate(assert_regex, data, + non_strict_mode_expected_result, **kwargs): + data = self.type2test(data) with self.assertRaisesRegex(binascii.Error, assert_regex): - binascii.a2b_base64(self.type2test(data), strict_mode=True) - self.assertEqual(binascii.a2b_base64(self.type2test(data), strict_mode=False), + binascii.a2b_base64(data, strict_mode=True, **kwargs) + self.assertEqual(binascii.a2b_base64(data, strict_mode=False), non_strict_mode_expected_result) - self.assertEqual(binascii.a2b_base64(self.type2test(data)), + self.assertEqual(binascii.a2b_base64(data, strict_mode=True, + ignorechars=b' ='), + non_strict_mode_expected_result) + self.assertEqual(binascii.a2b_base64(data), non_strict_mode_expected_result) - def assertExcessData(data, non_strict_mode_expected_result: bytes): - _assertRegexTemplate(r'(?i)Excess data', data, non_strict_mode_expected_result) - - def assertNonBase64Data(data, non_strict_mode_expected_result: bytes): - _assertRegexTemplate(r'(?i)Only base64 data', data, non_strict_mode_expected_result) + def assertLeadingPadding(*args, **kwargs): + _assertRegexTemplate(r'(?i)Leading padding', *args, **kwargs) - def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): - _assertRegexTemplate(r'(?i)Leading padding', data, non_strict_mode_expected_result) + def assertDiscontinuousPadding(*args): + _assertRegexTemplate(r'(?i)Discontinuous padding', *args) - def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): - _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(*args): + _assertRegexTemplate(r'(?i)Excess padding', *args) - def assertExcessPadding(data, non_strict_mode_expected_result: bytes): - _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + def assertInvalidLength(data, *args, length=None, **kwargs): + if length is None: + length = len(data.split(b'=', 1)[0].replace(b' ', b'')) + assert_regex = fr"(?i)Invalid.+number of data characters \({length}\)" + _assertRegexTemplate(assert_regex, data, *args, **kwargs) - # Test excess data exceptions assertExcessPadding(b'ab===', b'i') assertExcessPadding(b'ab====', b'i') - assertNonBase64Data(b'ab==:', b'i') - assertExcessData(b'abc=a', b'i\xb7\x1a') - assertNonBase64Data(b'abc=:', b'i\xb7') - assertNonBase64Data(b'ab==\n', b'i') assertExcessPadding(b'abc==', b'i\xb7') assertExcessPadding(b'abc===', b'i\xb7') assertExcessPadding(b'abc====', b'i\xb7') assertExcessPadding(b'abc=====', b'i\xb7') - # Test non-base64 data exceptions - assertNonBase64Data(b'\nab==', b'i') - assertNonBase64Data(b'ab:(){:|:&};:==', b'i') - assertNonBase64Data(b'a\nb==', b'i') - assertNonBase64Data(b'a\x00b==', b'i') - - # Test malformed padding assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') assertLeadingPadding(b'====', b'') assertLeadingPadding(b'=====', b'') + assertLeadingPadding(b'=abcd', b'i\xb7\x1d') + assertLeadingPadding(b'==abcd', b'i\xb7\x1d') + assertLeadingPadding(b'===abcd', b'i\xb7\x1d') + assertLeadingPadding(b'====abcd', b'i\xb7\x1d') + assertLeadingPadding(b'=====abcd', b'i\xb7\x1d') + assertLeadingPadding(b' =abcd', b'i\xb7\x1d', ignorechars=b' ') + + assertInvalidLength(b'a=b==', b'i') + assertInvalidLength(b'a=bc=', b'i\xb7') + assertInvalidLength(b'a=bc==', b'i\xb7') + assertInvalidLength(b'a=bcd', b'i\xb7\x1d') + assertInvalidLength(b'a=bcd=', b'i\xb7\x1d') + assertInvalidLength(b' a=b==', b'i', ignorechars=b' ') + assertInvalidLength(b'abcde=f==', b'i\xb7\x1dy') + assertInvalidLength(b' abcde=f==', b'i\xb7\x1dy', ignorechars=b' ') + assertDiscontinuousPadding(b'ab=c=', b'i\xb7') - assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertDiscontinuousPadding(b'ab=cd', b'i\xb7\x1d') + assertDiscontinuousPadding(b'ab=cd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') assertExcessPadding(b'abcd==', b'i\xb7\x1d') assertExcessPadding(b'abcd===', b'i\xb7\x1d') assertExcessPadding(b'abcd====', b'i\xb7\x1d') assertExcessPadding(b'abcd=====', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=efgh', b'i\xb7\x1dy\xf8!') + assertExcessPadding(b'abcd==efgh', b'i\xb7\x1dy\xf8!') + assertExcessPadding(b'abcd===efgh', b'i\xb7\x1dy\xf8!') + assertExcessPadding(b'abcd====efgh', b'i\xb7\x1dy\xf8!') + assertExcessPadding(b'abcd=====efgh', b'i\xb7\x1dy\xf8!') + + def test_a2b_base64_padded(self): + a2b_base64 = binascii.a2b_base64 + t = self.type2test + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + a2b_base64(t(data), padded=False, strict_mode=True) + self.assertEqual(a2b_base64(t(data), padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(a2b_base64(t(data), padded=False), expected) + self.assertEqual(a2b_base64(t(data), padded=False, strict_mode=True), + expected) + + check(b'', b'') + check(b'YQ==', b'a') + check(b'YWI=', b'ab') + check(b'YWJj', b'abc') + check(b'Y=WJj', b'abc') + check(b'YW=Jj', b'abc') + check(b'YWJ=j', b'abc') + + def _common_test_ignorechars(self, func): + eq = self.assertEqual + empty = self.type2test(b'') + data = self.type2test(b'\n \n') + ignorechars = self.type2test(b' \n') + eq(func(empty, ignorechars=ignorechars), b'') + eq(func(empty, ignorechars=empty), b'') + eq(func(data, ignorechars=ignorechars), b'') + with self.assertRaises(binascii.Error): + func(data, ignorechars=empty) + with self.assertRaises(binascii.Error): + func(data, ignorechars=ignorechars[1:]) + with self.assertRaises(binascii.Error): + func(data, ignorechars=ignorechars[:-1]) + with self.assertRaises(TypeError): + func(empty, ignorechars='') + with self.assertRaises(TypeError): + func(empty, ignorechars=[]) + with self.assertRaises(TypeError): + func(empty, ignorechars=None) + + def test_base64_invalidchars(self): + # Test non-base64 data exceptions + self._common_test_ignorechars(binascii.a2b_base64) + def assertNonBase64Data(data, expected, ignorechars): + data = self.type2test(data) + assert_regex = r'(?i)Only base64 data' + self.assertEqual(binascii.a2b_base64(data), expected) + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_base64(data, strict_mode=True) + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_base64(data, ignorechars=b'') + self.assertEqual(binascii.a2b_base64(data, ignorechars=ignorechars), + expected) + self.assertEqual(binascii.a2b_base64(data, strict_mode=False, ignorechars=b''), + expected) + + assertNonBase64Data(b'\nab==', b'i', ignorechars=b'\n') + assertNonBase64Data(b'ab:(){:|:&};:==', b'i', ignorechars=b':;(){}|&') + assertNonBase64Data(b'a\nb==', b'i', ignorechars=b'\n') + assertNonBase64Data(b'a\x00b==', b'i', ignorechars=b'\x00') + assertNonBase64Data(b'ab:==', b'i', ignorechars=b':') + assertNonBase64Data(b'ab=:=', b'i', ignorechars=b':') + assertNonBase64Data(b'ab==:', b'i', ignorechars=b':') + assertNonBase64Data(b'abc=:', b'i\xb7', ignorechars=b':') + assertNonBase64Data(b'ab==\n', b'i', ignorechars=b'\n') + assertNonBase64Data(b'a\nb==', b'i', ignorechars=bytearray(b'\n')) + assertNonBase64Data(b'a\nb==', b'i', ignorechars=memoryview(b'\n')) + + self.assertEqual(binascii.a2b_base64(b'+A-/B_', ignorechars=b'+/-_'), + b'\xf8\x0f\xc1') + self.assertEqual(binascii.a2b_base64(b'+A-/B_', ignorechars=b'+/-_', + alphabet=binascii.URLSAFE_BASE64_ALPHABET), + b'\x03\xe0\x7f') + + # Same cell in the cache: '\r' >> 3 == '\n' >> 3. + data = self.type2test(b'\r\n') + with self.assertRaises(binascii.Error): + binascii.a2b_base64(data, ignorechars=b'\r') + self.assertEqual(binascii.a2b_base64(data, ignorechars=b'\r\n'), b'') + # Same bit mask in the cache: '*' & 31 == '\n' & 31. + data = self.type2test(b'*\n') + with self.assertRaises(binascii.Error): + binascii.a2b_base64(data, ignorechars=b'*') + self.assertEqual(binascii.a2b_base64(data, ignorechars=b'*\n'), b'') def test_base64_excess_data(self): # Test excess data exceptions @@ -183,6 +330,9 @@ def assertExcessData(data, expected): binascii.a2b_base64(data, strict_mode=True) self.assertEqual(binascii.a2b_base64(data, strict_mode=False), expected) + self.assertEqual(binascii.a2b_base64(data, strict_mode=True, + ignorechars=b'='), + expected) self.assertEqual(binascii.a2b_base64(data), expected) assertExcessData(b'ab==c=', b'i\xb7') @@ -191,9 +341,15 @@ def assertExcessData(data, expected): def test_base64errors(self): # Test base64 with invalid padding - def assertIncorrectPadding(data): + def assertIncorrectPadding(data, strict_mode=True): + data = self.type2test(data) + with self.assertRaisesRegex(binascii.Error, r'(?i)Incorrect padding'): + binascii.a2b_base64(data) with self.assertRaisesRegex(binascii.Error, r'(?i)Incorrect padding'): - binascii.a2b_base64(self.type2test(data)) + binascii.a2b_base64(data, strict_mode=False) + if strict_mode: + with self.assertRaisesRegex(binascii.Error, r'(?i)Incorrect padding'): + binascii.a2b_base64(data, strict_mode=True) assertIncorrectPadding(b'ab') assertIncorrectPadding(b'ab=') @@ -201,16 +357,22 @@ def assertIncorrectPadding(data): assertIncorrectPadding(b'abcdef') assertIncorrectPadding(b'abcdef=') assertIncorrectPadding(b'abcdefg') - assertIncorrectPadding(b'a=b=') - assertIncorrectPadding(b'a\nb=') + assertIncorrectPadding(b'a=b=', strict_mode=False) + assertIncorrectPadding(b'a\nb=', strict_mode=False) # Test base64 with invalid number of valid characters (1 mod 4) - def assertInvalidLength(data): + def assertInvalidLength(data, strict_mode=True): n_data_chars = len(re.sub(br'[^A-Za-z0-9/+]', br'', data)) + data = self.type2test(data) expected_errmsg_re = \ r'(?i)Invalid.+number of data characters.+' + str(n_data_chars) with self.assertRaisesRegex(binascii.Error, expected_errmsg_re): - binascii.a2b_base64(self.type2test(data)) + binascii.a2b_base64(data) + with self.assertRaisesRegex(binascii.Error, expected_errmsg_re): + binascii.a2b_base64(data, strict_mode=False) + if strict_mode: + with self.assertRaisesRegex(binascii.Error, expected_errmsg_re): + binascii.a2b_base64(data, strict_mode=True) assertInvalidLength(b'a') assertInvalidLength(b'a=') @@ -218,7 +380,911 @@ def assertInvalidLength(data): assertInvalidLength(b'a===') assertInvalidLength(b'a' * 5) assertInvalidLength(b'a' * (4 * 87 + 1)) - assertInvalidLength(b'A\tB\nC ??DE') # only 5 valid characters + assertInvalidLength(b'A\tB\nC ??DE', # only 5 valid characters + strict_mode=False) + + def test_base64_canonical(self): + # https://datatracker.ietf.org/doc/html/rfc4648.html#section-3.5 + # Decoders MAY reject encoded data if the pad bits are not zero. + + # Without canonical=True, non-zero padding bits are accepted + self.assertEqual(binascii.a2b_base64(self.type2test(b'AB==')), b'\x00') + self.assertEqual(binascii.a2b_base64(self.type2test(b'AB=='), + strict_mode=True), b'\x00') + + # 2 data chars + "==": last char has 4 padding bits + # 'A' = 0, 'B' = 1 -> leftover 0001 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AB=='), canonical=True) + # 'A' = 0, 'P' = 15 -> leftover 1111 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AP=='), canonical=True) + + # 3 data chars + "=": last char has 2 padding bits + # 'A' = 0, 'A' = 0, 'B' = 1 -> leftover 01 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AAB='), canonical=True) + # 'A' = 0, 'A' = 0, 'D' = 3 -> leftover 11 (non-zero) + with self.assertRaises(binascii.Error): + binascii.a2b_base64(self.type2test(b'AAD='), canonical=True) + + # Verify that zero padding bits are accepted + binascii.a2b_base64(self.type2test(b'AA=='), canonical=True) + binascii.a2b_base64(self.type2test(b'AAA='), canonical=True) + + # Full quads with no padding have no leftover bits -- always valid + binascii.a2b_base64(self.type2test(b'AAAA'), canonical=True) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_base64_canonical_roundtrip(self, payload): + # The encoder must always produce canonical output. + encoded = binascii.b2a_base64(payload, newline=False) + decoded = binascii.a2b_base64(encoded, canonical=True) + self.assertEqual(decoded, payload) + + def test_base64_alphabet(self): + alphabet = (b'!"#$%&\'()*+,-012345689@' + b'ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr') + data = self.type2test(self.rawdata) + encoded = binascii.b2a_base64(data, alphabet=alphabet) + trans = bytes.maketrans(binascii.BASE64_ALPHABET, alphabet) + expected = binascii.b2a_base64(data).translate(trans) + self.assertEqual(encoded, expected) + self.assertEqual(binascii.a2b_base64(encoded, alphabet=alphabet), self.rawdata) + self.assertEqual(binascii.b2a_base64(data, alphabet=self.type2test(alphabet)), expected) + + data = self.type2test(b'') + self.assertEqual(binascii.b2a_base64(data, alphabet=alphabet), b'\n') + self.assertEqual(binascii.a2b_base64(data, alphabet=alphabet), b'') + + for func in binascii.b2a_base64, binascii.a2b_base64: + with self.assertRaises(TypeError): + func(data, alphabet=None) + with self.assertRaises(TypeError): + func(data, alphabet=alphabet.decode()) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet[:-1]) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet+b'?') + with self.assertRaises(TypeError): + binascii.a2b_base64(data, alphabet=bytearray(alphabet)) + + def test_ascii85_valid(self): + # Test Ascii85 with valid data + ASCII85_PREFIX = b"<~" + ASCII85_SUFFIX = b"~>" + + # Interleave blocks of 4 null bytes and 4 spaces into test data + rawdata = bytearray() + rawlines, i = [], 0 + for k in range(1, len(self.rawdata) + 1): + b = b"\0\0\0\0" if k & 1 else b" " + b = b + self.rawdata[i:i + k] + b = b" " if k & 1 else b"\0\0\0\0" + rawdata += b + rawlines.append(b) + i += k + if i >= len(self.rawdata): + break + + # Test core parameter combinations + params = (False, False), (False, True), (True, False), (True, True) + for foldspaces, adobe in params: + lines = [] + for rawline in rawlines: + b = self.type2test(rawline) + a = binascii.b2a_ascii85(b, foldspaces=foldspaces, adobe=adobe) + lines.append(a) + res = bytearray() + for line in lines: + a = self.type2test(line) + b = binascii.a2b_ascii85(a, foldspaces=foldspaces, adobe=adobe) + res += b + self.assertEqual(res, rawdata) + + # Inputs with length 1 mod 5 end with a 1-char group, which is + # an encoding violation per the PLRM spec. + error_params = [ + (b"a", False, False, b""), + (b"xbw", False, False, b"wx"), + (b"<~c~>", False, True, b""), + (b"{d ~>", False, True, b" {"), + (b"ye", True, False, b""), + (b"z\x01y\x00f", True, False, b"\x00\x01"), + (b"<~FCfN8yg~>", True, True, b""), + (b"FE;\x03#8zFCf\x02N8yh~>", True, True, b"\x02\x03"), + ] + for a, foldspaces, adobe, ignorechars in error_params: + kwargs = {"foldspaces": foldspaces, "adobe": adobe, "ignorechars": ignorechars} + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(self.type2test(a), **kwargs) + + def test_ascii85_invalid(self): + # Test Ascii85 with invalid characters interleaved + lines, i = [], 0 + for k in range(1, len(self.rawdata) + 1): + b = self.type2test(self.rawdata[i:i + k]) + a = binascii.b2a_ascii85(b) + lines.append(a) + i += k + if i >= len(self.rawdata): + break + + fillers = bytearray() + valid = b"!\"#$%&'()*+,-./0123456789:;<=>?@" \ + b"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu" + b"z" + for i in range(256): + if i not in valid: + fillers.append(i) + def addnoise(line): + res = bytearray() + for i in range(len(line)): + res.append(line[i]) + for j in range(i, len(fillers), len(line)): + res.append(fillers[j]) + return res + res = bytearray() + for line in map(addnoise, lines): + a = self.type2test(line) + b = binascii.a2b_ascii85(a, ignorechars=fillers) + res += b + self.assertEqual(res, self.rawdata) + + # Test Ascii85 with only invalid characters + fillers = self.type2test(fillers) + b = binascii.a2b_ascii85(fillers, ignorechars=fillers) + self.assertEqual(b, b"") + + def test_ascii85_errors(self): + def _assertRegexTemplate(assert_regex, data, **kwargs): + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_ascii85(self.type2test(data), **kwargs) + + def assertMissingDelimiter(data): + _assertRegexTemplate(r"(?i)end with b'~>'", data, adobe=True) + + def assertOverflow(data): + _assertRegexTemplate(r"(?i)Ascii85 overflow", data) + + def assertInvalidSpecial(data): + _assertRegexTemplate(r"(?i)'[yz]'.+5-tuple", data, foldspaces=True) + + def assertInvalidChar(data, **kwargs): + _assertRegexTemplate(r"(?i)Non-Ascii85 digit", data, **kwargs) + + # Test Ascii85 with missing delimiters + assertMissingDelimiter(b"") + assertMissingDelimiter(b"a") + assertMissingDelimiter(b"<~") + assertMissingDelimiter(b"<~!~") + assertMissingDelimiter(b"<~abc>") + assertMissingDelimiter(b"<~has delimiter but not terminal~> !") + + # Test Ascii85 with out-of-range encoded value + assertOverflow(b"t") + assertOverflow(b"s9") + assertOverflow(b"s8X") + assertOverflow(b"s8W.") + assertOverflow(b's8W-"') + assertOverflow(b"s8W-!u") + assertOverflow(b"s8W-!s8W-!zs8X") + + # Test Ascii85 with misplaced short form groups + assertInvalidSpecial(b"ay") + assertInvalidSpecial(b"az") + assertInvalidSpecial(b"aby") + assertInvalidSpecial(b"ayz") + assertInvalidSpecial(b"abcz") + assertInvalidSpecial(b"abcdy") + assertInvalidSpecial(b"y!and!z!then!!y") + + # Test Ascii85 with non-ignored invalid characters + assertInvalidChar(b"j\n") + assertInvalidChar(b" ", ignorechars=b"") + assertInvalidChar(b" valid\x02until\x03", ignorechars=b"\x00\x01\x02\x04") + assertInvalidChar(b"\tFCb", ignorechars=b"\n") + assertInvalidChar(b"xxxB\nP\thU'D v/F+", ignorechars=b" \n\tv") + + def _common_test_wrapcol(self, func): + eq = self.assertEqual + data = self.data + expected = func(data) + eq(func(data, wrapcol=0), expected) + eq(func(data, wrapcol=8000), expected) + eq(func(b'', wrapcol=0), func(b'')) + eq(func(b'', wrapcol=1), func(b'')) + eq(func(data, wrapcol=sys.maxsize), expected) + if check_impl_detail(): + eq(func(data, wrapcol=sys.maxsize*2), expected) + with self.assertRaises(OverflowError): + func(data, wrapcol=2**1000) + with self.assertRaises(ValueError): + func(data, wrapcol=-80) + with self.assertRaises(TypeError): + func(data, wrapcol=80.0) + with self.assertRaises(TypeError): + func(data, wrapcol='80') + with self.assertRaises(TypeError): + func(data, wrapcol=None) + + def test_ascii85_wrapcol(self): + # Test Ascii85 splitting lines + self._common_test_wrapcol(binascii.b2a_ascii85) + + def assertEncode(a_expected, data, n, adobe=False): + b = self.type2test(data) + a = binascii.b2a_ascii85(b, adobe=adobe, wrapcol=n) + self.assertEqual(a, a_expected) + + def assertDecode(data, b_expected, adobe=False): + a = self.type2test(data) + b = binascii.a2b_ascii85(a, adobe=adobe, ignorechars=b"\n") + self.assertEqual(b, b_expected) + + tests = [ + (b"", 0, b"", b"<~~>"), + (b"", 1, b"", b"<~\n~>"), + (b"a", 0, b"@/", b"<~@/~>"), + (b"a", 1, b"@\n/", b"<~\n@/\n~>"), + (b"a", 2, b"@/", b"<~\n@/\n~>"), + (b"a", 3, b"@/", b"<~@\n/~>"), + (b"a", 4, b"@/", b"<~@/\n~>"), + (b"a", 5, b"@/", b"<~@/\n~>"), + (b"a", 6, b"@/", b"<~@/~>"), + (b"a", 7, b"@/", b"<~@/~>"), + (b"a", 123, b"@/", b"<~@/~>"), + (b"this is a test", 7, b"FD,B0+D\nGm>@3BZ\n'F*%", + b"<~FD,B0\n+DGm>@3\nBZ'F*%\n~>"), + (b"a test!!!!!!! ", 11, b"@3BZ'F*&QK+\nX&!P+WqmM+9", + b"<~@3BZ'F*&Q\nK+X&!P+WqmM\n+9~>"), + (b"\0" * 56, 7, b"zzzzzzz\nzzzzzzz", b"<~zzzzz\nzzzzzzz\nzz~>"), + ] + for b, n, a, a_wrap in tests: + assertEncode(a, b, n) + assertEncode(a_wrap, b, n, adobe=True) + assertDecode(a, b) + assertDecode(a_wrap, b, adobe=True) + + def test_ascii85_pad(self): + # Test Ascii85 with encode padding + rawdata = b"n1n3tee\n ch@rAcTer$" + for i in range(1, len(rawdata) + 1): + padding = -i % 4 + b = rawdata[:i] + a_pad = binascii.b2a_ascii85(self.type2test(b), pad=True) + b_pad = binascii.a2b_ascii85(self.type2test(a_pad)) + b_pad_expected = b + b"\0" * padding + self.assertEqual(b_pad, b_pad_expected) + + # Test Ascii85 short form groups with encode padding + def assertShortPad(data, expected, **kwargs): + data = self.type2test(data) + res = binascii.b2a_ascii85(data, **kwargs) + self.assertEqual(res, expected) + + assertShortPad(b"\0", b"!!", pad=False) + assertShortPad(b"\0", b"z", pad=True) + assertShortPad(b"\0" * 2, b"z", pad=True) + assertShortPad(b"\0" * 3, b"z", pad=True) + assertShortPad(b"\0" * 4, b"z", pad=True) + assertShortPad(b"\0" * 5, b"zz", pad=True) + assertShortPad(b"\0" * 6, b"z!!!") + assertShortPad(b" " * 7, b"y+", + foldspaces=True, adobe=True, pad=True) + assertShortPad(b"\0\0\0\0abcd \0\0", b"<~z@:E_Wy\nz~>", + foldspaces=True, adobe=True, wrapcol=9, pad=True) + + def test_ascii85_ignorechars(self): + # Test Ascii85 with ignored characters + def assertIgnore(data, expected, ignorechars=b"", **kwargs): + data = self.type2test(data) + ignore = self.type2test(ignorechars) + with self.assertRaisesRegex(binascii.Error, r"(?i)Non-Ascii85 digit"): + binascii.a2b_ascii85(data, **kwargs) + res = binascii.a2b_ascii85(data, ignorechars=ignorechars, **kwargs) + self.assertEqual(res, expected) + + assertIgnore(b"\n", b"", ignorechars=b"\n") + assertIgnore(b"<~ ~>", b"", ignorechars=b" ", adobe=True) + assertIgnore(b"z|z", b"\0" * 8, ignorechars=b"|||") # repeats don't matter + assertIgnore(b"zz!!|", b"\0" * 9, ignorechars=b"|!z") # ignore only if invalid + assertIgnore(b"<~B P~@~>", b"hi", ignorechars=b" <~>", adobe=True) + assertIgnore(b"zy}", b"\0\0\0\0", ignorechars=b"zy}") + assertIgnore(b"zy}", b"\0\0\0\0 ", ignorechars=b"zy}", foldspaces=True) + + def test_base85_valid(self): + # Test base85 with valid data + lines, i = [], 0 + for k in range(1, len(self.rawdata) + 1): + b = self.type2test(self.rawdata[i:i + k]) + a = binascii.b2a_base85(b) + lines.append(a) + i += k + if i >= len(self.rawdata): + break + res = bytes() + for line in lines: + a = self.type2test(line) + b = binascii.a2b_base85(a) + res += b + self.assertEqual(res, self.rawdata) + + # Test decoding inputs with different length + # 1-char groups are rejected (encoding violation) + with self.assertRaises(binascii.Error): + binascii.a2b_base85(self.type2test(b'a')) + self.assertEqual(binascii.a2b_base85(self.type2test(b'ab')), b'q') + self.assertEqual(binascii.a2b_base85(self.type2test(b'abc')), b'qa') + self.assertEqual(binascii.a2b_base85(self.type2test(b'abcd')), + b'qa\x9e') + self.assertEqual(binascii.a2b_base85(self.type2test(b'abcde')), + b'qa\x9e\xb6') + # 6-char input = full 5-char group + trailing 1-char group (rejected) + with self.assertRaises(binascii.Error): + binascii.a2b_base85(self.type2test(b'abcdef')) + self.assertEqual(binascii.a2b_base85(self.type2test(b'abcdefg')), + b'qa\x9e\xb6\x81') + + def test_base85_errors(self): + def _assertRegexTemplate(assert_regex, data, **kwargs): + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_base85(self.type2test(data), **kwargs) + + def assertNonBase85Data(data): + _assertRegexTemplate(r"(?i)bad base85 character", data) + + def assertOverflow(data): + _assertRegexTemplate(r"(?i)base85 overflow", data) + + assertNonBase85Data(b"\xda") + assertNonBase85Data(b"00\0\0") + assertNonBase85Data(b"Z )*") + assertNonBase85Data(b"bY*jNb0Hyq\n") + + # Test base85 with out-of-range encoded value + assertOverflow(b"}") + assertOverflow(b"|O") + assertOverflow(b"|Nt") + assertOverflow(b"|NsD") + assertOverflow(b"|NsC1") + assertOverflow(b"|NsC0~") + assertOverflow(b"|NsC0|NsC0|NsD0") + + def test_base85_wrapcol(self): + self._common_test_wrapcol(binascii.b2a_base85) + b = self.type2test(b'www.python.org') + self.assertEqual(binascii.b2a_base85(b, wrapcol=10), + b'cXxL#aCvlS\nZ*DGca%T') + self.assertEqual(binascii.b2a_base85(b, wrapcol=14), + b'cXxL#aCvlS\nZ*DGca%T') + self.assertEqual(binascii.b2a_base85(b, wrapcol=1), + b'cXxL#\naCvlS\nZ*DGc\na%T') + + def test_base85_pad(self): + # Test base85 with encode padding + rawdata = b"n1n3Tee\n ch@rAc\te\r$" + for i in range(1, len(rawdata) + 1): + padding = -i % 4 + b = rawdata[:i] + a_pad = binascii.b2a_base85(self.type2test(b), pad=True) + b_pad = binascii.a2b_base85(self.type2test(a_pad)) + b_pad_expected = b + b"\0" * padding + self.assertEqual(b_pad, b_pad_expected) + + def test_base85_ignorechars(self): + a2b_base85 = binascii.a2b_base85 + self._common_test_ignorechars(a2b_base85) + eq = self.assertEqual + eq(a2b_base85(b'VPa\n !s\n', ignorechars=b' \n'), b'abcd') + eq(a2b_base85(b'VPa\n !s\n', ignorechars=bytearray(b' \n')), b'abcd') + + eq(a2b_base85(b'A~[B];C', ignorechars=b';[]~'), b'"1\xa3\x15') + eq(a2b_base85(b'A~[B];C', ignorechars=b';[]~', + alphabet=binascii.Z85_ALPHABET), b'r\xd8dv') + + def test_base85_alphabet(self): + alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') + data = self.type2test(self.rawdata) + encoded = binascii.b2a_base85(data, alphabet=alphabet) + trans = bytes.maketrans(binascii.BASE85_ALPHABET, alphabet) + expected = binascii.b2a_base85(data).translate(trans) + self.assertEqual(encoded, expected) + self.assertEqual(binascii.a2b_base85(encoded, alphabet=alphabet), self.rawdata) + self.assertEqual(binascii.b2a_base85(data, alphabet=self.type2test(alphabet)), expected) + + data = self.type2test(b'') + self.assertEqual(binascii.b2a_base85(data, alphabet=alphabet), b'') + self.assertEqual(binascii.a2b_base85(data, alphabet=alphabet), b'') + + for func in binascii.b2a_base85, binascii.a2b_base85: + with self.assertRaises(TypeError): + func(data, alphabet=None) + with self.assertRaises(TypeError): + func(data, alphabet=alphabet.decode()) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet[:-1]) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet+b'?') + with self.assertRaises(TypeError): + binascii.a2b_base64(data, alphabet=bytearray(alphabet)) + + def test_base85_canonical(self): + # Non-canonical encodings are accepted without canonical=True + self.assertEqual(binascii.a2b_base85(b'VF'), b'a') + + # 1-char partial groups are always rejected (encoding violation: + # no conforming encoder produces them) + with self.assertRaises(binascii.Error): + binascii.a2b_base85(b'V') + with self.assertRaises(binascii.Error): + binascii.a2b_base85(b'0') + + # Verify round-trip: encode then decode with canonical=True works + for data in [b'a', b'ab', b'abc', b'abcd', b'abcde', + b'\x00', b'\xff', b'\x00\x00', b'\xff\xff\xff']: + encoded = binascii.b2a_base85(data) + decoded = binascii.a2b_base85(encoded, canonical=True) + self.assertEqual(decoded, data) + + # Test non-canonical rejection for each partial group size + # (2-char/1-byte, 3-char/2-byte, 4-char/3-byte). + # Incrementing the last digit by 1 produces a non-canonical + # encoding. For 4-char groups (n_pad=1) a +1 can change the + # output byte, so we use b'ab\x00' whose canonical form allows + # a +1 that still decodes to the same 3 bytes. + for data in [b'a', b'ab', b'ab\x00']: + canonical_enc = binascii.b2a_base85(data) + non_canonical = (canonical_enc[:-1] + + bytes([canonical_enc[-1] + 1])) + # Same decoded output without canonical check + self.assertEqual(binascii.a2b_base85(non_canonical), data) + # Rejected with canonical=True + with self.assertRaises(binascii.Error): + binascii.a2b_base85(non_canonical, canonical=True) + + # Boundary bytes: \x00 and \xff for each partial group size + for data in [b'\x00', b'\x00\x00', b'\x00\x00\x00', + b'\xff', b'\xff\xff', b'\xff\xff\xff']: + canonical_enc = binascii.b2a_base85(data) + binascii.a2b_base85(canonical_enc, canonical=True) + + # Full 5-char groups are always canonical (no padding bits) + self.assertEqual( + binascii.a2b_base85(b'VPa!s', canonical=True), b'abcd') + + # Empty input is valid + self.assertEqual(binascii.a2b_base85(b'', canonical=True), b'') + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_base85_canonical_roundtrip(self, payload): + encoded = binascii.b2a_base85(payload) + decoded = binascii.a2b_base85(encoded, canonical=True) + self.assertEqual(decoded, payload) + + @hypothesis.given(payload=hypothesis.strategies.binary(min_size=1, max_size=3)) + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff') + @hypothesis.example(b'ab\x00') + def test_base85_canonical_unique(self, payload): + # For a partial group, sweeping all 85 last-digit values should + # yield exactly one encoding that both decodes to the original + # payload AND passes canonical=True. + hypothesis.assume(len(payload) % 4 != 0) + canonical_enc = binascii.b2a_base85(payload) + table = binascii.BASE85_ALPHABET + accepted = [] + for digit in table: + candidate = canonical_enc[:-1] + bytes([digit]) + try: + result = binascii.a2b_base85(candidate, canonical=True) + if result == payload: + accepted.append(candidate) + except binascii.Error: + pass + self.assertEqual(accepted, [canonical_enc]) + + def test_ascii85_canonical(self): + # Non-canonical encodings are accepted without canonical=True + self.assertEqual(binascii.a2b_ascii85(b'@0'), b'a') + + # 1-char partial groups are always rejected (PLRM encoding violation) + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'@') + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'!') + + # Verify round-trip: encode then decode with canonical=True works + for data in [b'a', b'ab', b'abc', b'abcd', b'abcde', + b'\x00', b'\xff', b'\x00\x00', b'\xff\xff\xff']: + encoded = binascii.b2a_ascii85(data) + decoded = binascii.a2b_ascii85(encoded, canonical=True) + self.assertEqual(decoded, data) + + # Test non-canonical rejection for each partial group size. + # See test_base85_canonical for why b'ab\x00' is used for 3 bytes. + for data in [b'a', b'ab', b'ab\x00']: + canonical_enc = binascii.b2a_ascii85(data) + non_canonical = (canonical_enc[:-1] + + bytes([canonical_enc[-1] + 1])) + self.assertEqual(binascii.a2b_ascii85(non_canonical), data) + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(non_canonical, canonical=True) + + # Full 5-char groups are always canonical + self.assertEqual( + binascii.a2b_ascii85(b'@:E_W', canonical=True), b'abcd') + + # 'z' is the canonical form for all-zero groups per the PLRM. + # '!!!!!' decodes identically but is non-canonical. + self.assertEqual(binascii.a2b_ascii85(b'!!!!!'), b'\x00' * 4) + self.assertEqual(binascii.a2b_ascii85(b'z'), b'\x00' * 4) + self.assertEqual( + binascii.a2b_ascii85(b'z', canonical=True), b'\x00' * 4) + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'!!!!!', canonical=True) + # Multiple groups: z + !!!!! should fail + with self.assertRaises(binascii.Error): + binascii.a2b_ascii85(b'z!!!!!', canonical=True) + # Multiple z groups are fine + self.assertEqual( + binascii.a2b_ascii85(b'zz', canonical=True), b'\x00' * 8) + + # Empty input is valid + self.assertEqual(binascii.a2b_ascii85(b'', canonical=True), b'') + + # Adobe-wrapped with canonical + self.assertEqual( + binascii.a2b_ascii85(b'<~@:E_W~>', canonical=True, adobe=True), + b'abcd') + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\x00\x00\x00\x00') # triggers z abbreviation + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_ascii85_canonical_roundtrip(self, payload): + encoded = binascii.b2a_ascii85(payload) + decoded = binascii.a2b_ascii85(encoded, canonical=True) + self.assertEqual(decoded, payload) + + @hypothesis.given(payload=hypothesis.strategies.binary(min_size=1, max_size=3)) + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff') + @hypothesis.example(b'ab\x00') + def test_ascii85_canonical_unique(self, payload): + hypothesis.assume(len(payload) % 4 != 0) + canonical_enc = binascii.b2a_ascii85(payload) + # Ascii85 alphabet: '!' (33) through 'u' (117) + accepted = [] + for digit in range(33, 118): + candidate = canonical_enc[:-1] + bytes([digit]) + try: + result = binascii.a2b_ascii85(candidate, canonical=True) + if result == payload: + accepted.append(candidate) + except binascii.Error: + pass + self.assertEqual(accepted, [canonical_enc]) + + def test_base32_valid(self): + # Test base32 with valid data + lines = [] + step = 0 + i = 0 + while i < len(self.rawdata): + b = self.type2test(self.rawdata[i:i + step]) + a = binascii.b2a_base32(b) + lines.append(a) + i += step + step += 1 + res = bytes() + for line in lines: + a = self.type2test(line) + b = binascii.a2b_base32(a) + res += b + self.assertEqual(res, self.rawdata) + + def test_base32_errors(self): + def _fixPadding(data): + fixed = data.replace(b"=", b"") + len_8 = len(fixed) % 8 + p = 8 - len_8 if len_8 else 0 + return fixed + b"=" * p + + def _assertRegexTemplate(assert_regex, data, good_padding_result=None, **kwargs): + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_base32(self.type2test(data), **kwargs) + if good_padding_result: + fixed = self.type2test(_fixPadding(data)) + self.assertEqual(binascii.a2b_base32(fixed), good_padding_result) + + def assertNonBase32Data(*args): + _assertRegexTemplate(r"(?i)Only base32 data", *args) + + def assertExcessData(*args): + _assertRegexTemplate(r"(?i)Excess data", *args) + + def assertExcessPadding(*args): + _assertRegexTemplate(r"(?i)Excess padding", *args) + + def assertLeadingPadding(*args, **kwargs): + _assertRegexTemplate(r"(?i)Leading padding", *args, **kwargs) + + def assertIncorrectPadding(*args): + _assertRegexTemplate(r"(?i)Incorrect padding", *args) + + def assertDiscontinuousPadding(*args): + _assertRegexTemplate(r"(?i)Discontinuous padding", *args) + + def assertInvalidLength(data, *args, length=None, **kwargs): + if length is None: + length = len(data.split(b'=', 1)[0].replace(b' ', b'')) + assert_regex = fr"(?i)Invalid.+number of data characters \({length}\)" + _assertRegexTemplate(assert_regex, data, *args, **kwargs) + + assertNonBase32Data(b"a") + assertNonBase32Data(b"AA-") + assertNonBase32Data(b"ABCDE==!") + assertNonBase32Data(b"ab:(){:|:&};:==") + + assertExcessData(b"AB======C") + assertExcessData(b"AB======CD") + assertExcessData(b"ABCD====E") + assertExcessData(b"ABCDE===FGH") + assertExcessData(b"ABCDEFG=H") + assertExcessData(b"432Z====55555555") + + assertExcessData(b"BE======EF", b"\t\x08") + assertExcessData(b"BEEF====C", b"\t\x08Q") + assertExcessData(b"BEEFC===AK", b"\t\x08Q\x01") + assertExcessData(b"BEEFCAK=E", b"\t\x08Q\x01D") + + assertExcessPadding(b"BE=======", b"\t") + assertExcessPadding(b"BE========", b"\t") + assertExcessPadding(b"BEEF=====", b"\t\x08") + assertExcessPadding(b"BEEF======", b"\t\x08") + assertExcessPadding(b"BEEFC====", b"\t\x08Q") + assertExcessPadding(b"BEEFC=====", b"\t\x08Q") + assertExcessPadding(b"BEEFCAK==", b"\t\x08Q\x01") + assertExcessPadding(b"BEEFCAK===", b"\t\x08Q\x01") + assertExcessPadding(b"BEEFCAKE=", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE==", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE===", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE====", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE=====", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE======", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE=======", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE========", b"\t\x08Q\x01D") + assertExcessPadding(b"BEEFCAKE=========", b"\t\x08Q\x01D") + + assertLeadingPadding(b"=", b"") + assertLeadingPadding(b"==", b"") + assertLeadingPadding(b"===", b"") + assertLeadingPadding(b"====", b"") + assertLeadingPadding(b"=====", b"") + assertLeadingPadding(b"======", b"") + assertLeadingPadding(b"=======", b"") + assertLeadingPadding(b"========", b"") + assertLeadingPadding(b"=========", b"") + assertLeadingPadding(b"=BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"==BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"===BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"====BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"=====BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"======BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"=======BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"========BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b"=========BEEFCAKE", b"\t\x08Q\x01D") + assertLeadingPadding(b" =BEEFCAKE", ignorechars=b' ') + + assertIncorrectPadding(b"AB") + assertIncorrectPadding(b"ABCD") + assertIncorrectPadding(b"ABCDE") + assertIncorrectPadding(b"ABCDEFG") + + assertIncorrectPadding(b"BE=", b"\t") + assertIncorrectPadding(b"BE==", b"\t") + assertIncorrectPadding(b"BE===", b"\t") + assertIncorrectPadding(b"BE====", b"\t") + assertIncorrectPadding(b"BE=====", b"\t") + assertIncorrectPadding(b"BEEF=", b"\t\x08") + assertIncorrectPadding(b"BEEF==", b"\t\x08") + assertIncorrectPadding(b"BEEF===", b"\t\x08") + assertIncorrectPadding(b"BEEFC=", b"\t\x08Q") + assertIncorrectPadding(b"BEEFC==", b"\t\x08Q") + + assertDiscontinuousPadding(b"BE=EF===", b"\t\x08") + assertDiscontinuousPadding(b"BE==EF==", b"\t\x08") + assertDiscontinuousPadding(b"BEEF=C==", b"\t\x08Q") + assertDiscontinuousPadding(b"BEEFC=AK", b"\t\x08Q\x01") + + assertInvalidLength(b"A") + assertInvalidLength(b"ABC") + assertInvalidLength(b"ABCDEF") + assertInvalidLength(b"ABCDEFGHI") + assertInvalidLength(b"ABCDEFGHIJK") + assertInvalidLength(b"ABCDEFGHIJKLMN") + + assertInvalidLength(b"A=") + assertInvalidLength(b"A==") + assertInvalidLength(b"A===") + assertInvalidLength(b"A====") + assertInvalidLength(b"A=====") + assertInvalidLength(b"A======") + assertInvalidLength(b"ABC=") + assertInvalidLength(b"ABC==") + assertInvalidLength(b"ABC===") + assertInvalidLength(b"ABC====") + assertInvalidLength(b"ABCDEF=") + + assertInvalidLength(b"B=E=====", b"\t") + assertInvalidLength(b"B==E====", b"\t") + assertInvalidLength(b"BEE=F===", b"\t\x08") + assertInvalidLength(b"BEE==F==", b"\t\x08") + assertInvalidLength(b"BEEFCA=K", b"\t\x08Q\x01") + assertInvalidLength(b"BEEFCA=====K", b"\t\x08Q\x01") + + assertInvalidLength(b" A", ignorechars=b' ') + assertInvalidLength(b" ABC", ignorechars=b' ') + assertInvalidLength(b" ABCDEF", ignorechars=b' ') + assertInvalidLength(b" ABCDEFGHI", ignorechars=b' ') + assertInvalidLength(b" ABCDEFGHIJK", ignorechars=b' ') + assertInvalidLength(b" ABCDEFGHIJKLMN", ignorechars=b' ') + assertInvalidLength(b" A=======", ignorechars=b' ') + assertInvalidLength(b" ABC=====", ignorechars=b' ') + assertInvalidLength(b" ABCDEF==", ignorechars=b' ') + + def test_base32_canonical(self): + # https://datatracker.ietf.org/doc/html/rfc4648.html#section-3.5 + # Decoders MAY reject encoded data if the pad bits are not zero. + + # Without canonical=True, non-zero padding bits are accepted + self.assertEqual(binascii.a2b_base32(self.type2test(b'AB======')), + b'\x00') + + # 2 data chars + "======": last char has 2 padding bits + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AB======'), canonical=True) + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AD======'), canonical=True) + + # 4 data chars + "====": last char has 4 padding bits + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAB===='), canonical=True) + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAP===='), canonical=True) + + # 5 data chars + "===": last char has 1 padding bit + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAAB==='), canonical=True) + + # 7 data chars + "=": last char has 3 padding bits + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAAAAB='), canonical=True) + with self.assertRaises(binascii.Error): + binascii.a2b_base32(self.type2test(b'AAAAAAH='), canonical=True) + + # Verify that zero padding bits are accepted + binascii.a2b_base32(self.type2test(b'AA======'), canonical=True) + binascii.a2b_base32(self.type2test(b'AAAA===='), canonical=True) + binascii.a2b_base32(self.type2test(b'AAAAA==='), canonical=True) + binascii.a2b_base32(self.type2test(b'AAAAAAA='), canonical=True) + + # Full octet with no padding -- always valid + binascii.a2b_base32(self.type2test(b'AAAAAAAA'), canonical=True) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'') + @hypothesis.example(b'\x00') + @hypothesis.example(b'\xff\xff') + @hypothesis.example(b'abc') + def test_base32_canonical_roundtrip(self, payload): + encoded = binascii.b2a_base32(payload) + decoded = binascii.a2b_base32(encoded, canonical=True) + self.assertEqual(decoded, payload) + + def test_a2b_base32_padded(self): + a2b_base32 = binascii.a2b_base32 + t = self.type2test + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + a2b_base32(t(data), padded=False) + self.assertEqual(a2b_base32(t(data), padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(a2b_base32(t(data), padded=False), expected) + + check(b'', b'') + check(b'ME======', b'a') + check(b'MFRA====', b'ab') + check(b'MFRGG===', b'abc') + check(b'MFRGGZA=', b'abcd') + check(b'MFRGGZDF', b'abcde') + check(b'M=FRGGZDF', b'abcde') + check(b'MF=RGGZDF', b'abcde') + check(b'MFR=GGZDF', b'abcde') + check(b'MFRG=GZDF', b'abcde') + check(b'MFRGG=ZDF', b'abcde') + check(b'MFRGGZ=DF', b'abcde') + check(b'MFRGGZD=F', b'abcde') + + def test_b2a_base32_padded(self): + b2a_base32 = binascii.b2a_base32 + t = self.type2test + self.assertEqual(b2a_base32(t(b''), padded=False), b'') + self.assertEqual(b2a_base32(t(b'a'), padded=False), b'ME') + self.assertEqual(b2a_base32(t(b'ab'), padded=False), b'MFRA') + self.assertEqual(b2a_base32(t(b'abc'), padded=False), b'MFRGG') + self.assertEqual(b2a_base32(t(b'abcd'), padded=False), b'MFRGGZA') + self.assertEqual(b2a_base32(t(b'abcde'), padded=False), b'MFRGGZDF') + + def test_base32_wrapcol(self): + self._common_test_wrapcol(binascii.b2a_base32) + b = self.type2test(b'www.python.org') + self.assertEqual(binascii.b2a_base32(b, wrapcol=16), + b'O53XOLTQPF2GQ33O\nFZXXEZY=') + self.assertEqual(binascii.b2a_base32(b, wrapcol=23), + b'O53XOLTQPF2GQ33O\nFZXXEZY=') + self.assertEqual(binascii.b2a_base32(b, wrapcol=1), + b'O53XOLTQ\nPF2GQ33O\nFZXXEZY=') + + def test_base32_ignorechars(self): + a2b_base32 = binascii.a2b_base32 + self._common_test_ignorechars(a2b_base32) + eq = self.assertEqual + eq(a2b_base32(b'MFRG\n GZDF\n', ignorechars=b' \n'), b'abcde') + eq(a2b_base32(b'MFRG\n GZDF\n', ignorechars=bytearray(b' \n')), b'abcde') + eq(a2b_base32(b'M=======FRGGZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MF======RGGZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFR=====GGZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRG====GZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGG===ZDF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGGZ==DF', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGGZD=F', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRGGZDF=', ignorechars=b'='), b'abcde') + eq(a2b_base32(b'MFRA======', ignorechars=b'='), b'ab') + + eq(a2b_base32(b'A1B3C5W7Z9', ignorechars=b'19WZ'), b'\x00v.\xdb\xf9') + eq(a2b_base32(b'A1B3C5W7Z9', ignorechars=b'19WZ', + alphabet=binascii.BASE32HEX_ALPHABET), b'PV6\x14\xe9') + + def test_base32_alphabet(self): + alphabet = b'0Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8Ii9JjKk' + data = self.type2test(self.rawdata) + encoded = binascii.b2a_base32(data, alphabet=alphabet) + trans = bytes.maketrans(binascii.BASE32_ALPHABET, alphabet) + expected = binascii.b2a_base32(data).translate(trans) + self.assertEqual(encoded, expected) + self.assertEqual(binascii.a2b_base32(encoded, alphabet=alphabet), self.rawdata) + self.assertEqual(binascii.b2a_base32(data, alphabet=self.type2test(alphabet)), expected) + + data = self.type2test(b'') + self.assertEqual(binascii.b2a_base32(data, alphabet=alphabet), b'') + self.assertEqual(binascii.a2b_base32(data, alphabet=alphabet), b'') + + for func in binascii.b2a_base32, binascii.a2b_base32: + with self.assertRaises(TypeError): + func(data, alphabet=None) + with self.assertRaises(TypeError): + func(data, alphabet=alphabet.decode()) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet[:-1]) + with self.assertRaises(ValueError): + func(data, alphabet=alphabet+b'?') + with self.assertRaises(TypeError): + binascii.a2b_base32(data, alphabet=bytearray(alphabet)) def test_uu(self): MAX_UU = 45 @@ -325,6 +1391,15 @@ def test_hex_separator(self): expected1 = s.hex(':').encode('ascii') self.assertEqual(binascii.b2a_hex(self.type2test(s), ':'), expected1) + def test_hex_ignorechars(self): + a2b_hex = binascii.a2b_hex + self._common_test_ignorechars(a2b_hex) + self._common_test_ignorechars(binascii.unhexlify) + eq = self.assertEqual + eq(a2b_hex(b'A B\nC D\n', ignorechars=b' \n'), b'\xab\xcd') + eq(a2b_hex(b'A B\nC D\n', ignorechars=bytearray(b' \n')), b'\xab\xcd') + eq(a2b_hex(b'aBcD', ignorechars=b'ac'), b'\xab\xcd') + def test_qp(self): type2test = self.type2test a2b_qp = binascii.a2b_qp @@ -499,6 +1574,32 @@ def test_b2a_base64_newline(self): b'aGVsbG8=\n') self.assertEqual(binascii.b2a_base64(b, newline=False), b'aGVsbG8=') + b = self.type2test(b'') + self.assertEqual(binascii.b2a_base64(b), b'\n') + self.assertEqual(binascii.b2a_base64(b, newline=True), b'\n') + self.assertEqual(binascii.b2a_base64(b, newline=False), b'') + + def test_b2a_base64_padded(self): + b2a_base64 = binascii.b2a_base64 + t = self.type2test + self.assertEqual(b2a_base64(t(b''), padded=False), b'\n') + self.assertEqual(b2a_base64(t(b'a'), padded=False), b'YQ\n') + self.assertEqual(b2a_base64(t(b'ab'), padded=False), b'YWI\n') + self.assertEqual(b2a_base64(t(b'abc'), padded=False), b'YWJj\n') + + def test_b2a_base64_wrapcol(self): + self._common_test_wrapcol(binascii.b2a_base64) + b = self.type2test(b'www.python.org') + self.assertEqual(binascii.b2a_base64(b, wrapcol=8), + b'd3d3LnB5\ndGhvbi5v\ncmc=\n') + self.assertEqual(binascii.b2a_base64(b, wrapcol=11), + b'd3d3LnB5\ndGhvbi5v\ncmc=\n') + self.assertEqual(binascii.b2a_base64(b, wrapcol=8, newline=False), + b'd3d3LnB5\ndGhvbi5v\ncmc=') + self.assertEqual(binascii.b2a_base64(b, wrapcol=1), + b'd3d3\nLnB5\ndGhv\nbi5v\ncmc=\n') + b = self.type2test(b'') + self.assertEqual(binascii.b2a_base64(b, wrapcol=8, newline=False), b'') @hypothesis.given( binary=hypothesis.strategies.binary(), diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 25247e4d010..ea31322575e 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -10,7 +10,7 @@ license.workspace = true [features] default = ["std"] -binascii = ["dep:base64", "dep:crc32fast"] +binascii = ["dep:crc32fast"] bz2 = ["std", "dep:bzip2"] cjk-codecs = [] inet = ["std"] @@ -39,7 +39,6 @@ rustpython-unicode = { workspace = true } rustpython-wtf8 = { workspace = true } ascii = { workspace = true } -base64 = { workspace = true, optional = true } bitflags = { workspace = true } blake2b_simd = { workspace = true, optional = true } blake2s_simd = { workspace = true, optional = true } diff --git a/crates/common/src/binascii.rs b/crates/common/src/binascii.rs index 2a262d3a28b..73a17c1b01e 100644 --- a/crates/common/src/binascii.rs +++ b/crates/common/src/binascii.rs @@ -7,7 +7,6 @@ //! the caller's responsibility (see `mod.rs`). use alloc::{vec, vec::Vec}; -use base64::Engine; /// The base64 pad character `=`. pub const PAD: u8 = 61u8; @@ -31,13 +30,17 @@ pub enum Error { TooLong, /// `a2b_base64`: base64 decoding failed. Base64(Base64DecodeError), + /// The base64 output would exceed the maximum Python bytes length. + TooMuchBase64Data, + /// Reserving the transform output failed. + Memory, } /// Mirrors CPython's `binascii_a2b_base64_impl` (`Modules/binascii.c`) error /// paths one-for-one, so the caller can build the exact Python error message. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Base64DecodeError { - /// A `=` at `quad_pos == 0, i == 0`, strict mode. + /// Padding before any decoded output, in strict mode. LeadingPaddingNotAllowed, /// A `=` outside the padding-completion window, strict mode. ExcessPaddingNotAllowed, @@ -51,6 +54,10 @@ pub enum Base64DecodeError { InvalidLastSymbol { index: usize }, /// `quad_pos != 0` and the trailing padding does not complete the quad. IncorrectPadding, + /// Padding is forbidden when decoding unpadded input in strict mode. + PaddingNotAllowed, + /// The final symbol contains non-zero unused bits. + NonZeroPaddingBits, } const fn hex_nibble(n: u8) -> u8 { @@ -230,45 +237,53 @@ pub fn crc_hqx(bytes: &[u8], init: u32) -> u32 { crc } -/// `a2b_base64`. A line-by-line port of `binascii_a2b_base64_impl` -/// (CPython `Modules/binascii.c`): padding characters within the completion -/// window (`quad_pos >= 2 && quad_pos + pads <= 4`) are always skipped — -/// even in strict mode — and non-strict mode additionally skips *every* -/// pad and every non-alphabet byte outside that window. Errors therefore -/// only fire in strict mode (mid-loop) or from the two post-loop length -/// checks, which apply unconditionally. -pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result, Error> { - // Converts between ASCII and base-64 characters. The index of a given number yields the - // number in ASCII while the value of said index yields the number in base-64. For example - // "=" is 61 in ASCII but 0 (since it's the pad character) in base-64, so BASE64_TABLE[61] == 0 - #[rustfmt::skip] - const BASE64_TABLE: [i8; 256] = [ - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, - 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, /* Note PAD->0 */ - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, - 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1, - -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, - 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1, - - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - ]; +/// RFC 4648's standard base64 alphabet. +pub const BASE64_ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/// Build the decoder table, with later duplicate symbols taking precedence. +#[must_use] +pub const fn base64_reverse_table(alphabet: &[u8; 64]) -> [u8; 256] { + let mut table = [u8::MAX; 256]; + let mut index = 0; + while index < 64 { + table[alphabet[index] as usize] = index as u8; + index += 1; + } + // Padding always takes precedence, even in a custom alphabet. + table[PAD as usize] = 64; + table +} - let mut decoded: Vec = Vec::with_capacity(b.len().div_ceil(4) * 3); +/// Options for CPython 3.15's base64 decoder. +pub struct Base64DecodeOptions<'a> { + pub strict_mode: bool, + pub padded: bool, + pub table: &'a [u8; 256], + pub ignored: &'a [bool; 256], + pub canonical: bool, +} - let mut quad_pos: u32 = 0; // position in the quad, 0..=3 - let mut pads: u32 = 0; +/// `a2b_base64`, following CPython 3.15's `Modules/binascii.c`. +pub fn a2b_base64(b: &[u8], options: Base64DecodeOptions<'_>) -> Result, Error> { + let Base64DecodeOptions { + strict_mode, + padded, + table, + ignored, + canonical, + } = options; + + let mut decoded: Vec = Vec::new(); + decoded + .try_reserve_exact(b.len().div_ceil(4) * 3) + .map_err(|_| Error::Memory)?; + + let mut quad_pos: usize = 0; // position in the quad, 0..=3 + let mut pads: usize = 0; let mut left_char: u8 = 0; - for (i, &el) in b.iter().enumerate() { - if el == PAD { + for &el in b { + if padded && el == PAD { pads += 1; if quad_pos >= 2 && quad_pos + pads <= 4 { continue; @@ -276,7 +291,7 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result, Error> { // RFC 4648 section-3.3: implementations MAY ignore a pad // character present before the end of the encoded data, and // MAY ignore excess pad characters. Non-strict mode does. - if !strict_mode { + if !strict_mode || ignored[PAD as usize] { continue; } if quad_pos == 1 { @@ -284,22 +299,26 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result, Error> { // `quad_pos == 1` check below. break; } - return Err(Error::Base64(if quad_pos == 0 && i == 0 { + return Err(Error::Base64(if quad_pos == 0 && decoded.is_empty() { Base64DecodeError::LeadingPaddingNotAllowed } else { Base64DecodeError::ExcessPaddingNotAllowed })); } - let binary_char = BASE64_TABLE[el as usize]; - if binary_char < 0 { - if strict_mode { - return Err(Error::Base64(Base64DecodeError::OnlyBase64DataAllowed)); + let binary_char = table[el as usize]; + if binary_char >= 64 { + if strict_mode && !ignored[el as usize] { + return Err(Error::Base64(if el == PAD { + Base64DecodeError::PaddingNotAllowed + } else { + Base64DecodeError::OnlyBase64DataAllowed + })); } continue; } - if pads > 0 && strict_mode { + if pads > 0 && strict_mode && !ignored[PAD as usize] { return Err(Error::Base64(if quad_pos + pads == 4 { Base64DecodeError::ExcessDataAfterPadding } else { @@ -312,21 +331,21 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result, Error> { match quad_pos { 0 => { quad_pos = 1; - left_char = binary_char as u8; + left_char = binary_char; } 1 => { quad_pos = 2; - decoded.push((left_char << 2) | (binary_char >> 4) as u8); - left_char = (binary_char & 0x0f) as u8; + decoded.push((left_char << 2) | (binary_char >> 4)); + left_char = binary_char & 0x0f; } 2 => { quad_pos = 3; - decoded.push((left_char << 4) | (binary_char >> 2) as u8); - left_char = (binary_char & 0x03) as u8; + decoded.push((left_char << 4) | (binary_char >> 2)); + left_char = binary_char & 0x03; } 3 => { quad_pos = 0; - decoded.push((left_char << 6) | binary_char as u8); + decoded.push((left_char << 6) | binary_char); left_char = 0; } _ => unsafe { @@ -344,24 +363,72 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result, Error> { })); } - if quad_pos != 0 && quad_pos + pads < 4 { + if padded && quad_pos != 0 && quad_pos + pads < 4 { return Err(Error::Base64(Base64DecodeError::IncorrectPadding)); } + if canonical && left_char != 0 { + return Err(Error::Base64(Base64DecodeError::NonZeroPaddingBits)); + } Ok(decoded) } -/// `b2a_base64`. -#[must_use] -pub fn b2a_base64(data: &[u8], newline: bool) -> Vec { - // https://stackoverflow.com/questions/63916821 - let mut encoded = base64::engine::general_purpose::STANDARD - .encode(data) - .into_bytes(); +/// `b2a_base64`, including CPython 3.15's wrapping and custom alphabets. +pub fn b2a_base64( + data: &[u8], + padded: bool, + wrapcol: usize, + newline: bool, + alphabet: &[u8; 64], +) -> Result, Error> { + let pads = (3 - data.len() % 3) % 3; + let mut size = data + .len() + .div_ceil(3) + .checked_mul(4) + .ok_or(Error::TooMuchBase64Data)?; + if !padded { + size -= pads; + } + let width = if wrapcol == 0 { + usize::MAX + } else { + wrapcol.max(4) / 4 * 4 + }; + let newlines = size.saturating_sub(1) / width; + size = size + .checked_add(newlines) + .and_then(|n| n.checked_add(usize::from(newline))) + .filter(|&n| isize::try_from(n).is_ok()) + .ok_or(Error::TooMuchBase64Data)?; + let mut encoded = Vec::new(); + encoded.try_reserve_exact(size).map_err(|_| Error::Memory)?; + let mut column = 0; + let mut emit = |symbol| { + if column == width { + encoded.push(b'\n'); + column = 0; + } + encoded.push(symbol); + column += 1; + }; + for chunk in data.chunks(3) { + let value = (u32::from(chunk[0]) << 16) + | (u32::from(*chunk.get(1).unwrap_or(&0)) << 8) + | u32::from(*chunk.get(2).unwrap_or(&0)); + for index in 0..=chunk.len() { + emit(alphabet[((value >> (18 - index * 6)) & 63) as usize]); + } + if padded { + for _ in chunk.len() + 1..4 { + emit(PAD); + } + } + } if newline { encoded.push(b'\n'); } - encoded + Ok(encoded) } #[inline] diff --git a/crates/stdlib/src/binascii.rs b/crates/stdlib/src/binascii.rs index 73c20506dd7..715410f14b6 100644 --- a/crates/stdlib/src/binascii.rs +++ b/crates/stdlib/src/binascii.rs @@ -1,4 +1,6 @@ -// spell-checker:ignore hexlify unhexlify uuencodes rlecode rledecode +// spell-checker:ignore hexlify unhexlify uuencodes rlecode rledecode ABCDEFGHIJKLMNPQRSTUVXYZ abcdefhijklmpqr + +mod base_n; pub(super) use decl::crc32; pub(crate) use decl::module_def; @@ -8,14 +10,138 @@ use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef}; #[pymodule(name = "binascii")] mod decl { - use super::new_binascii_error; + use super::{base_n, new_binascii_error}; use crate::vm::{ - PyResult, VirtualMachine, - builtins::PyTypeRef, - function::{ArgAsciiBuffer, ArgBytesLike, ArgIndex, OptionalArg}, + AsObject, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyBytesRef, PyDict, PyModule, PyTypeRef}, + function::{ + ArgAsciiBuffer, ArgBytesLike, ArgIndex, ArgIntoBool, ItemDoc, OptionalArg, PyMethodDef, + PyMethodFlags, + }, }; use rustpython_common::binascii; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + module.set_native_state(vm.ctx.new_dict().into()); + const DECODE: &PyMethodDef = &PyMethodDef::new_const( + "a2b_base64", + a2b_base64, + PyMethodFlags::empty(), + ItemDoc::static_text( + "a2b_base64($module, data, /, *, strict_mode=, padded=True, alphabet=BASE64_ALPHABET, ignorechars=, canonical=False)\n--\n\nDecode a line of base64 data.", + ), + ); + const DECODE32: &PyMethodDef = &PyMethodDef::new_const( + "a2b_base32", + a2b_base32, + PyMethodFlags::empty(), + ItemDoc::static_text( + "a2b_base32($module, data, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False)\n--\n\nDecode a line of base32 data.", + ), + ); + const DECODE85: &PyMethodDef = &PyMethodDef::new_const( + "a2b_base85", + a2b_base85, + PyMethodFlags::empty(), + ItemDoc::static_text( + "a2b_base85($module, data, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False)\n--\n\nDecode a line of Base85 data.", + ), + ); + for (name, definition) in [ + ("a2b_base64", DECODE), + ("a2b_base32", DECODE32), + ("a2b_base85", DECODE85), + ] { + let function = definition.build_bound_function(&vm.ctx, module.to_owned().into()); + function + .as_object() + .set_attr("__module__", vm.ctx.new_str("binascii"), vm)?; + module.dict().set_item(name, function.into(), vm)?; + } + Ok(()) + } + + fn reverse_table( + module: &Py, + alphabet: &PyBytesRef, + size: usize, + padchar: Option, + vm: &VirtualMachine, + ) -> PyResult<[u8; 256]> { + let bytes = alphabet.as_bytes(); + if bytes.len() != size { + return Err(vm.new_value_error(format!("alphabet must have length {size}"))); + } + let cache = module + .native_state() + .ok_or_else(|| vm.new_runtime_error("Missing binascii state"))? + .downcast::() + .map_err(|_| vm.new_runtime_error("Broken binascii cache"))?; + // The actual bytes object is the key: subclass hash/equality callbacks + // are part of CPython's cache lookup and insertion semantics. + if let Some(value) = cache.get_item_opt(alphabet.as_object(), vm)? { + return value + .downcast_ref::() + .and_then(|value| value.as_bytes().try_into().ok()) + .ok_or_else(|| vm.new_runtime_error("Broken binascii cache")); + } + let table = base_n::reverse_table(bytes, padchar); + cache.set_item( + alphabet.as_object(), + vm.ctx.new_bytes(table.to_vec()).into(), + vm, + )?; + Ok(table) + } + + #[pyattr(name = "CRYPT_ALPHABET")] + fn crypt_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx + .new_bytes(b"./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".to_vec()) + } + + #[pyattr(name = "UU_ALPHABET")] + fn uu_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx.new_bytes( + b" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_".to_vec(), + ) + } + + #[pyattr(name = "BINHEX_ALPHABET")] + fn binhex_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx.new_bytes( + b"!\"#$%&'()*+,-012345689@ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr".to_vec(), + ) + } + + #[pyattr(name = "BASE85_ALPHABET")] + fn base85_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx.new_bytes(b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~".to_vec()) + } + + #[pyattr(name = "ASCII85_ALPHABET")] + fn ascii85_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx.new_bytes(b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu".to_vec()) + } + + #[pyattr(name = "Z85_ALPHABET")] + fn z85_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx.new_bytes(b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#".to_vec()) + } + + #[pyattr(name = "BASE32_ALPHABET")] + fn base32_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx + .new_bytes(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567".to_vec()) + } + + #[pyattr(name = "BASE32HEX_ALPHABET")] + fn base32hex_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx + .new_bytes(b"0123456789ABCDEFGHIJKLMNOPQRSTUV".to_vec()) + } + #[pyattr(name = "Error", once)] pub(super) fn error_type(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.new_exception_type( @@ -54,12 +180,24 @@ mod decl { Ok(data.with_ref(|bytes| binascii::hexlify(bytes, sep, bytes_per_sep))) } + #[derive(FromArgs)] + struct UnhexlifyArgs { + #[pyarg(positional)] + hexstr: ArgAsciiBuffer, + #[pyarg(named, optional, py_default = "b''")] + ignorechars: OptionalArg, + } + #[pyfunction(name = "a2b_hex")] #[pyfunction] - fn unhexlify(hexstr: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult> { - hexstr - .with_ref(binascii::unhexlify) - .map_err(|e| new_binascii_error(e, vm)) + fn unhexlify(args: UnhexlifyArgs, vm: &VirtualMachine) -> PyResult> { + let ignorechars = ignored_argument(&args.ignorechars); + let result = args + .hexstr + .with_ref(|data| base_n::decode_hex(data, &ignorechars)); + drop(args.hexstr); + drop(args.ignorechars); + base_n_result(result, vm) } #[derive(FromArgs)] @@ -86,30 +224,418 @@ mod decl { data.with_ref(|bytes| binascii::crc_hqx(bytes, crc.into_int_ref().as_u32_mask())) } - #[derive(FromArgs)] - struct NewlineArg { - #[pyarg(named, default = true)] - newline: bool, + #[pyattr(name = "BASE64_ALPHABET")] + fn base64_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx.new_bytes(binascii::BASE64_ALPHABET.to_vec()) + } + + #[pyattr(name = "URLSAFE_BASE64_ALPHABET")] + fn urlsafe_base64_alphabet(vm: &VirtualMachine) -> PyBytesRef { + vm.ctx + .new_bytes(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".to_vec()) } #[derive(FromArgs)] struct A2bBase64Args { #[pyarg(positional)] data: ArgAsciiBuffer, - #[pyarg(named, default)] - strict_mode: bool, + #[pyarg(named, optional)] + strict_mode: OptionalArg, + #[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")] + padded: ArgIntoBool, + #[pyarg(named, optional, py_default = "BASE64_ALPHABET")] + alphabet: OptionalArg, + #[pyarg(named, optional)] + ignorechars: OptionalArg, + #[pyarg(named, optional, py_default = "False")] + canonical: OptionalArg, + } + + fn a2b_base64( + module: PyRef, + args: A2bBase64Args, + vm: &VirtualMachine, + ) -> PyResult> { + let A2bBase64Args { + data, + strict_mode, + padded, + alphabet, + ignorechars, + canonical, + } = args; + let strict_mode = match strict_mode { + OptionalArg::Present(value) => value.into_bool(), + OptionalArg::Missing => matches!(&ignorechars, OptionalArg::Present(_)), + }; + let result = (|| { + // Convert the last Clinic argument while both acquired buffers are alive. + let canonical = match canonical { + OptionalArg::Present(value) => value.try_to_bool(vm)?, + OptionalArg::Missing => false, + }; + const DEFAULT_TABLE: [u8; 256] = + binascii::base64_reverse_table(binascii::BASE64_ALPHABET); + let table = match &alphabet { + OptionalArg::Present(value) => reverse_table(&module, value, 64, Some(b'='), vm)?, + OptionalArg::Missing => DEFAULT_TABLE, + }; + let mut ignored = [false; 256]; + if strict_mode && let OptionalArg::Present(value) = &ignorechars { + value.with_ref(|bytes| { + for &symbol in bytes { + ignored[symbol as usize] = true; + } + }); + } + data.with_ref(|bytes| { + binascii::a2b_base64( + bytes, + binascii::Base64DecodeOptions { + strict_mode, + padded: padded.into_bool(), + table: &table, + ignored: &ignored, + canonical, + }, + ) + }) + .map_err(|e| new_binascii_error(e, vm)) + })(); + // Match Argument Clinic cleanup order, including failures in __bool__. + drop(data); + drop(ignorechars); + result + } + + #[derive(FromArgs)] + struct B2aBase64Args { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")] + padded: ArgIntoBool, + #[pyarg(named, default = 0)] + wrapcol: usize, + #[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")] + newline: ArgIntoBool, + #[pyarg(named, optional, py_default = "BASE64_ALPHABET")] + alphabet: OptionalArg, } #[pyfunction] - fn a2b_base64(args: A2bBase64Args, vm: &VirtualMachine) -> PyResult> { - let A2bBase64Args { data, strict_mode } = args; - data.with_ref(|b| binascii::a2b_base64(b, strict_mode)) + fn b2a_base64(args: B2aBase64Args, vm: &VirtualMachine) -> PyResult> { + let B2aBase64Args { + data, + padded, + wrapcol, + newline, + alphabet, + } = args; + let copy_alphabet = |bytes: &[u8]| -> PyResult<[u8; 64]> { + bytes + .try_into() + .map_err(|_| vm.new_value_error("alphabet must have length 64")) + }; + let alphabet_bytes = match &alphabet { + OptionalArg::Present(value) => value.with_ref(copy_alphabet), + OptionalArg::Missing => Ok(*binascii::BASE64_ALPHABET), + }; + let result = alphabet_bytes.and_then(|alphabet_bytes| { + data.with_ref(|bytes| { + binascii::b2a_base64( + bytes, + padded.into_bool(), + wrapcol, + newline.into_bool(), + &alphabet_bytes, + ) + }) .map_err(|e| new_binascii_error(e, vm)) + }); + drop(data); + drop(alphabet); + result + } + + fn base_n_result( + result: Result, base_n::Error>, + vm: &VirtualMachine, + ) -> PyResult> { + result.map_err(|error| match error { + base_n::Error::InvalidAlphabet(size) => { + vm.new_value_error(format!("alphabet must have length {size}")) + } + base_n::Error::Codec(message) => vm.new_exception_msg(error_type(vm), message.into()), + base_n::Error::Memory => vm.no_memory_error(), + }) + } + + fn encoder_alphabet( + argument: &OptionalArg, + default: &[u8], + ) -> Result, base_n::Error> { + match argument { + OptionalArg::Present(value) => { + value.with_ref(|bytes| base_n::copy_alphabet(bytes, default.len())) + } + OptionalArg::Missing => base_n::copy_alphabet(default, default.len()), + } + } + + fn ignored_argument(argument: &OptionalArg) -> [bool; 256] { + match argument { + OptionalArg::Present(value) => value.with_ref(base_n::ignored), + OptionalArg::Missing => [false; 256], + } + } + + fn canonical_argument( + argument: &OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + match argument { + OptionalArg::Present(value) => value.try_to_bool(vm), + OptionalArg::Missing => Ok(false), + } + } + + #[derive(FromArgs)] + struct B2aBase32Args { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")] + padded: ArgIntoBool, + #[pyarg(named, default = 0)] + wrapcol: usize, + #[pyarg(named, optional, py_default = "BASE32_ALPHABET")] + alphabet: OptionalArg, + } + + #[derive(FromArgs)] + struct A2bBase32Args { + #[pyarg(positional)] + data: ArgAsciiBuffer, + #[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")] + padded: ArgIntoBool, + #[pyarg(named, optional, py_default = "BASE32_ALPHABET")] + alphabet: OptionalArg, + #[pyarg(named, optional, py_default = "b''")] + ignorechars: OptionalArg, + #[pyarg(named, optional, py_default = "False")] + canonical: OptionalArg, + } + + fn a2b_base32( + module: PyRef, + args: A2bBase32Args, + vm: &VirtualMachine, + ) -> PyResult> { + let A2bBase32Args { + data, + padded, + alphabet, + ignorechars, + canonical, + } = args; + let result = (|| { + // This is the last Clinic conversion. Keep acquired exports alive + // and reach ordered cleanup even when __bool__ raises. + let canonical = canonical_argument(&canonical, vm)?; + let table = match &alphabet { + OptionalArg::Present(value) => reverse_table(&module, value, 32, Some(b'='), vm)?, + OptionalArg::Missing => base_n::reverse_table(base_n::BASE32, Some(b'=')), + }; + let ignore = ignored_argument(&ignorechars); + let result = data.with_ref(|data| { + base_n::decode32(data, &table, &ignore, padded.into_bool(), canonical) + }); + base_n_result(result, vm) + })(); + // Match Argument Clinic cleanup: release data before auxiliary exports. + drop(data); + drop(ignorechars); + result } #[pyfunction] - fn b2a_base64(data: ArgBytesLike, NewlineArg { newline }: NewlineArg) -> Vec { - data.with_ref(|bytes| binascii::b2a_base64(bytes, newline)) + fn b2a_base32(args: B2aBase32Args, vm: &VirtualMachine) -> PyResult> { + let B2aBase32Args { + data, + padded, + wrapcol, + alphabet, + } = args; + let result = encoder_alphabet(&alphabet, base_n::BASE32).and_then(|alphabet_bytes| { + data.with_ref(|data| { + base_n::encode32(data, padded.into_bool(), wrapcol, &alphabet_bytes) + }) + }); + drop(data); + drop(alphabet); + base_n_result(result, vm) + } + + #[derive(FromArgs)] + struct B2aBase85Args { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")] + pad: ArgIntoBool, + #[pyarg(named, default = 0)] + wrapcol: usize, + #[pyarg(named, optional, py_default = "BASE85_ALPHABET")] + alphabet: OptionalArg, + } + + #[derive(FromArgs)] + struct A2bBase85Args { + #[pyarg(positional)] + data: ArgAsciiBuffer, + #[pyarg(named, optional, py_default = "BASE85_ALPHABET")] + alphabet: OptionalArg, + #[pyarg(named, optional, py_default = "b''")] + ignorechars: OptionalArg, + #[pyarg(named, optional, py_default = "False")] + canonical: OptionalArg, + } + + fn a2b_base85( + module: PyRef, + args: A2bBase85Args, + vm: &VirtualMachine, + ) -> PyResult> { + let A2bBase85Args { + data, + alphabet, + ignorechars, + canonical, + } = args; + let result = (|| { + // This is the last Clinic conversion. Keep acquired exports alive + // and reach ordered cleanup even when __bool__ raises. + let canonical = canonical_argument(&canonical, vm)?; + let table = match &alphabet { + OptionalArg::Present(value) => reverse_table(&module, value, 85, None, vm)?, + OptionalArg::Missing => base_n::reverse_table(base_n::BASE85, None), + }; + let ignore = ignored_argument(&ignorechars); + let result = data.with_ref(|data| { + base_n::decode85(data, &table, &ignore, canonical, false, false, false) + }); + base_n_result(result, vm) + })(); + drop(data); + drop(ignorechars); + result + } + + #[pyfunction] + fn b2a_base85(args: B2aBase85Args, vm: &VirtualMachine) -> PyResult> { + let B2aBase85Args { + data, + pad, + wrapcol, + alphabet, + } = args; + let result = encoder_alphabet(&alphabet, base_n::BASE85).and_then(|alphabet_bytes| { + data.with_ref(|data| { + base_n::encode85( + data, + pad.into_bool(), + wrapcol, + &alphabet_bytes, + false, + false, + false, + ) + }) + }); + drop(data); + drop(alphabet); + base_n_result(result, vm) + } + + #[derive(FromArgs)] + struct B2aAscii85Args { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")] + foldspaces: ArgIntoBool, + #[pyarg(named, default = 0)] + wrapcol: usize, + #[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")] + pad: ArgIntoBool, + #[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")] + adobe: ArgIntoBool, + } + + #[derive(FromArgs)] + struct A2bAscii85Args { + #[pyarg(positional)] + data: ArgAsciiBuffer, + #[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")] + foldspaces: ArgIntoBool, + #[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")] + adobe: ArgIntoBool, + #[pyarg(named, optional, py_default = "b''")] + ignorechars: OptionalArg, + #[pyarg(named, optional, py_default = "False")] + canonical: OptionalArg, + } + + #[pyfunction] + fn a2b_ascii85(args: A2bAscii85Args, vm: &VirtualMachine) -> PyResult> { + let A2bAscii85Args { + data, + foldspaces, + adobe, + ignorechars, + canonical, + } = args; + let result = (|| { + // This is the last Clinic conversion. Keep acquired exports alive + // and reach ordered cleanup even when __bool__ raises. + let canonical = canonical_argument(&canonical, vm)?; + let ignore = ignored_argument(&ignorechars); + let result = data.with_ref(|data| { + base_n::decode85( + data, + &base_n::reverse_table(base_n::ASCII85, None), + &ignore, + canonical, + true, + foldspaces.into_bool(), + adobe.into_bool(), + ) + }); + base_n_result(result, vm) + })(); + drop(data); + drop(ignorechars); + result + } + + #[pyfunction] + fn b2a_ascii85(args: B2aAscii85Args, vm: &VirtualMachine) -> PyResult> { + let B2aAscii85Args { + data, + foldspaces, + wrapcol, + pad, + adobe, + } = args; + let result = data.with_ref(|data| { + base_n::encode85( + data, + pad.into_bool(), + wrapcol, + base_n::ASCII85, + true, + foldspaces.into_bool(), + adobe.into_bool(), + ) + }); + base_n_result(result, vm) } #[derive(FromArgs)] @@ -182,6 +708,8 @@ fn new_binascii_error(error: Error, vm: &VirtualMachine) -> PyBaseExceptionRef { Error::TrailingGarbage => "Trailing garbage".to_owned(), Error::TooLong => "At most 45 bytes at once".to_owned(), Error::Base64(e) => base64_message(e), + Error::TooMuchBase64Data => "Too much data for base64".to_owned(), + Error::Memory => return vm.no_memory_error(), }; vm.new_exception_msg(decl::error_type(vm), message.into()) } @@ -199,5 +727,7 @@ fn base64_message(error: Base64DecodeError) -> String { "Invalid base64-encoded string: number of data characters ({index}) cannot be 1 more than a multiple of 4" ), Base64DecodeError::IncorrectPadding => "Incorrect padding".to_owned(), + Base64DecodeError::PaddingNotAllowed => "Padding not allowed".to_owned(), + Base64DecodeError::NonZeroPaddingBits => "Non-zero padding bits".to_owned(), } } diff --git a/crates/stdlib/src/binascii/base_n.rs b/crates/stdlib/src/binascii/base_n.rs new file mode 100644 index 00000000000..fb966a9c3d4 --- /dev/null +++ b/crates/stdlib/src/binascii/base_n.rs @@ -0,0 +1,431 @@ +// Scalar native equivalents of CPython 3.15 Modules/binascii.c base-N codecs. + +pub(super) const BASE32: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; +pub(super) const BASE85: &[u8] = + b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; +pub(super) const ASCII85: &[u8] = + b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu"; + +#[derive(Debug)] +pub(super) enum Error { + InvalidAlphabet(usize), + Codec(String), + Memory, +} + +type Result = core::result::Result; + +fn codec(message: impl Into) -> Error { + Error::Codec(message.into()) +} + +fn buffer(size: usize, too_large: &str) -> Result> { + if size > isize::MAX as usize { + return Err(codec(too_large)); + } + let mut result = Vec::new(); + result.try_reserve_exact(size).map_err(|_| Error::Memory)?; + Ok(result) +} + +fn checked_alphabet(alphabet: &[u8], size: usize) -> Result<&[u8]> { + if alphabet.len() != size { + return Err(Error::InvalidAlphabet(size)); + } + Ok(alphabet) +} + +pub(super) fn copy_alphabet(alphabet: &[u8], size: usize) -> Result> { + checked_alphabet(alphabet, size)?; + let mut out = buffer(size, "Too large alphabet")?; + out.extend_from_slice(alphabet); + Ok(out) +} + +pub(super) fn reverse_table(alphabet: &[u8], padchar: Option) -> [u8; 256] { + let mut table = [u8::MAX; 256]; + for (digit, &ch) in alphabet.iter().enumerate() { + // Duplicate characters are allowed; their last occurrence wins. + table[usize::from(ch)] = digit as u8; + } + if let Some(padchar) = padchar { + table[usize::from(padchar)] = alphabet.len() as u8; + } + table +} + +pub(super) fn ignored(ignorechars: &[u8]) -> [bool; 256] { + let mut table = [false; 256]; + for &ch in ignorechars { + table[usize::from(ch)] = true; + } + table +} + +fn rounded_width(width: usize, quantum: usize) -> usize { + if width == 0 { + 0 + } else { + width.max(quantum) / quantum * quantum + } +} + +fn wrapped_size(size: usize, width: usize, newline: bool, too_large: &str) -> Result { + let breaks = if width == 0 || size == 0 { + 0 + } else { + (size - 1) / width + }; + size.checked_add(breaks) + .and_then(|n| n.checked_add(usize::from(newline))) + .filter(|n| isize::try_from(*n).is_ok()) + .ok_or_else(|| codec(too_large)) +} + +fn wrap(mut input: Vec, width: usize, newline: bool, too_large: &str) -> Result> { + let size = wrapped_size(input.len(), width, newline, too_large)?; + if width != 0 && input.len() > width { + let mut out = buffer(size, too_large)?; + for (index, chunk) in input.chunks(width).enumerate() { + if index != 0 { + out.push(b'\n'); + } + out.extend_from_slice(chunk); + } + input = out; + } + if newline { + input.try_reserve(1).map_err(|_| Error::Memory)?; + input.push(b'\n'); + } + Ok(input) +} + +pub(super) fn encode32( + data: &[u8], + padded: bool, + width: usize, + alphabet: &[u8], +) -> Result> { + let (size, in_group, out_group, too_large) = (32, 5, 8, "Too much data for base32"); + let bits = 5; + checked_alphabet(alphabet, size)?; + let full_size = data + .len() + .div_ceil(in_group) + .checked_mul(out_group) + .ok_or_else(|| codec(too_large))?; + let pads = (in_group - data.len() % in_group) % in_group * out_group / in_group; + let encoded_size = full_size - if padded { 0 } else { pads }; + let width = rounded_width(width, out_group); + wrapped_size(encoded_size, width, false, too_large)?; + let mut out = buffer(encoded_size, too_large)?; + let mut value = 0u32; + let mut available = 0u32; + let mask = (1u32 << bits) - 1; + for &byte in data { + value = (value << 8) | u32::from(byte); + available += 8; + while available >= bits { + available -= bits; + out.push(alphabet[((value >> available) & mask) as usize]); + } + value &= (1u32 << available) - 1; + } + if available != 0 { + out.push(alphabet[((value << (bits - available)) & mask) as usize]); + } + if padded { + out.resize(encoded_size, b'='); + } + wrap(out, width, false, too_large) +} + +pub(super) fn decode32( + data: &[u8], + table: &[u8; 256], + ignorechars: &[bool; 256], + padded: bool, + canonical: bool, +) -> Result> { + let (size, group, decoded_group, name) = (32, 8, 5, "base32"); + let bits = 5; + let ignore = ignorechars; + let capacity = data + .len() + .div_ceil(group) + .checked_mul(decoded_group) + .ok_or_else(|| codec(format!("Too much {name} data")))?; + let mut out = buffer(capacity, &format!("Too much {name} data"))?; + let mut pos = 0usize; + let mut pads = 0usize; + let mut value = 0u32; + let mut available = 0u32; + for &ch in data { + if padded && ch == b'=' { + pads += 1; + let allowed_pos = matches!(pos, 2 | 4 | 5 | 7); + if allowed_pos && pos + pads <= group { + continue; + } + if ignore[usize::from(b'=')] { + continue; + } + if matches!(pos, 1 | 3 | 6) { + break; + } + return Err(codec(if pos == 0 && out.is_empty() { + "Leading padding not allowed" + } else { + "Excess padding not allowed" + })); + } + let digit = table[usize::from(ch)]; + if usize::from(digit) >= size { + if !ignore[usize::from(ch)] { + return Err(codec(if ch == b'=' { + "Padding not allowed".to_owned() + } else { + format!("Only {name} data is allowed") + })); + } + continue; + } + if pads != 0 && !ignore[usize::from(b'=')] { + return Err(codec(if pos + pads == group { + "Excess data after padding" + } else { + "Discontinuous padding not allowed" + })); + } + value = (value << bits) | u32::from(digit); + available += bits; + if available >= 8 { + available -= 8; + out.push((value >> available) as u8); + value &= (1u32 << available) - 1; + } + pos += 1; + if pos == group { + pos = 0; + pads = 0; + } + } + if matches!(pos, 1 | 3 | 6) { + let count = out.len() / decoded_group * group + pos; + return Err(codec(format!( + "Invalid base32-encoded string: number of data characters ({count}) cannot be 1, 3, or 6 more than a multiple of 8" + ))); + } + if padded && pos != 0 && pos + pads < group { + return Err(codec("Incorrect padding")); + } + if canonical && value != 0 { + return Err(codec("Non-zero padding bits")); + } + Ok(out) +} + +fn digits85(mut value: u32, alphabet: &[u8]) -> [u8; 5] { + let mut out = [0; 5]; + for slot in out.iter_mut().rev() { + *slot = alphabet[(value % 85) as usize]; + value /= 85; + } + out +} + +pub(super) fn encode85( + data: &[u8], + pad: bool, + width: usize, + alphabet: &[u8], + ascii85: bool, + foldspaces: bool, + adobe: bool, +) -> Result> { + checked_alphabet(alphabet, 85)?; + let too_large = if ascii85 { + "Too much data for Ascii85" + } else { + "Too much data for Base85" + }; + let full_size = data + .len() + .div_ceil(4) + .checked_mul(5) + .and_then(|n| n.checked_add(if adobe { 4 } else { 0 })) + .ok_or_else(|| codec(too_large))?; + let size = full_size + - if !pad && !data.len().is_multiple_of(4) { + 4 - data.len() % 4 + } else { + 0 + }; + let width = if ascii85 { + if adobe && width == 1 { 2 } else { width } + } else { + rounded_width(width, 5) + }; + wrapped_size(size, width, false, too_large)?; + let mut out = buffer(size, too_large)?; + if adobe { + out.extend_from_slice(b"<~"); + } + for chunk in data.chunks(4) { + let mut block = [0; 4]; + block[..chunk.len()].copy_from_slice(chunk); + let value = u32::from_be_bytes(block); + if ascii85 && value == 0 && (pad || chunk.len() == 4) { + out.push(b'z'); + } else if ascii85 && foldspaces && value == 0x20202020 && chunk.len() == 4 { + out.push(b'y'); + } else { + let digits = digits85(value, alphabet); + let count = if pad || chunk.len() == 4 { + 5 + } else { + chunk.len() + 1 + }; + out.extend_from_slice(&digits[..count]); + } + } + if adobe { + out.extend_from_slice(b"~>"); + } + let mut out = wrap(out, width, false, too_large)?; + if adobe && out.len() >= 3 && out[out.len() - 2] == b'\n' { + let len = out.len(); + out[len - 3] = b'\n'; + out[len - 2] = b'~'; + } + Ok(out) +} + +pub(super) fn decode85( + mut data: &[u8], + table: &[u8; 256], + ignorechars: &[bool; 256], + canonical: bool, + ascii85: bool, + foldspaces: bool, + adobe: bool, +) -> Result> { + if adobe { + if !data.ends_with(b"~>") { + return Err(codec("Ascii85 encoded byte sequences must end with b'~>'")); + } + data = &data[..data.len() - 2]; + if data.starts_with(b"<~") { + data = &data[2..]; + } + } + let ignore = ignorechars; + let shortcuts = if ascii85 { + data.iter().filter(|ch| matches!(ch, b'y' | b'z')).count() + } else { + 0 + }; + let size = (data.len() - shortcuts) + .div_ceil(5) + .checked_mul(4) + .and_then(|n| shortcuts.checked_mul(4).and_then(|m| n.checked_add(m))) + .ok_or_else(|| codec("Too much Ascii85 data"))?; + let mut out = buffer(size, "Too much Ascii85 data")?; + let mut value = 0u32; + let mut pos = 0usize; + let mut from_z = false; + let mut index = 0usize; + let mut tail_len = 4usize; + while index < data.len() || pos != 0 { + let (ch, digit) = if index < data.len() { + let ch = data[index]; + (ch, table[usize::from(ch)]) + } else { + tail_len -= 1; + (0, 84) + }; + if digit < 85 { + value = value + .checked_mul(85) + .and_then(|n| n.checked_add(u32::from(digit))) + .ok_or_else(|| { + codec(if ascii85 { + "Ascii85 overflow".to_owned() + } else { + format!("Base85 overflow in hunk starting at byte {}", index / 5 * 5) + }) + })?; + pos += 1; + } else if ascii85 && (ch == b'z' || (ch == b'y' && foldspaces)) { + if pos != 0 { + return Err(codec(format!( + "'{}' inside Ascii85 5-tuple", + char::from(ch) + ))); + } + value = if ch == b'y' { 0x20202020 } else { 0 }; + from_z = ch == b'z'; + pos = 5; + } else if !ignore[usize::from(ch)] { + return Err(codec(if ascii85 { + format!("Non-Ascii85 digit found: {}", char::from(ch)) + } else { + format!("bad Base85 character at position {index}") + })); + } + index += 1; + if pos != 5 { + continue; + } + if tail_len == 0 { + return Err(codec(if ascii85 { + "Incomplete Ascii85 group" + } else { + "Incomplete Base85 group" + })); + } + out.extend_from_slice(&value.to_be_bytes()[..tail_len]); + if canonical { + if ascii85 && tail_len == 4 && value == 0 && !from_z { + return Err(codec("Non-canonical encoding, use 'z' for all-zero groups")); + } + if tail_len < 4 { + let pads = 4 - tail_len; + let top = value >> (pads * 8) << (pads * 8); + let divisor = 85u32.pow(pads as u32); + if top / divisor != value / divisor { + return Err(codec("Non-zero padding bits")); + } + } + } + pos = 0; + value = 0; + from_z = false; + } + Ok(out) +} + +pub(super) fn decode_hex(data: &[u8], ignorechars: &[bool; 256]) -> Result> { + let ignore = ignorechars; + let mut out = buffer(data.len() / 2, "Too much hexadecimal data")?; + let mut high = None; + for &ch in data { + let digit = match ch { + b'0'..=b'9' => ch - b'0', + b'a'..=b'f' => ch - b'a' + 10, + b'A'..=b'F' => ch - b'A' + 10, + _ if ignore[usize::from(ch)] => continue, + _ => return Err(codec("Non-hexadecimal digit found")), + }; + if let Some(hi) = high.take() { + out.push((hi << 4) | digit); + } else { + high = Some(digit); + } + } + if high.is_some() { + return Err(codec("Odd number of hexadecimal digits")); + } + Ok(out) +} diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index ac89021b9f6..61e93ad031b 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -140,7 +140,7 @@ impl Representable for PyNativeFunction { if let Some(bound) = zelf .zelf .as_ref() - .filter(|b| !b.class().is(vm.ctx.types.module_type)) + .filter(|b| !b.class().is_subtype(vm.ctx.types.module_type)) { Ok(format!( "", @@ -169,7 +169,7 @@ impl PyNativeFunction { fn __qualname__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult { let zelf = zelf.0; let qualname = if let Some(bound) = &zelf.zelf { - if bound.class().is(vm.ctx.types.module_type) { + if bound.class().is_subtype(vm.ctx.types.module_type) { return Ok(vm.ctx.intern_str(zelf.value.name).to_owned()); } let prefix = if bound.class().is(vm.ctx.types.type_type) { diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index ac01c901c17..e7f9e2bfc9f 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -82,7 +82,7 @@ impl PyModuleDef { } } -#[pyclass(module = false, name = "module")] +#[pyclass(module = false, name = "module", traverse = "manual")] // The dict lives in the object extension, not the payload. The offset is // the dict cell at the front of that extension. #[pymember(name = "__dict__", offset = ::rustpython_vm::object::dict_member_offset())] @@ -90,12 +90,26 @@ impl PyModuleDef { pub struct PyModule { // PyObject *md_dict; pub def: Option<&'static PyModuleDef>, - // state: Any + native_state: crate::object::PyAtomicRef>, // weaklist // for logging purposes after md_dict is cleared pub name: Option<&'static PyStrInterned>, } +// SAFETY: Native state is the only owned Python reference in the payload. +// The module dictionary is visited separately by the object extension. +unsafe impl crate::object::Traverse for PyModule { + fn traverse(&self, tracer: &mut crate::object::TraverseFn<'_>) { + self.native_state.traverse(tracer); + } + + fn clear(&mut self, out: &mut Vec) { + if let Some(state) = self.native_state.store(None) { + out.push(state); + } + } +} + impl PyPayload for PyModule { #[inline] fn class(ctx: &Context) -> &'static Py { @@ -119,6 +133,7 @@ impl PyModule { pub const fn new() -> Self { Self { def: None, + native_state: crate::object::PyAtomicRef::new_empty(), name: None, } } @@ -127,10 +142,22 @@ impl PyModule { pub const fn from_def(def: &'static PyModuleDef) -> Self { Self { def: Some(def), + native_state: crate::object::PyAtomicRef::new_empty(), name: Some(def.name), } } + /// Interpreter-local native state, retained and traversed with the module. + #[must_use] + pub fn native_state(&self) -> Option { + self.native_state.load_owned() + } + + /// Replace native state without exposing it in the Python module dictionary. + pub fn set_native_state(&self, state: PyObjectRef) { + drop(self.native_state.store(Some(state))); + } + pub fn __init_dict_from_def(vm: &VirtualMachine, module: &Py) { let doc = module.def.unwrap().doc.map(|doc| doc.to_owned()); module.init_dict(module.name.unwrap(), doc, vm); From e8cb0c7355911bd31d70d4c0bf00be3050d94c9e Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 00:13:03 -0700 Subject: [PATCH 117/229] Implement Python 3.15 Windows event-log APIs (#9086) * Implement Python 3.15 Windows event-log APIs Extract RegisterEventSource, DeregisterEventSource, ReportEvent and six constants from the existing Python 3.15 work. Match final CPython conversion and Win32 call contracts, with Windows-only argument rejection coverage that cannot perform host operations. Assisted-by: OpenAI Codex:model identifier unavailable * Match final unsigned event-log argument errors Keep every pre-host rejection case and distinguish negative ValueError from positive OverflowError, as the final Python 3.15 Clinic converters do. Assisted-by: OpenAI Codex:model identifier unavailable * Rename Windows API regressions to the shared snippet Preserve all existing regression bodies and coverage while following the requested snippet organization. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/host_env/Cargo.toml | 1 + crates/host_env/src/winapi.rs | 42 ++++++++++++++++++++++++ crates/vm/src/stdlib/_winapi.rs | 46 +++++++++++++++++++++++++++ extra_tests/snippets/stdlib_winapi.py | 39 +++++++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 extra_tests/snippets/stdlib_winapi.py diff --git a/crates/host_env/Cargo.toml b/crates/host_env/Cargo.toml index 029c2860f33..7f9634213fc 100644 --- a/crates/host_env/Cargo.toml +++ b/crates/host_env/Cargo.toml @@ -98,6 +98,7 @@ windows-sys = { workspace = true, features = [ "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_Environment", + "Win32_System_EventLog", "Win32_System_Ole", "Win32_System_IO", "Win32_System_Ioctl", diff --git a/crates/host_env/src/winapi.rs b/crates/host_env/src/winapi.rs index bf62cfaafa0..420c10882da 100644 --- a/crates/host_env/src/winapi.rs +++ b/crates/host_env/src/winapi.rs @@ -39,6 +39,10 @@ pub use windows_sys::Win32::{ }, System::{ Console::{STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}, + EventLog::{ + EVENTLOG_AUDIT_FAILURE, EVENTLOG_AUDIT_SUCCESS, EVENTLOG_ERROR_TYPE, + EVENTLOG_INFORMATION_TYPE, EVENTLOG_SUCCESS, EVENTLOG_WARNING_TYPE, + }, Memory::{ FILE_MAP_ALL_ACCESS, FILE_MAP_COPY, FILE_MAP_EXECUTE, FILE_MAP_READ, FILE_MAP_WRITE, MEM_COMMIT, MEM_FREE, MEM_IMAGE, MEM_MAPPED, MEM_PRIVATE, MEM_RESERVE, PAGE_EXECUTE, @@ -86,6 +90,44 @@ pub const LCMAP_HASH_FLAG: u32 = windows_sys::Win32::Globalization::LCMAP_HASH; pub const LCMAP_SORTHANDLE_FLAG: u32 = windows_sys::Win32::Globalization::LCMAP_SORTHANDLE; pub const LCMAP_SORTKEY_FLAG: u32 = windows_sys::Win32::Globalization::LCMAP_SORTKEY; +pub fn register_event_source_w( + server: Option<&widestring::WideCStr>, + source: &widestring::WideCStr, +) -> io::Result { + let server = server.map_or(core::ptr::null(), |name| name.as_ptr()); + unsafe { windows_sys::Win32::System::EventLog::RegisterEventSourceW(server, source.as_ptr()) } + .check_nonnull() +} + +pub fn deregister_event_source(handle: HANDLE) -> io::Result<()> { + unsafe { windows_sys::Win32::System::EventLog::DeregisterEventSource(handle) } + .check_win32_bool() +} + +pub fn report_event_w( + handle: HANDLE, + event_type: u16, + category: u16, + event_id: u32, + message: &widestring::WideCStr, +) -> io::Result<()> { + let message = message.as_ptr(); + unsafe { + windows_sys::Win32::System::EventLog::ReportEventW( + handle, + event_type, + category, + event_id, + core::ptr::null_mut(), + 1, + 0, + &message, + core::ptr::null(), + ) + } + .check_win32_bool() +} + pub struct PeekNamedPipeResult { pub data: Option>, pub available: u32, diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs index 36bd9afbdfa..c47ea682379 100644 --- a/crates/vm/src/stdlib/_winapi.rs +++ b/crates/vm/src/stdlib/_winapi.rs @@ -38,6 +38,8 @@ mod _winapi { ERROR_ALREADY_EXISTS, ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_NETNAME_DELETED, ERROR_NO_DATA, ERROR_NO_SYSTEM_RESOURCES, ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, ERROR_PIPE_CONNECTED, ERROR_PRIVILEGE_NOT_HELD, ERROR_SEM_TIMEOUT, + EVENTLOG_AUDIT_FAILURE, EVENTLOG_AUDIT_SUCCESS, EVENTLOG_ERROR_TYPE, + EVENTLOG_INFORMATION_TYPE, EVENTLOG_SUCCESS, EVENTLOG_WARNING_TYPE, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_MAP_ALL_ACCESS, FILE_MAP_COPY, FILE_MAP_EXECUTE, FILE_MAP_READ, FILE_MAP_WRITE, FILE_TYPE_CHAR, FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_REMOTE, FILE_TYPE_UNKNOWN, @@ -74,6 +76,50 @@ mod _winapi { WindowsSysResult(host_winapi::close_handle(handle.0)) } + #[pyfunction] + fn RegisterEventSource( + server: Option, + source: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let server = server + .map(|server| server.as_wtf8().to_wide_cstring()) + .transpose() + .map_err(|_| nul_char_error(vm))?; + let source = source + .as_wtf8() + .to_wide_cstring() + .map_err(|_| nul_char_error(vm))?; + vm.allow_threads(|| host_winapi::register_event_source_w(server.as_deref(), &source)) + .map(WinHandle) + .map_err(|error| error.to_pyexception(vm)) + } + + #[pyfunction] + fn DeregisterEventSource(handle: WinHandle, vm: &VirtualMachine) -> PyResult<()> { + vm.allow_threads(|| host_winapi::deregister_event_source(handle.0)) + .map_err(|error| error.to_pyexception(vm)) + } + + #[pyfunction] + fn ReportEvent( + handle: WinHandle, + event_type: u16, + category: u16, + event_id: u32, + message: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let message = message + .as_wtf8() + .to_wide_cstring() + .map_err(|_| nul_char_error(vm))?; + vm.allow_threads(|| { + host_winapi::report_event_w(handle.0, event_type, category, event_id, &message) + }) + .map_err(|error| error.to_pyexception(vm)) + } + /// CreateFile - Create or open a file or I/O device. #[expect( clippy::too_many_arguments, diff --git a/extra_tests/snippets/stdlib_winapi.py b/extra_tests/snippets/stdlib_winapi.py new file mode 100644 index 00000000000..ef1cad44c37 --- /dev/null +++ b/extra_tests/snippets/stdlib_winapi.py @@ -0,0 +1,39 @@ +"""Check Windows event-log conversions without invoking host operations.""" + +import sys + +from testutils import assert_raises + +if sys.platform == "win32": + import _winapi + + # Every call fails before a host operation. ReportEvent overflow cases + # also retain an embedded NUL as an independent safety barrier. + assert _winapi.EVENTLOG_SUCCESS == 0 + assert _winapi.EVENTLOG_ERROR_TYPE == 1 + assert _winapi.EVENTLOG_WARNING_TYPE == 2 + assert _winapi.EVENTLOG_INFORMATION_TYPE == 4 + assert _winapi.EVENTLOG_AUDIT_SUCCESS == 8 + assert _winapi.EVENTLOG_AUDIT_FAILURE == 16 + + assert_raises(TypeError, _winapi.RegisterEventSource, 1, "source\0") + assert_raises(TypeError, _winapi.RegisterEventSource, None, 1) + assert_raises(TypeError, _winapi.RegisterEventSource, None, None) + assert_raises(ValueError, _winapi.RegisterEventSource, None, "source\0") + assert_raises(ValueError, _winapi.RegisterEventSource, "server\0", "source\0") + assert_raises(ValueError, _winapi.RegisterEventSource, "", "source\0") + assert_raises(TypeError, _winapi.DeregisterEventSource, "invalid handle") + assert_raises(TypeError, _winapi.DeregisterEventSource, None) + assert_raises( + TypeError, _winapi.ReportEvent, "invalid handle", 0, 0, 0, "message\0" + ) + assert_raises(ValueError, _winapi.ReportEvent, 0, 0, 0, 0, "message\0") + assert_raises(TypeError, _winapi.ReportEvent, 0, 0, 0, 0, None) + for index, bits in ((1, 16), (2, 16), (3, 32)): + for invalid, exception in ((-1, ValueError), (1 << bits, OverflowError)): + arguments = [0, 0, 0, 0, "message\0"] + arguments[index] = invalid + assert_raises(exception, _winapi.ReportEvent, *arguments) + arguments = [0, 0, 0, 0, "message\0"] + arguments[index] = "invalid integer" + assert_raises(TypeError, _winapi.ReportEvent, *arguments) From c0694d955cb08c5acee72a3585fd07c5037ea80e Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 00:15:01 -0700 Subject: [PATCH 118/229] Reject stale GenericAlias parameters on Python 3.15 (#9069) * Reject GenericAlias parameters added after caching Validate stale cached parameters and retain the Python regression without obsolete reference prerequisites. Assisted-by: OpenAI Codex:model identifier unavailable * Compute GenericAlias parameters lazily for Python 3.15 Populate the parameter cache on first lookup or subscription. Preserve cached identity and outer lookup replacement, propagate descriptor failures, and release displaced parameters outside the cache lock. Add Python regressions for timing, retries, reentrancy, recursion and concurrent callbacks. Assisted-by: OpenAI Codex:model identifier unavailable * Check GenericAlias recursion at lazy parameter materialization Align the existing cyclic and deeply nested fixtures with certified CPython 3.15. Keep RecursionError assertions, cover both parameter lookup and subscription on both interpreters, and retain the separate deep-hash regression. Assisted-by: OpenAI Codex:model identifier unavailable * Keep finite GenericAlias stack stress native-specific Preserve shared cyclic recursion checks and all RustPython stress assertions. Finite acyclic depth is not a portable CPython C-stack limit across optimized builds. Assisted-by: OpenAI Codex:model identifier unavailable * Materialize union parameters lazily like GenericAlias Share the reentrant-safe parameter cache between aliases and unions. Preserve parameter identity without applying union argument equality or flattening. Cover nested alias timing, descriptor retry, reentrant replacement, empty caches and concurrent callbacks in Python regressions checked against final CPython 3.15. Assisted-by: OpenAI Codex:model identifier unavailable * Consolidate union regressions in the typing snippet Preserve all existing regression bodies and coverage while following the requested snippet organization. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/builtins/genericalias.rs | 71 ++++-- crates/vm/src/builtins/union.rs | 35 ++- extra_tests/snippets/stdlib_genericalias.py | 226 ++++++++++++++++++++ extra_tests/snippets/stdlib_types.py | 26 ++- extra_tests/snippets/stdlib_typing.py | 214 ++++++++++++++++++ 5 files changed, 521 insertions(+), 51 deletions(-) create mode 100644 extra_tests/snippets/stdlib_genericalias.py diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index 9c6a2485c5c..5afa70dc142 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -1,5 +1,5 @@ // spell-checker:ignore iparam gaiterobject -use crate::common::lock::LazyLock; +use crate::common::lock::{LazyLock, PyMutex}; use super::type_; use crate::{ @@ -40,7 +40,7 @@ pub struct PyGenericAlias { origin: PyObjectRef, #[pymember(name = "__args__")] args: PyTupleRef, - parameters: PyTupleRef, + parameters: PyMutex>, #[pymember(name = "__unpacked__")] starred: bool, // for __unpacked__ attribute } @@ -105,13 +105,12 @@ impl PyGenericAlias { origin: impl Into, args: PyTupleRef, starred: bool, - vm: &VirtualMachine, + _vm: &VirtualMachine, ) -> PyResult { - let parameters = make_parameters(&args, vm)?; Ok(Self { origin: origin.into(), args, - parameters, + parameters: PyMutex::new(None), starred, }) } @@ -130,6 +129,10 @@ impl PyGenericAlias { Self::new(origin, args, false, vm) } + fn parameters(&self, vm: &VirtualMachine) -> PyResult { + cached_parameters(&self.parameters, &self.args, vm) + } + fn repr(&self, vm: &VirtualMachine) -> PyResult { fn repr_item(obj: &PyObject, vm: &VirtualMachine) -> PyResult { if obj.is(&vm.ctx.ellipsis) { @@ -206,7 +209,8 @@ impl PyGenericAlias { } fn __getitem__(zelf: &Py, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let new_args = subs_parameters(zelf.as_object(), &zelf.args, &zelf.parameters, needle, vm)?; + let parameters = zelf.parameters(vm)?; + let new_args = subs_parameters(zelf.as_object(), &zelf.args, ¶meters, needle, vm)?; Ok(Self::new(zelf.origin.clone(), new_args, false, vm)?.into_pyobject(vm)) } @@ -236,8 +240,8 @@ impl PyGenericAlias { )] impl Py { #[pygetset] - fn __parameters__(&self) -> PyObjectRef { - self.parameters.clone().into() + fn __parameters__(&self, vm: &VirtualMachine) -> PyResult { + self.parameters(vm) } #[pygetset] @@ -312,11 +316,36 @@ impl Py { } } +pub(crate) fn cached_parameters( + cache: &PyMutex>, + args: &Py, + vm: &VirtualMachine, +) -> PyResult { + let cached = cache.lock().clone(); + if let Some(parameters) = cached { + return Ok(parameters); + } + // Attribute lookup can re-enter this alias or union. Compute without the + // lock, then replace even a cache populated by the re-entrant lookup. + let parameters = make_parameters(args, vm); + let previous = core::mem::replace(&mut *cache.lock(), parameters.as_ref().ok().cloned()); + // A failed outer lookup clears the cache populated by a nested lookup too. + // Cached parameters can own objects with Python finalizers. + drop(previous); + parameters +} + pub(crate) fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyResult { make_parameters_from_slice(args.as_slice(), vm) } fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyResult { + vm.with_recursion(" in __parameter__ calculation", || { + make_parameters_inner(args, vm) + }) +} + +fn make_parameters_inner(args: &[PyObjectRef], vm: &VirtualMachine) -> PyResult { let mut parameters: Vec = Vec::with_capacity(args.len()); for arg in args { @@ -326,16 +355,21 @@ fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyRe } // Check for __typing_subst__ attribute - if arg.get_attr(identifier!(vm, __typing_subst__), vm).is_ok() { + if vm + .get_attribute_opt(arg, identifier!(vm, __typing_subst__))? + .is_some() + { if tuple_index(¶meters, arg).is_none() { parameters.push(arg.clone()); } - } else if let Ok(subparams) = arg.get_attr(identifier!(vm, __parameters__), vm) - && let Ok(sub_params) = subparams.try_to_ref::(vm) + } else if let Some(subparams) = + vm.get_attribute_opt(arg, identifier!(vm, __parameters__))? { - for sub_param in sub_params { - if tuple_index(¶meters, sub_param).is_none() { - parameters.push(sub_param.clone()); + if let Some(sub_params) = subparams.downcast_ref::() { + for sub_param in sub_params { + if tuple_index(¶meters, sub_param).is_none() { + parameters.push(sub_param.clone()); + } } } } else if arg.try_to_ref::(vm).is_ok() || arg.try_to_ref::(vm).is_ok() { @@ -346,9 +380,7 @@ fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyRe let list = arg.downcast_ref::().unwrap(); list.borrow_vec().to_vec() }; - let sub = vm.with_recursion(" while computing __parameters__", || { - make_parameters_from_slice(&items, vm) - })?; + let sub = make_parameters_from_slice(&items, vm)?; for sub_param in sub.as_slice() { if tuple_index(¶meters, sub_param).is_none() { parameters.push(sub_param.clone()); @@ -558,7 +590,10 @@ pub(crate) fn subs_parameters( if let Some(iparam) = tuple_index(parameters.as_slice(), arg) { subst.call((arg_items[iparam].clone(),), vm)? } else { - subs_tvars(arg.clone(), parameters, &arg_items, vm)? + return Err(vm.new_type_error(format!( + "argument {} with __typing_subst__ was not found in __parameters__", + arg.repr(vm)? + ))); } } else { subs_tvars(arg.clone(), parameters, &arg_items, vm)? diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs index 42d21d144ac..8a3711fb29b 100644 --- a/crates/vm/src/builtins/union.rs +++ b/crates/vm/src/builtins/union.rs @@ -1,5 +1,5 @@ use super::{genericalias, type_}; -use crate::common::lock::LazyLock; +use crate::common::lock::{LazyLock, PyMutex}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, @@ -24,7 +24,7 @@ pub struct PyUnion { hashable_args: Option>, /// Tuple of initially unhashable args, or None if all args were hashable unhashable_args: Option, - parameters: PyTupleRef, + parameters: PyMutex>, } impl fmt::Debug for PyUnion { @@ -42,14 +42,13 @@ impl PyPayload for PyUnion { impl PyUnion { /// Create a new union from dedup result (internal use) - fn from_components(result: UnionComponents, vm: &VirtualMachine) -> PyResult { - let parameters = make_parameters(&result.args, vm)?; - Ok(Self { + fn from_components(result: UnionComponents) -> Self { + Self { args: result.args, hashable_args: result.hashable_args, unhashable_args: result.unhashable_args, - parameters, - }) + parameters: PyMutex::new(None), + } } /// Direct access to args field (_Py_union_args) @@ -127,8 +126,8 @@ impl Py { } #[pygetset] - fn __parameters__(&self) -> PyObjectRef { - self.parameters.clone().into() + fn __parameters__(&self, vm: &VirtualMachine) -> PyResult { + genericalias::cached_parameters(&self.parameters, &self.args, vm) } #[pymethod] @@ -239,12 +238,6 @@ pub(crate) fn or_op(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) make_union(&tuple, vm) } -fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyResult { - let parameters = genericalias::make_parameters(args, vm)?; - let result = dedup_and_flatten_args(¶meters, vm)?; - Ok(result.args) -} - fn flatten_args(args: &Py, vm: &VirtualMachine) -> PyTupleRef { let mut total_args = 0; for arg in args { @@ -372,19 +365,15 @@ pub fn make_union(args: &Py, vm: &VirtualMachine) -> PyResult { let result = dedup_and_flatten_args(args, vm)?; Ok(match result.args.as_slice().len() { 1 => result.args.as_slice()[0].to_owned(), - _ => PyUnion::from_components(result, vm)?.to_pyobject(vm), + _ => PyUnion::from_components(result).to_pyobject(vm), }) } impl PyUnion { fn getitem(zelf: &Py, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let new_args = genericalias::subs_parameters( - zelf.as_object(), - &zelf.args, - &zelf.parameters, - needle, - vm, - )?; + let parameters = genericalias::cached_parameters(&zelf.parameters, &zelf.args, vm)?; + let new_args = + genericalias::subs_parameters(zelf.as_object(), &zelf.args, ¶meters, needle, vm)?; Ok(if new_args.as_slice().is_empty() { make_union(&new_args, vm)? diff --git a/extra_tests/snippets/stdlib_genericalias.py b/extra_tests/snippets/stdlib_genericalias.py new file mode 100644 index 00000000000..ddfb6d5587a --- /dev/null +++ b/extra_tests/snippets/stdlib_genericalias.py @@ -0,0 +1,226 @@ +from types import GenericAlias + +from testutils import assert_raises + + +class Parameter: + def __typing_subst__(self, value): + return value + + +parameter = Parameter() + + +# Construction does not inspect arguments. Both entry points populate the cache +# using the attributes present at the first lookup, rather than construction. +class LateParameter: + __parameters__ = (parameter,) + + def __repr__(self): + return "late_parameter" + + +for materialize in (False, True): + late = LateParameter() + alias = GenericAlias(list, late) + if materialize: + cached = alias.__parameters__ + assert cached == (parameter,) + late.__typing_subst__ = lambda value: value + if materialize: + with assert_raises(TypeError) as caught: + alias[int] + assert str(caught.exception) == ( + "argument late_parameter with __typing_subst__ was not found in __parameters__" + ) + assert alias.__parameters__ is cached + else: + assert alias[int] == list[int] + cached = alias.__parameters__ + assert cached == (late,) + assert alias.__parameters__ is cached + +late = LateParameter() +alias = GenericAlias(list, late) +late.__typing_subst__ = lambda value: value +assert alias.__parameters__ == (late,) +assert alias[int] == list[int] + +# Empty tuples are cached too, even if parameters appear later. +empty = LateParameter() +empty.__parameters__ = () +alias = GenericAlias(list, empty) +cached = alias.__parameters__ +empty.__typing_subst__ = lambda value: value +assert alias.__parameters__ is cached +assert cached == () +with assert_raises(TypeError): + alias[int] + + +# Lookup errors happen lazily and leave the cache uninitialized for a retry. +class FailingParameter: + def __init__(self, failing_attribute): + self.failing_attribute = failing_attribute + self.fail = True + self.lookups = [] + + def __getattribute__(self, name): + if name in ("__typing_subst__", "__parameters__"): + self.lookups.append(name) + if name == self.failing_attribute and self.fail: + raise RuntimeError("parameter lookup failed") + return object.__getattribute__(self, name) + + __parameters__ = (parameter,) + + +for attribute in ("__typing_subst__", "__parameters__"): + for subscribe in (False, True): + failing = FailingParameter(attribute) + alias = GenericAlias(list, failing) + assert failing.lookups == [] + with assert_raises(RuntimeError) as caught: + if subscribe: + alias[int] + else: + alias.__parameters__ + assert str(caught.exception) == "parameter lookup failed" + failing.fail = False + cached = alias.__parameters__ + assert cached == (parameter,) + count = len(failing.lookups) + assert alias.__parameters__ is cached + assert len(failing.lookups) == count + + +# A nested lookup may populate the cache, but the outer calculation replaces it. +inner_parameter = Parameter() +outer_parameter = Parameter() + + +class ReentrantParameter: + entered = False + + @property + def __parameters__(self): + if not self.entered: + self.entered = True + self.inner = self.alias.__parameters__ + return (outer_parameter,) + return (inner_parameter,) + + +reentrant = ReentrantParameter() +reentrant.alias = GenericAlias(list, reentrant) +cached = reentrant.alias.__parameters__ +assert reentrant.inner == (inner_parameter,) +assert cached == (outer_parameter,) +assert cached is reentrant.alias.__parameters__ +assert cached is not reentrant.inner + + +# A cyclic parameter lookup raises rather than deadlocking or overflowing Rust. +class RecursiveParameter: + recurse = True + + @property + def __parameters__(self): + if self.recurse: + return self.alias.__parameters__ + return (parameter,) + + +recursive = RecursiveParameter() +recursive.alias = GenericAlias(list, recursive) +with assert_raises(RecursionError): + recursive.alias.__parameters__ +recursive.recurse = False +assert recursive.alias.__parameters__ == (parameter,) + +# Substitution and unpacking construct aliases with their own lazy caches. +failing = FailingParameter("__parameters__") +result = GenericAlias(list, parameter)[failing] +assert failing.lookups == [] +with assert_raises(RuntimeError): + result.__parameters__ + +failing = FailingParameter("__parameters__") +original = GenericAlias(tuple, failing) +starred = next(iter(original)) +assert failing.lookups == [] +with assert_raises(RuntimeError): + starred.__parameters__ +failing.fail = False +assert starred.__parameters__ == (parameter,) +failing.__parameters__ = () +assert original.__parameters__ == () +assert starred.__parameters__ == (parameter,) + +# Concurrent descriptor callbacks can overlap without holding the cache lock. +import threading + +barrier = threading.Barrier(2) + + +class ConcurrentParameter: + @property + def __parameters__(self): + barrier.wait(timeout=10) + return (parameter,) + + +concurrent = GenericAlias(list, ConcurrentParameter()) +results = [] +errors = [] + + +def read_parameters(): + try: + results.append(concurrent.__parameters__) + except BaseException as error: + errors.append(error) + + +threads = [threading.Thread(target=read_parameters) for _ in range(2)] +for thread in threads: + thread.start() +for thread in threads: + thread.join(timeout=15) +assert all(not thread.is_alive() for thread in threads) +assert errors == [] +assert results == [(parameter,), (parameter,)] +cached = concurrent.__parameters__ +assert any(cached is result for result in results) +assert concurrent.__parameters__ is cached + + +# Failure in the outer calculation clears an inner successful cache as well. +class ReentrantFailure: + entered = False + fail = True + + @property + def __parameters__(self): + if not self.entered: + self.entered = True + self.inner = self.alias.__parameters__ + if self.fail: + raise RuntimeError("outer lookup failed") + return (inner_parameter,) if self.fail else (outer_parameter,) + + +for subscribe in (False, True): + failure = ReentrantFailure() + failure.alias = GenericAlias(list, failure) + with assert_raises(RuntimeError) as caught: + if subscribe: + failure.alias[int] + else: + failure.alias.__parameters__ + assert str(caught.exception) == "outer lookup failed" + assert failure.inner == (inner_parameter,) + failure.fail = False + cached = failure.alias.__parameters__ + assert cached == (outer_parameter,) + assert failure.alias.__parameters__ is cached diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index 892daeb4ba5..bc9017b881b 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -43,21 +43,27 @@ def _run_missing_type_params_regression(): _run_missing_type_params_regression() -if sys.implementation.name == "rustpython": - # __parameters__ is computed when the alias is built, and the walk descends - # into every list and tuple argument, so a self-referential or deeply - # nested argument must be caught. CPython, which also runs this snippet, - # does not walk into plain lists at all. - self_referential = [] - self_referential.append(self_referential) - with assert_raises(RecursionError): - list[self_referential] +# GenericAlias construction is lazy. Python 3.15 walks nested lists and tuples +# when parameters are first needed, including during subscription. +self_referential = [] +self_referential.append(self_referential) +alias = list[self_referential] +with assert_raises(RecursionError): + alias.__parameters__ +with assert_raises(RecursionError): + alias[int] +if sys.implementation.name == "rustpython": + # Finite-depth stack exhaustion depends on the interpreter build and OS stack. + # Keep this native stress case separate from the shared cyclic contract above. nested = [0] for _ in range(100_000): nested = [nested] + alias = list[nested] + with assert_raises(RecursionError): + alias.__parameters__ with assert_raises(RecursionError): - list[nested] + alias[int] # hashing an alias walks the same shape. Release inlining of the hash # slot needs more depth than debug to trip the native stack guard. diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py index 4bebfae4540..c4096051934 100644 --- a/extra_tests/snippets/stdlib_typing.py +++ b/extra_tests/snippets/stdlib_typing.py @@ -89,3 +89,217 @@ def __module__(self): ) assert alias_namespace["alias"].__module__ is None assert alias_module_lookups == [] + + +# GenericAlias rejects parameters added after its parameter tuple was cached. +from types import GenericAlias + + +class LateParameter: + def __repr__(self): + return "late_parameter" + + +first = LateParameter() +first.__typing_subst__ = lambda value: value +late = LateParameter() +alias = GenericAlias(dict, (first, late)) +assert alias.__parameters__ == (first,) +late.__typing_subst__ = lambda value: value +with assert_raises(TypeError) as caught: + alias[0] +assert str(caught.exception) == ( + "argument late_parameter with __typing_subst__ was not found in __parameters__" +) + + +def test_union_lazy_parameters(): + import threading + from types import GenericAlias + from typing import Union + + from testutils import assert_raises + + class Parameter: + def __typing_subst__(self, value): + return value + + parameter = Parameter() + + class LateParameter: + __parameters__ = (parameter,) + + def __repr__(self): + return "late_parameter" + + # Building a union must not freeze the nested alias's parameter cache. + for materialize in (False, True): + for subscribe_first in (False, True): + late = LateParameter() + alias = GenericAlias(list, late) + union = alias | int + if materialize: + cached = union.__parameters__ + assert cached == (parameter,) + late.__typing_subst__ = lambda value: value + if materialize: + with assert_raises(TypeError) as caught: + union[str] + assert str(caught.exception) == ( + "argument late_parameter with __typing_subst__ was not found in __parameters__" + ) + assert union.__parameters__ is cached + else: + if not subscribe_first: + assert union.__parameters__ == (late,) + assert union[str] == (list[str] | int) + assert alias[str] == list[str] + cached = union.__parameters__ + assert cached == (late,) + assert union.__parameters__ is cached + + # typing.Union and flattening an existing union preserve the same lazy boundary. + for make_union in ( + lambda alias: Union[alias, int], + lambda alias: (alias | bytes) | int, + ): + late = LateParameter() + alias = GenericAlias(list, late) + union = make_union(alias) + late.__typing_subst__ = lambda value: value + assert union[str] == make_union(list[str]) + assert alias[str] == list[str] + + # Nested descriptor failures occur on first use and do not poison either cache. + class FailingParameter: + fail = True + lookups = 0 + + @property + def __parameters__(self): + self.lookups += 1 + if self.fail: + raise RuntimeError("lazy union failure") + return (parameter,) + + for subscribe in (False, True): + failing = FailingParameter() + alias = GenericAlias(list, failing) + union = alias | int + assert failing.lookups == 0 + with assert_raises(RuntimeError) as caught: + if subscribe: + union[str] + else: + union.__parameters__ + assert str(caught.exception) == "lazy union failure" + failing.fail = False + cached = union.__parameters__ + assert cached == (parameter,) + count = failing.lookups + assert union.__parameters__ is cached + assert failing.lookups == count + + # Parameter collection uses identity, not union argument hash/equality rules. + parameter_comparisons = [] + + class EqualParameter(Parameter): + def __hash__(self): + parameter_comparisons.append("hash") + return 0 + + def __eq__(self, other): + parameter_comparisons.append("eq") + return True + + first = EqualParameter() + second = EqualParameter() + union = GenericAlias(tuple, (first, second)) | int + parameter_comparisons.clear() + cached = union.__parameters__ + assert len(cached) == 2 + assert cached[0] is first and cached[1] is second + assert parameter_comparisons == [] + assert union.__parameters__ is cached + + # Empty caches are also stable when a nested alias's descriptor changes. + state = {"parameters": ()} + + class ChangingAlias(GenericAlias): + @property + def __parameters__(self): + return state["parameters"] + + union = ChangingAlias(list, int) | str + cached = union.__parameters__ + assert cached == () + state["parameters"] = (parameter,) + assert union.__parameters__ is cached + with assert_raises(TypeError): + union[bytes] + + # Reentrant callbacks may populate the cache. The outer calculation replaces + # that cache on success and clears it on failure, allowing another attempt. + inner_parameter = Parameter() + outer_parameter = Parameter() + + for fail in (False, True): + state = {"entered": False, "parameters": (inner_parameter,)} + + class ReentrantAlias(GenericAlias): + @property + def __parameters__(self): + if not state["entered"]: + state["entered"] = True + state["inner"] = state["union"].__parameters__ + if fail: + raise RuntimeError("outer union failure") + return (outer_parameter,) + return state["parameters"] + + state["union"] = ReentrantAlias(list, int) | str + union = state["union"] + if fail: + with assert_raises(RuntimeError) as caught: + union.__parameters__ + assert str(caught.exception) == "outer union failure" + state["parameters"] = (outer_parameter,) + cached = union.__parameters__ + assert state["inner"] == (inner_parameter,) + assert cached == (outer_parameter,) + assert cached is not state["inner"] + assert union.__parameters__ is cached + + # Concurrent callbacks must run without holding the union's parameter lock. + barrier = threading.Barrier(2) + + class ConcurrentAlias(GenericAlias): + @property + def __parameters__(self): + barrier.wait(timeout=10) + return (parameter,) + + union = ConcurrentAlias(list, int) | str + results = [] + errors = [] + + def read_parameters(): + try: + results.append(union.__parameters__) + except BaseException as error: + errors.append(error) + + threads = [threading.Thread(target=read_parameters) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=15) + assert all(not thread.is_alive() for thread in threads) + assert errors == [] + assert results == [(parameter,), (parameter,)] + cached = union.__parameters__ + assert any(cached is result for result in results) + assert union.__parameters__ is cached + + +test_union_lazy_parameters() From cae11f696599e82b1ba35031a33e4666e7400f81 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 06:17:26 -0700 Subject: [PATCH 119/229] Sync email, mock, logging, and console behavior with Python 3.15 (#9079) * Sync library behavior with CPython 3.15.0 Import exact d220409a13b77de7a6c30abe16ea90db01528edc sources through scripts/update_lib migrate, preserving the existing logging test marker. Retain focused Python license and mock property regressions outside Lib. This later Python-stage change requires native binascii wrapcol support; listener DNS runtime validation remains out of scope. Assisted-by: OpenAI Codex:model identifier unavailable * Keep the library update limited to upstream files Remove the two PR-added extra_tests snippets as requested in review. Preserve the script-migrated CPython library files and their canonical tests unchanged. Assisted-by: OpenAI Codex:model identifier unavailable --- Lib/_sitebuiltins.py | 34 +++--- Lib/email/contentmanager.py | 25 ++-- Lib/encodings/aliases.py | 81 +++++++++++-- Lib/logging/__init__.py | 60 ++++++++-- Lib/logging/config.py | 21 +++- Lib/logging/handlers.py | 91 ++++++++------- Lib/tabnanny.py | 13 ++- Lib/test/support/warnings_helper.py | 36 ++++-- Lib/test/test_email/test_contentmanager.py | 60 ++++++++++ Lib/test/test_logging.py | 127 +++++++++++++++++++-- Lib/test/test_tabnanny.py | 26 +++-- Lib/unittest/mock.py | 18 +-- 12 files changed, 450 insertions(+), 142 deletions(-) diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index c66269a5719..84551e3546e 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -36,7 +36,7 @@ def __init__(self, name, data, files=(), dirs=()): import os self.__name = name self.__data = data - self.__lines = None + self.__lines = [] self.__filenames = [os.path.join(dir, filename) for dir in dirs for filename in files] @@ -65,24 +65,22 @@ def __repr__(self): return "Type %s() to see the full %s text" % ((self.__name,)*2) def __call__(self): - self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: + try: + from _pyrepl.pager import get_pager + except ModuleNotFoundError: try: - for i in range(lineno, lineno + self.MAXLINES): - print(self.__lines[i]) - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break + from pydoc import get_pager + except ModuleNotFoundError: + def get_pager(): + def _print(text, title=None): + print(text) + return _print + + self.__setup() + + pager = get_pager() + text = "\n".join(self.__lines) + pager(text, title=self.__name) class _Helper(object): diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index 11d1536db27..c0090af7165 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -129,19 +129,6 @@ def _finalize_set(msg, disposition, filename, cid, params): msg.set_param(key, value) -# XXX: This is a cleaned-up version of base64mime.body_encode (including a bug -# fix in the calculation of unencoded_bytes_per_line). It would be nice to -# drop both this and quoprimime.body_encode in favor of enhanced binascii -# routines that accepted a max_line_length parameter. -def _encode_base64(data, max_line_length): - encoded_lines = [] - unencoded_bytes_per_line = max_line_length // 4 * 3 - for i in range(0, len(data), unencoded_bytes_per_line): - thisline = data[i:i+unencoded_bytes_per_line] - encoded_lines.append(binascii.b2a_base64(thisline).decode('ascii')) - return ''.join(encoded_lines) - - def _encode_text(string, charset, cte, policy): # If max_line_length is 0 or None, there is no limit. maxlen = policy.max_line_length or sys.maxsize @@ -176,7 +163,7 @@ def normal_body(lines): return b'\n'.join(lines) + b'\n' data = quoprimime.body_encode(normal_body(lines).decode('latin-1'), maxlen) elif cte == 'base64': - data = _encode_base64(embedded_body(lines), maxlen) + data = binascii.b2a_base64(embedded_body(lines), wrapcol=maxlen).decode('ascii') else: raise ValueError("Unknown content transfer encoding {}".format(cte)) return cte, data @@ -186,11 +173,12 @@ def set_text_content(msg, string, subtype="plain", charset='utf-8', cte=None, disposition=None, filename=None, cid=None, params=None, headers=None): _prepare_set(msg, 'text', subtype, headers) + + cs = email.charset.Charset(charset) + charset = cs.output_charset cte, payload = _encode_text(string, charset, cte, msg.policy) msg.set_payload(payload) - msg.set_param('charset', - email.charset.ALIASES.get(charset, charset), - replace=True) + msg.set_param('charset', charset, replace=True) msg['Content-Transfer-Encoding'] = cte _finalize_set(msg, disposition, filename, cid, params) raw_data_manager.add_set_handler(str, set_text_content) @@ -234,7 +222,8 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64', params=None, headers=None): _prepare_set(msg, maintype, subtype, headers) if cte == 'base64': - data = _encode_base64(data, max_line_length=msg.policy.max_line_length) + data = binascii.b2a_base64(data, wrapcol=msg.policy.max_line_length) + data = data.decode('ascii') elif cte == 'quoted-printable': # XXX: quoprimime.body_encode won't encode newline characters in data, # so we can't use it. This means max_line_length is ignored. Another diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index 8b175772c58..ef51168d755 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -17,7 +17,7 @@ """ aliases = { - # Please keep this list sorted alphabetically by value ! + # Please keep this list sorted alphabetically by value! # ascii codec '646' : 'ascii', @@ -43,6 +43,7 @@ # big5hkscs codec 'big5_hkscs' : 'big5hkscs', + 'csbig5hkscs' : 'big5hkscs', 'hkscs' : 'big5hkscs', # bz2_codec codec @@ -71,6 +72,7 @@ # cp1140 codec '1140' : 'cp1140', + 'ccsid01140' : 'cp1140', 'cp01140' : 'cp1140', 'csibm01140' : 'cp1140', 'ebcdic_us_37_euro' : 'cp1140', @@ -79,38 +81,47 @@ # cp1250 codec '1250' : 'cp1250', + 'cswindows1250' : 'cp1250', 'windows_1250' : 'cp1250', # cp1251 codec '1251' : 'cp1251', + 'cswindows1251' : 'cp1251', 'windows_1251' : 'cp1251', # cp1252 codec '1252' : 'cp1252', + 'cswindows1252' : 'cp1252', 'windows_1252' : 'cp1252', # cp1253 codec '1253' : 'cp1253', + 'cswindows1253' : 'cp1253', 'windows_1253' : 'cp1253', # cp1254 codec '1254' : 'cp1254', + 'cswindows1254' : 'cp1254', 'windows_1254' : 'cp1254', # cp1255 codec '1255' : 'cp1255', + 'cswindows1255' : 'cp1255', 'windows_1255' : 'cp1255', # cp1256 codec '1256' : 'cp1256', + 'cswindows1256' : 'cp1256', 'windows_1256' : 'cp1256', # cp1257 codec '1257' : 'cp1257', + 'cswindows1257' : 'cp1257', 'windows_1257' : 'cp1257', # cp1258 codec '1258' : 'cp1258', + 'cswindows1258' : 'cp1258', 'windows_1258' : 'cp1258', # cp273 codec @@ -163,6 +174,7 @@ # cp858 codec '858' : 'cp858', + 'ccsid00858' : 'cp858', 'cp00858' : 'cp858', 'csibm00858' : 'cp858', 'csibm858' : 'cp858', @@ -214,11 +226,13 @@ # cp874 codec '874' : 'cp874', + 'cswindows874' : 'cp874', 'ms874' : 'cp874', 'windows_874' : 'cp874', # cp932 codec '932' : 'cp932', + 'cswindows31j' : 'cp932', 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', @@ -226,6 +240,16 @@ # cp949 codec '949' : 'cp949', + 'csksc56011987' : 'cp949', + 'iso_ir_149' : 'cp949', + 'korean' : 'cp949', + 'ks_c_5601_1987' : 'cp949', + 'ks_c_5601_1989' : 'cp949', + 'ksc5601' : 'cp949', + 'ks_c_5601' : 'cp949', + 'ksc_5601' : 'cp949', + 'ksx1001' : 'cp949', + 'ks_x_1001' : 'cp949', 'ms949' : 'cp949', 'uhc' : 'cp949', @@ -242,47 +266,47 @@ 'eucjisx0213' : 'euc_jisx0213', # euc_jp codec + 'cseucpkdfmtjapanese' : 'euc_jp', 'eucjp' : 'euc_jp', 'ujis' : 'euc_jp', 'u_jis' : 'euc_jp', # euc_kr codec - 'euckr' : 'euc_kr', - 'korean' : 'euc_kr', - 'ksc5601' : 'euc_kr', - 'ks_c_5601' : 'euc_kr', - 'ks_c_5601_1987' : 'euc_kr', - 'ksx1001' : 'euc_kr', - 'ks_x_1001' : 'euc_kr', 'cseuckr' : 'euc_kr', + 'euckr' : 'euc_kr', # gb18030 codec + 'csgb18030' : 'gb18030', 'gb18030_2000' : 'gb18030', # gb2312 codec 'chinese' : 'gb2312', + 'csgb2312' : 'gb2312', 'csiso58gb231280' : 'gb2312', 'euc_cn' : 'gb2312', 'euccn' : 'gb2312', 'eucgb2312_cn' : 'gb2312', 'gb2312_1980' : 'gb2312', 'gb2312_80' : 'gb2312', + 'gb_2312_80' : 'gb2312', 'iso_ir_58' : 'gb2312', # gbk codec '936' : 'gbk', 'cp936' : 'gbk', + 'csgbk' : 'gbk', 'ms936' : 'gbk', + 'windows_936' : 'gbk', # hex_codec codec 'hex' : 'hex_codec', # hp_roman8 codec - 'roman8' : 'hp_roman8', - 'r8' : 'hp_roman8', - 'csHPRoman8' : 'hp_roman8', 'cp1051' : 'hp_roman8', + 'cshproman8' : 'hp_roman8', 'ibm1051' : 'hp_roman8', + 'r8' : 'hp_roman8', + 'roman8' : 'hp_roman8', # hz codec 'hzgb' : 'hz', @@ -299,6 +323,7 @@ 'iso_2022_jp_1' : 'iso2022_jp_1', # iso2022_jp_2 codec + 'csiso2022jp2' : 'iso2022_jp_2', 'iso2022jp_2' : 'iso2022_jp_2', 'iso_2022_jp_2' : 'iso2022_jp_2', @@ -326,6 +351,7 @@ 'iso_ir_157' : 'iso8859_10', 'l6' : 'iso8859_10', 'latin6' : 'iso8859_10', + 'latin_6' : 'iso8859_10', # iso8859_11 codec 'thai' : 'iso8859_11', @@ -333,29 +359,37 @@ 'iso_8859_11_2001' : 'iso8859_11', # iso8859_13 codec + 'csiso885913' : 'iso8859_13', 'iso_8859_13' : 'iso8859_13', 'l7' : 'iso8859_13', 'latin7' : 'iso8859_13', + 'latin_7' : 'iso8859_13', # iso8859_14 codec + 'csiso885914' : 'iso8859_14', 'iso_8859_14' : 'iso8859_14', 'iso_8859_14_1998' : 'iso8859_14', 'iso_celtic' : 'iso8859_14', 'iso_ir_199' : 'iso8859_14', 'l8' : 'iso8859_14', 'latin8' : 'iso8859_14', + 'latin_8' : 'iso8859_14', # iso8859_15 codec + 'csiso885915' : 'iso8859_15', 'iso_8859_15' : 'iso8859_15', 'l9' : 'iso8859_15', 'latin9' : 'iso8859_15', + 'latin_9' : 'iso8859_15', # iso8859_16 codec + 'csiso885916' : 'iso8859_16', 'iso_8859_16' : 'iso8859_16', 'iso_8859_16_2001' : 'iso8859_16', 'iso_ir_226' : 'iso8859_16', 'l10' : 'iso8859_16', 'latin10' : 'iso8859_16', + 'latin_10' : 'iso8859_16', # iso8859_2 codec 'csisolatin2' : 'iso8859_2', @@ -364,6 +398,7 @@ 'iso_ir_101' : 'iso8859_2', 'l2' : 'iso8859_2', 'latin2' : 'iso8859_2', + 'latin_2' : 'iso8859_2', # iso8859_3 codec 'csisolatin3' : 'iso8859_3', @@ -372,6 +407,7 @@ 'iso_ir_109' : 'iso8859_3', 'l3' : 'iso8859_3', 'latin3' : 'iso8859_3', + 'latin_3' : 'iso8859_3', # iso8859_4 codec 'csisolatin4' : 'iso8859_4', @@ -380,6 +416,7 @@ 'iso_ir_110' : 'iso8859_4', 'l4' : 'iso8859_4', 'latin4' : 'iso8859_4', + 'latin_4' : 'iso8859_4', # iso8859_5 codec 'csisolatincyrillic' : 'iso8859_5', @@ -408,6 +445,8 @@ 'iso_ir_126' : 'iso8859_7', # iso8859_8 codec + 'csiso88598e' : 'iso8859_8', + 'csiso88598i' : 'iso8859_8', 'csisolatinhebrew' : 'iso8859_8', 'hebrew' : 'iso8859_8', 'iso_8859_8' : 'iso8859_8', @@ -423,6 +462,7 @@ 'iso_ir_148' : 'iso8859_9', 'l5' : 'iso8859_9', 'latin5' : 'iso8859_9', + 'latin_5' : 'iso8859_9', # johab codec 'cp1361' : 'johab', @@ -431,7 +471,11 @@ # koi8_r codec 'cskoi8r' : 'koi8_r', + # koi8_u codec + 'cskoi8u' : 'koi8_u', + # kz1048 codec + 'cskz1048' : 'kz1048', 'kz_1048' : 'kz1048', 'rk1048' : 'kz1048', 'strk1048_2002' : 'kz1048', @@ -471,7 +515,9 @@ 'maclatin2' : 'mac_latin2', # mac_roman codec + 'csmacintosh' : 'mac_roman', 'macintosh' : 'mac_roman', + 'mac' : 'mac_roman', 'macroman' : 'mac_roman', # mac_turkish codec @@ -512,6 +558,7 @@ 's_jisx0213' : 'shift_jisx0213', # tis_620 codec + 'cstis620' : 'tis_620', 'tis620' : 'tis_620', 'tis_620_0' : 'tis_620', 'tis_620_2529_0' : 'tis_620', @@ -519,33 +566,42 @@ 'iso_ir_166' : 'tis_620', # utf_16 codec + 'csutf16' : 'utf_16', 'u16' : 'utf_16', 'utf16' : 'utf_16', # utf_16_be codec + 'csutf16be' : 'utf_16_be', 'unicodebigunmarked' : 'utf_16_be', 'utf_16be' : 'utf_16_be', # utf_16_le codec + 'csutf16le' : 'utf_16_le', 'unicodelittleunmarked' : 'utf_16_le', 'utf_16le' : 'utf_16_le', # utf_32 codec + 'csutf32' : 'utf_32', 'u32' : 'utf_32', 'utf32' : 'utf_32', # utf_32_be codec + 'csutf32be' : 'utf_32_be', 'utf_32be' : 'utf_32_be', # utf_32_le codec + 'csutf32le' : 'utf_32_le', 'utf_32le' : 'utf_32_le', # utf_7 codec + 'csunicode11utf7' : 'utf_7', + 'csutf7' : 'utf_7', 'u7' : 'utf_7', 'utf7' : 'utf_7', 'unicode_1_1_utf_7' : 'utf_7', # utf_8 codec + 'csutf8' : 'utf_8', 'u8' : 'utf_8', 'utf' : 'utf_8', 'utf8' : 'utf_8', @@ -553,6 +609,9 @@ 'utf8_ucs4' : 'utf_8', 'cp65001' : 'utf_8', + # utf_8_sig codec + 'utf8_sig' : 'utf_8_sig', + # uu_codec codec 'uu' : 'uu_codec', diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 9005f1ef865..80ce896a780 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -45,9 +45,6 @@ __author__ = "Vinay Sajip " __status__ = "production" -# The following module attributes are no longer updated. -__version__ = "0.5.1.2" -__date__ = "07 February 2010" #--------------------------------------------------------------------------- # Miscellaneous module data @@ -625,6 +622,9 @@ def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *, self._fmt = self._style._fmt self.datefmt = datefmt + def __repr__(self): + return '<%s (%s)>' % (self.__class__.__name__, self._fmt) + default_time_format = '%Y-%m-%d %H:%M:%S' default_msec_format = '%s,%03d' @@ -797,6 +797,9 @@ def __init__(self, name=''): self.name = name self.nlen = len(name) + def __repr__(self): + return '<%s (%s)>' % (self.__class__.__name__, self.name) + def filter(self, record): """ Determine if the specified record is to be logged. @@ -1694,7 +1697,11 @@ def removeHandler(self, hdlr): """ with _lock: if hdlr in self.handlers: - self.handlers.remove(hdlr) + # Replace the list instead of mutating it in place, so that + # callHandlers() can iterate it without a lock (gh-79366). + handlers = self.handlers.copy() + handlers.remove(hdlr) + self.handlers = handlers def hasHandlers(self): """ @@ -2047,6 +2054,15 @@ def basicConfig(**kwargs): created FileHandler, causing it to be used when the file is opened in text mode. If not specified, the default value is `backslashreplace`. + formatter If specified, set this formatter instance for all involved + handlers. + If not specified, the default is to create and use an instance of + `logging.Formatter` based on arguments 'format', 'datefmt' and + 'style'. + When 'formatter' is specified together with any of the three + arguments 'format', 'datefmt' and 'style', a `ValueError` + is raised to signal that these arguments would lose meaning + otherwise. Note that you could specify a stream created using open(filename, mode) rather than passing the filename and mode in. However, it should be @@ -2069,6 +2085,9 @@ def basicConfig(**kwargs): .. versionchanged:: 3.9 Added the ``encoding`` and ``errors`` parameters. + + .. versionchanged:: 3.15 + Added the ``formatter`` parameter. """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads @@ -2104,13 +2123,19 @@ def basicConfig(**kwargs): stream = kwargs.pop("stream", None) h = StreamHandler(stream) handlers = [h] - dfs = kwargs.pop("datefmt", None) - style = kwargs.pop("style", '%') - if style not in _STYLES: - raise ValueError('Style must be one of: %s' % ','.join( - _STYLES.keys())) - fs = kwargs.pop("format", _STYLES[style][1]) - fmt = Formatter(fs, dfs, style) + fmt = kwargs.pop("formatter", None) + if fmt is None: + dfs = kwargs.pop("datefmt", None) + style = kwargs.pop("style", '%') + if style not in _STYLES: + raise ValueError('Style must be one of: %s' % ','.join( + _STYLES.keys())) + fs = kwargs.pop("format", _STYLES[style][1]) + fmt = Formatter(fs, dfs, style) + else: + for forbidden_key in ("datefmt", "format", "style"): + if forbidden_key in kwargs: + raise ValueError(f"{forbidden_key!r} should not be specified together with 'formatter'") for h in handlers: if h.formatter is None: h.setFormatter(fmt) @@ -2324,3 +2349,16 @@ def captureWarnings(capture): if _warnings_showwarning is not None: warnings.showwarning = _warnings_showwarning _warnings_showwarning = None + + +def __getattr__(name): + if name in ("__version__", "__date__"): + from warnings import _deprecated + + _deprecated(name, remove=(3, 20)) + return { # Do not change + "__version__": "0.5.1.2", + "__date__": "07 February 2010", + }[name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 3d9aa00fa52..e0524eb5e3d 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -32,6 +32,7 @@ import os import queue import re +import socket import struct import threading import traceback @@ -1022,6 +1023,15 @@ class ConfigSocketReceiver(ThreadingTCPServer): def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): + # The host can have no IPv4 address, for example if "localhost" + # is only aliased to ::1. Leave resolution errors to the server. + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError: + pass + else: + if not any(info[0] == socket.AF_INET for info in infos): + self.address_family = infos[0][0] ThreadingTCPServer.__init__(self, (host, port), handler) with logging._lock: self.abort = 0 @@ -1053,9 +1063,14 @@ def __init__(self, rcvr, hdlr, port, verify): self.ready = threading.Event() def run(self): - server = self.rcvr(port=self.port, handler=self.hdlr, - ready=self.ready, - verify=self.verify) + try: + server = self.rcvr(port=self.port, handler=self.hdlr, + ready=self.ready, + verify=self.verify) + except BaseException: + # Do not leave the caller waiting for ready forever. + self.ready.set() + raise if self.port == 0: self.port = server.server_address[1] self.ready.set() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 4a07258f8d6..575f2babbc4 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1129,7 +1129,7 @@ class NTEventLogHandler(logging.Handler): """ A handler class which sends events to the NT Event Log. Adds a registry entry for the specified application name. If no dllname is - provided, win32service.pyd (which contains some basic message + provided and pywin32 installed, win32service.pyd (which contains some basic message placeholders) is used. Note that use of these placeholders will make your event logs big, as the entire message source is held in the log. If you want slimmer logs, you have to pass in the name of your own DLL @@ -1137,38 +1137,46 @@ class NTEventLogHandler(logging.Handler): """ def __init__(self, appname, dllname=None, logtype="Application"): logging.Handler.__init__(self) - try: - import win32evtlogutil, win32evtlog - self.appname = appname - self._welu = win32evtlogutil - if not dllname: - dllname = os.path.split(self._welu.__file__) + import _winapi + self._winapi = _winapi + self.appname = appname + if not dllname: + # backward compatibility + try: + import win32evtlogutil + dllname = os.path.split(win32evtlogutil.__file__) dllname = os.path.split(dllname[0]) dllname = os.path.join(dllname[0], r'win32service.pyd') - self.dllname = dllname - self.logtype = logtype - # Administrative privileges are required to add a source to the registry. - # This may not be available for a user that just wants to add to an - # existing source - handle this specific case. - try: - self._welu.AddSourceToRegistry(appname, dllname, logtype) - except Exception as e: - # This will probably be a pywintypes.error. Only raise if it's not - # an "access denied" error, else let it pass - if getattr(e, 'winerror', None) != 5: # not access denied - raise - self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE - self.typemap = { - logging.DEBUG : win32evtlog.EVENTLOG_INFORMATION_TYPE, - logging.INFO : win32evtlog.EVENTLOG_INFORMATION_TYPE, - logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE, - logging.ERROR : win32evtlog.EVENTLOG_ERROR_TYPE, - logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE, - } - except ImportError: - print("The Python Win32 extensions for NT (service, event "\ - "logging) appear not to be available.") - self._welu = None + except ImportError: + pass + self.dllname = dllname + self.logtype = logtype + # Administrative privileges are required to add a source to the registry. + # This may not be available for a user that just wants to add to an + # existing source - handle this specific case. + try: + self._add_source_to_registry(appname, dllname, logtype) + except PermissionError: + pass + self.deftype = _winapi.EVENTLOG_ERROR_TYPE + self.typemap = { + logging.DEBUG: _winapi.EVENTLOG_INFORMATION_TYPE, + logging.INFO: _winapi.EVENTLOG_INFORMATION_TYPE, + logging.WARNING: _winapi.EVENTLOG_WARNING_TYPE, + logging.ERROR: _winapi.EVENTLOG_ERROR_TYPE, + logging.CRITICAL: _winapi.EVENTLOG_ERROR_TYPE, + } + + @staticmethod + def _add_source_to_registry(appname, dllname, logtype): + import winreg + + key_path = f"SYSTEM\\CurrentControlSet\\Services\\EventLog\\{logtype}\\{appname}" + + with winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, key_path) as key: + if dllname: + winreg.SetValueEx(key, "EventMessageFile", 0, winreg.REG_EXPAND_SZ, dllname) + winreg.SetValueEx(key, "TypesSupported", 0, winreg.REG_DWORD, 7) # All types are supported def getMessageID(self, record): """ @@ -1209,15 +1217,20 @@ def emit(self, record): Determine the message ID, event category and event type. Then log the message in the NT event log. """ - if self._welu: + try: + id = self.getMessageID(record) + cat = self.getEventCategory(record) + type = self.getEventType(record) + msg = self.format(record) + + # Get a handle to the event log + handle = self._winapi.RegisterEventSource(None, self.appname) try: - id = self.getMessageID(record) - cat = self.getEventCategory(record) - type = self.getEventType(record) - msg = self.format(record) - self._welu.ReportEvent(self.appname, id, cat, type, [msg]) - except Exception: - self.handleError(record) + self._winapi.ReportEvent(handle, type, cat, id, msg) + finally: + self._winapi.DeregisterEventSource(handle) + except Exception: + self.handleError(record) def close(self): """ diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index c0097351b26..04c5baf7b52 100644 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -16,8 +16,6 @@ # XXX The API needs to undergo changes however; the current code is too # XXX script-like. This will be addressed later. -__version__ = "6" - import os import sys import tokenize @@ -49,7 +47,7 @@ def main(): if o == '-v': verbose = verbose + 1 if not args: - errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...") + errprint(f"Usage: {sys.executable} -m tabnanny [-v] file_or_directory ...") for arg in args: check(arg) @@ -334,5 +332,14 @@ def _process_tokens(tokens): raise NannyNag(start[0], msg, line) +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "6" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + if __name__ == '__main__': main() diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py index 5f6f14afd74..c046f39fbff 100644 --- a/Lib/test/support/warnings_helper.py +++ b/Lib/test/support/warnings_helper.py @@ -1,11 +1,11 @@ import contextlib -import functools import importlib import re import sys import warnings + def import_deprecated(name): """Import *name* while suppressing DeprecationWarning.""" with warnings.catch_warnings(): @@ -42,20 +42,32 @@ def check_syntax_warning(testcase, statement, errtext='', testcase.assertEqual(warns, []) -def ignore_warnings(*, category): +@contextlib.contextmanager +def ignore_warnings(*, category, message=''): """Decorator to suppress warnings. - Use of context managers to hide warnings make diffs - more noisy and tools like 'git blame' less useful. + Can also be used as a context manager. This is not preferred, + because it makes diffs more noisy and tools like 'git blame' less useful. + But, it's useful for async functions. """ - def decorator(test): - @functools.wraps(test) - def wrapper(self, *args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=category) - return test(self, *args, **kwargs) - return wrapper - return decorator + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=category, message=message) + yield + + +@contextlib.contextmanager +def ignore_fork_in_thread_deprecation_warnings(): + """Suppress deprecation warnings related to forking in multi-threaded code. + + See gh-135427 + + Can be used as decorator (preferred) or context manager. + """ + with ignore_warnings( + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning, + ): + yield class WarningsRecorder(object): diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py index dceb54f15e4..3115941f870 100644 --- a/Lib/test/test_email/test_contentmanager.py +++ b/Lib/test/test_email/test_contentmanager.py @@ -342,6 +342,66 @@ def test_set_text_charset_latin_1(self): self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) self.assertEqual(m.get_content(), content) + def test_set_text_charset_cp949(self): + m = self._make_message() + content = "\ud55c\uad6d\uc5b4\n\uac02\n" + raw_data_manager.set_content(m, content, charset='cp949') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="ks_c_5601-1987" + Content-Transfer-Encoding: base64 + + x9Gxub7uCoFBCg== + """)) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="ks_c_5601-1987" + Content-Transfer-Encoding: 8bit + + \ud55c\uad6d\uc5b4 + \uac02 + """).encode('ks_c_5601-1987')) + self.assertEqual(m.get_payload(decode=True), content.encode('ks_c_5601-1987')) + self.assertEqual(m.get_content(), content) + + def test_set_text_charset_shift_jis(self): + m = self._make_message() + content = "\u65e5\u672c\u8a9e\n" + raw_data_manager.set_content(m, content, charset='shift_jis') + self.assertEqual(m['Content-Type'], 'text/plain; charset="iso-2022-jp"') + self.assertEqual(m.get_payload(decode=True), content.encode('iso-2022-jp')) + self.assertEqual(m.get_content(), content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \x1b$BF|K\\8l\x1b(B + """)) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \u65e5\u672c\u8a9e + """).encode('iso-2022-jp')) + + def test_set_text_charset_euc_jp(self): + m = self._make_message() + content = "\u65e5\u672c\u8a9e\n" + raw_data_manager.set_content(m, content, charset='euc-jp') + self.assertEqual(m['Content-Type'], 'text/plain; charset="iso-2022-jp"') + self.assertEqual(m.get_payload(decode=True), content.encode('iso-2022-jp')) + self.assertEqual(m.get_content(), content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \x1b$BF|K\\8l\x1b(B + """)) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \u65e5\u672c\u8a9e + """).encode('iso-2022-jp')) + def test_set_text_plain_long_line_heuristics(self): m = self._make_message() content = ("Simple but long message that is over 78 characters" diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 69364b0575d..9effc375851 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -62,7 +62,7 @@ import weakref from http.server import HTTPServer, BaseHTTPRequestHandler -from unittest.mock import patch +from unittest.mock import call, Mock, patch from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) @@ -404,6 +404,20 @@ def test_empty_filter(self): r = logging.makeLogRecord({'name': 'spam.eggs'}) self.assertTrue(f.filter(r)) + def test_filter_repr(self): + f = logging.Filter('myapp') + self.assertEqual(repr(f), '') + + def test_filter_repr_empty(self): + f = logging.Filter() + self.assertEqual(repr(f), '') + + def test_filter_repr_subclass(self): + class MyFilter(logging.Filter): + pass + f = MyFilter('myapp') + self.assertEqual(repr(f), '') + # # First, we define our levels. There can be as many as you want - the only # limitations are that they should be integers, the lowest should be > 0 and @@ -731,6 +745,7 @@ def remove_loop(fname, tries): # based on os.fork existing because that is what users and this test use. # This helps ensure that when fork exists (the important concept) that the # register_at_fork mechanism is also present and used. + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork @@ -799,6 +814,23 @@ def lock_holder_thread_fn(): support.wait_process(pid, exitcode=0) + def test_remove_handler_while_emitting(self): + # Removing a handler while callHandlers() iterates over the handlers + # should not cause the following handlers to be skipped (gh-79366). + logger = logging.Logger('test_remove_handler_while_emitting') + calls = [] + class RemovingHandler(logging.Handler): + def emit(self, record): + calls.append('removing') + logger.removeHandler(self) + class CountingHandler(logging.Handler): + def emit(self, record): + calls.append('counting') + logger.addHandler(RemovingHandler()) + logger.addHandler(CountingHandler()) + logger.error('spam') + self.assertEqual(calls, ['removing', 'counting']) + class BadStream(object): def write(self, data): @@ -3634,14 +3666,14 @@ def setup_via_listener(self, text, verify=None): # Ask for a randomly assigned port (by using port 0) t = logging.config.listen(0, verify) t.start() - t.ready.wait() + self.assertTrue(t.ready.wait(support.LONG_TIMEOUT), + msg='the listener did not start') # Now get the port allocated port = t.port t.ready.clear() try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2.0) - sock.connect(('localhost', port)) + # The server can listen on IPv6, so do not force a family. + sock = socket.create_connection(('localhost', port), timeout=2.0) slen = struct.pack('>L', len(text)) s = slen + text @@ -3756,6 +3788,18 @@ def verify_reverse(stuff): ('ERROR', '2'), ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + @support.requires_working_socket() + def test_listen_server_error(self): + # The "ready" event should be set even if the server fails to start. + t = logging.config.listen(-1) + t.daemon = True + with threading_helper.catch_threading_exception() as cm: + t.start() + self.assertTrue(t.ready.wait(support.SHORT_TIMEOUT), + msg='the listener did not report the failure') + threading_helper.join_thread(t) + self.assertIs(cm.exc_type, OverflowError) + def test_bad_format(self): self.assertRaises(ValueError, self.apply_config, self.bad_format) @@ -4046,6 +4090,7 @@ def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_man self._apply_simple_queue_listener_configuration(qspec) manager.assert_not_called() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork @support.requires_subprocess() @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" @@ -4068,6 +4113,7 @@ def test_config_reject_simple_queue_handler_multiprocessing_context(self): with self.assertRaises(ValueError): self._apply_simple_queue_listener_configuration(qspec) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork @support.requires_subprocess() @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" @@ -4108,6 +4154,7 @@ def _mpinit_issue121723(qspec, message_to_log): # log a message (this creates a record put in the queue) logging.getLogger().info(message_to_log) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): @@ -4911,6 +4958,20 @@ def test_relativeCreated_has_higher_precision(self): # After PR gh-102412, precision (places) increases from 3 to 7 self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) + def test_formatter_repr(self): + f = logging.Formatter('%(message)s') + self.assertEqual(repr(f), '') + + def test_formatter_repr_default(self): + f = logging.Formatter() + self.assertEqual(repr(f), '') + + def test_formatter_repr_subclass(self): + class MyFormatter(logging.Formatter): + pass + f = MyFormatter('%(message)s') + self.assertEqual(repr(f), '') + class TestBufferingFormatter(logging.BufferingFormatter): def formatHeader(self, records): @@ -5339,6 +5400,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork def test_multiprocessing(self): support.skip_if_broken_multiprocessing_synchronize() @@ -5574,12 +5636,19 @@ def test_incompatible(self): assertRaises = self.assertRaises handlers = [logging.StreamHandler()] stream = sys.stderr + formatter = logging.Formatter() assertRaises(ValueError, logging.basicConfig, filename='test.log', stream=stream) assertRaises(ValueError, logging.basicConfig, filename='test.log', handlers=handlers) assertRaises(ValueError, logging.basicConfig, stream=stream, handlers=handlers) + assertRaises(ValueError, logging.basicConfig, formatter=formatter, + format='%(message)s') + assertRaises(ValueError, logging.basicConfig, formatter=formatter, + datefmt='%H:%M:%S') + assertRaises(ValueError, logging.basicConfig, formatter=formatter, + style='%') # Issue 23207: test for invalid kwargs assertRaises(ValueError, logging.basicConfig, loglevel=logging.INFO) # Should pop both filename and filemode even if filename is None @@ -5714,6 +5783,20 @@ def dummy_handle_error(record): # didn't write anything due to the encoding error self.assertEqual(data, r'') + def test_formatter_given(self): + mock_formatter = Mock() + mock_handler = Mock(formatter=None) + with patch("logging.Formatter") as mock_formatter_init: + logging.basicConfig(formatter=mock_formatter, handlers=[mock_handler]) + self.assertEqual(mock_handler.setFormatter.call_args_list, [call(mock_formatter)]) + self.assertEqual(mock_formatter_init.call_count, 0) + + def test_formatter_not_given(self): + mock_handler = Mock(formatter=None) + with patch("logging.Formatter") as mock_formatter_init: + logging.basicConfig(handlers=[mock_handler]) + self.assertEqual(mock_formatter_init.call_count, 1) + @support.requires_working_socket() def test_log_taskName(self): async def log_record(): @@ -6276,11 +6359,12 @@ def setUp(self): self.rmfiles = [] def tearDown(self): - for fn in self.rmfiles: - os.unlink(fn) - if os.path.exists(self.fn): - os.unlink(self.fn) - BaseTest.tearDown(self) + try: + for fn in self.rmfiles: + os_helper.unlink(fn) + os_helper.unlink(self.fn) + finally: + BaseTest.tearDown(self) def assertLogFile(self, filename): "Assert a log file is there and register it for deletion" @@ -7223,8 +7307,8 @@ def test_compute_rollover(self, when=when, interval=interval, exp=exp): setattr(TimedRotatingFileHandlerTest, name, test_compute_rollover) -@unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') class NTEventLogHandlerTest(BaseTest): + @unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') def test_basic(self): logtype = 'Application' elh = win32evtlog.OpenEventLog(None, logtype) @@ -7258,6 +7342,17 @@ def test_basic(self): msg = 'Record not found in event log, went back %d records' % GO_BACK self.assertTrue(found, msg=msg) + @unittest.skipUnless(sys.platform == "win32", "Windows required for this test") + def test_without_pywin32(self): + h = logging.handlers.NTEventLogHandler('python_test') + self.addCleanup(h.close) + + # Verify that the handler uses _winapi module + self.assertIsNotNone(h._winapi, "_winapi module should be available") + + r = logging.makeLogRecord({'msg': 'Hello!'}) + h.emit(r) + class MiscTestCase(unittest.TestCase): def test__all__(self): @@ -7269,6 +7364,16 @@ def test__all__(self): support.check__all__(self, logging, not_exported=not_exported) +class TestModule(unittest.TestCase): + def test_deprecated__version__and__date__(self): + msg = "is deprecated and slated for removal in Python 3.20" + for attr in ("__version__", "__date__"): + with self.subTest(attr=attr): + with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + getattr(logging, attr) + self.assertEqual(cm.filename, __file__) + + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index 11b3f8ea016..85e3e76e900 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -3,19 +3,18 @@ Glossary: * errored : Whitespace related problems present in file. """ -from unittest import TestCase, mock + +from unittest import TestCase, main, mock import errno import os +import sys import tabnanny import tokenize import tempfile import textwrap -from test.support import (captured_stderr, captured_stdout, script_helper, - findfile) +from test.support import captured_stderr, captured_stdout, script_helper from test.support.os_helper import unlink -import unittest # XXX: RUSTPYTHON; importing to be able to skip tests - SOURCE_CODES = { "incomplete_expression": ( @@ -329,8 +328,7 @@ def test_with_error_free_file(self): def test_command_usage(self): """Should display usage on no arguments.""" - path = findfile('tabnanny.py') - stderr = f"Usage: {path} [-v] file_or_directory ..." + stderr = f"Usage: {sys.executable} -m tabnanny [-v] file_or_directory ..." self.validate_cmd(stderr=stderr, expect_failure=True) def test_quiet_flag(self): @@ -354,3 +352,17 @@ def test_double_verbose_mode(self): "offending line: '\\tprint(\"world\")'" ).strip() self.validate_cmd("-vv", path, stdout=stdout, partial=True) + + +class TestModule(TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(tabnanny, "__version__") + self.assertEqual(cm.filename, __file__) + + +if __name__ == "__main__": + main() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 5a8fccf59dc..aad4182f304 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -597,7 +597,7 @@ def __set_return_value(self, value): __return_value_doc = "The value to be returned when the mock is called." return_value = property(__get_return_value, __set_return_value, - __return_value_doc) + doc=__return_value_doc) @property @@ -701,7 +701,7 @@ def __getattr__(self, name): if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')) or name in _ATTRIB_DENY_LIST: raise AttributeError( f"{name!r} is not a valid assertion. Use a spec " - f"for the mock if {name!r} is meant to be an attribute.") + f"for the mock if {name!r} is meant to be an attribute") with NonCallableMock._lock: result = self._mock_children.get(name) @@ -937,7 +937,7 @@ def _call_matcher(self, _call): return _call def assert_not_called(self): - """assert that the mock was never called. + """Assert that the mock was never called. """ if self.call_count != 0: msg = ("Expected '%s' to not have been called. Called %s times.%s" @@ -947,7 +947,7 @@ def assert_not_called(self): raise AssertionError(msg) def assert_called(self): - """assert that the mock was called at least once + """Assert that the mock was called at least once. """ if self.call_count == 0: msg = ("Expected '%s' to have been called." % @@ -955,7 +955,7 @@ def assert_called(self): raise AssertionError(msg) def assert_called_once(self): - """assert that the mock was called only once. + """Assert that the mock was called only once. """ if not self.call_count == 1: msg = ("Expected '%s' to have been called once. Called %s times.%s" @@ -965,7 +965,7 @@ def assert_called_once(self): raise AssertionError(msg) def assert_called_with(self, /, *args, **kwargs): - """assert that the last call was made with the specified arguments. + """Assert that the last call was made with the specified arguments. Raises an AssertionError if the args and keyword args passed in are different to the last call to the mock.""" @@ -987,7 +987,7 @@ def _error_message(): def assert_called_once_with(self, /, *args, **kwargs): - """assert that the mock was called exactly once and that that call was + """Assert that the mock was called exactly once and that call was with the specified arguments.""" if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times.%s" @@ -999,7 +999,7 @@ def assert_called_once_with(self, /, *args, **kwargs): def assert_has_calls(self, calls, any_order=False): - """assert the mock has been called with the specified calls. + """Assert the mock has been called with the specified calls. The `mock_calls` list is checked for the calls. If `any_order` is False (the default) then the calls must be @@ -1044,7 +1044,7 @@ def assert_has_calls(self, calls, any_order=False): def assert_any_call(self, /, *args, **kwargs): - """assert the mock has been called with the specified arguments. + """Assert the mock has been called with the specified arguments. The assert passes if the mock has *ever* been called, unlike `assert_called_with` and `assert_called_once_with` that only pass if From 129330f95c5cf2fbd4c115e96bdbcb094f64618a Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 06:28:05 -0700 Subject: [PATCH 120/229] Report Windows signal wakeup socket errors (#9023) Capture Winsock failures with bounded atomic storage and report them through sys.unraisablehook on the interpreter thread. Honor warn_on_full_buffer for WSAEWOULDBLOCK. Include bounded Windows socket regressions without changing Unix or CRT wakeup writes. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/host_env/src/signal.rs | 33 ++++++-- crates/vm/src/signal.rs | 10 +++ crates/vm/src/stdlib/_signal.rs | 63 +++++++++++++-- extra_tests/snippets/stdlib_signal.py | 112 ++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 10 deletions(-) diff --git a/crates/host_env/src/signal.rs b/crates/host_env/src/signal.rs index 60b06d63e37..f5205d84190 100644 --- a/crates/host_env/src/signal.rs +++ b/crates/host_env/src/signal.rs @@ -395,13 +395,28 @@ pub fn wakeup_fd_is_socket(fd: libc::SOCKET) -> io::Result { Ok(false) } +#[cfg(windows)] +pub struct WakeupSendError(pub i32); + +#[cfg(windows)] +impl WakeupSendError { + pub fn is_would_block(&self) -> bool { + self.0 == windows_sys::Win32::Networking::WinSock::WSAEWOULDBLOCK + } + + // Only convert to an allocating error outside the OS signal handler. + pub fn into_io_error(self) -> io::Error { + io::Error::from_raw_os_error(self.0) + } +} + #[cfg(windows)] pub fn notify_signal( signum: i32, wakeup_fd: libc::SOCKET, wakeup_is_socket: bool, sigint_event: Option, -) { +) -> Option { if signum == libc::SIGINT && let Some(handle) = sigint_event { @@ -411,23 +426,31 @@ pub fn notify_signal( } if wakeup_fd == INVALID_SOCKET { - return; + return None; } let sigbyte = signum as u8; if wakeup_is_socket { - unsafe { - let _ = windows_sys::Win32::Networking::WinSock::send( + let result = unsafe { + windows_sys::Win32::Networking::WinSock::send( wakeup_fd, &sigbyte as *const u8 as *const _, 1, 0, - ); + ) + }; + if result < 0 { + Some(WakeupSendError(unsafe { + windows_sys::Win32::Networking::WinSock::WSAGetLastError() + })) + } else { + None } } else { unsafe { let _ = libc::write(wakeup_fd as _, &sigbyte as *const u8 as *const _, 1); } + None } } diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index eda531bbc42..8bef84d33da 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -103,6 +103,16 @@ fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { } let _guard = SignalHandlerGuard; + #[cfg(all(feature = "host_env", windows))] + if vm.is_main_thread() && vm.state.is_main_interpreter() { + // Report before a Python signal handler can raise out of the current + // error-reporting scope. + crate::stdlib::_signal::_signal::report_wakeup_send_errors(vm); + } else if crate::stdlib::_signal::_signal::wakeup_send_errors_pending() { + // A different interpreter/thread must leave the report for the main one. + set_triggered(); + } + let signal_handlers = vm .signal_handlers .get() diff --git a/crates/vm/src/stdlib/_signal.rs b/crates/vm/src/stdlib/_signal.rs index 1c2bc792539..348f5bc921d 100644 --- a/crates/vm/src/stdlib/_signal.rs +++ b/crates/vm/src/stdlib/_signal.rs @@ -73,6 +73,13 @@ pub(crate) mod _signal { } } + #[cfg(windows)] + static WARN_ON_FULL_BUFFER: atomic::AtomicBool = atomic::AtomicBool::new(true); + + // Bounded pending-call storage: signal handlers cannot allocate or lock. + #[cfg(windows)] + static WAKEUP_SEND_ERRORS: [atomic::AtomicI32; 32] = [const { atomic::AtomicI32::new(0) }; 32]; + #[cfg(any(unix, windows, target_os = "wasi"))] #[allow(unused_imports)] pub use host_signal::SIG_ERR; @@ -344,7 +351,7 @@ pub(crate) mod _signal { #[pyfunction] fn set_wakeup_fd(args: SetWakeupFdArgs, vm: &VirtualMachine) -> PyResult { - // TODO: implement warn_on_full_buffer + // TODO: implement warn_on_full_buffer for the remaining wakeup write paths. let _ = args.warn_on_full_buffer; let fd = cfg_select! { windows => args.fd.0, @@ -383,7 +390,10 @@ pub(crate) mod _signal { let old_fd = WAKEUP.swap(fd, Ordering::Relaxed); #[cfg(windows)] - WAKEUP_IS_SOCKET.store(is_socket, Ordering::Relaxed); + { + WARN_ON_FULL_BUFFER.store(args.warn_on_full_buffer, Ordering::Relaxed); + WAKEUP_IS_SOCKET.store(is_socket, Ordering::Relaxed); + } #[cfg(windows)] if old_fd == INVALID_WAKEUP { @@ -532,14 +542,57 @@ pub(crate) mod _signal { signal::TRIGGERS[signum as usize].store(true, Ordering::Relaxed); signal::set_triggered(); - host_signal::notify_signal( + #[cfg(not(windows))] + host_signal::notify_signal(signum, WAKEUP.load(Ordering::Relaxed)); + + #[cfg(windows)] + let error = host_signal::notify_signal( signum, WAKEUP.load(Ordering::Relaxed), - #[cfg(windows)] WAKEUP_IS_SOCKET.load(Ordering::Relaxed), - #[cfg(windows)] signal::get_sigint_event(), ); + #[cfg(windows)] + if let Some(error) = error + && (WARN_ON_FULL_BUFFER.load(Ordering::Relaxed) || !error.is_would_block()) + { + for pending in &WAKEUP_SEND_ERRORS { + if pending + .compare_exchange(0, error.0, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + // The VM may have handled the signal on another thread + // before the failed wakeup send completed. + signal::set_triggered(); + break; + } + } + } + } + + #[cfg(windows)] + pub(crate) fn wakeup_send_errors_pending() -> bool { + WAKEUP_SEND_ERRORS + .iter() + .any(|error| error.load(Ordering::Relaxed) != 0) + } + + #[cfg(windows)] + pub(crate) fn report_wakeup_send_errors(vm: &VirtualMachine) { + for pending in &WAKEUP_SEND_ERRORS { + let error = pending.swap(0, Ordering::Relaxed); + if error != 0 { + vm.run_unraisable( + host_signal::WakeupSendError(error) + .into_io_error() + .into_pyexception(vm), + Some( + "Exception ignored while trying to send to the signal wakeup fd".to_owned(), + ), + vm.ctx.none(), + ); + } + } } /// Reset wakeup fd after fork in child process. diff --git a/extra_tests/snippets/stdlib_signal.py b/extra_tests/snippets/stdlib_signal.py index 39f103157fa..c03bc70ab30 100644 --- a/extra_tests/snippets/stdlib_signal.py +++ b/extra_tests/snippets/stdlib_signal.py @@ -106,3 +106,115 @@ def arm_other(signum, frame): signal.signal(signal.SIGALRM, signal.SIG_DFL) signal.signal(signal.SIGUSR1, signal.SIG_DFL) signal.signal(signal.SIGUSR2, signal.SIG_DFL) + + +def check_windows_wakeup_reporting(): + if sys.platform != "win32": + return + + import errno + import socket + + def fill_socket(write): + # A nonblocking Windows send can briefly fail before TCP has filled + # the receiving socket. Require repeated one-byte failures to settle. + deadline = time.monotonic() + 5.0 + written = 0 + blocked = 0 + for _ in range(4096): + assert time.monotonic() < deadline, "socket fill timed out" + assert written < 16 * 1024 * 1024, "socket fill exceeded byte bound" + try: + written += write.send(b"x" * 65536) + except BlockingIOError: + try: + written += write.send(b"x") + except BlockingIOError: + blocked += 1 + if blocked == 4: + return + time.sleep(0.050) + continue + blocked = 0 + raise AssertionError("socket did not fill within the attempt bound") + + # Use numeric loopback addresses and bounded connection/receive calls. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.settimeout(5.0) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as write: + write.settimeout(5.0) + write.connect(listener.getsockname()) + read, _ = listener.accept() + with read: + read.settimeout(5.0) + read.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096) + write.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 4096) + write.setblocking(False) + old_handler = signal.getsignal(signal.SIGINT) + old_hook = sys.unraisablehook + old_fd = signal.set_wakeup_fd(-1) + errors = [] + delivered = [] + + def raise_wakeup_signal(): + delivered.clear() + # Reinstall for each raise: Windows CRT handlers can reset. + signal.signal( + signal.SIGINT, lambda signum, frame: delivered.append(signum) + ) + signal.raise_signal(signal.SIGINT) + assert delivered == [signal.SIGINT] + + try: + sys.unraisablehook = errors.append + signal.set_wakeup_fd(write.fileno()) + raise_wakeup_signal() + assert read.recv(1) == bytes([signal.SIGINT]) + assert not errors + + for options in ( + {}, + {"warn_on_full_buffer": True}, + {"warn_on_full_buffer": False}, + {}, + ): + fill_socket(write) + signal.set_wakeup_fd(write.fileno(), **options) + raise_wakeup_signal() + if options.get("warn_on_full_buffer", True): + assert len(errors) == 1, errors + error = errors.pop() + assert error.exc_type is BlockingIOError + assert error.exc_value.winerror == errno.WSAEWOULDBLOCK + assert error.exc_value.errno in ( + errno.EAGAIN, + errno.EWOULDBLOCK, + ) + assert error.object is None + assert error.exc_traceback is error.exc_value.__traceback__ + assert error.err_msg == ( + "Exception ignored while trying to send " + "to the signal wakeup fd" + ) + else: + assert not errors + + # False suppresses a full buffer, not other send failures. + signal.set_wakeup_fd(write.fileno(), warn_on_full_buffer=False) + write.close() + raise_wakeup_signal() + assert len(errors) == 1, errors + assert errors[0].exc_type is OSError + assert errors[0].exc_value.winerror == errno.WSAENOTSOCK + assert errors[0].err_msg == ( + "Exception ignored while trying to send to the signal wakeup fd" + ) + finally: + signal.set_wakeup_fd(old_fd) + signal.signal(signal.SIGINT, old_handler) + sys.unraisablehook = old_hook + + +check_windows_wakeup_reporting() From 85d5956dbbc17b359b103f9988276589b126a08a Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:29:24 +0300 Subject: [PATCH 121/229] Release unused capacity after short reads (#9028) Assisted-by: Codex:gpt-6 --- crates/stdlib/src/socket.rs | 6 +++++- crates/stdlib/src/ssl.rs | 3 ++- crates/vm/src/stdlib/_io.rs | 2 ++ crates/vm/src/stdlib/os.rs | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index db22e6a1069..e3745d1090c 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -1931,6 +1931,7 @@ mod _socket { .recv_with_flags(buffer.spare_capacity_mut(), flags) })?; unsafe { buffer.set_len(n) }; + buffer.shrink_to_fit(); Ok(buffer) } @@ -1984,6 +1985,7 @@ mod _socket { .recv_from_with_flags(buffer.spare_capacity_mut(), flags) })?; unsafe { buffer.set_len(n) }; + buffer.shrink_to_fit(); Ok(( buffer, get_addr_tuple(&addr, zelf.proto.load(Ordering::Relaxed), vm), @@ -2239,7 +2241,7 @@ mod _socket { let ancbufsize = ancbufsize as usize; let flags = flags.unwrap_or(0); - let msg = zelf + let mut msg = zelf .sock_op(vm, SockWaitKind::Read, || { let sock = zelf.sock_snapshot()?; let fd = unsafe { std::os::fd::BorrowedFd::borrow_raw(sock_fileno(&sock)) }; @@ -2247,6 +2249,8 @@ mod _socket { }) .map_err(|e| e.into_pyexception(vm))?; + msg.data.shrink_to_fit(); + // Build ancdata list let ancdata = Self::parse_ancillary_data(&msg.ancdata, vm); diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 74750cced43..3f18d93c37a 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -3198,7 +3198,7 @@ mod _ssl { } // Helper function to handle return value based on buffer presence - let return_data = |data: Vec, + let return_data = |mut data: Vec, buffer_arg: &OptionalArg, vm: &VirtualMachine| -> PyResult { @@ -3216,6 +3216,7 @@ mod _ssl { } OptionalArg::Missing => { // Return bytes object + data.shrink_to_fit(); Ok(vm.ctx.new_bytes(data).into()) } } diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index d20cb367999..9bf3b3cfb02 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -1439,6 +1439,7 @@ mod _io { match ($x, written > 0) { (Some(0), _) | (None, true) => { out.truncate(written); + out.shrink_to_fit(); return Ok(Some(out)); } (Some(r), _) => r, @@ -6250,6 +6251,7 @@ mod fileio { } }; bytes.truncate(n); + bytes.shrink_to_fit(); bytes } else { let mut bytes = vec![]; diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 7419c8f9260..d096157d092 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -403,6 +403,7 @@ pub(super) mod _os { match vm.allow_threads(|| crt_fd::read(fd, &mut buffer)) { Ok(n) => { buffer.truncate(n); + buffer.shrink_to_fit(); return Ok(vm.ctx.new_bytes(buffer)); } Err(e) if e.raw_os_error() == Some(libc::EINTR) => { From c3e1276229b9b29b0f830193707aba9b4e551acd Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:33:55 +0900 Subject: [PATCH 122/229] Release unused capacity after more short reads (#9089) Call shrink_to_fit on the result buffer in os.pread, os.getrandom, WindowsConsoleIO.read, _winapi.PeekNamedPipe, _winapi.ReadFile, _multiprocessing.recv and the openssl MemoryBIO.read. Assisted-by: Claude:claude-opus-5-5 Assisted-by: Grok:grok-4.7 --- crates/stdlib/src/multiprocessing.rs | 7 +++++-- crates/stdlib/src/openssl.rs | 1 + crates/vm/src/stdlib/_io.rs | 1 + crates/vm/src/stdlib/_winapi.rs | 6 ++++-- crates/vm/src/stdlib/posix.rs | 2 ++ 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/stdlib/src/multiprocessing.rs b/crates/stdlib/src/multiprocessing.rs index 5c335649509..c992f90f737 100644 --- a/crates/stdlib/src/multiprocessing.rs +++ b/crates/stdlib/src/multiprocessing.rs @@ -408,8 +408,11 @@ mod _multiprocessing { #[pyfunction] fn recv(socket: usize, size: usize, vm: &VirtualMachine) -> PyResult> { - host_multiprocessing::recv_socket(socket as host_multiprocessing::RawSocket, size) - .map_err(|_| vm.new_last_os_error()) + let mut buf = + host_multiprocessing::recv_socket(socket as host_multiprocessing::RawSocket, size) + .map_err(|_| vm.new_last_os_error())?; + buf.shrink_to_fit(); + Ok(buf) } #[pyfunction] diff --git a/crates/stdlib/src/openssl.rs b/crates/stdlib/src/openssl.rs index 5470f788462..bba64f702e1 100644 --- a/crates/stdlib/src/openssl.rs +++ b/crates/stdlib/src/openssl.rs @@ -3706,6 +3706,7 @@ mod _ssl { } buf.truncate(nbytes as usize); + buf.shrink_to_fit(); Ok(buf) } } diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 9bf3b3cfb02..6c1fa064a27 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -6949,6 +6949,7 @@ mod winconsoleio { } buf.truncate(read_len); + buf.shrink_to_fit(); Ok(vm.ctx.new_bytes(buf).into()) } diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs index c47ea682379..6f495299a99 100644 --- a/crates/vm/src/stdlib/_winapi.rs +++ b/crates/vm/src/stdlib/_winapi.rs @@ -768,7 +768,8 @@ mod _winapi { if size > 0 { let result = host_winapi::peek_named_pipe(handle.0, Some(size as u32)) .map_err(|e| e.to_pyexception(vm))?; - let buf = result.data.unwrap_or_default(); + let mut buf = result.data.unwrap_or_default(); + buf.shrink_to_fit(); let bytes: PyObjectRef = vm.ctx.new_bytes(buf).into(); Ok(vm .ctx @@ -886,9 +887,10 @@ mod _winapi { return Ok(result.into()); } - let result = vm + let mut result = vm .allow_threads(|| host_winapi::read_file(handle.0, size)) .map_err(|e| e.to_pyexception(vm))?; + result.data.shrink_to_fit(); Ok(vm .ctx .new_tuple(vec![ diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 388771acbc6..26371c87e0a 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1214,6 +1214,7 @@ pub mod module { match vm.allow_threads(|| rustpython_host_env::posix::pread(fd, &mut buffer, offset)) { Ok(n) => { buffer.truncate(n); + buffer.shrink_to_fit(); return Ok(vm.ctx.new_bytes(buffer)); } Err(err) if err.raw_os_error() == Some(libc::EINTR) => { @@ -2976,6 +2977,7 @@ pub mod module { .map_err(|_| vm.new_last_os_error())?; buf.set_len(len); } + buf.shrink_to_fit(); Ok(buf) } From 140761e80b9b8f8ab77255dc462cadaa6a2312c3 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 18:47:23 -0700 Subject: [PATCH 123/229] Accept dict subclasses as warning module globals (#9062) Preserve overridable loader lookup while reading module names from stored dictionary data. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/vm/src/stdlib/_warnings.rs | 4 +-- crates/vm/src/warn.rs | 2 +- extra_tests/snippets/stdlib_warnings.py | 47 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/stdlib/_warnings.rs b/crates/vm/src/stdlib/_warnings.rs index 35aca82d176..9387c16ebe3 100644 --- a/crates/vm/src/stdlib/_warnings.rs +++ b/crates/vm/src/stdlib/_warnings.rs @@ -21,7 +21,7 @@ pub fn warn( mod _warnings { use crate::{ AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyDictRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, + builtins::{PyDict, PyDictRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, convert::TryFromObject, function::OptionalArg, }; @@ -180,7 +180,7 @@ mod _warnings { let source_line = if let Some(mg) = args.module_globals.into_option() { if vm.is_none(&mg) { None - } else if !mg.class().is(vm.ctx.types.dict_type) { + } else if mg.downcast_ref::().is_none() { return Err(vm.new_type_error(format!( "module_globals must be a dict, not '{}'", mg.class().name() diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs index 3d8e7cb6fb1..14ae7199a9b 100644 --- a/crates/vm/src/warn.rs +++ b/crates/vm/src/warn.rs @@ -201,7 +201,7 @@ pub(crate) fn get_source_line( }; let module_name = if let Some(dict) = module_globals.downcast_ref::() { - match dict.get_item_opt(identifier!(vm, __name__), vm)? { + match dict.inner_getitem_opt(identifier!(vm, __name__), vm)? { Some(name) => name, None => return Ok(None), } diff --git a/extra_tests/snippets/stdlib_warnings.py b/extra_tests/snippets/stdlib_warnings.py index 73ccc512ea7..974899d8b9e 100644 --- a/extra_tests/snippets/stdlib_warnings.py +++ b/extra_tests/snippets/stdlib_warnings.py @@ -18,3 +18,50 @@ class CustomWarning(UserWarning): assert len(caught) == 1 assert caught[0].message is message assert caught[0].category is CustomWarning + + +from importlib.machinery import ModuleSpec + +events = [] + + +class WarningLoader: + def get_source(self, name): + events.append(("source", name)) + return "source line\n" + + +class WarningGlobals(dict): + def get(self, key, default=None): + events.append(("get", key)) + return super().get(key, default) + + def __getitem__(self, key): + raise AssertionError("module name must use stored dict lookup") + + __missing__ = __getitem__ + + +loader = WarningLoader() +module_globals = WarningGlobals( + __name__="stored_name", + __loader__=loader, + __spec__=ModuleSpec("stored_name", loader), +) +for has_name in (True, False): + events.clear() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + warnings.warn_explicit( + "subclass globals", + UserWarning, + "fixed_ascii.py", + 1, + module_globals=module_globals, + ) + assert len(caught) == 1 + expected = [("get", "__loader__"), ("get", "__spec__")] + if has_name: + expected.append(("source", "stored_name")) + del module_globals["__name__"] + assert events == expected From 85edb52b6532c4f028871766fffda9cf8d5b0407 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 18:50:51 -0700 Subject: [PATCH 124/229] Update enum from CPython 3.15.0rc2 (#8942) Sync enum and its regression tests from CPython v3.15.0rc2 (435c9e5a798c99653e3ab64ce29baed0e4f3dfee). Preserve the two existing RustPython expected-failure decorators. Keep this first migration slice limited to enum without changing the declared Python target or native runtime. AI assistance: Codex/OpenAI reconstructed the upstream sync, reviewed the diff, and ran validation. The runtime did not expose the exact model version. Assisted-by: Codex:model-version-unavailable Co-authored-by: youknowdot <336763669+youknowdot@users.noreply.github.com> --- Lib/enum.py | 43 ++++++++++++++++++++++++++++++++------- Lib/test/test_enum.py | 47 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 87b2f2ab7eb..f536a3eae2b 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -10,14 +10,15 @@ 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', 'global_flag_repr', 'global_enum_repr', 'global_str', 'global_enum', 'EnumCheck', 'CONTINUOUS', 'NAMED_FLAGS', 'UNIQUE', - 'pickle_by_global_name', 'pickle_by_enum_name', + 'pickle_by_global_name', 'pickle_by_enum_name', 'show_flag_values', + 'bin', ] # Dummy value for Enum and Flag as there are explicit checks for them # before they have been created. # This is also why there are checks in EnumType like `if Enum is not None` -Enum = Flag = EJECT = _stdlib_enums = ReprEnum = None +Enum = Flag = EJECT = ReprEnum = None class nonmember(object): """ @@ -536,7 +537,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) # - # Flag structures (will be removed if final class is not a Flag + # Flag structures (will be removed if final class is not a Flag) classdict['_boundary_'] = ( boundary or getattr(first_enum, '_boundary_', None) @@ -545,6 +546,29 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_singles_mask_'] = 0 classdict['_all_bits_'] = 0 classdict['_inverted_'] = None + # check for negative flag values and invert if found (using _proto_members) + if Flag is not None and bases and issubclass(bases[-1], Flag): + bits = 0 + inverted = [] + for n in member_names: + p = classdict[n] + if isinstance(p.value, int): + if p.value < 0: + inverted.append(p) + else: + bits |= p.value + elif p.value is None: + pass + elif isinstance(p.value, tuple) and p.value and isinstance(p.value[0], int): + if p.value[0] < 0: + inverted.append(p) + else: + bits |= p.value[0] + for p in inverted: + if isinstance(p.value, int): + p.value = bits & p.value + else: + p.value = (bits & p.value[0], ) + p.value[1:] try: classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) @@ -752,12 +776,16 @@ def __delattr__(cls, attr): super().__delattr__(attr) def __dir__(cls): + if issubclass(cls, Flag): + members = list(cls._member_map_.keys()) + else: + members = cls._member_names_ interesting = set([ '__class__', '__contains__', '__doc__', '__getitem__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', ] - + cls._member_names_ + + members ) if cls._new_member_ is not object.__new__: interesting.add('__new__') @@ -1489,7 +1517,10 @@ def _missing_(cls, value): ) if value < 0: neg_value = value - value = all_bits + 1 + value + if cls._boundary_ in (EJECT, KEEP): + value = all_bits + 1 + value + else: + value = singles_mask & value # get members and unknown unknown = value & ~flag_mask aliases = value & ~singles_mask @@ -2165,5 +2196,3 @@ def _old_convert_(etype, name, module, filter, source=None, *, boundary=None): members.sort(key=lambda t: t[0]) cls = etype(name, members, module=module, boundary=boundary or KEEP) return cls - -_stdlib_enums = IntEnum, StrEnum, IntFlag diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index dc7628945d3..7bc8b4c6ede 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1002,12 +1002,18 @@ class OpenAB(self.enum_type): self.assertIs(~(A|B), OpenAB(252)) self.assertIs(~AB_MASK, OpenAB(0)) self.assertIs(~OpenAB(0), AB_MASK) + self.assertIs(OpenAB(~4), OpenAB(251)) else: self.assertIs(~A, B) self.assertIs(~B, A) + self.assertIs(OpenAB(~1), B) + self.assertIs(OpenAB(~2), A) self.assertIs(~(A|B), OpenAB(0)) self.assertIs(~AB_MASK, OpenAB(0)) self.assertIs(~OpenAB(0), (A|B)) + self.assertIs(OpenAB(~3), OpenAB(0)) + self.assertIs(OpenAB(~4), OpenAB(3)) + self.assertIs(OpenAB(~33), B) # class OpenXYZ(self.enum_type): X = 4 @@ -1031,6 +1037,9 @@ class OpenXYZ(self.enum_type): self.assertIs(~X, Y|Z) self.assertIs(~Y, X|Z) self.assertIs(~Z, X|Y) + self.assertIs(OpenXYZ(~4), Y|Z) + self.assertIs(OpenXYZ(~2), X|Z) + self.assertIs(OpenXYZ(~1), X|Y) self.assertIs(~(X|Y), Z) self.assertIs(~(X|Z), Y) self.assertIs(~(Y|Z), X) @@ -1038,6 +1047,28 @@ class OpenXYZ(self.enum_type): self.assertIs(~XYZ_MASK, OpenXYZ(0)) self.assertTrue(~OpenXYZ(0), (X|Y|Z)) + def test_assigned_negative_value(self): + class X(self.enum_type): + A = auto() + B = auto() + C = A | B + D = ~A + self.assertEqual(list(X), [X.A, X.B]) + self.assertIs(~X.A, X.B) + self.assertIs(X.D, X.B) + self.assertEqual(X.D.value, 2) + # + class Y(self.enum_type): + A = auto() + B = auto() + C = A | B + D = ~A + E = auto() + self.assertEqual(list(Y), [Y.A, Y.B, Y.E]) + self.assertIs(~Y.A, Y.B|Y.E) + self.assertIs(Y.D, Y.B|Y.E) + self.assertEqual(Y.D.value, 6) + class TestPlainEnumClass(_EnumTests, _PlainOutputTests, unittest.TestCase): enum_type = Enum @@ -3681,6 +3712,8 @@ class SkipFlag(enum.Flag): C = 4 | B # self.assertTrue(SkipFlag.C in (SkipFlag.A|SkipFlag.C)) + self.assertTrue(SkipFlag.B in SkipFlag.C) + self.assertIs(SkipFlag(~1), SkipFlag.B) self.assertRaisesRegex(ValueError, 'SkipFlag.. invalid value 42', SkipFlag, 42) # class SkipIntFlag(enum.IntFlag): @@ -3689,6 +3722,8 @@ class SkipIntFlag(enum.IntFlag): C = 4 | B # self.assertTrue(SkipIntFlag.C in (SkipIntFlag.A|SkipIntFlag.C)) + self.assertTrue(SkipIntFlag.B in SkipIntFlag.C) + self.assertIs(SkipIntFlag(~1), SkipIntFlag.B|SkipIntFlag.C) self.assertEqual(SkipIntFlag(42).value, 42) # class MethodHint(Flag): @@ -4728,6 +4763,8 @@ class Color(Flag): BLUE = 4 WHITE = -1 # no error means success + self.assertEqual(list(Color.WHITE), [Color.RED, Color.GREEN, Color.BLUE]) + self.assertEqual(Color.WHITE.value, 7) class TestInternals(unittest.TestCase): @@ -4851,7 +4888,7 @@ class Color(Enum): def _generate_next_value_(name, start, count, last): return name - def test_auto_order_wierd(self): + def test_auto_order_weird(self): weird_auto = auto() weird_auto.value = 'pathological case' class Color(Enum): @@ -5288,7 +5325,7 @@ def __new__(cls, value, label): class MiscTestCase(unittest.TestCase): def test__all__(self): - support.check__all__(self, enum, not_exported={'bin', 'show_flag_values'}) + support.check__all__(self, enum) @cpython_only def test_lazy_import(self): @@ -5493,12 +5530,16 @@ def test_enum_dict_standalone(self): # helpers def enum_dir(cls): + if issubclass(cls, Flag): + members = list(cls._member_map_.keys()) + else: + members = cls._member_names_ interesting = set([ '__class__', '__contains__', '__doc__', '__getitem__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', ] - + cls._member_names_ + + members ) if cls._new_member_ is not object.__new__: interesting.add('__new__') From 8b66b7c48f03b4960478b7e130935641bd30db12 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 22:11:37 -0700 Subject: [PATCH 125/229] Unify PyObjectRef ownership with extensible PyRef views (#9080) Represent PyObjectRef through PyRef while preserving typed ownership, payload projection, and drop behavior. Use the sealed target selector for borrowed set/frozenset union views and typed tuple elements. Keep tuple storage erased behind invariant typed slice views, with checked conversions and reference-count ownership tests. Preserve exact tuple identity for normalized nonempty full slices. Integrate the ownership contract into the public object module documentation and retain focused Rust ownership/layout tests without modifying bundled Lib sources. Spell the set/frozenset union owner directly as PyRef without a second owned alias; retain the sealed internal view registration. Assisted-by: OpenAI Codex:model identifier unavailable --- Cargo.lock | 1 + crates/capi/Cargo.toml | 1 + crates/capi/src/pyerrors.rs | 1 - crates/capi/src/util.rs | 22 +- crates/stdlib/src/_sqlite3.rs | 1 - crates/vm/src/builtins/descriptor.rs | 6 - crates/vm/src/builtins/frame_locals_proxy.rs | 4 +- crates/vm/src/builtins/set.rs | 365 +++++++++++++---- crates/vm/src/builtins/tuple.rs | 135 ++++--- crates/vm/src/object/core.rs | 395 +++++++++++++------ crates/vm/src/object/ext.rs | 8 +- crates/vm/src/object/mod.rs | 75 ++++ 12 files changed, 762 insertions(+), 252 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf5e0f9c0cd..f541d0c3eeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3725,6 +3725,7 @@ dependencies = [ "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", + "static_assertions", ] [[package]] diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml index 671c8d1640c..08c102b4a64 100644 --- a/crates/capi/Cargo.toml +++ b/crates/capi/Cargo.toml @@ -22,6 +22,7 @@ rustpython-stdlib = {workspace = true, features = ["threading"] } rustpython-pylib = { workspace = true } [dev-dependencies] +static_assertions = { workspace = true } pyo3 = { workspace = true, features = ["auto-initialize", "abi3t"] } [lints] diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs index e43358ae348..0bf843aa442 100644 --- a/crates/capi/src/pyerrors.rs +++ b/crates/capi/src/pyerrors.rs @@ -5,7 +5,6 @@ use core::convert::Infallible; use core::ffi::{c_char, c_int}; use core::slice; use rustpython_vm::builtins::{PyBaseException, PyTraceback, PyTuple, PyType}; -use rustpython_vm::convert::IntoObject; use rustpython_vm::exceptions::ExceptionZoo; use rustpython_vm::{AsObject, PyResult}; diff --git a/crates/capi/src/util.rs b/crates/capi/src/util.rs index 5d0e2c26f5a..d8926f44d28 100644 --- a/crates/capi/src/util.rs +++ b/crates/capi/src/util.rs @@ -40,20 +40,12 @@ where impl FfiResult<*mut Py> for PyRef where - Self: Into, + T: rustpython_vm::object::PyRefTarget>, { const ERR_VALUE: *mut Py = core::ptr::null_mut(); fn into_output(self, _vm: &VirtualMachine) -> *mut Py { - self.into().into_raw().as_ptr().cast() - } -} - -impl FfiResult<*mut PyObject> for PyObjectRef { - const ERR_VALUE: *mut PyObject = core::ptr::null_mut(); - - fn into_output(self, _vm: &VirtualMachine) -> *mut PyObject { - self.into_raw().as_ptr() + PyObjectRef::from(self).into_raw().as_ptr().cast() } } @@ -324,7 +316,7 @@ impl FfiPtrExt for *mut Py { "Attempted to dereference NULL {}", type_name::() ); - unsafe { PyRef::from_raw(self.cast_const()) } + unsafe { PyRef::::from_raw(self.cast_const()) } } #[inline] @@ -384,6 +376,14 @@ mod tests { use core::ffi::{c_longlong, c_ulonglong}; use core::fmt::Debug; + #[test] + fn pyref_object_ffi_targets() { + use rustpython_vm::builtins::PyInt; + static_assertions::assert_impl_all!(PyObjectRef: FfiResult<*mut PyObject>); + static_assertions::assert_impl_all!(PyRef: FfiResult<*mut PyObject>, FfiResult<*mut Py>); + static_assertions::assert_not_impl_any!(PyObjectRef: FfiResult<*mut Py>); + } + #[test] fn ffi_result_err_value() { fn assert_error_value(value: Output) diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs index d0bb95a6c9f..08bf755a23d 100644 --- a/crates/stdlib/src/_sqlite3.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -61,7 +61,6 @@ mod _sqlite3 { PyInt, PyIntRef, PyModule, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8Str, PyUtf8StrRef, }, - convert::IntoObject, function::{ ArgCallable, ArgIterable, FsPath, FuncArgs, KwArgs, OptionalArg, PyComparisonValue, PySetterValue, TimeoutSeconds, diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index 79afc8cb5a8..ae8f0f6a1af 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -543,12 +543,6 @@ impl MemberCell for core::sync::atomic::AtomicU32 {} // One object pointer. Readonly members may be a plain pointer. Writable // members are an atomic cell: `PyAtomicRef` when the pointer is // never null, or `PyAtomicRef>` when it may be. -impl MemberLayout for PyObjectRef { - const KIND: MemberKind = MemberKind::Object; -} -impl MemberLayout for Option { - const KIND: MemberKind = MemberKind::Object; -} impl MemberLayout for crate::object::PyObjectCell { const KIND: MemberKind = MemberKind::Object; const ATOMIC: bool = true; diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs index cbd21527da3..28da2341f64 100644 --- a/crates/vm/src/builtins/frame_locals_proxy.rs +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -145,10 +145,10 @@ impl FrameLocalsProxy { return Ok(vm.ctx.not_implemented()); } let result = self.snapshot(vm)?; - if other.downcast_ref::().is_some() { + if let Some(dict) = other.downcast_ref::() { // PyDict_Update reads a dict subclass's stored entries directly; // it does not dispatch to overridden mapping methods. - result.merge_dict(&other.downcast().unwrap(), true, vm)?; + result.merge_dict(dict, true, vm)?; } else { result.merge_object(other, vm)?; } diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index 992f14ac2f4..9c2255d0084 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -16,7 +16,7 @@ use crate::{ rc::PyRc, wtf8::Wtf8Buf, }, - convert::ToPyResult, + convert::{ToPyResult, TransmuteFromObject}, dict_inner::{self, DictSize}, function::{ ArgIterable, FuncArgs, NameOthers, OptionalArg, PosArgs, PyArithmeticValue, @@ -70,7 +70,7 @@ impl PySet { fn op( &self, - other: AnySet, + other: PyRef, op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult, vm: &VirtualMachine, ) -> PyResult { @@ -132,7 +132,7 @@ impl PyFrozenSet { fn op( &self, - other: AnySet, + other: PyRef, op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult, vm: &VirtualMachine, ) -> PyResult { @@ -537,14 +537,14 @@ impl PySetInner { } fn update_internal(&self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // check AnySet - if let Ok(any_set) = AnySet::try_from_object(vm, iterable.to_owned()) { + // check set or frozenset + if let Ok(any_set) = PyRef::::try_from_object(vm, iterable.to_owned()) { self.merge_set(any_set, vm) // check Dict } else if let Ok(dict) = iterable.to_owned().downcast_exact::(vm) { self.merge_dict(&dict, vm) } else { - // add iterable that is not AnySet or Dict + // add iterable that is not a set, frozenset, or dict for item in iterable.try_into_value::(vm)?.iter(vm)? { let item = item?; self.add(&item, vm)?; @@ -553,7 +553,7 @@ impl PySetInner { } } - fn merge_set(&self, any_set: AnySet, vm: &VirtualMachine) -> PyResult<()> { + fn merge_set(&self, any_set: PyRef, vm: &VirtualMachine) -> PyResult<()> { for (item, hash) in any_set.as_inner().content.keys_with_hashes() { self.add_known_hash(&item, hash, vm)?; } @@ -752,7 +752,7 @@ impl PySet { } fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(self.op( other, PySetInner::union, @@ -768,7 +768,7 @@ impl PySet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(Self { inner: self.inner.intersection(other.into_iterable(vm)?, vm)?, })) @@ -782,7 +782,7 @@ impl PySet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(Self { inner: self.inner.difference_new(other.into_iterable(vm)?, vm)?, })) @@ -796,7 +796,7 @@ impl PySet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(Self { inner: other .as_inner() @@ -812,7 +812,7 @@ impl PySet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(self.op( other, PySetInner::symmetric_difference, @@ -823,12 +823,20 @@ impl PySet { } } - fn __ior__(zelf: PyRef, set: AnySet, vm: &VirtualMachine) -> PyResult> { + fn __ior__( + zelf: PyRef, + set: PyRef, + vm: &VirtualMachine, + ) -> PyResult> { zelf.inner.merge_set(set, vm)?; Ok(zelf) } - fn __iand__(zelf: PyRef, set: AnySet, vm: &VirtualMachine) -> PyResult> { + fn __iand__( + zelf: PyRef, + set: PyRef, + vm: &VirtualMachine, + ) -> PyResult> { if !set.is(zelf.as_object()) { zelf.inner .intersection_update(core::iter::once(set.into_iterable(vm)?), vm)?; @@ -836,7 +844,11 @@ impl PySet { Ok(zelf) } - fn __isub__(zelf: PyRef, set: AnySet, vm: &VirtualMachine) -> PyResult> { + fn __isub__( + zelf: PyRef, + set: PyRef, + vm: &VirtualMachine, + ) -> PyResult> { if set.is(zelf.as_object()) { zelf.inner.clear(); } else { @@ -846,7 +858,11 @@ impl PySet { Ok(zelf) } - fn __ixor__(zelf: PyRef, set: AnySet, vm: &VirtualMachine) -> PyResult> { + fn __ixor__( + zelf: PyRef, + set: PyRef, + vm: &VirtualMachine, + ) -> PyResult> { if set.is(zelf.as_object()) { zelf.inner.clear(); } else { @@ -1068,10 +1084,7 @@ impl Comparable for PySet { impl Iterable for PySet { fn iter(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - Ok(PySetIterator::new(AnySet { - object: zelf.into(), - }) - .into_pyobject(vm)) + Ok(PySetIterator::new(zelf.into()).into_pyobject(vm)) } } @@ -1081,7 +1094,7 @@ impl AsNumber for PySet { // Binary ops check both operands are sets (like CPython's set_sub, etc.) // This is needed because __rsub__ swaps operands: a.__rsub__(b) calls subtract(b, a) subtract: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1096,7 +1109,7 @@ impl AsNumber for PySet { } }), and: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1110,7 +1123,7 @@ impl AsNumber for PySet { } }), xor: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1124,7 +1137,7 @@ impl AsNumber for PySet { } }), or: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1139,32 +1152,48 @@ impl AsNumber for PySet { }), inplace_subtract: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::() { - PySet::__isub__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) + PySet::__isub__( + a.to_owned(), + PyRef::::try_from_object(vm, b.to_owned())?, + vm, + ) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), inplace_and: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::() { - PySet::__iand__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) + PySet::__iand__( + a.to_owned(), + PyRef::::try_from_object(vm, b.to_owned())?, + vm, + ) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), inplace_xor: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::() { - PySet::__ixor__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) + PySet::__ixor__( + a.to_owned(), + PyRef::::try_from_object(vm, b.to_owned())?, + vm, + ) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } }), inplace_or: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::() { - PySet::__ior__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) + PySet::__ior__( + a.to_owned(), + PyRef::::try_from_object(vm, b.to_owned())?, + vm, + ) + .to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -1279,7 +1308,7 @@ impl PyFrozenSet { } fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult> { - if let Ok(set) = AnySet::try_from_object(vm, other) { + if let Ok(set) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(self.op( set, PySetInner::union, @@ -1295,7 +1324,7 @@ impl PyFrozenSet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(Self { inner: self.inner.intersection(other.into_iterable(vm)?, vm)?, ..Default::default() @@ -1310,7 +1339,7 @@ impl PyFrozenSet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(Self { inner: self.inner.difference_new(other.into_iterable(vm)?, vm)?, ..Default::default() @@ -1325,7 +1354,7 @@ impl PyFrozenSet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(Self { inner: other .as_inner() @@ -1342,7 +1371,7 @@ impl PyFrozenSet { other: PyObjectRef, vm: &VirtualMachine, ) -> PyResult> { - if let Ok(other) = AnySet::try_from_object(vm, other) { + if let Ok(other) = PyRef::::try_from_object(vm, other) { Ok(PyArithmeticValue::Implemented(self.op( other, PySetInner::symmetric_difference, @@ -1520,10 +1549,7 @@ impl Comparable for PyFrozenSet { impl Iterable for PyFrozenSet { fn iter(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - Ok(PySetIterator::new(AnySet { - object: zelf.into(), - }) - .into_pyobject(vm)) + Ok(PySetIterator::new(zelf.into()).into_pyobject(vm)) } } @@ -1533,7 +1559,7 @@ impl AsNumber for PyFrozenSet { // Binary ops check both operands are sets (like CPython's set_sub, etc.) // __rsub__ swaps operands. Result type follows first operand's type. subtract: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1546,7 +1572,7 @@ impl AsNumber for PyFrozenSet { } }), and: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1558,7 +1584,7 @@ impl AsNumber for PyFrozenSet { } }), xor: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1570,7 +1596,7 @@ impl AsNumber for PyFrozenSet { } }), or: Some(|a, b, vm| { - if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + if !PyRef::::check(a, vm) || !PyRef::::check(b, vm) { return Ok(vm.ctx.not_implemented()); } if let Some(a) = a.downcast_ref::() { @@ -1604,27 +1630,78 @@ impl Representable for PyFrozenSet { } } -struct AnySet { - object: PyObjectRef, +/// A borrowed view of either a set or a frozenset, including subclasses. +/// +/// This is an erased header view, not a shared payload layout: every payload +/// access still downcasts to the actual alternative. It cannot be allocated as +/// a payload or constructed without checking the object. +#[repr(transparent)] +pub(crate) struct PyAnySet { + object: PyObject, } -impl Borrow for AnySet { +impl Borrow for PyAnySet { #[inline(always)] fn borrow(&self) -> &PyObject { &self.object } } -impl AnySet { - /// Check if object is a set or frozenset (including subclasses) - /// Equivalent to CPython's PyAnySet_Check +impl Borrow for PyRef { + fn borrow(&self) -> &PyAnySet { + self + } +} + +impl ToOwned for PyAnySet { + type Owned = PyRef; + + fn to_owned(&self) -> PyRef { + // SAFETY: this borrowed view already guarantees a set payload. + unsafe { PyRef::::from_object_unchecked(self.object.to_owned()) } + } +} + +impl From> for PyRef { + fn from(set: PyRef) -> Self { + // SAFETY: the typed owner guarantees the set alternative. + unsafe { Self::from_object_unchecked(set.into()) } + } +} + +impl From> for PyRef { + fn from(set: PyRef) -> Self { + // SAFETY: the typed owner guarantees the frozenset alternative. + unsafe { Self::from_object_unchecked(set.into()) } + } +} + +impl From> for PyObjectRef { + fn from(set: PyRef) -> Self { + set.into_object() + } +} + +impl PyAnySet { + /// Check if object is a set or frozenset (including subclasses). + /// Equivalent to CPython's PyAnySet_Check. fn check(obj: &PyObject, vm: &VirtualMachine) -> bool { let ctx = &vm.ctx; obj.fast_isinstance(ctx.types.set_type) || obj.fast_isinstance(ctx.types.frozenset_type) } + fn as_inner(&self) -> &PySetInner { + extract_set(&self.object).expect("PyAnySet is always PySet or PyFrozenSet") + } +} + +impl PyRef { + fn check(obj: &PyObject, vm: &VirtualMachine) -> bool { + PyAnySet::check(obj, vm) + } + fn into_iterable(self, vm: &VirtualMachine) -> PyResult { - self.object.try_into_value(vm) + self.into_object().try_into_value(vm) } fn into_iterable_iter( @@ -1633,26 +1710,28 @@ impl AnySet { ) -> PyResult> { Ok(core::iter::once(self.into_iterable(vm)?)) } +} - fn as_inner(&self) -> &PySetInner { - match_class!(match self.object.as_object() { - ref set @ PySet => &set.inner, - ref frozen @ PyFrozenSet => &frozen.inner, - _ => unreachable!("AnySet is always PySet or PyFrozenSet"), // should not be called. - }) +// SAFETY: PyRef is a transparent PyRef owner. A successful check proves +// both the Python class and the actual set/frozenset payload of this object. +unsafe impl TransmuteFromObject for PyRef { + fn check(vm: &VirtualMachine, obj: &PyObject) -> PyResult<()> { + if PyAnySet::check(obj, vm) && extract_set(obj).is_some() { + Ok(()) + } else { + Err(vm.new_type_error(format!( + "{} is not a subtype of set or frozenset", + obj.class() + ))) + } } } -impl TryFromObject for AnySet { +impl TryFromObject for PyRef { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - let class = obj.class(); - if class.fast_issubclass(vm.ctx.types.set_type) - || class.fast_issubclass(vm.ctx.types.frozenset_type) - { - Ok(Self { object: obj }) - } else { - Err(vm.new_type_error(format!("{class} is not a subtype of set or frozenset"))) - } + ::check(vm, &obj)?; + // SAFETY: the shared checker proved the union view's payload invariant. + Ok(unsafe { Self::from_object_unchecked(obj) }) } } @@ -1663,7 +1742,7 @@ pub(crate) struct PySetIterator { /// records by writing a size no set can have. Sticky: what it makes the /// iterator answer, it answers from then on. changed: PyAtomic, - internal: PyMutex>, + internal: PyMutex>>, } impl fmt::Debug for PySetIterator { @@ -1681,7 +1760,7 @@ impl PyPayload for PySetIterator { } impl PySetIterator { - fn new(set: AnySet) -> Self { + fn new(set: PyRef) -> Self { Self { size: set.as_inner().content.size(), changed: Radium::new(false), @@ -1811,3 +1890,151 @@ pub(crate) fn init(context: &'static Context) { PySetIterator::extend_class(context, context.types.set_iterator_type); } + +#[cfg(test)] +mod reference_tests { + use super::*; + + #[test] + fn anyset_reference_layout_and_ownership() { + static_assertions::assert_not_impl_any!(PyAnySet: PyPayload); + static_assertions::assert_type_eq_all!( + ::Target, + PyAnySet + ); + static_assertions::assert_eq_size!(PyRef, PyObjectRef, Option>); + static_assertions::assert_eq_align!(PyRef, PyObjectRef); + static_assertions::assert_eq_size!(PyAnySet, PyObject); + static_assertions::assert_eq_align!(PyAnySet, PyObject); + #[cfg(feature = "threading")] + static_assertions::assert_impl_all!(PyRef: Send, Sync); + #[cfg(not(feature = "threading"))] + static_assertions::assert_not_impl_any!(PyRef: Send, Sync); + + let ctx = Context::genesis(); + let set: PyRef = PySet::default().into_ref(ctx).into(); + let frozen: PyRef = PyFrozenSet::default().into_ref(ctx).into(); + for owner in [set, frozen] { + let object = owner.as_object() as *const PyObject; + assert_eq!(owner.as_object().strong_count(), 1); + let borrowed: &PyAnySet = &owner; + assert!(core::ptr::eq(borrowed.as_object(), object)); + assert!(core::ptr::eq( + borrowed.as_inner(), + extract_set(owner.as_object()).unwrap() + )); + let copy = borrowed.to_owned(); + assert_eq!(owner.as_object().strong_count(), 2); + let clone = owner.clone(); + assert_eq!(owner.as_object().strong_count(), 3); + let erased = clone.into_object(); + assert_eq!(erased.strong_count(), 3); + assert!(core::ptr::eq(&*erased, object)); + drop(copy); + drop(owner); + assert_eq!(erased.strong_count(), 1); + } + } + + #[test] + fn anyset_checked_reference_conversion() { + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let set = PySet::default().into_pyobject(vm); + let frozen = PyFrozenSet::default().into_pyobject(vm); + for object in [set, frozen] { + let owner = PyRef::::try_from_object(vm, object.clone()).unwrap(); + assert_eq!(object.strong_count(), 2); + assert!(core::ptr::eq(owner.as_object(), &*object)); + let erased: PyObjectRef = owner.into(); + assert_eq!(object.strong_count(), 2); + assert!(core::ptr::eq(&*erased, &*object)); + drop(erased); + assert_eq!(object.strong_count(), 1); + } + let unrelated: PyObjectRef = vm.ctx.new_int(1000).into(); + let error = PyRef::::try_from_object(vm, unrelated.clone()).unwrap_err(); + assert!(error.fast_isinstance(vm.ctx.exceptions.type_error)); + assert_eq!(unrelated.strong_count(), 1); + }); + } + + #[test] + fn anyset_typed_tuple_views_and_refcounts() { + use crate::builtins::PyTuple; + type Typed = PyTuple>; + type Nested = PyTuple>; + static_assertions::assert_not_impl_any!(PyObject: crate::object::PyTypedRefTarget); + static_assertions::assert_impl_all!(PyAnySet: crate::object::PyTypedRefTarget); + static_assertions::assert_impl_all!(Typed: crate::object::PyTypedRefTarget); + static_assertions::assert_eq_size!(Typed, PyTuple); + static_assertions::assert_eq_align!(Typed, PyTuple); + static_assertions::assert_eq_size!(Py, Py); + static_assertions::assert_eq_align!(Py, Py); + + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let set: PyRef = PySet::default().into_ref(&vm.ctx).into(); + let frozen: PyRef = PyFrozenSet::default().into_ref(&vm.ctx).into(); + let tuple = Typed::new_ref_typed(vec![set.clone(), frozen.clone()], &vm.ctx); + assert_eq!(set.as_object().strong_count(), 2); + assert_eq!(frozen.as_object().strong_count(), 2); + assert!( + tuple.as_slice()[0] + .as_object() + .downcast_ref::() + .is_some() + ); + assert!( + tuple.as_slice()[1] + .as_object() + .downcast_ref::() + .is_some() + ); + assert!(core::ptr::eq( + tuple.as_slice()[0].as_inner(), + set.as_inner() + )); + assert!(core::ptr::eq( + tuple.as_slice()[1].as_inner(), + frozen.as_inner() + )); + let erased = tuple.clone().into_untyped(); + assert!(core::ptr::eq(tuple.as_untyped(), &*erased)); + assert_eq!(erased.as_object().strong_count(), 2); + let checked = erased.try_into_typed::(vm).unwrap(); + assert!(core::ptr::eq(&*checked, &*tuple)); + assert_eq!(checked.as_object().strong_count(), 2); + let borrowed_copy = (*checked).to_owned(); + assert_eq!(tuple.as_object().strong_count(), 3); + let nested: PyRef = PyTuple::new_ref_typed(vec![borrowed_copy], &vm.ctx); + assert!(core::ptr::eq(&*nested.as_slice()[0], &*tuple)); + let untyped: PyTupleRef = nested.into(); + assert_eq!(tuple.as_object().strong_count(), 3); + drop(untyped); + drop(checked); + assert_eq!(tuple.as_object().strong_count(), 1); + drop(tuple); + assert_eq!(set.as_object().strong_count(), 1); + assert_eq!(frozen.as_object().strong_count(), 1); + }); + } + + #[test] + fn anyset_typed_tuple_rejects_invalid_element_without_leaking() { + use crate::builtins::PyTuple; + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let set = PySet::default().into_pyobject(vm); + let wrong: PyObjectRef = vm.ctx.new_int(1000).into(); + let tuple = PyTuple::new_ref(vec![set.clone(), wrong.clone()], &vm.ctx); + let error = tuple.clone().try_into_typed::(vm).unwrap_err(); + assert!(error.fast_isinstance(vm.ctx.exceptions.type_error)); + assert_eq!(tuple.as_object().strong_count(), 1); + assert!(core::ptr::eq(&*tuple.as_slice()[0], &*set)); + assert!(core::ptr::eq(&*tuple.as_slice()[1], &*wrong)); + assert_eq!(set.strong_count(), 2); + assert_eq!(wrong.strong_count(), 2); + drop(tuple); + assert_eq!(set.strong_count(), 1); + assert_eq!(wrong.strong_count(), 1); + }); + } +} diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index c8647057330..4421a67e818 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -4,7 +4,7 @@ use super::{ }; use crate::common::lock::LazyLock; use crate::common::{hash, hash::PyHash, lock::PyMutex, wtf8::wtf8_concat}; -use crate::object::{Traverse, TraverseFn}; +use crate::object::{PyTypedRefTarget, Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, @@ -25,9 +25,12 @@ use crate::{ }; use alloc::fmt; use core::cell::{Cell, UnsafeCell}; +use core::marker::PhantomData; +use core::mem::ManuallyDrop; use core::ptr::NonNull; #[pyclass(module = false, name = "tuple", traverse = "manual")] +#[repr(transparent)] pub struct PyTuple { elements: TupleElements, } @@ -35,30 +38,41 @@ pub struct PyTuple { /// Tuple storage is immutable after publication, but marshal must publish a /// tuple in its reference table before recursively reading its children. /// This mirrors CPython's `PyTuple_New` followed by `PyTuple_SET_ITEM`. -struct TupleElements(UnsafeCell>); +// Every allocation stores erased owners, including when borrowed through a +// typed tuple view. The marker preserves the old storage's invariance and +// ownership relationship to R. +// Safe constructors and checked view conversions only admit R = PyRef, +// and establish each element's target invariant before publishing that view. +#[repr(transparent)] +struct TupleElements(UnsafeCell>, PhantomData>); unsafe impl Send for TupleElements {} unsafe impl Sync for TupleElements {} impl TupleElements { - const fn new(elements: Box<[R]>) -> Self { - Self(UnsafeCell::new(elements)) + fn as_slice(&self) -> &[R] { + // SAFETY: all admitted R are transparent PyRef owners with the same + // size/alignment as PyObjectRef. Their element invariants were proven + // by construction or checked conversion. Published elements cannot be + // replaced, and the returned shared slice borrows this storage. + let elements = unsafe { &*self.0.get() }; + unsafe { core::slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) } } +} - fn as_slice(&self) -> &[R] { - // SAFETY: initialization writes happen only while the tuple is owned by - // the synchronous marshal decoder; afterwards the storage is immutable. - unsafe { &*self.0.get() } +impl TupleElements { + const fn new(elements: Box<[PyObjectRef]>) -> Self { + Self(UnsafeCell::new(elements), PhantomData) } - fn get_mut(&mut self) -> &mut Box<[R]> { + fn get_mut(&mut self) -> &mut Box<[PyObjectRef]> { self.0.get_mut() } /// # Safety /// The tuple must still be in its private initialization phase, and each /// placeholder index must be replaced at most once before it is observable. - unsafe fn set_initializing(&self, index: usize, value: R) { + unsafe fn set_initializing(&self, index: usize, value: PyObjectRef) { unsafe { (*self.0.get())[index] = value }; } } @@ -398,6 +412,38 @@ impl Py> { } impl Py { + fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + match SequenceIndex::try_from_borrowed_object::(vm, needle)? { + SequenceIndex::Int(i) => { + let index = self + .elements + .wrap_index(i) + .ok_or_else(|| vm.new_index_error("tuple index out of range"))?; + Ok(self.elements[index].clone()) + } + SequenceIndex::Slice(slice) => { + let len = self.as_slice().len(); + let (range, step, slice_len) = slice.adjust_indices(len); + if slice_len != 0 + && range.start == 0 + && step == 1 + && slice_len == len + && self.class().is(vm.ctx.types.tuple_type) + { + Ok(self.to_owned().into()) + } else { + self.elements + .getitem_by_slice(vm, slice) + .map(|x| vm.ctx.new_tuple(x).into()) + } + } + } + } + + fn __getitem__(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + self._getitem(needle, vm) + } + pub fn extract_tuple<'a, T: FromPyTuple<'a>>(&'a self, vm: &VirtualMachine) -> PyResult { T::from_pytuple(self, vm) } @@ -410,8 +456,14 @@ impl PyTuple> { ) -> PyRef { // SAFETY: PyRef has the same layout as PyObjectRef. unsafe { - let elements: Vec = - core::mem::transmute::>, Vec>(elements); + let mut elements = ManuallyDrop::new(elements); + // Preserve the buffer, length and capacity without assuming Vec's + // field order. Each transparent owner transfers its one reference. + let elements = Vec::from_raw_parts( + elements.as_mut_ptr().cast::(), + elements.len(), + elements.capacity(), + ); let tuple = PyRef::new_ref( PyTuple::new_unchecked(elements.into_boxed_slice()), tuple_type, @@ -424,8 +476,14 @@ impl PyTuple> { pub fn new_ref_typed(elements: Vec>, ctx: &Context) -> PyRef { // SAFETY: PyRef has the same layout as PyObjectRef unsafe { - let elements: Vec = - core::mem::transmute::>, Vec>(elements); + let mut elements = ManuallyDrop::new(elements); + // Preserve the buffer, length and capacity without assuming Vec's + // field order. Each transparent owner transfers its one reference. + let elements = Vec::from_raw_parts( + elements.as_mut_ptr().cast::(), + elements.len(), + elements.capacity(), + ); let tuple = PyTuple::::new_ref(elements, ctx); core::mem::transmute::, PyRef>(tuple) } @@ -469,26 +527,6 @@ impl PyTuple { Self::repeat(zelf, value, vm) } - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object::(vm, needle)? { - SequenceIndex::Int(i) => { - let index = self - .elements - .wrap_index(i) - .ok_or_else(|| vm.new_index_error("tuple index out of range"))?; - Ok(self.elements[index].clone()) - } - SequenceIndex::Slice(slice) => self - .elements - .getitem_by_slice(vm, slice) - .map(|x| vm.ctx.new_tuple(x).into()), - } - } - - fn __getitem__(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - self._getitem(needle, vm) - } - fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { for element in &self.elements { if vm.identical_or_equal(element, needle)? { @@ -565,9 +603,9 @@ impl AsMapping for PyTuple { length: atomic_func!(|mapping, _vm| { Ok(PyTuple::mapping_downcast(mapping).as_slice().len()) }), - subscript: atomic_func!(|mapping, needle, vm| PyTuple::mapping_downcast(mapping) - .payload - ._getitem(needle, vm)), + subscript: atomic_func!( + |mapping, needle, vm| PyTuple::mapping_downcast(mapping)._getitem(needle, vm) + ), ..PyMappingMethods::NOT_IMPLEMENTED }); &AS_MAPPING @@ -686,35 +724,42 @@ impl Representable for PyTuple { } impl PyRef> { - pub fn try_into_typed( + pub fn try_into_typed( self, vm: &VirtualMachine, - ) -> PyResult>>> { + ) -> PyResult>>> + where + PyRef: TransmuteFromObject, + { // Check that all elements are of the correct type for elem in self.as_slice() { as TransmuteFromObject>::check(vm, elem)?; } - // SAFETY: We just verified all elements are of type T + // SAFETY: every element satisfies T's view invariant. Transparent + // tuple wrappers retain the same erased storage for every marker; + // this transfers the owner without changing its allocation or count. Ok(unsafe { core::mem::transmute::>>>(self) }) } } -impl PyRef>> { +impl PyRef>> { #[must_use] pub fn into_untyped(self) -> PyRef { - // SAFETY: PyTuple> has the same layout as PyTuple + // SAFETY: transparent tuple wrappers have identical erased storage; + // forgetting the element guarantee does not change the allocation. unsafe { core::mem::transmute::>(self) } } } -impl Py>> { +impl Py>> { pub fn as_untyped(&self) -> &Py { - // SAFETY: PyTuple> has the same layout as PyTuple + // SAFETY: transparent tuple wrappers have identical erased storage; + // forgetting the element guarantee does not change the allocation. unsafe { core::mem::transmute::<&Self, &Py>(self) } } } -impl From>>> for PyTupleRef { +impl From>>> for PyTupleRef { #[inline] fn from(tup: PyRef>>) -> Self { tup.into_untyped() diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 1eed9b712a0..d22d6ef7475 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -1,15 +1,3 @@ -//! Essential types for object models -//! -//! +-------------------------+--------------+-----------------------+ -//! | Management | Typed | Untyped | -//! +-------------------------+------------------+-------------------+ -//! | Interpreter-independent | [`Py`] | [`PyObject`] | -//! | Reference-counted | [`PyRef`] | [`PyObjectRef`] | -//! | Weak | [`PyWeakRef`] | [`PyRef`] | -//! +-------------------------+--------------+-----------------------+ -//! -//! [`PyRef`] may looking like to be called as PyObjectWeak by the rule, -//! but not to do to remember it is a PyRef object. use super::{ PyAtomicRef, ext::{AsObject, PyRefExact, PyResult}, @@ -708,7 +696,7 @@ unsafe fn try_reuse_weakref(ptr: *mut Py) -> Option> { let node = unsafe { &*ptr }; node.ref_count .safe_inc() - .then(|| unsafe { PyRef::from_raw(ptr) }) + .then(|| unsafe { PyRef::::from_raw(ptr) }) } impl WeakRefList { @@ -889,7 +877,7 @@ impl WeakRefList { // Collect callback only if we can still acquire a strong ref. if wr.ref_count.safe_inc() { - let wr_ref = unsafe { PyRef::from_raw(wr as *const Py) }; + let wr_ref = unsafe { PyRef::::from_raw(wr as *const Py) }; let cb = unsafe { wr.payload.callback.get().replace(None) }; if let Some(cb) = cb { callbacks.push((wr_ref, cb)); @@ -940,7 +928,7 @@ impl WeakRefList { // Collect callback without invoking only if we can keep weakref alive. if wr.ref_count.safe_inc() { - let wr_ref = unsafe { PyRef::from_raw(wr as *const Py) }; + let wr_ref = unsafe { PyRef::::from_raw(wr as *const Py) }; let cb = unsafe { wr.payload.callback.get().replace(None) }; if let Some(cb) = cb { callbacks.push((wr_ref, cb)); @@ -974,7 +962,7 @@ impl WeakRefList { while let Some(node) = current { let wr = unsafe { node.as_ref() }; if wr.ref_count.safe_inc() { - v.push(unsafe { PyRef::from_raw(wr as *const Py) }); + v.push(unsafe { PyRef::::from_raw(wr as *const Py) }); } current = unsafe { WeakLink::pointers(node).as_ref().get_next() }; } @@ -1001,7 +989,7 @@ unsafe impl Link for WeakLink { #[inline(always)] unsafe fn from_raw(ptr: NonNull) -> Self::Handle { - unsafe { PyRef::from_raw(ptr.as_ptr()) } + unsafe { PyRef::::from_raw(ptr.as_ptr()) } } #[inline(always)] @@ -1513,38 +1501,11 @@ impl core::ops::DerefMut for FreeList { /// this reference counting is accounted for by this type. Use the `.clone()` /// method to create a new reference and increment the amount of references /// to the python object by 1. -#[repr(transparent)] -pub struct PyObjectRef { - ptr: NonNull, -} - -impl Clone for PyObjectRef { - #[inline(always)] - fn clone(&self) -> Self { - (**self).to_owned() - } -} - -cfg_select! { - feature = "threading" => { - unsafe impl Send for PyObjectRef {} - unsafe impl Sync for PyObjectRef {} - } - _ => {} -} +pub type PyObjectRef = PyRef; #[repr(transparent)] pub struct PyObject(Py); -impl Deref for PyObjectRef { - type Target = PyObject; - - #[inline(always)] - fn deref(&self) -> &PyObject { - unsafe { self.ptr.as_ref() } - } -} - impl ToOwned for PyObject { type Owned = PyObjectRef; @@ -1552,7 +1513,8 @@ impl ToOwned for PyObject { fn to_owned(&self) -> Self::Owned { self.0.ref_count.inc(); PyObjectRef { - ptr: NonNull::from(self), + ptr: NonNull::from(self).cast(), + marker: PhantomData, } } } @@ -1566,7 +1528,8 @@ impl PyObject { pub fn try_to_owned(&self) -> Option { if self.0.ref_count.safe_inc() { Some(PyObjectRef { - ptr: NonNull::from(self), + ptr: NonNull::from(self).cast(), + marker: PhantomData, }) } else { None @@ -1590,6 +1553,7 @@ impl PyObject { if ref_count.safe_inc() { Some(PyObjectRef { ptr: unsafe { NonNull::new_unchecked(ptr) }, + marker: PhantomData, }) } else { None @@ -1621,7 +1585,10 @@ impl PyObjectRef { #[inline(always)] #[must_use] pub const unsafe fn from_raw(ptr: NonNull) -> Self { - Self { ptr } + Self { + ptr: ptr.cast(), + marker: PhantomData, + } } /// Attempt to downcast this reference to a subclass. @@ -1653,7 +1620,8 @@ impl PyObjectRef { // manual impl to avoid assertion let obj = ManuallyDrop::new(self); PyRef { - ptr: obj.ptr.cast(), + ptr: obj.ptr, + marker: PhantomData, } } @@ -1698,7 +1666,7 @@ impl PyObject { } else { let head = unsafe { &*head_ptr }; if head.ref_count.safe_inc() { - Some(unsafe { PyRef::from_raw(head_ptr) }.into()) + Some(unsafe { PyRef::::from_raw(head_ptr) }.into()) } else { None } @@ -2400,20 +2368,6 @@ impl PyObject { } } -impl Borrow for PyObjectRef { - #[inline(always)] - fn borrow(&self) -> &PyObject { - self - } -} - -impl AsRef for PyObjectRef { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self - } -} - impl<'a, T: PyPayload> From<&'a Py> for &'a PyObject { #[inline(always)] fn from(py_ref: &'a Py) -> Self { @@ -2421,15 +2375,6 @@ impl<'a, T: PyPayload> From<&'a Py> for &'a PyObject { } } -impl Drop for PyObjectRef { - #[inline] - fn drop(&mut self) { - if self.0.ref_count.dec() { - unsafe { PyObject::drop_slow(self.ptr) } - } - } -} - impl fmt::Debug for PyObject { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // SAFETY: the vtable contains functions that accept payload types that always match up @@ -2438,12 +2383,6 @@ impl fmt::Debug for PyObject { } } -impl fmt::Debug for PyObjectRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.as_object().fmt(f) - } -} - const STACKREF_BORROW_TAG: usize = 1; /// A tagged stack reference to a Python object. @@ -2655,14 +2594,15 @@ impl Py { } } -impl ToOwned for Py { +impl> ToOwned for Py { type Owned = PyRef; #[inline(always)] fn to_owned(&self) -> Self::Owned { self.ref_count.inc(); PyRef { - ptr: NonNull::from(self), + ptr: NonNull::from(self).cast(), + marker: PhantomData, } } } @@ -2721,6 +2661,63 @@ impl fmt::Debug for Py { } } +mod ref_target_sealed { + pub trait Sealed {} + impl Sealed for T {} + impl Sealed for super::PyObject {} + impl Sealed for crate::builtins::set::PyAnySet {} + impl Sealed for super::PyTuple> {} +} + +/// Selects the borrowed view of a Python reference. +/// +/// Payloads borrow `Py`; `PyObject` borrows the erased object itself. +/// This trait is sealed so safe code cannot choose an unrelated layout. +/// Typed tuples are immutable views of the ordinary tuple allocation. +/// Internal union views borrow an erased header and dispatch on the payload; +/// they must not assume that their alternative payloads have the same layout. +/// +/// An arbitrary borrowed layout cannot be registered by downstream code: +/// +/// ```compile_fail +/// use rustpython_vm::object::PyRefTarget; +/// struct Unrelated; +/// impl PyRefTarget for Unrelated { +/// type Target = u8; +/// } +/// ``` +pub trait PyRefTarget: ref_target_sealed::Sealed { + type Target; +} + +impl PyRefTarget for T { + type Target = Py; +} + +impl PyRefTarget for PyObject { + type Target = Self; +} + +/// A non-erased reference target that can be an element of a typed tuple view. +/// +/// This is sealed through [`PyRefTarget`]. Payloads, registered union views and +/// recursively typed tuples qualify; `PyObject` does not, since ordinary tuples +/// already implement [`PyPayload`]. This marker selects a layout, not a runtime +/// type check: checked element conversion still requires +/// [`crate::convert::TransmuteFromObject`]. +pub trait PyTypedRefTarget: PyRefTarget {} + +impl>> PyTypedRefTarget for T {} +impl PyTypedRefTarget for crate::builtins::set::PyAnySet {} + +impl PyRefTarget for crate::builtins::set::PyAnySet { + type Target = Self; +} + +impl PyRefTarget for PyTuple> { + type Target = Py; +} + /// A reference to a Python object. /// /// Note that a `PyRef` can only deref to a shared / immutable reference. @@ -2732,7 +2729,9 @@ impl fmt::Debug for Py { /// where a reference to the same object must be returned. #[repr(transparent)] pub struct PyRef { - ptr: NonNull>, + ptr: NonNull, + // Preserve the variance and auto traits of the former typed pointer. + marker: PhantomData>>, } cfg_select! { @@ -2743,17 +2742,17 @@ cfg_select! { _ => {} } -impl fmt::Debug for PyRef { +impl fmt::Debug for PyRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (**self).fmt(f) + self.as_ref().fmt(f) } } impl Drop for PyRef { #[inline] fn drop(&mut self) { - if self.ref_count.dec() { - unsafe { PyObject::drop_slow(self.ptr.cast::()) } + if unsafe { self.ptr.as_ref() }.0.ref_count.dec() { + unsafe { PyObject::drop_slow(self.ptr) } } } } @@ -2761,14 +2760,48 @@ impl Drop for PyRef { impl Clone for PyRef { #[inline(always)] fn clone(&self) -> Self { - (**self).to_owned() + unsafe { self.ptr.as_ref() }.0.ref_count.inc(); + Self { + ptr: self.ptr, + marker: PhantomData, + } + } +} + +impl PyRef { + /// Transfers ownership from an erased reference to a registered view. + /// + /// # Safety + /// The object must satisfy all invariants of `T::Target`, including its + /// payload restrictions. Registering a target establishes its layout only; + /// the caller must validate this particular object before the transfer. + #[inline] + pub(crate) unsafe fn from_object_unchecked(object: PyObjectRef) -> Self { + let object = ManuallyDrop::new(object); + Self { + ptr: object.ptr, + marker: PhantomData, + } + } +} + +impl PyRef { + /// Erases the borrowed view, transferring the existing strong reference. + #[inline] + #[must_use] + pub fn into_object(self) -> PyObjectRef { + let object = ManuallyDrop::new(self); + PyObjectRef { + ptr: object.ptr, + marker: PhantomData, + } } } impl PyRef { #[inline(always)] pub(super) const fn into_non_null(self) -> NonNull> { - let ptr = self.ptr; + let ptr = self.ptr.cast(); core::mem::forget(self); ptr } @@ -2778,7 +2811,10 @@ impl PyRef { #[must_use] #[inline(always)] pub const unsafe fn from_non_null(ptr: NonNull>) -> Self { - Self { ptr } + Self { + ptr: ptr.cast(), + marker: PhantomData, + } } /// # Safety @@ -2792,15 +2828,13 @@ impl PyRef { #[inline(always)] unsafe fn from_obj_unchecked(obj: PyObjectRef) -> Self { debug_assert!(obj.downcast_ref::().is_some()); - let obj = ManuallyDrop::new(obj); - Self { - ptr: obj.ptr.cast(), - } + // SAFETY: the caller guarantees the payload type. + unsafe { Self::from_object_unchecked(obj) } } #[must_use] pub const fn leak(pyref: Self) -> &'static Py { - let ptr = pyref.ptr; + let ptr = pyref.ptr.cast::>(); core::mem::forget(pyref); unsafe { ptr.as_ref() } } @@ -2858,7 +2892,10 @@ impl PyRef { } } - Self { ptr } + Self { + ptr: ptr.cast(), + marker: PhantomData, + } } } @@ -2915,54 +2952,47 @@ impl Py { } } -impl Borrow for PyRef -where - T: PyPayload, -{ +impl Borrow for PyRef { #[inline(always)] fn borrow(&self) -> &PyObject { - (**self).as_object() + unsafe { self.ptr.as_ref() } } } -impl AsRef for PyRef -where - T: PyPayload, -{ +impl AsRef for PyRef { #[inline(always)] fn as_ref(&self) -> &PyObject { self.borrow() } } -impl From> for PyObjectRef { +impl>> From> for PyObjectRef { #[inline] fn from(value: PyRef) -> Self { - let me = ManuallyDrop::new(value); - Self { ptr: me.ptr.cast() } + value.into_object() } } -impl Borrow> for PyRef { +impl>> Borrow> for PyRef { #[inline(always)] fn borrow(&self) -> &Py { self } } -impl AsRef> for PyRef { +impl>> AsRef> for PyRef { #[inline(always)] fn as_ref(&self) -> &Py { self } } -impl Deref for PyRef { - type Target = Py; +impl Deref for PyRef { + type Target = T::Target; #[inline(always)] - fn deref(&self) -> &Py { - unsafe { self.ptr.as_ref() } + fn deref(&self) -> &Self::Target { + unsafe { self.ptr.cast::().as_ref() } } } @@ -3060,12 +3090,12 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy { } unsafe fn initial_ref(ptr: *mut Py) -> PyRef { - unsafe { PyRef::from_raw(ptr.cast()) } + unsafe { PyRef::::from_raw(ptr.cast()) } } unsafe fn clone_raw_ref(ptr: *mut Py) -> PyRef { unsafe { &*ptr::addr_of!((*ptr).ref_count) }.inc(); - unsafe { PyRef::from_raw(ptr.cast()) } + unsafe { PyRef::::from_raw(ptr.cast()) } } unsafe fn into_type_tuple(tuple: PyTupleRef) -> PyTypeTupleRef { @@ -3245,6 +3275,141 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy { mod tests { use super::*; + #[derive(Debug)] + struct RefProbe(alloc::sync::Arc); + + impl MaybeTraverse for RefProbe { + fn try_traverse(&self, _traverse_fn: &mut TraverseFn<'_>) {} + } + + impl PyPayload for RefProbe { + fn class(ctx: &crate::Context) -> &'static Py { + ctx.types.object_type + } + } + + impl Drop for RefProbe { + fn drop(&mut self) { + self.0.fetch_add(1, core::sync::atomic::Ordering::SeqCst); + } + } + + #[test] + fn pyref_object_layout_and_views() { + static_assertions::assert_type_eq_all!(PyObjectRef, PyRef); + static_assertions::assert_not_impl_any!(PyObject: PyPayload); + static_assertions::assert_impl_all!(PyRef: Unpin); + static_assertions::assert_eq_size!( + PyObjectRef, + NonNull, + Option, + PyRef, + Option>, + PyRef>> + ); + static_assertions::assert_eq_align!( + PyObjectRef, + NonNull, + PyRef, + PyRef>> + ); + #[cfg(feature = "threading")] + static_assertions::assert_impl_all!(PyRef: Send, Sync); + #[cfg(not(feature = "threading"))] + static_assertions::assert_not_impl_any!(PyRef: Send, Sync); + + fn erased_view(reference: &PyRef) -> &PyObject { + reference + } + fn payload_view(reference: &PyRef) -> &Py { + reference + } + fn tuple_view( + reference: &PyRef>>, + ) -> &Py>> { + reference + } + type NestedTuple = PyTuple>>>; + fn nested_view(reference: &PyRef) -> &Py { + reference + } + fn covariance<'long: 'short, 'short>(reference: PyRef<&'long ()>) -> PyRef<&'short ()> { + reference + } + fn into_object>() {} + into_object::>(); + let _ = ( + erased_view, + payload_view, + tuple_view, + nested_view, + covariance, + ); + } + + #[test] + fn pyref_object_ownership_and_raw_roundtrip() { + let drops = alloc::sync::Arc::new(core::sync::atomic::AtomicUsize::new(0)); + let ctx = crate::Context::genesis(); + let typed = PyRef::new_ref( + RefProbe(drops.clone()), + ctx.types.object_type.to_owned(), + None, + ); + let pointer = typed.as_object() as *const PyObject; + assert_eq!(typed.as_object().strong_count(), 1); + let erased: PyRef = typed.clone().into(); + assert_eq!(erased.strong_count(), 2); + assert!(core::ptr::eq(&*erased, pointer)); + let raw = erased.into_raw(); + assert_eq!(typed.as_object().strong_count(), 2); + let erased = unsafe { PyRef::::from_raw(raw) }; + let erased = erased.downcast::().unwrap_err(); + assert_eq!(erased.strong_count(), 2); + let restored = erased.downcast::().unwrap(); + assert!(core::ptr::eq(&*typed, &*restored)); + let raw = restored.into_non_null(); + let restored = unsafe { PyRef::::from_non_null(raw) }; + drop(typed); + assert_eq!(restored.as_object().strong_count(), 1); + assert_eq!(drops.load(core::sync::atomic::Ordering::SeqCst), 0); + drop(restored); + assert_eq!(drops.load(core::sync::atomic::Ordering::SeqCst), 1); + } + + #[test] + fn pyref_typed_tuple_erased_storage_drops_once() { + let ctx = crate::Context::genesis(); + // Both constructors transfer a Vec allocation into erased tuple storage. + for explicit_type in [false, true] { + let drops = alloc::sync::Arc::new(core::sync::atomic::AtomicUsize::new(0)); + let item = PyRef::new_ref( + RefProbe(drops.clone()), + ctx.types.object_type.to_owned(), + None, + ); + let mut elements = Vec::with_capacity(4); + elements.push(item.clone()); + let tuple = if explicit_type { + PyTuple::new_ref_typed_with_type(elements, ctx.types.tuple_type.to_owned()) + } else { + PyTuple::new_ref_typed(elements, ctx) + }; + assert_eq!(tuple.as_slice()[0].as_object().strong_count(), 2); + let copy = (*tuple).to_owned(); + let object: PyObjectRef = tuple.into(); + let untyped = copy.into_untyped(); + assert_eq!(object.strong_count(), 2); + assert!(core::ptr::eq(untyped.as_object(), &*object)); + assert_eq!(untyped.as_slice()[0].strong_count(), 2); + drop(object); + drop(item); + assert_eq!(drops.load(core::sync::atomic::Ordering::SeqCst), 0); + drop(untyped); + assert_eq!(drops.load(core::sync::atomic::Ordering::SeqCst), 1); + } + } + #[test] fn native_type_basicsize_includes_payload_padding() { use crate::class::PyClassDef; diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index 97309722a19..68fea76d4ff 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -360,7 +360,7 @@ impl PyAtomicRef { pub unsafe fn swap(&self, pyref: PyRef) -> PyRef { let py = PyRef::leak(pyref) as *const Py as *mut _; let old = Radium::swap(&self.inner, py, Ordering::AcqRel); - unsafe { PyRef::from_raw(old.cast()) } + unsafe { PyRef::::from_raw(old.cast()) } } pub fn swap_to_temporary_refs(&self, pyref: PyRef, vm: &VirtualMachine) { @@ -439,7 +439,11 @@ impl PyAtomicRef> { pub unsafe fn swap(&self, opt_ref: Option>) -> Option> { let val = opt_ref.map_or(null_mut(), |x| PyRef::leak(x) as *const Py as *mut _); let old = Radium::swap(&self.inner, val, Ordering::AcqRel); - unsafe { old.cast::>().as_ref().map(|x| PyRef::from_raw(x)) } + unsafe { + old.cast::>() + .as_ref() + .map(|x| PyRef::::from_raw(x)) + } } pub fn swap_to_temporary_refs(&self, opt_ref: Option>, vm: &VirtualMachine) { diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index 4f028338d4b..1921e3bec67 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -1,3 +1,78 @@ +//! Essential types for object models +//! +//! +-------------------------+--------------+-----------------------+ +//! | Management | Typed | Untyped | +//! +-------------------------+------------------+-------------------+ +//! | Interpreter-independent | [`Py`] | [`PyObject`] | +//! | Reference-counted | [`PyRef`] | [`PyObjectRef`] | +//! | Weak | [`PyWeakRef`] | [`PyRef`] | +//! +-------------------------+--------------+-----------------------+ +//! +//! [`PyRef`] may looking like to be called as PyObjectWeak by the rule, +//! but not to do to remember it is a PyRef object. +//! +//! # Reference ownership +//! +//! `PyObjectRef` is an alias for `PyRef`. Typed payload references, +//! erased references, union views, and typed tuple views share the same owning representation. +//! +//! ## Representation and invariants +//! +//! - `PyRef` stores `NonNull` and a zero-sized +//! `PhantomData>>`. The latter preserves the previous typed pointer's +//! covariance and auto-trait/`Unpin` shape. The transparent owner and its `Option` +//! remain one pointer wide with pointer alignment. +//! - The sealed `object::PyRefTarget` selects `Deref::Target`: payload `T` maps to +//! `Py`, `PyObject` maps to itself, and immutable typed tuples map recursively +//! to their existing typed `Py` view. Downstream safe code cannot register an +//! unrelated layout. `PyObject` does not implement `PyPayload`. +//! - `PyAnySet` is an internal transparent `PyObject` view. Its owner is +//! `PyRef`, whose construction validates set/frozenset (including +//! subclasses). Payload access downcasts to the actual alternative; neither +//! `Py` nor a shared set/frozenset payload layout is assumed. +//! `ArgIterable` instead stores a cached iterator function beside its owner, +//! so it remains a wrapper rather than an allocation view. +//! - [`PyTypedRefTarget`] admits non-erased element targets for typed tuple +//! views, including union views and recursively typed tuples. It does not +//! perform validation: `try_into_typed` checks every element before changing +//! the view. Typed constructors instead receive already-typed owners. +//! All tuple views retain erased element storage with transparent wrappers; +//! typed slices reinterpret only the guaranteed transparent reference +//! elements, never a generic `Vec` or `Box` representation. Private marshal +//! initialization finishes before publication; published elements are immutable. +//! - The owner itself has no new generic bound. `Clone` and `Drop` share one +//! implementation over the existing object header. Object allocation, vtable +//! destruction, weakrefs, GC, and trashcan handling are unchanged. +//! - Typed erasure uses `T: PyRefTarget>`. This excludes the erased +//! type from the conversion blanket, avoiding overlap with Rust's identity +//! `From for T`, while retaining typed and nested tuple conversions. +//! Union views provide their own checked conversions; `PyRef::into_object` +//! erases any owning view without adjusting its strong count. +//! - Ownership transfers use `ManuallyDrop` or `forget`, without an extra incref or +//! decrement. The existing raw APIs retain their ownership-transfer contracts. +//! - C-API typed output uses the same target equality. An erased owner supports +//! `*mut PyObject`, but cannot produce the fictitious `*mut Py` output. +//! Duplicate erased owner/member-layout/FFI implementations are removed. +//! - Feature-dependent `Send`/`Sync` behavior is retained. The non-threading erased +//! owner is neither; the threading owner implements both. +//! +//! ## API compatibility considerations +//! +//! Existing payload `PyRef` and `PyObjectRef` use remains supported, but aliasing +//! makes some inference less direct: +//! +//! - Distinct raw constructor signatures remain: erased +//! `PyObjectRef::from_raw(NonNull)` and typed +//! `PyRef::::from_raw(*const Py)`. Unqualified `PyRef::from_raw` can now be +//! ambiguous and require the explicit type parameter. +//! - Associated-target dereferencing can require an explicit downcast type or a +//! `PyRefTarget` equality bound in generic typed-view code. Arbitrary non-payload +//! views no longer automatically gain typed dereferencing; supported internal +//! views must be registered in the sealed selector. +//! - Rust `Debug` formatting now consistently uses the allocation's vtable rather +//! than the statically selected payload's formatter. This is a Rust diagnostic +//! difference, not a Python semantic assertion. + mod core; mod ext; mod payload; From 8f47ced2c7df646e19e401afba0236ed3482c2ca Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:12:22 +0900 Subject: [PATCH 126/229] Stabilize Windows all-thread traceback reads with stop-the-world (#9092) Use the existing normal dump_traceback stop-the-world path on Windows before traversing raw interpreter frames. Preserve registry-lock release before resuming threads. Signal and watchdog paths are unchanged. Assisted-by: OpenAI Codex:model identifier unavailable Co-authored-by: dot --- crates/stdlib/src/faulthandler.rs | 47 ++----------------------------- 1 file changed, 2 insertions(+), 45 deletions(-) diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 267326933c2..8e4572088bd 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -252,10 +252,10 @@ mod decl { #[cfg(any(unix, windows))] fn dump_all_threads(fd: i32, vm: &VirtualMachine) { // Get all threads' frame stacks from the shared registry - // unix: stop-the-world so every other thread is parked at a safepoint + // Stop the world so every other thread is parked at a safepoint // and its frame chain is quiescent and alive while we walk it (matches // faulthandler.dump_traceback running with the GIL held). - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] { use core::sync::atomic::Ordering; let current_tid = rustpython_vm::stdlib::_thread::get_ident(); @@ -302,49 +302,6 @@ mod decl { dump_live_frames(fd); } - #[cfg(all(not(unix), feature = "threading"))] - { - let current_tid = rustpython_vm::stdlib::_thread::get_ident(); - let registry = vm.state.thread_frames.lock(); - - // Dump non-current threads using top_iframe, which includes - // both FrameObject and stack-allocated frames. - #[expect( - clippy::iter_over_hash_type, - reason = "Iteration order doesn't matter here" - )] - for (&tid, slot) in registry.iter() { - if tid == current_tid { - continue; - } - - let iframe_ptr = slot.top_iframe.load(core::sync::atomic::Ordering::Relaxed) - as *const rustpython_vm::frame::InterpreterFrame; - write_thread_id(fd, tid, false); - if iframe_ptr.is_null() { - puts(fd, " \n"); - } else { - let mut cur = iframe_ptr; - let mut depth = 0; - while !cur.is_null() && depth < 100 { - let iframe = unsafe { &*cur }; - dump_iframe(fd, iframe); - depth += 1; - cur = iframe.previous(); - } - if depth >= 100 && !cur.is_null() { - puts(fd, " ...\n"); - } - } - puts(fd, "\n"); - } - - // Now dump current thread from its live frame chain - // (includes stack-allocated iframes without FrameObject). - write_thread_id(fd, current_tid, true); - dump_live_frames(fd); - } - #[cfg(not(feature = "threading"))] { let _ = vm; From e09846b4eb0ecbbfec41e4448ff5c3f8e90c2125 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Mon, 12 Oct 2026 22:19:27 -0700 Subject: [PATCH 127/229] Fix bytearray membership racing with resizing (#9091) * Fix bytearray membership racing with resizing Search exact bytes and int needles under the data read lock instead of taking a temporary export. This prevents a concurrent slice replacement from receiving BufferError for a callback-free membership operation. Preserve export protection for generic index/buffer callbacks and actual memoryviews. Add Python coverage for these contracts. AI assistance: Codex/OpenAI diagnosed the race, implemented and reviewed the patch and regression snippet, and ran the reported validation. The runtime did not expose an exact model version. Assisted-by: Codex:model-version-unavailable * Handle bool needles and consolidate bytearray regression coverage Include exact builtin bool in the callback-free integer membership path. Keep int subclasses and generic conversion paths under export protection. Move this PR's regression coverage into the existing bytearray snippet and cover True/False membership as requested in review. AI assistance: Codex/OpenAI implemented and independently reviewed the review follow-up and ran the reported validation. The runtime did not expose an exact model version. Assisted-by: Codex:model-version-unavailable --- crates/vm/src/builtins/bytearray.rs | 19 +++++- extra_tests/snippets/builtin_bytearray.py | 83 +++++++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index c463489df61..fc046460b70 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -1,6 +1,6 @@ //! Implementation of the python bytearray object. use super::{ - PositionIterInternal, PyBytes, PyDictRef, PyGenericAlias, PyStrRef, PyTuple, PyTupleRef, + PositionIterInternal, PyBytes, PyDictRef, PyGenericAlias, PyInt, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, iter::builtins_iter, locked_next, }; use crate::{ @@ -12,7 +12,7 @@ use crate::{ bytes_inner::{ ByteInnerFindOptions, ByteInnerHexOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerReplaceOptions, ByteInnerSplitOptions, ByteInnerStripOptions, ByteInnerSub, - ByteInnerTranslateOptions, BytesJoin, DecodeArgs, PyBytesInner, bytes_decode, + ByteInnerTranslateOptions, ByteOr, BytesJoin, DecodeArgs, PyBytesInner, bytes_decode, }, class::PyClassImpl, common::{ @@ -317,6 +317,21 @@ impl PyByteArray { } fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Exact builtins cannot call Python while preparing the needle. The + // read lock alone protects their search; a temporary export would make + // a concurrent resize fail instead of waiting for that lock. + if let Some(bytes) = needle.downcast_ref_if_exact::(vm) { + return Ok(self.borrow_buf().contains_str(bytes.as_bytes())); + } + let int = if needle.class().is(vm.ctx.types.bool_type) { + needle.downcast_ref::() + } else { + needle.downcast_ref_if_exact::(vm) + }; + if let Some(int) = int { + let byte = int.as_bigint().byte_or(vm)?; + return Ok(self.borrow_buf().contains(&byte)); + } // Converting the needle runs Python, which must not resize this bytearray. self.exports.fetch_add(1, Ordering::Release); scopeguard::defer! { self.exports.fetch_sub(1, Ordering::Release); } diff --git a/extra_tests/snippets/builtin_bytearray.py b/extra_tests/snippets/builtin_bytearray.py index 2f2793a76cf..1ea800b64db 100644 --- a/extra_tests/snippets/builtin_bytearray.py +++ b/extra_tests/snippets/builtin_bytearray.py @@ -1504,3 +1504,86 @@ def __index__(self): test_bytearray_callback_exports() + + +def test_bytearray_contains_builtin_and_callback_exports(): + for value in (b"", b"a", b"ab\x00\xff", b"\x01"): + target = bytearray(value) + for needle in ( + b"", + b"a", + b"ab", + b"z", + b"\x00", + b"\xff", + 0, + 97, + 255, + False, + True, + ): + assert (needle in target) == (needle in value) + assert target in target + + def must_raise_buffer_error(action): + try: + action() + except BufferError: + return + raise AssertionError("resize was not rejected") + + target = bytearray(b"a") + view = memoryview(target) + try: + assert b"a" in target + assert 97 in target + must_raise_buffer_error(lambda: target.append(0)) + target[:] = b"b" + assert b"b" in target + assert 98 in target + finally: + view.release() + target.append(0) + assert target == b"b\x00" + + class IndexNeedle: + def __index__(self): + must_raise_buffer_error(lambda: target.append(0)) + target[0] = 98 + return 98 + + class BytesNeedle(bytes): + __index__ = IndexNeedle.__index__ + + for needle in (IndexNeedle(), BytesNeedle(b"\xdd")): + target[:] = b"a" + assert needle in target + assert target == b"b" + target.append(0) + assert target == b"b\x00" + + for needle in (-1, 256): + with assert_raises(ValueError): + needle in target + + events = [0, 0] + + class BufferNeedle: + def __buffer__(self, flags): + must_raise_buffer_error(lambda: target.append(0)) + target[0] = 98 + events[0] += 1 + return memoryview(b"b") + + def __release_buffer__(self, view): + must_raise_buffer_error(lambda: target.append(0)) + events[1] += 1 + + target[:] = b"a" + assert BufferNeedle() in target + assert events == [1, 1] + target.append(0) + assert target == b"b\x00" + + +test_bytearray_contains_builtin_and_callback_exports() From 10cb49430a1f9c73f6ebdce69a982cd470904639 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:47:12 +0900 Subject: [PATCH 128/229] Detach during blocking Windows multiprocessing socket I/O (#9093) Snapshot locked send buffers before waiting and preserve the host-captured Winsock error across VM reattachment. Assisted-by: OpenAI Codex: model identifier unavailable Co-authored-by: dot --- crates/stdlib/src/multiprocessing.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/stdlib/src/multiprocessing.rs b/crates/stdlib/src/multiprocessing.rs index c992f90f737..c9a0790396b 100644 --- a/crates/stdlib/src/multiprocessing.rs +++ b/crates/stdlib/src/multiprocessing.rs @@ -408,19 +408,22 @@ mod _multiprocessing { #[pyfunction] fn recv(socket: usize, size: usize, vm: &VirtualMachine) -> PyResult> { - let mut buf = - host_multiprocessing::recv_socket(socket as host_multiprocessing::RawSocket, size) - .map_err(|_| vm.new_last_os_error())?; + let mut buf = vm + .allow_threads(|| { + host_multiprocessing::recv_socket(socket as host_multiprocessing::RawSocket, size) + }) + .map_err(|error| error.to_pyexception(vm))?; buf.shrink_to_fit(); Ok(buf) } #[pyfunction] fn send(socket: usize, buf: ArgBytesLike, vm: &VirtualMachine) -> PyResult { - buf.with_ref(|b| { - host_multiprocessing::send_socket(socket as host_multiprocessing::RawSocket, b) + let buf = buf.borrow_buf_unlocked(vm)?; + vm.allow_threads(|| { + host_multiprocessing::send_socket(socket as host_multiprocessing::RawSocket, &buf) }) - .map_err(|_| vm.new_last_os_error()) + .map_err(|error| error.to_pyexception(vm)) } } From 7c3c7a73c705c6581c24c8027cb27874bd195368 Mon Sep 17 00:00:00 2001 From: Jiwoo Ahn Date: Tue, 13 Oct 2026 20:47:53 +0900 Subject: [PATCH 129/229] ci: run all passing extra_tests snippets on wasm-wasi via exclude list (#9083) - Problem: WASM_WASI_SNIPPETS only declared 2 of 244 extra_tests/snippets files, so every newly-fixed snippet needed a manual ci.yaml edit to gain wasm-wasi coverage - Fix: loop over extra_tests/snippets/*.py directly and skip only the 57 files that still fail, hang, or are intentionally-failing (WASM_WASI_SNIPPET_EXCLUDE), so fixed and future snippets run automatically - Verified: built wasm32-wasip1 from current upstream/main, ran all 244 snippets; the 187 non-excluded files pass 3x repeated runs with no flakiness (~48s total, well under the 30-minute job budget); the 57 excluded files were categorized (missing modules, threading disabled in this build, 32-bit overflow, no process support on WASI, wasm call-stack limits, socket stub, sandbox path issues, and a couple of individual bugs) Assisted-by: Claude Code:claude-sonnet-5 Signed-off-by: Jiwoo Ahn --- .github/workflows/ci.yaml | 81 +++++++++++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c48bd99a105..8f8783da1f4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,6 +77,7 @@ jobs: ':examples/**' \ ':benches/**' \ ':example_projects/**' \ + ':extra_tests/snippets/**' \ ':.github/workflows/ci.yaml' \ ; then echo "changed=false" >> "$GITHUB_OUTPUT" @@ -1009,10 +1010,71 @@ jobs: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true RUN_WASM_WASI: ${{ needs.determine_changes.outputs.rust_code == 'true' || github.ref == 'refs/heads/main' }} - # Passing snippets under wasm32-wasip1. - WASM_WASI_SNIPPETS: >- - stdlib_random - stdlib_time + # extra_tests/snippets files that still fail or hang under wasm32-wasip1; + # every other snippet is discovered and run automatically. + WASM_WASI_SNIPPET_EXCLUDE: >- + builtin_bytearray + builtin_bytes + builtin_hash + builtin_int + builtin_iter + builtin_list + builtin_map + builtin_memoryview + builtin_open + builtin_posixshmem + builtin_slice + builtin_str + builtin_thread + cli_int_max_str_digits + protocol_index_argument + recursion + stdlib_ast + stdlib_asyncio + stdlib_collections_deque + stdlib_contextvars + stdlib_ctypes + stdlib_ctypes_byvalue + stdlib_ctypes_calls + stdlib_ctypes_from_buffer + stdlib_genericalias + stdlib_io + stdlib_io_blocking_buffer + stdlib_itertools + stdlib_json + stdlib_mmap + stdlib_multiprocessing_condition + stdlib_multiprocessing_fork_sharer + stdlib_multiprocessing_sharedctypes + stdlib_os + stdlib_pwd + stdlib_queue_signal + stdlib_select + stdlib_socket + stdlib_sqlite + stdlib_ssl + stdlib_ssl_short_recv + stdlib_struct + stdlib_subinterpreters + stdlib_subprocess + stdlib_sys + stdlib_sys_audit + stdlib_threading_current_frames + stdlib_threading_gc_frame_race + stdlib_threading_gc_import + stdlib_threading_generator + stdlib_threading_set_repr + stdlib_threading_type_cache + stdlib_thread_native_id + stdlib_types + stdlib_typing + stdlib_urllib_https_misaligned_recv + stdlib_zoneinfo_cache + syntax_async + syntax_async_comprehension + syntax_deep_nesting + vm_frame_locals + xfail_assert # Passing Lib/test files under wasm32-wasip1 WASM_WASI_TEST_FILES: >- test_int @@ -1075,9 +1137,14 @@ jobs: - name: run snippets if: env.RUN_WASM_WASI == 'true' run: | - for name in $WASM_WASI_SNIPPETS; do - echo "::group::extra_tests/snippets/${name}.py" - wasmer run --volume "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/${name}.py" + exclude=" $WASM_WASI_SNIPPET_EXCLUDE " + for path in extra_tests/snippets/*.py; do + name=$(basename "$path" .py) + case "$exclude" in + *" $name "*) continue ;; + esac + echo "::group::${path}" + wasmer run --volume "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/${path}" echo "::endgroup::" done - name: run cpython unittest From 0803a55df573e8db5ce8aa941d0562df1161194b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:50:13 +0900 Subject: [PATCH 130/229] Convert native base references without consulting the Python class (#9090) PyRef::into_base and Py::to_base cast the pointer directly instead of going through a class-based downcast. into_base no longer reaches unreachable_unchecked when the object's MRO omits the native base, and to_base drops the equivalent debug assertion. Extracted from #8963. Assisted-by: Codex:gpt-6 Assisted-by: Claude:claude-opus-5-5 Co-authored-by: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> --- crates/vm/src/object/core.rs | 48 ++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index d22d6ef7475..9cd52c32d3b 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -2904,16 +2904,14 @@ where T::Base: core::fmt::Debug, { /// Converts this reference to the base type (ownership transfer). - /// # Safety - /// T and T::Base must have compatible layouts in size_of::() bytes. #[inline] #[must_use] pub fn into_base(self) -> PyRef { - let obj: PyObjectRef = self.into(); - match obj.downcast() { - Ok(base_ref) => base_ref, - Err(_) => unsafe { core::hint::unreachable_unchecked() }, - } + // PySubclass establishes the physical prefix; Python may have changed + // the object's class or MRO since this typed reference was constructed. + let ptr = self.ptr.cast(); + core::mem::forget(self); + PyRef { ptr } } #[inline] #[must_use] @@ -2934,9 +2932,8 @@ impl Py { /// Converts `&Py` to `&Py`. #[inline] pub fn to_base(&self) -> &Py { - debug_assert!(self.as_object().downcast_ref::().is_some()); - // SAFETY: T is #[repr(transparent)] over T::Base, - // so Py and Py have the same layout. + // SAFETY: PySubclass guarantees a valid base prefix with matching + // payload offsets and compatible object alignment. unsafe { &*(self as *const Self as *const Py) } } @@ -3410,6 +3407,37 @@ mod tests { } } + #[test] + fn native_base_conversions_ignore_python_mro() { + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let class = vm.ctx.new_class( + None, + "HiddenOSErrorBase", + vm.ctx.exceptions.os_error.to_owned(), + Default::default(), + ); + let marker: PyObjectRef = vm.ctx.new_int(42).into(); + let error = vm.new_os_subtype_error(class.clone(), None, marker.clone()); + // A custom MRO can omit native bases without changing the allocation. + *class.mro.write() = vec![class.clone(), vm.ctx.types.object_type.to_owned()]; + assert!( + !error + .as_object() + .downcastable::() + ); + + assert!(error.to_base().to_base().get_arg(0).unwrap().is(&marker)); + assert!( + error + .into_base() + .into_base() + .get_arg(0) + .unwrap() + .is(&marker) + ); + }); + } + #[test] fn native_type_basicsize_includes_payload_padding() { use crate::class::PyClassDef; From 723dee027eb119a50f13c41d60e40e9b36afca41 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:58:28 +0900 Subject: [PATCH 131/229] Initialize PyRef marker in native base conversion (#9096) Fix the missing field in the new into_base initializer without changing its pointer or ownership transfer. Assisted-by: OpenAI Codex: model identifier unavailable Co-authored-by: dot --- crates/vm/src/object/core.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 9cd52c32d3b..8dacc13d303 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -2911,7 +2911,10 @@ where // the object's class or MRO since this typed reference was constructed. let ptr = self.ptr.cast(); core::mem::forget(self); - PyRef { ptr } + PyRef { + ptr, + marker: PhantomData, + } } #[inline] #[must_use] From bbb0f78e972d171f0659fa9d0d85edc12018239b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:54:42 +0900 Subject: [PATCH 132/229] build(deps): bump j178/prek-action from 3.0.0 to 3.0.1 (#9097) Bumps [j178/prek-action](https://github.com/j178/prek-action) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/4e14d07f9231acabce116ccfca13b13dd9755ece...1382ad72dea72f9d53832f2bcb65b3e87a947204) --- updated-dependencies: - dependency-name: j178/prek-action dependency-version: 3.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8f8783da1f4..6d192b432c4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -776,7 +776,7 @@ jobs: - name: install prek id: prek - uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 + uses: j178/prek-action@1382ad72dea72f9d53832f2bcb65b3e87a947204 # v3.0.1 with: cache: false show-verbose-logs: false From 2aacf9dba91f39e59a776e3b6bcbf1f0f5bcc21e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:54:54 +0900 Subject: [PATCH 133/229] build(deps): bump reviewdog/action-actionlint from 1.77.0 to 1.79.1 (#9098) Bumps [reviewdog/action-actionlint](https://github.com/reviewdog/action-actionlint) from 1.77.0 to 1.79.1. - [Release notes](https://github.com/reviewdog/action-actionlint/releases) - [Commits](https://github.com/reviewdog/action-actionlint/compare/2085657ab2c7f48c58edcc767fba576f63bea76b...1758727be30241166609dad4c97588b6ad411f32) --- updated-dependencies: - dependency-name: reviewdog/action-actionlint dependency-version: 1.79.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6d192b432c4..b4fd35cfa24 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -763,7 +763,7 @@ jobs: components: rustfmt - name: actionlint - uses: reviewdog/action-actionlint@2085657ab2c7f48c58edcc767fba576f63bea76b # v1.77.0 + uses: reviewdog/action-actionlint@1758727be30241166609dad4c97588b6ad411f32 # v1.79.1 - name: zizmor uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 From 10b1110608543b8cf8952d1f7ecfd756fa2a1d66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:33:48 +0900 Subject: [PATCH 134/229] build(deps): bump https://github.com/astral-sh/ruff-pre-commit (#9102) Bumps [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit) from v0.16.9 to 0.16.10. - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.16.9...v0.16.10) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.16.10 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a65592fac6c..bf511f5e656 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: priority: 0 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.9 + rev: v0.16.10 hooks: - id: ruff-format priority: 0 From 29e3531295d024557a6528807eebcef1202bdce3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:34:20 +0900 Subject: [PATCH 135/229] build(deps): bump https://github.com/streetsidesoftware/cspell-cli (#9103) Bumps [https://github.com/streetsidesoftware/cspell-cli](https://github.com/streetsidesoftware/cspell-cli) from v10.2.0 to 10.3.0. - [Release notes](https://github.com/streetsidesoftware/cspell-cli/releases) - [Changelog](https://github.com/streetsidesoftware/cspell-cli/blob/main/CHANGELOG.md) - [Commits](https://github.com/streetsidesoftware/cspell-cli/compare/v10.2.0...v10.3.0) --- updated-dependencies: - dependency-name: https://github.com/streetsidesoftware/cspell-cli dependency-version: 10.3.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf511f5e656..a78da8d2e6a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,7 +63,7 @@ repos: - manual - repo: https://github.com/streetsidesoftware/cspell-cli - rev: v10.2.0 + rev: v10.3.0 hooks: - id: cspell types: [rust] From c0c6306233374e2be26fd6cbf2f546027128e140 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:35:14 +0900 Subject: [PATCH 136/229] build(deps): bump github/gh-aw/actions/setup from 0.89.21 to 0.91.3 (#9099) Bumps [github/gh-aw/actions/setup](https://github.com/github/gh-aw) from 0.89.21 to 0.91.3. - [Release notes](https://github.com/github/gh-aw/releases) - [Changelog](https://github.com/github/gh-aw/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/gh-aw/compare/c35393777e5604a63721d09512263b1383301d4f...382f4b025fefaa7d6b2492c3608f098340bf6536) --- updated-dependencies: - dependency-name: github/gh-aw/actions/setup dependency-version: 0.91.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/upgrade-pylib.lock.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml index 6fa651d5035..dfdc331c8a1 100644 --- a/.github/workflows/upgrade-pylib.lock.yml +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -58,7 +58,7 @@ jobs: comment_repo: "" steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 + uses: github/gh-aw/actions/setup@382f4b025fefaa7d6b2492c3608f098340bf6536 # v0.91.3 with: destination: /opt/gh-aw/actions - name: Check workflow file timestamps @@ -99,7 +99,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 + uses: github/gh-aw/actions/setup@382f4b025fefaa7d6b2492c3608f098340bf6536 # v0.91.3 with: destination: /opt/gh-aw/actions - name: Checkout repository @@ -804,7 +804,7 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 + uses: github/gh-aw/actions/setup@382f4b025fefaa7d6b2492c3608f098340bf6536 # v0.91.3 with: destination: /opt/gh-aw/actions - name: Download agent output artifact @@ -925,7 +925,7 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 + uses: github/gh-aw/actions/setup@382f4b025fefaa7d6b2492c3608f098340bf6536 # v0.91.3 with: destination: /opt/gh-aw/actions - name: Download agent artifacts @@ -1037,7 +1037,7 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@c35393777e5604a63721d09512263b1383301d4f # v0.89.21 + uses: github/gh-aw/actions/setup@382f4b025fefaa7d6b2492c3608f098340bf6536 # v0.91.3 with: destination: /opt/gh-aw/actions - name: Download agent output artifact From e157f645a2a130bf2b3de3fc0117ac843af9dd3a Mon Sep 17 00:00:00 2001 From: Leesoo Ahn Date: Wed, 14 Oct 2026 09:36:18 +0900 Subject: [PATCH 137/229] Fix grp.getgrnam raising OSError for unknown groups (#9101) Some NSS modules (e.g. sss, systemd) return ENOENT from getgrnam_r/getgrgid_r for a missing group. Treat any lookup error as "not found" and raise KeyError, matching CPython. This fixes test_shutil's test_chown and test_make_archive_owner_group. Assisted-by: Antigravity:claude-opus-5-5 --- crates/stdlib/src/grp.rs | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/crates/stdlib/src/grp.rs b/crates/stdlib/src/grp.rs index 39e5285f6e2..fe04813964a 100644 --- a/crates/stdlib/src/grp.rs +++ b/crates/stdlib/src/grp.rs @@ -6,7 +6,7 @@ mod grp { use crate::vm::{ PyObjectRef, PyResult, VirtualMachine, builtins::{PyListRef, PyUtf8StrRef}, - convert::{IntoPyException, ToPyObject}, + convert::ToPyObject, exceptions, function::ArgIndex, types::PyStructSequence, @@ -54,17 +54,14 @@ mod grp { let gr_gid = gid.as_bigint(); let gid = host_grp::gid_t::try_from(gr_gid).ok(); let group = gid - .map(host_grp::getgrgid) - .transpose() - .map_err(|err| err.into_pyexception(vm))? - .flatten(); - let group = group.ok_or_else(|| { - vm.new_key_error( - vm.ctx - .new_str(format!("getgrgid: group id {gr_gid} not found")) - .into(), - ) - })?; + .and_then(|gid| host_grp::getgrgid(gid).ok().flatten()) + .ok_or_else(|| { + vm.new_key_error( + vm.ctx + .new_str(format!("getgrgid: group id {gr_gid} not found")) + .into(), + ) + })?; Ok(GroupData::from_group(group, vm)) } @@ -82,8 +79,7 @@ mod grp { return Err(exceptions::nul_char_error(vm)); } let gr_name = name.as_str(); - let group = host_grp::getgrnam(gr_name).map_err(|err| err.into_pyexception(vm))?; - let group = group.ok_or_else(|| { + let group = host_grp::getgrnam(gr_name).ok().flatten().ok_or_else(|| { vm.new_key_error( vm.ctx .new_str(format!("getgrnam: group name {gr_name} not found")) From e61d23b32e4fc3deae6f8d7cc985810ebddc005d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:42:31 +0900 Subject: [PATCH 138/229] Serialize stop-the-world transactions across interpreters (#9094) * Serialize stop-the-world transactions across interpreters Coordinate individual stop requests with process-wide collection, retain snapshot owners through rollback, and release transferred ownership after restart. Assisted-by: OpenAI Codex: model identifier unavailable * Reject reentrant stop-the-world transactions Use non-reused logical requester identities to fail before waiting on the same thread, while preserving cross-thread restart and fork ownership. Assisted-by: OpenAI Codex: model identifier unavailable --------- Co-authored-by: dot --- crates/vm/src/gc_state.rs | 45 +++++++---- crates/vm/src/vm/mod.rs | 158 ++++++++++++++++++++++++-------------- 2 files changed, 129 insertions(+), 74 deletions(-) diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index 774f2ff39ba..d78d9c1b932 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -207,22 +207,24 @@ type GcMap = std::collections::HashMap; /// incref. Reading a refcount that another interpreter is changing is what /// makes an object look unreachable when it is not, so every live interpreter /// is stopped, not just the collecting one. Stopping in `runtime` id order -/// keeps exclusion acquisition ordered; the `collecting` mutex additionally -/// serializes collections process-wide, so no second collector can take these -/// exclusions in another order. +/// is deterministic. One process-wide STW transaction covers the entire group, +/// preventing a single-interpreter requester from stopping a collector while +/// that collector waits for a different interpreter. #[cfg(feature = "threading")] struct CollectStopTheWorld { - /// Stopped interpreter states, in stop order. Held as strong references so - /// an interpreter cannot be dropped between stop and restart, and kept past - /// the restart so that releasing the last one — which frees that - /// interpreter's objects, and so removes them from these lists — happens - /// after the collection has let go of the generation locks. - stopped: Vec>, + /// Entire interpreter snapshot, retained through restart even on unwind. + /// Dropping the last state can release Python objects and run finalizers, + /// so both stopped and not-yet-stopped states must outlive the transaction + /// and the collection's generation locks. + states: Vec>, + stopped_count: usize, /// Keeps interpreters from registering between the snapshot below and the - /// restart. One registered in that window would be missing from `stopped`, + /// restart. One registered in that window would be missing from `states`, /// so its bootstrap would keep running — and mutating the shared generation /// lists — while this collection reads them. admission: Option>, + /// Acquired before admission and retained until every state is restarted. + transaction: Option, restarted: bool, } @@ -237,8 +239,10 @@ impl CollectStopTheWorld { // keep the historical no-barrier fallback. if !crate::vm::thread::current_vm_is_set() { return Self { - stopped: Vec::new(), + states: Vec::new(), + stopped_count: 0, admission: None, + transaction: None, restarted: true, }; } @@ -247,14 +251,20 @@ impl CollectStopTheWorld { // `stop_the_world` unwinds, dropping this guard restarts the // interpreters already stopped, instead of leaving their threads parked // and their exclusion held forever. + let transaction = crate::vm::StopTheWorldTransaction::acquire(); let mut guard = Self { - stopped: Vec::new(), + states: Vec::new(), + stopped_count: 0, admission: Some(crate::vm::runtime::lock_admission_for_stop()), + transaction: Some(transaction), restarted: false, }; - for state in crate::vm::runtime::live_interpreter_states() { - state.stop_the_world.stop_the_world(&state); - guard.stopped.push(state); + guard.states = crate::vm::runtime::live_interpreter_states(); + for state in &guard.states { + state + .stop_the_world + .stop_with_transaction(state, guard.transaction.as_ref().unwrap()); + guard.stopped_count += 1; } guard } @@ -267,17 +277,18 @@ impl CollectStopTheWorld { self.restarted = true; // Reverse of the stop order. The references stay until this guard is // dropped; see the field comment. - for state in self.stopped.iter().rev() { + for state in self.states[..self.stopped_count].iter().rev() { state.stop_the_world.start_the_world(state); } // Nothing is parked any more, so registration may resume. self.admission = None; + self.transaction = None; } /// Whether this collection actually stopped the world. #[cfg(all(unix, debug_assertions))] fn is_stopped(&self) -> bool { - !self.stopped.is_empty() + self.stopped_count != 0 } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 0c5f8e2b088..28d50958a6a 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -189,6 +189,68 @@ impl Default for ExceptionStack { } } +/// Serializes complete stop/restart transactions across interpreters. +/// A collector holds one permit while stopping every interpreter; other callers +/// transfer their permit to the stopped state until the matching restart. +#[cfg(feature = "threading")] +pub(crate) struct StopTheWorldTransaction { + _private: (), +} + +#[cfg(feature = "threading")] +static STW_TRANSACTION: AtomicU64 = AtomicU64::new(0); + +#[cfg(feature = "threading")] +static NEXT_STW_REQUESTER: AtomicU64 = AtomicU64::new(1); + +#[cfg(feature = "threading")] +thread_local! { + // OS thread IDs can be reused after a requester exits, even while another + // thread still owes its matching restart. Logical IDs are never reused. + static STW_REQUESTER: u64 = NEXT_STW_REQUESTER + .try_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1)) + .expect("stop-the-world requester IDs exhausted"); +} + +#[cfg(feature = "threading")] +impl StopTheWorldTransaction { + pub(crate) fn acquire() -> Self { + let requester = STW_REQUESTER.with(|id| *id); + loop { + match STW_TRANSACTION.compare_exchange( + 0, + requester, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => return Self { _private: () }, + Err(owner) => assert_ne!( + owner, requester, + "stop-the-world transactions cannot be nested on the same thread" + ), + } + let wait = || { + while STW_TRANSACTION.load(Ordering::Acquire) != 0 { + std::thread::yield_now(); + } + }; + // Wait detached from the CURRENT interpreter, not the interpreter + // we intend to stop. Reattach without owning the permit, then retry: + // a suspended or shutting-down waiter must not strand ownership. + if thread::try_with_current_vm(|vm| vm.allow_threads(wait)).is_none() { + wait(); + } + } + } +} + +#[cfg(feature = "threading")] +impl Drop for StopTheWorldTransaction { + fn drop(&mut self) { + STW_TRANSACTION.store(0, Ordering::Release); + } +} + /// Stop-the-world state for fork safety. Before `fork()`, the requester /// stops all other Python threads so they are not holding internal locks. #[cfg(feature = "threading")] @@ -199,10 +261,9 @@ pub struct StopTheWorldState { world_stopped: AtomicBool, /// Ident of the thread that requested the stop (like `stw->requester`) requester: AtomicU64, - /// Single exclusion held for the whole stop→start span. Fork and GC are - /// both stop-the-world requesters driving this shared state; only one may - /// hold it at a time. Acquired before any stop bookkeeping (see - /// `acquire_exclusion`) and released by `start_the_world`/`reset_after_fork`. + /// Ownership transferred by a successful public stop. GC retains its + /// transaction externally and leaves this false. Only a matching restart + /// or the forking owner's child reset may release this ownership. exclusion: AtomicBool, /// Signaled by suspending threads when their state transitions to SUSPENDED notify_mutex: std::sync::Mutex<()>, @@ -401,50 +462,11 @@ impl StopTheWorldState { forced_parks != 0 && self.thread_countdown.load(Ordering::Acquire) == 0 } - /// Acquire the single stop-the-world exclusion in a park-friendly way. - /// - /// Fork and GC both request stop-the-world through the same shared state; - /// without this exclusion their `requester`/`requested`/countdown words - /// could be clobbered by an interleaving requester, so the completion - /// check could never converge and a requester would wait on itself forever. - /// - /// The acquire must be park-friendly. While another requester's stop is in - /// progress it sets this thread's stop bit and waits for it to suspend; - /// blocking on a plain lock here would keep this thread from ever reaching - /// that safepoint, so the active requester would wait for this thread while - /// this thread waits for the lock — a deadlock swap. Instead we poll and - /// honor the suspend request between tries. Suspending here is safe as long - /// as any lock a spinning requester still holds is never acquired - /// attached-blocking by another thread. The fork requester holds IMP_LOCK, - /// but its acquisition detaches (`allow_threads`), so no attached thread - /// blocks on it; the GC requester holds only the `collecting` mutex, which - /// is only ever `try_lock`'d. The active requester therefore force-parks - /// this thread, finishes its whole stop→start span, releases the exclusion, - /// and only then does this thread resume and acquire it. - fn acquire_exclusion(&self, state: &PyGlobalState) { - if self - .exclusion - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { - return; - } - loop { - crate::vm::thread::suspend_if_needed(state); - std::thread::yield_now(); - if self - .exclusion - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { - return; - } - } - } - - /// Release the stop-the-world exclusion taken by `acquire_exclusion`. + /// Release only a transaction owned by this state, after every wake/clear. fn release_exclusion(&self) { - self.exclusion.store(false, Ordering::Release); + if self.exclusion.swap(false, Ordering::AcqRel) { + drop(StopTheWorldTransaction { _private: () }); + } } /// Stop all non-requester threads (`stop_the_world`). @@ -454,11 +476,27 @@ impl StopTheWorldState { /// 3. Waits (polling with 1 ms condvar timeout) for attached threads /// to self-suspend in `check_signals`. /// - /// Takes the shared exclusion first so at most one requester (fork or GC) - /// drives the stop→start span at a time; it is released by - /// `start_the_world`/`reset_after_fork`. + /// Public calls own one process-wide transaction until the matching start. + /// The caller must finish stop before invoking start, including when those + /// calls occur on different threads; concurrent duplicate starts are invalid. + /// Transactions are non-reentrant: a nested stop on the requesting thread + /// panics before changing any stop state or releasing the outer transaction. pub fn stop_the_world(&self, state: &PyGlobalState) { - self.acquire_exclusion(state); + let transaction = StopTheWorldTransaction::acquire(); + self.stop_with_transaction(state, &transaction); + self.exclusion.store(true, Ordering::Release); + core::mem::forget(transaction); + } + + /// Stop one state within a transaction retained by the caller (notably GC). + pub(crate) fn stop_with_transaction( + &self, + state: &PyGlobalState, + _transaction: &StopTheWorldTransaction, + ) { + // A partial stop must clear requests and wake parked slots before its + // caller releases the transaction or rolls back previously stopped VMs. + let rollback = scopeguard::guard((), |_| self.resume_threads(state, false)); let start = std::time::Instant::now(); let requester_ident = crate::stdlib::_thread::get_ident(); self.requester.store(requester_ident, Ordering::Relaxed); @@ -478,6 +516,7 @@ impl StopTheWorldState { stw_trace(format_args!( "stop end requester={requester_ident} wait_ns=0 polls=0" )); + scopeguard::ScopeGuard::into_inner(rollback); return; } @@ -526,10 +565,17 @@ impl StopTheWorldState { stw_trace(format_args!( "stop end requester={requester_ident} wait_ns={wait_ns} polls={polls}" )); + scopeguard::ScopeGuard::into_inner(rollback); } /// Resume all suspended threads (`start_the_world`). pub fn start_the_world(&self, state: &PyGlobalState) { + self.resume_threads(state, true); + self.release_exclusion(); + } + + /// Rollback also permits threads that never reached SUSPENDED. + fn resume_threads(&self, state: &PyGlobalState, fully_stopped: bool) { use thread::ThreadState; let requester = self.requester.load(Ordering::Relaxed); stw_trace(format_args!("start begin requester={requester}")); @@ -560,7 +606,7 @@ impl StopTheWorldState { continue; } debug_assert!( - state == ThreadState::Suspended as i32, + !fully_stopped || state == ThreadState::Suspended as i32, "non-requester thread not suspended at start-the-world: id={id} state={state}" ); if state == ThreadState::Suspended as i32 { @@ -579,9 +625,6 @@ impl StopTheWorldState { crate::signal::clear_stop_bit(); #[cfg(debug_assertions)] self.debug_assert_all_non_requester_detached(state); - // Release the exclusion last, ending the stop→start span so the next - // requester (fork or GC) can proceed. - self.release_exclusion(); stw_trace(format_args!("start end requester={requester}")); } @@ -595,8 +638,9 @@ impl StopTheWorldState { // Only one thread survives fork; any stop-the-world bit inherited // from the parent is stale. crate::signal::clear_stop_bit(); - // The surviving child thread inherited the exclusion taken by the - // pre-fork `stop_the_world`; release it (no start_the_world runs here). + // before_fork owns the sole transaction. Release it exactly once via + // that state's ownership flag; resetting other interpreters must not + // release a later transaction. No unconditional global reset is needed. self.release_exclusion(); stw_trace(format_args!("reset-after-fork")); } From bef501fcd113513c98a444d83ebfeb4ccaa40a5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:53:11 +0900 Subject: [PATCH 139/229] build(deps): bump taiki-e/install-action from 2.87.21 to 2.87.26 (#9100) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.87.21 to 2.87.26. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/4cef1412cce204788f482e778a0b9187f9626a29...f7e5d7c961414b23f5b25b2da9294395d08513ad) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.87.26 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- .github/workflows/cron-ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b4fd35cfa24..c5cccd14703 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -734,7 +734,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 + - uses: taiki-e/install-action@f7e5d7c961414b23f5b25b2da9294395d08513ad # v2.87.26 with: tool: cargo-shear fallback: none diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index f5e7bddc29e..96c175fed65 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -33,7 +33,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 + - uses: taiki-e/install-action@f7e5d7c961414b23f5b25b2da9294395d08513ad # v2.87.26 with: tool: cargo-llvm-cov From 28393e932d46ac4df4e55ff1e393d3f4dc231bac Mon Sep 17 00:00:00 2001 From: youknowdot Date: Wed, 14 Oct 2026 00:42:56 -0700 Subject: [PATCH 140/229] Implement PEP 814 frozendict (#8951) * Restore consolidated frozendict implementation on current main Preserve reverse dictionary iterator fixes and the 3.15 frame imports. Keep library refreshes separate pending the runtime dependency sequencing decision. Assisted-by: OpenAI Codex:model identifier unavailable * Support scoped canonical test updates in update_lib Index conditional test classes and methods without crossing namespaces, reject ambiguous targets, and preserve inherited override indentation. Add hash-pinned whole-test copying that verifies execution context and exact source bytes, with focused Python fixtures. Assisted-by: OpenAI Codex:model identifier unavailable * Align frozendict copy support and tests with CPython 3.15 Import exact-final copy.py with update_lib; defer Mapping ABC registration to the later library stage. Generate only proven marker corrections, retain observed debug recursion-limit expected failures, and copy the complete canonical builtin-doc discovery test using hash-pinned update_lib copy-test. Assisted-by: OpenAI Codex:model identifier unavailable * Preserve decorator rationale during canonical test copying Reject differing decorator source blocks even when their ASTs match. Cover inline and multiline marker comments, guarded async methods, and CLI no-write behavior. Assisted-by: OpenAI Codex:model identifier unavailable * Finish frozendict with borrowed PyAnyDict views Use the landed PyRef target model for a transparent dict/frozendict header view, retaining objects explicitly with PyRef. Keep targets sealed and preserve mutable-only entry points. Borrow frame globals from the object header rather than movable reference wrappers. Unify exact-TypeError key diagnostics with final CPython semantics, including callback failure propagation and no explicit cause. Keep the Python regressions in builtin_dict and test reference layout/ownership in Rust. Preserve existing canonical library updates. Assisted-by: OpenAI Codex:model identifier unavailable * Align frozendict with CPython and use CPython 3.15 tests - Name the target type in unhashable key errors through PyDict::update_arg (dict_update_arg), replacing the separate PyFrozenDict::merge_into copy of the merge logic. - Split the | operator into dict_or (_PyDict_Or) and frozendict_or. - Return the argument from frozendict(frozendict) only when the called type is frozendict. - Call through the vectorcall slot in PyCallable::invoke when there are no keyword arguments, instead of special-casing frozendict. - Add ensure_builtins (_PyEval_EnsureBuiltins) for eval/exec, and insert __builtins__ for source strings regardless of globals type. - Read __warningregistry__ and __name__ from stored globals entries and accept only None or str for the module name in setup_context. - Remove the unused PyFrozenDict::from_object. - Migrate test_dict.py, mapping_tests.py, test_copy.py and test_builtin.py to v3.15.0 with update_lib, and add test_import/data/syntax_warnings.py. - Drop builtin_dict.py frozendict tests covered by those modules, keeping the descriptor and namespace integration tests. Assisted-by: Claude Code:claude-opus-5-5 * Update crates/vm/src/builtins/dict.rs --------- Co-authored-by: Jeong YunWon Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- Lib/copy.py | 35 +- Lib/test/mapping_tests.py | 133 +- Lib/test/pickletester.py | 4 - Lib/test/test_builtin.py | 229 +++- Lib/test/test_copy.py | 46 +- Lib/test/test_dict.py | 276 ++++- Lib/test/test_doctest/test_doctest.py | 2 +- Lib/test/test_dynamic.py | 2 +- Lib/test/test_import/data/syntax_warnings.py | 21 + Lib/test/test_pickle.py | 12 + .../test_unittest/testmock/testhelpers.py | 1 - crates/capi/src/ceval.rs | 13 +- crates/capi/src/dictobject.rs | 114 +- crates/capi/src/pyframe.rs | 4 +- crates/vm/src/builtins/dict.rs | 1073 ++++++++++++++--- crates/vm/src/builtins/frame.rs | 4 +- crates/vm/src/builtins/function.rs | 32 +- crates/vm/src/builtins/mappingproxy.rs | 6 + crates/vm/src/builtins/mod.rs | 2 +- crates/vm/src/builtins/type.rs | 45 +- crates/vm/src/frame.rs | 105 +- crates/vm/src/function/protocol.rs | 10 +- crates/vm/src/object/core.rs | 6 + crates/vm/src/protocol/callable.rs | 5 + crates/vm/src/protocol/mapping.rs | 6 +- crates/vm/src/scope.rs | 15 +- crates/vm/src/stdlib/_testinternalcapi.rs | 38 +- crates/vm/src/stdlib/builtins.rs | 62 +- crates/vm/src/types/zoo.rs | 2 + crates/vm/src/vm/crossinterp.rs | 11 +- crates/vm/src/vm/mod.rs | 6 +- crates/vm/src/vm/vm_new.rs | 7 +- crates/vm/src/warn.rs | 13 +- extra_tests/snippets/builtin_dict.py | 294 +++++ scripts/update_lib/__main__.py | 11 + scripts/update_lib/cmd_copy_test.py | 134 ++ scripts/update_lib/patch_spec.py | 73 +- scripts/update_lib/tests/test_copy_test.py | 206 ++++ scripts/update_lib/tests/test_patch_spec.py | 132 ++ src/shell/helper.rs | 8 +- 40 files changed, 2705 insertions(+), 493 deletions(-) create mode 100644 Lib/test/test_import/data/syntax_warnings.py create mode 100644 scripts/update_lib/cmd_copy_test.py create mode 100644 scripts/update_lib/tests/test_copy_test.py diff --git a/Lib/copy.py b/Lib/copy.py index c64fc076179..6149301ad13 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -100,14 +100,14 @@ def copy(x): return _reconstruct(x, None, *rv) -_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple, - bytes, frozenset, type, range, slice, property, +_copy_atomic_types = frozenset({types.NoneType, int, float, bool, complex, str, tuple, + bytes, frozendict, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, - weakref.ref, super} -_copy_builtin_containers = {list, dict, set, bytearray} + weakref.ref, super}) +_copy_builtin_containers = frozenset({list, dict, set, bytearray}) -def deepcopy(x, memo=None, _nil=[]): +def deepcopy(x, memo=None): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. @@ -122,8 +122,8 @@ def deepcopy(x, memo=None, _nil=[]): if memo is None: memo = {} else: - y = memo.get(d, _nil) - if y is not _nil: + y = memo.get(d, None) + if y is not None: return y copier = _deepcopy_dispatch.get(cls) @@ -162,9 +162,9 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y -_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, +_atomic_types = frozenset({types.NoneType, types.EllipsisType, types.NotImplementedType, int, float, bool, complex, bytes, str, types.CodeType, type, range, - types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property}) _deepcopy_dispatch = d = {} @@ -203,6 +203,21 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy): return y d[dict] = _deepcopy_dict +def _deepcopy_frozendict(x, memo, deepcopy=deepcopy): + y = {} + for key, value in x.items(): + y[deepcopy(key, memo)] = deepcopy(value, memo) + + # We're not going to put the frozendict in the memo, but it's still + # important we check for it, in case the frozendict contains recursive + # mutable structures. + try: + return memo[id(x)] + except KeyError: + pass + return frozendict(y) +d[frozendict] = _deepcopy_frozendict + def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) d[types.MethodType] = _deepcopy_method @@ -230,7 +245,7 @@ def _reconstruct(x, memo, func, args, *, deepcopy=deepcopy): deep = memo is not None if deep and args: - args = (deepcopy(arg, memo) for arg in args) + args = [deepcopy(arg, memo) for arg in args] y = func(*args) if deep: memo[id(x)] = y diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 1358200add5..ae2fb3f5f44 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -4,7 +4,7 @@ from test import support -class BasicTestMappingProtocol(unittest.TestCase): +class BasicTestImmutableMappingProtocol(unittest.TestCase): # This base class can be used to check that an object conforms to the # mapping protocol @@ -20,12 +20,9 @@ def _empty_mapping(self): """Return an empty mapping object""" return self.type2test() def _full_mapping(self, data): - """Return a mapping object with the value contained in data + """Return a mapping object with the values contained in data dictionary""" - x = self._empty_mapping() - for key, value in data.items(): - x[key] = value - return x + return self.type2test(data) def __init__(self, *args, **kw): unittest.TestCase.__init__(self, *args, **kw) @@ -88,6 +85,72 @@ def check_iterandlist(iter, lst, ref): self.assertEqual(d.get(knownkey, knownvalue), knownvalue) self.assertNotIn(knownkey, d) + def test_constructor(self): + self.assertEqual(self._empty_mapping(), self._empty_mapping()) + + def test_bool(self): + self.assertTrue(not self._empty_mapping()) + self.assertTrue(self.reference) + self.assertFalse(bool(self._empty_mapping())) + self.assertTrue(bool(self.reference)) + + def test_keys(self): + d = self._empty_mapping() + self.assertEqual(list(d.keys()), []) + d = self.reference + self.assertIn(list(self.inmapping.keys())[0], d.keys()) + self.assertNotIn(list(self.other.keys())[0], d.keys()) + self.assertRaises(TypeError, d.keys, None) + + def test_values(self): + d = self._empty_mapping() + self.assertEqual(list(d.values()), []) + + self.assertRaises(TypeError, d.values, None) + + def test_items(self): + d = self._empty_mapping() + self.assertEqual(list(d.items()), []) + + self.assertRaises(TypeError, d.items, None) + + def test_len(self): + d = self._empty_mapping() + self.assertEqual(len(d), 0) + + def test_getitem(self): + d = self.reference + self.assertEqual(d[list(self.inmapping.keys())[0]], + list(self.inmapping.values())[0]) + + self.assertRaises(TypeError, d.__getitem__) + + # no test_fromkeys or test_copy as both os.environ and selves don't support it + + def test_get(self): + d = self._empty_mapping() + self.assertIsNone(d.get(list(self.other.keys())[0])) + self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) + d = self.reference + self.assertIsNone(d.get(list(self.other.keys())[0])) + self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) + self.assertEqual(d.get(list(self.inmapping.keys())[0]), + list(self.inmapping.values())[0]) + self.assertEqual(d.get(list(self.inmapping.keys())[0], 3), + list(self.inmapping.values())[0]) + self.assertRaises(TypeError, d.get) + self.assertRaises(TypeError, d.get, None, None, None) + + +class BasicTestMappingProtocol(BasicTestImmutableMappingProtocol): + def _full_mapping(self, data): + """Return a mapping object with the values contained in data + dictionary""" + x = self._empty_mapping() + for key, value in data.items(): + x[key] = value + return x + def test_write(self): # Test for write operations on mapping p = self._empty_mapping() @@ -130,46 +193,6 @@ def test_write(self): p=self._empty_mapping() self.assertRaises(KeyError, p.popitem) - def test_constructor(self): - self.assertEqual(self._empty_mapping(), self._empty_mapping()) - - def test_bool(self): - self.assertTrue(not self._empty_mapping()) - self.assertTrue(self.reference) - self.assertTrue(bool(self._empty_mapping()) is False) - self.assertTrue(bool(self.reference) is True) - - def test_keys(self): - d = self._empty_mapping() - self.assertEqual(list(d.keys()), []) - d = self.reference - self.assertIn(list(self.inmapping.keys())[0], d.keys()) - self.assertNotIn(list(self.other.keys())[0], d.keys()) - self.assertRaises(TypeError, d.keys, None) - - def test_values(self): - d = self._empty_mapping() - self.assertEqual(list(d.values()), []) - - self.assertRaises(TypeError, d.values, None) - - def test_items(self): - d = self._empty_mapping() - self.assertEqual(list(d.items()), []) - - self.assertRaises(TypeError, d.items, None) - - def test_len(self): - d = self._empty_mapping() - self.assertEqual(len(d), 0) - - def test_getitem(self): - d = self.reference - self.assertEqual(d[list(self.inmapping.keys())[0]], - list(self.inmapping.values())[0]) - - self.assertRaises(TypeError, d.__getitem__) - def test_update(self): # mapping argument d = self._empty_mapping() @@ -265,22 +288,6 @@ def __next__(self): self.assertRaises(ValueError, d.update, [(1, 2, 3)]) - # no test_fromkeys or test_copy as both os.environ and selves don't support it - - def test_get(self): - d = self._empty_mapping() - self.assertTrue(d.get(list(self.other.keys())[0]) is None) - self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) - d = self.reference - self.assertTrue(d.get(list(self.other.keys())[0]) is None) - self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) - self.assertEqual(d.get(list(self.inmapping.keys())[0]), - list(self.inmapping.values())[0]) - self.assertEqual(d.get(list(self.inmapping.keys())[0], 3), - list(self.inmapping.values())[0]) - self.assertRaises(TypeError, d.get) - self.assertRaises(TypeError, d.get, None, None, None) - def test_setdefault(self): d = self._empty_mapping() self.assertRaises(TypeError, d.setdefault) @@ -622,7 +629,7 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) - @support.run_with_limited_c_stack() + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() @support.skip_if_sanitizer("requires deep stack", ub=True) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 41e4087bacb..75e2afa7ae6 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -4394,8 +4394,6 @@ def test_fast_save_enter_frozenset(self): def test_fast_save_enter_set(self): self.fast_save_enter(lambda i: set([i])) - # TODO: RUSTPYTHON; frozendict support: https://github.com/RustPython/RustPython/pull/8951 - @unittest.expectedFailure def test_fast_save_enter_frozendict(self): if self.py_version < (3, 15): self.skipTest('need frozendict') @@ -4443,8 +4441,6 @@ def test_deep_nested_struct_set(self): depth=FAST_NESTING_LIMIT+1, compare_equal=False) - # TODO: RUSTPYTHON; frozendict support: https://github.com/RustPython/RustPython/pull/8951 - @unittest.expectedFailure def test_deep_nested_struct_frozendict(self): if self.py_version < (3, 15): self.skipTest('need frozendict') diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 8fdee358acf..2a1a876660e 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -4,6 +4,7 @@ import builtins import collections import contextlib +import copy import decimal import fractions import gc @@ -21,6 +22,7 @@ import typing import unittest import warnings +import weakref from contextlib import ExitStack from functools import partial from inspect import CO_COROUTINE @@ -31,6 +33,7 @@ from test import support from test.support import cpython_only, swap_attr from test.support import async_yield, run_yielding_async_fn +from test.support import warnings_helper from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok @@ -51,6 +54,10 @@ # used as proof of globals being used A_GLOBAL_VALUE = 123 +A_SENTINEL = sentinel("A_SENTINEL") + +class SentinelContainer: + CLASS_SENTINEL = sentinel("SentinelContainer.CLASS_SENTINEL") class Squares: @@ -245,7 +252,7 @@ def test_any(self): S = [10, 20, 30] self.assertEqual(any(x > 42 for x in S), False) - def test_all_any_tuple_optimization(self): + def test_all_any_tuple_list_set_optimization(self): def f_all(): return all(x-2 for x in [1,2,3]) @@ -255,7 +262,13 @@ def f_any(): def f_tuple(): return tuple(2*x for x in [1,2,3]) - funcs = [f_all, f_any, f_tuple] + def f_list(): + return list(2*x for x in [1,2,3]) + + def f_set(): + return set(2*x for x in [1,2,3]) + + funcs = [f_all, f_any, f_tuple, f_list, f_set] for f in funcs: # check that generator code object is not duplicated @@ -265,33 +278,35 @@ def f_tuple(): # check the overriding the builtins works - global all, any, tuple - saved = all, any, tuple + global all, any, tuple, list, set + saved = all, any, tuple, list, set try: all = lambda x : "all" any = lambda x : "any" tuple = lambda x : "tuple" + list = lambda x : "list" + set = lambda x : "set" overridden_outputs = [f() for f in funcs] finally: - all, any, tuple = saved - - self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + all, any, tuple, list, set = saved + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set']) # Now repeat, overriding the builtins module as well - saved = all, any, tuple + saved = all, any, tuple, list, set try: builtins.all = all = lambda x : "all" builtins.any = any = lambda x : "any" builtins.tuple = tuple = lambda x : "tuple" + builtins.list = list = lambda x : "list" + builtins.set = set = lambda x : "set" overridden_outputs = [f() for f in funcs] finally: - all, any, tuple = saved - builtins.all, builtins.any, builtins.tuple = saved - - self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + all, any, tuple, list, set = saved + builtins.all, builtins.any, builtins.tuple, builtins.list, builtins.set = saved + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set']) def test_builtin_call_async_genexpr_no_crash(self): async def f_all(): @@ -527,6 +542,10 @@ async def sleep(delay, result=None): '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', '''a = [await sleep(0, x) async for x in arange(2)][1]''', + '''a = [await sleep(0, 1) for _ in [0]][0]''', + '''a = {await sleep(0, 1) for _ in [0]}.pop()''', + '''a = {0: await sleep(0, 1) for _ in [0]}[0]''', + '''a = (lambda x=[await sleep(0, 1) for _ in [0]]: x)()[0]''', # gh-121637: Make sure we correctly handle the case where the # async code is optimized away '''assert not await sleep(0); a = 1''', @@ -595,7 +614,26 @@ async def __aexit__(self, *exc_info): '''def f(): async with Lock() as l: a = 1 - ''' + ''', + '''class C: + [await x for x in y] + ''', + '''class C: + [x async for x in arange(10)] + ''', + '''async def f(): + class C: + [await x for x in y] + ''', + '''lambda: [await x for x in y]''', + '''class C: + def f(self, x=[await y for y in z]): + pass + ''', + '''type T = [await x for x in y]''', + '''async def f[T=[await x for x in y]](): + pass + ''', ] for mode, code_sample in product(modes, code_samples): source = dedent(code_sample) @@ -796,6 +834,16 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_frozendict(self): + ns = frozendict(x=1, data=[], __builtins__=__builtins__) + eval("data.append(x)", ns, ns) + self.assertEqual(ns['data'], [1]) + + ns = frozendict() + errmsg = "cannot assign __builtins__ to frozendict globals" + with self.assertRaisesRegex(TypeError, errmsg): + eval("", ns, ns) + def test_eval_kwargs(self): data = {"A_GLOBAL_VALUE": 456} self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) @@ -894,6 +942,21 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) + def test_exec_frozendict(self): + ns = frozendict(x=1, data=[], __builtins__=__builtins__) + exec("data.append(x)", ns, ns) + self.assertEqual(ns['data'], [1]) + + ns = frozendict(__builtins__=__builtins__) + errmsg = "'frozendict' object does not support item assignment" + with self.assertRaisesRegex(TypeError, errmsg): + exec("x = 1", ns, ns) + + ns = frozendict() + errmsg = "cannot assign __builtins__ to frozendict globals" + with self.assertRaisesRegex(TypeError, errmsg): + exec("", ns, ns) + def test_exec_kwargs(self): g = {} exec('global z\nz = 1', globals=g) @@ -1108,6 +1171,30 @@ def four_freevars(): three_freevars.__globals__, closure=my_closure) + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: invalid decimal literal + def test_exec_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + exec(source, {}) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package.module\z') + warnings.filterwarnings('error', module=r'') + exec(source, {'__name__': 'package.module', '__file__': filename}) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + def test_filter(self): self.assertEqual(list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')), list('elloorld')) @@ -1868,6 +1955,115 @@ class C: __repr__ = None self.assertRaises(TypeError, repr, C()) + def test_sentinel(self): + missing = sentinel("MISSING") + other = sentinel("MISSING") + + self.assertIsInstance(missing, sentinel) + self.assertIs(type(missing), sentinel) + self.assertEqual(missing.__name__, "MISSING") + self.assertEqual(missing.__module__, __name__) + self.assertIsNot(missing, other) + self.assertEqual(repr(missing), "MISSING") + self.assertTrue(missing) + self.assertIs(copy.copy(missing), missing) + self.assertIs(copy.deepcopy(missing), missing) + self.assertEqual(missing, missing) + self.assertNotEqual(missing, other) + self.assertRaises(TypeError, sentinel) + self.assertRaises(TypeError, sentinel, "MISSING", "EXTRA") + self.assertRaises(TypeError, sentinel, name="MISSING") + with self.assertRaisesRegex(TypeError, "must be str"): + sentinel(1) + self.assertTrue(sentinel.__flags__ & support._TPFLAGS_IMMUTABLETYPE) + self.assertTrue(sentinel.__flags__ & support._TPFLAGS_HAVE_GC) + self.assertFalse(sentinel.__flags__ & support._TPFLAGS_BASETYPE) + with self.assertRaises(TypeError): + class SubSentinel(sentinel): + pass + + def test_sentinel_attributes(self): + missing = sentinel("MISSING") + with self.assertRaises(TypeError): + sentinel.attribute = "value" + with self.assertRaises(AttributeError): + missing.attribute = "value" + with self.assertRaises(AttributeError): + missing.__name__ = "CHANGED" + missing.__module__ = "changed" + self.assertEqual(missing.__module__, "changed") + with self.assertRaises(AttributeError): + del missing.__name__ + del missing.__module__ + with self.assertRaises(AttributeError): + missing.__module__ + + def test_sentinel_repr(self): + with_repr = sentinel("WITH_REPR", repr="custom") + without_repr = sentinel("WITHOUT_REPR", repr=None) + self.assertEqual(repr(with_repr), "custom") + self.assertEqual(repr(without_repr), "WITHOUT_REPR") + self.assertEqual(str(with_repr), "custom") + self.assertEqual(str(without_repr), "WITHOUT_REPR") + + with self.assertRaisesRegex(TypeError, "repr.*str or None"): + sentinel("BAD_REPR", repr=42) + + def test_sentinel_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + self.assertIs( + pickle.loads(pickle.dumps(A_SENTINEL, protocol=proto)), + A_SENTINEL) + self.assertIs( + pickle.loads(pickle.dumps( + SentinelContainer.CLASS_SENTINEL, protocol=proto)), + SentinelContainer.CLASS_SENTINEL) + + missing = sentinel("MISSING") + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + with self.assertRaises(pickle.PicklingError): + pickle.dumps(missing, protocol=proto) + + def test_sentinel_str_subclass_name_cycle(self): + class Name(str): + pass + + name = Name("MISSING") + missing = sentinel(name) + self.assertIs(missing.__name__, name) + self.assertTrue(gc.is_tracked(missing)) + + name.missing = missing + ref = weakref.ref(name) + del name, missing + support.gc_collect() + self.assertIsNone(ref()) + + def test_sentinel_union(self): + missing = sentinel("MISSING") + + self.assertIsInstance(missing | int, typing.Union) + self.assertEqual((missing | int).__args__, (missing, int)) + self.assertIsInstance(int | missing, typing.Union) + self.assertEqual((int | missing).__args__, (int, missing)) + self.assertIs(missing | missing, missing) + self.assertEqual(repr(int | missing), "int | MISSING") + self.assertIsInstance(missing | None, typing.Union) + self.assertEqual((missing | None).__args__, (missing, type(None))) + self.assertIsInstance(None | missing, typing.Union) + self.assertEqual((None | missing).__args__, (type(None), missing)) + self.assertIsInstance(missing | list[int], typing.Union) + self.assertEqual((missing | list[int]).__args__, (missing, list[int])) + self.assertIsInstance(missing | (int | str), typing.Union) + self.assertEqual((missing | (int | str)).__args__, (missing, int, str)) + + with self.assertRaises(TypeError): + missing | 1 + with self.assertRaises(TypeError): + 1 | missing + def test_round(self): self.assertEqual(round(0.0), 0.0) self.assertEqual(type(round(0.0)), int) @@ -2594,6 +2790,7 @@ def run_child(self, child, terminal_input): finally: signal.signal(signal.SIGHUP, old_sighup) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def _run_child(self, child, terminal_input): r, w = os.pipe() # Pipe test results from child back to parent try: @@ -2983,6 +3180,12 @@ def test_type_doc(self): A.__doc__ = doc self.assertEqual(A.__doc__, doc) + def test_type_frozendict(self): + A = type('A', (), frozendict({'x': 4, 'y': 2})) + self.assertEqual(A.x, 4) + self.assertEqual(A.y, 2) + self.assertEqual(A.__name__, 'A') + def test_bad_args(self): with self.assertRaises(TypeError): type() diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index b452ddbef5c..13b7ce29565 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -60,7 +60,7 @@ class C(object): def __reduce_ex__(self, proto): c.append(1) return "" - def __reduce__(self): + def __reduce__(*args): self.fail("shouldn't call this") c = [] x = C() @@ -70,9 +70,15 @@ def __reduce__(self): def test_copy_reduce(self): class C(object): + def __reduce_ex__(*args): + self.fail("shouldn't call this") def __reduce__(self): c.append(1) return "" + def __getattribute__(self, name): + if name == "__reduce_ex__": + raise AttributeError(name) + return object.__getattribute__(self, name) c = [] x = C() y = copy.copy(x) @@ -133,6 +139,12 @@ def test_copy_dict(self): self.assertEqual(y, x) self.assertIsNot(y, x) + def test_copy_frozendict(self): + x = frozendict(x=1, y=2) + self.assertIs(copy.copy(x), x) + x = frozendict() + self.assertIs(copy.copy(x), x) + def test_copy_set(self): x = {1, 2, 3} y = copy.copy(x) @@ -323,7 +335,7 @@ class C(object): def __reduce_ex__(self, proto): c.append(1) return "" - def __reduce__(self): + def __reduce__(*args): self.fail("shouldn't call this") c = [] x = C() @@ -333,9 +345,15 @@ def __reduce__(self): def test_deepcopy_reduce(self): class C(object): + def __reduce_ex__(*args): + self.fail("shouldn't call this") def __reduce__(self): c.append(1) return "" + def __getattribute__(self, name): + if name == "__reduce_ex__": + raise AttributeError(name) + return object.__getattribute__(self, name) c = [] x = C() y = copy.deepcopy(x) @@ -421,6 +439,30 @@ def test_deepcopy_dict(self): self.assertIsNot(x, y) self.assertIsNot(x["foo"], y["foo"]) + def test_deepcopy_frozendict(self): + x = frozendict({"foo": [1, 2], "bar": 3}) + y = copy.deepcopy(x) + self.assertEqual(y, x) + self.assertIsNot(x, y) + self.assertIsNot(x["foo"], y["foo"]) + + # recursive frozendict + x = frozendict(foo=[]) + x['foo'].append(x) + y = copy.deepcopy(x) + self.assertEqual(y.keys(), x.keys()) + self.assertIsNot(x, y) + self.assertIsNot(x["foo"], y["foo"]) + self.assertIs(y['foo'][0], y) + + x = frozendict(foo=[]) + x['foo'].append(x) + x = x['foo'] + y = copy.deepcopy(x) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) + self.assertIs(y[0]['foo'], y) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 195466e6658..e0d2dbb92b3 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -678,7 +678,7 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) - @support.run_with_limited_c_stack() + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_repr_deep(self): @@ -951,7 +951,6 @@ def test_empty_presized_dict_in_freelist(self): 'f': None, 'g': None, 'h': None} d = {} - @unittest.expectedFailure # TODO: RUSTPYTHON def test_container_iterator(self): # Bug #3680: tp_traverse was not implemented for dictiter and # dictview objects. @@ -1264,7 +1263,6 @@ def test_free_after_iterating(self): support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_equal_operator_modifying_operand(self): # test fix for seg fault reported in bpo-27945 part 3. class X(): @@ -1426,6 +1424,32 @@ def test_reversed_dict_after_clear_and_restore(self): for it in iterators: self.assertEqual(list(it), []) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised + def test_reversed_dict_keys_changed_during_iteration(self): + d = dict.fromkeys(range(10)) + for i in range(7): + del d[i] + + iterators = ( + reversed(d), + reversed(d.keys()), + reversed(d.values()), + reversed(d.items()), + ) + for it in iterators: + next(it) + + # Same size as before, but with different keys below + # the iterators' current position. + d.clear() + d.update(dict.fromkeys(range(10))) + for i in range(3, 10): + del d[i] + + for it in iterators: + with self.assertRaisesRegex(RuntimeError, 'keys changed'): + list(it) + def test_dict_copy_order(self): # bpo-34320 od = collections.OrderedDict([('a', 1), ('b', 2)]) @@ -1634,7 +1658,7 @@ def check_unhashable_key(): with check_unhashable_key(): d.get(key) - # Only TypeError exception is overriden, + # Only TypeError exception is overridden, # other exceptions are left unchanged. class HashError: def __hash__(self): @@ -1830,6 +1854,250 @@ class Dict(dict): class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = Dict +class FrozenDictMappingTests(mapping_tests.BasicTestImmutableMappingProtocol): + type2test = frozendict + + +class FrozenDict(frozendict): + pass + + +class FrozenDictSlots(frozendict): + __slots__ = ('slot_attr',) + + +class FrozenDictTests(unittest.TestCase): + def test_constructor(self): + # frozendict.__init__() has no effect + d = frozendict(a=1, b=2, c=3) + d.__init__(x=1) + self.assertEqual(d, frozendict(a=1, b=2, c=3)) + + # dict constructor cannot be used on frozendict + with self.assertRaises(TypeError): + dict.__init__(d, x=1) + + # Avoid copy if it's frozendict type + d2 = frozendict(d) + self.assertIs(d2, d) + d2 = FrozenDict(d) + self.assertIsNot(d2, d) + self.assertEqual(d2, d) + + def test_copy(self): + d = frozendict(x=1, y=2) + d2 = d.copy() + self.assertIs(d2, d) + + d = FrozenDict(x=1, y=2) + d2 = d.copy() + self.assertIsNot(d2, d) + self.assertEqual(d2, frozendict(x=1, y=2)) + self.assertEqual(type(d2), frozendict) + + def test_merge(self): + # test "a | b" operator + self.assertEqual(frozendict(x=1) | frozendict(y=2), + frozendict({'x': 1, 'y': 2})) + self.assertEqual(frozendict(x=1) | dict(y=2), + frozendict({'x': 1, 'y': 2})) + self.assertEqual(frozendict(x=1, y=2) | frozendict(y=5), + frozendict({'x': 1, 'y': 5})) + self.assertEqual(FrozenDict(x=1, y=2) | FrozenDict(y=5), + frozendict({'x': 1, 'y': 5})) + + fd = frozendict(x=1, y=2) + self.assertIs(fd | frozendict(), fd) + self.assertIs(fd | {}, fd) + self.assertIs(frozendict() | fd, fd) + + fd = FrozenDict(x=1, y=2) + self.assertEqual(fd | frozendict(), fd) + self.assertEqual(fd | {}, fd) + self.assertEqual(frozendict() | fd, fd) + + # gh-149676: Test hash(frozendict | frozendict) + a = frozendict({"a": 1}) + b = frozendict({"b": 2}) + self.assertEqual(hash(a | b), hash(frozendict({"a": 1, "b": 2}))) + + def test_update(self): + # test "a |= b" operator + d = frozendict(x=1) + copy = d + self.assertIs(copy, d) + d |= frozendict(y=2) + self.assertIsNot(copy, d) + self.assertEqual(d, frozendict({'x': 1, 'y': 2})) + self.assertEqual(copy, frozendict({'x': 1})) + + def test_items_xor(self): + # test "a ^ b" operator on items views + res = frozendict(a=1, b=2).items() ^ frozendict(b=2, c=3).items() + self.assertEqual(res, {('a', 1), ('c', 3)}) + + def test_repr(self): + d = frozendict() + self.assertEqual(repr(d), "frozendict()") + + d = frozendict(x=1, y=2) + self.assertEqual(repr(d), "frozendict({'x': 1, 'y': 2})") + + d = FrozenDict(x=1, y=2) + self.assertEqual(repr(d), "FrozenDict({'x': 1, 'y': 2})") + + def test_hash(self): + # hash() doesn't rely on the items order + self.assertEqual(hash(frozendict(x=1, y=2)), + hash(frozendict(y=2, x=1))) + + # Check that hash() computes the hash of (key, value) pairs + cases = [ + frozendict(a=False, b=True, c=True), + frozendict(a=True, b=False, c=True), + frozendict(a=True, b=True, c=False), + frozendict({False: "a", "b": True, "c": True}), + frozendict({"a": "b", False: True, True: "c"}), + ] + hashes = {hash(fd) for fd in cases} + self.assertEqual(len(hashes), len(cases)) + + fd = frozendict(x=[1], y=[2]) + with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): + hash(fd) + + @support.cpython_only + def test_hash_cpython(self): + # Check that hash(frozendict) implementation is: + # hash(frozenset(fd.items())) + for fd in ( + frozendict(), + frozendict(x=1, y=2), + frozendict(y=2, x=1), + frozendict(a=False, b=True, c=True), + frozendict.fromkeys('abc'), + ): + with self.subTest(fd=fd): + self.assertEqual(hash(fd), hash(frozenset(fd.items()))) + + def test_fromkeys(self): + self.assertEqual(frozendict.fromkeys('abc'), + frozendict(a=None, b=None, c=None)) + + # Subclass which overrides the constructor + created = frozendict(x=1) + class FrozenDictSubclass(frozendict): + def __new__(cls, *args, **kwargs): + if args or kwargs: + return super().__new__(cls, *args, **kwargs) + else: + return created + + fd = FrozenDictSubclass.fromkeys("abc") + self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None)) + self.assertEqual(type(fd), FrozenDictSubclass) + self.assertEqual(created, frozendict(x=1)) + + fd = FrozenDictSubclass.fromkeys(frozendict(y=2)) + self.assertEqual(fd, frozendict(x=1, y=None)) + self.assertEqual(type(fd), FrozenDictSubclass) + self.assertEqual(created, frozendict(x=1)) + + # Dict subclass with a constructor which returns a frozendict + # by default + class DictSubclass(dict): + def __new__(cls, *args, **kwargs): + if args or kwargs: + return super().__new__(cls, *args, **kwargs) + else: + return created + + fd = DictSubclass.fromkeys("abc") + self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None)) + self.assertEqual(type(fd), DictSubclass) + self.assertEqual(created, frozendict(x=1)) + + # Subclass which doesn't override the constructor + class FrozenDictSubclass2(frozendict): + pass + + fd = FrozenDictSubclass2.fromkeys("abc") + self.assertEqual(fd, frozendict(a=None, b=None, c=None)) + self.assertEqual(type(fd), FrozenDictSubclass2) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for fd in ( + frozendict(), + frozendict(x=1, y=2), + FrozenDict(x=1, y=2), + FrozenDictSlots(x=1, y=2), + ): + if type(fd) == FrozenDict: + fd.attr = 123 + if type(fd) == FrozenDictSlots: + fd.slot_attr = 456 + with self.subTest(fd=fd, proto=proto): + if proto >= 2: + p = pickle.dumps(fd, proto) + fd2 = pickle.loads(p) + self.assertEqual(fd2, fd) + self.assertEqual(type(fd2), type(fd)) + if type(fd) == FrozenDict: + self.assertEqual(fd2.attr, 123) + if type(fd) == FrozenDictSlots: + self.assertEqual(fd2.slot_attr, 456) + else: + # protocol 0 and 1 don't support frozendict + with self.assertRaises(TypeError): + pickle.dumps(fd, proto) + + def test_pickle_iter(self): + fd = frozendict(c=1, b=2, a=3, d=4, e=5, f=6) + for method_name in (None, 'keys', 'values', 'items'): + if method_name is not None: + meth = getattr(fd, method_name) + else: + meth = lambda: fd + expected = list(meth())[1:] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(method_name=method_name, protocol=proto): + it = iter(meth()) + next(it) + p = pickle.dumps(it, proto) + unpickled = pickle.loads(p) + self.assertEqual(list(unpickled), expected) + self.assertEqual(list(it), expected) + + def test_unhashable_key(self): + d = frozendict(a=1) + key = [1, 2, 3] + + def check_unhashable_key(): + msg = "cannot use 'list' as a frozendict key (unhashable type: 'list')" + return self.assertRaisesRegex(TypeError, re.escape(msg)) + + with check_unhashable_key(): + key in d + with check_unhashable_key(): + d[key] + with check_unhashable_key(): + d.get(key) + + # Only TypeError exception is overridden, + # other exceptions are left unchanged. + class HashError: + def __hash__(self): + raise KeyError('error') + + key2 = HashError() + with self.assertRaises(KeyError): + key2 in d + with self.assertRaises(KeyError): + d[key2] + with self.assertRaises(KeyError): + d.get(key2) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 1945cd3f0dd..b9f67f07ebc 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -742,7 +742,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 750 < len(tests) < 800 # approximate number of objects with docstrings + >>> 750 < len(tests) < 850 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 948a2029f1e..1df010732c1 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -135,7 +135,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailureIf(sys.getrecursionlimit() == 256, "TODO: RUSTPYTHON; default debug recursion limit is 256") @skip_wasi_stack_overflow() def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 diff --git a/Lib/test/test_import/data/syntax_warnings.py b/Lib/test/test_import/data/syntax_warnings.py new file mode 100644 index 00000000000..103f07b6187 --- /dev/null +++ b/Lib/test/test_import/data/syntax_warnings.py @@ -0,0 +1,21 @@ +# Syntax warnings emitted in different parts of the Python compiler. + +# Parser/lexer/lexer.c +x = 1or 0 # line 4 + +# Parser/tokenizer/helpers.c +'\z' # line 7 + +# Parser/string_parser.c +'\400' # line 10 + +# _PyCompile_Warn() in Python/codegen.c +assert(x, 'message') # line 13 +x is 1 # line 14 + +# _PyErr_EmitSyntaxWarning() in Python/ast_preprocess.c +def f(): + try: + pass + finally: + return 42 # line 21 diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 71cd4baa942..1b85eceac17 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -107,6 +107,10 @@ def loads(self, buf, **kwds): def test_c_methods(self): return super().test_c_methods() + @unittest.expectedFailureIf(sys.getrecursionlimit() == 256, "TODO: RUSTPYTHON; default debug recursion limit is 256") + def test_deep_nested_struct_frozendict(self): + return super().test_deep_nested_struct_frozendict() + @@ -408,6 +412,10 @@ class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler + @unittest.expectedSuccess # TODO: RUSTPYTHON; native pickler passes at the default debug recursion limit + def test_deep_nested_struct_frozendict(self): + return super().test_deep_nested_struct_frozendict() + class CPersPicklerTests(PyPersPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler @@ -421,6 +429,10 @@ class CDumpPickle_LoadPickle(PyPicklerTests): pickler = _pickle.Pickler unpickler = pickle._Unpickler + @unittest.expectedSuccess # TODO: RUSTPYTHON; native pickler passes at the default debug recursion limit + def test_deep_nested_struct_frozendict(self): + return super().test_deep_nested_struct_frozendict() + class DumpPickle_CLoadPickle(PyPicklerTests): pickler = pickle._Pickler unpickler = _pickle.Unpickler diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index bcae14c5b3c..0e82c723ec3 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -45,7 +45,6 @@ def test_any_and_datetime(self): mock.assert_called_with(ANY, foo=ANY) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_any_mock_calls_comparison_order(self): mock = Mock() class Foo(object): diff --git a/crates/capi/src/ceval.rs b/crates/capi/src/ceval.rs index ace7896163c..036c259454b 100644 --- a/crates/capi/src/ceval.rs +++ b/crates/capi/src/ceval.rs @@ -3,10 +3,10 @@ use crate::pystate::with_vm; use crate::unicodeobject::decode_fsdefault_and_size; use crate::util::{CStrExt, FfiPtrExt}; use core::ffi::{CStr, c_char, c_int}; -use rustpython_vm::builtins::{PyCode, PyDict}; +use rustpython_vm::builtins::{PyAnyDict, PyCode}; use rustpython_vm::function::ArgMapping; use rustpython_vm::scope::Scope; -use rustpython_vm::{AsObject, PyObject, TryFromObject}; +use rustpython_vm::{AsObject, PyObject, PyRef, TryFromObject}; use rustpython_vm::{PyObjectRef, version}; #[unsafe(no_mangle)] @@ -32,12 +32,17 @@ pub unsafe extern "C" fn PyEval_EvalCode( ) -> *mut PyObject { with_vm(|vm| { let code = unsafe { co.assume_borrowed_and_cast::(vm) }?; - let globals = unsafe { globals.assume_borrowed_and_cast::(vm) }?; + let globals = PyRef::::try_from_object( + vm, + unsafe { globals.assume_borrowed() }.to_owned(), + )?; let locals = unsafe { locals.assume_borrowed_or_opt() } .map(|obj| ArgMapping::try_from_object(vm, obj.to_owned())) .transpose()?; - let scope = Scope::with_builtins(locals, globals.to_owned(), vm); + let locals = locals.or_else(|| Some(ArgMapping::from_anydict_exact(globals.clone()))); + // Unlike Python's eval/exec, this API does not insert __builtins__. + let scope = Scope::new(locals, globals); vm.run_code_obj(code.to_owned(), scope) }) diff --git a/crates/capi/src/dictobject.rs b/crates/capi/src/dictobject.rs index 2c3eedff3ba..1e6fab1b131 100644 --- a/crates/capi/src/dictobject.rs +++ b/crates/capi/src/dictobject.rs @@ -4,9 +4,8 @@ use crate::pystate::with_vm; use crate::util::{CStrExt, FfiPtrExt}; use core::ffi::{c_char, c_int}; use core::ptr::NonNull; -use rustpython_vm::AsObject; -use rustpython_vm::PyPayload; -use rustpython_vm::builtins::PyDict; +use rustpython_vm::builtins::{PyAnyDict, PyDict, PyFrozenDict}; +use rustpython_vm::{AsObject, Py, PyPayload, PyResult, VirtualMachine}; define_py_check!(fn PyDict_Check, types.dict_type); define_py_check!(exact fn PyDict_CheckExact, types.dict_type); @@ -14,6 +13,27 @@ define_py_check!(fn PyDictKeys_Check, types.dict_keys_type); define_py_check!(fn PyDictValues_Check, types.dict_values_type); define_py_check!(fn PyDictItems_Check, types.dict_items_type); +fn any_dict<'a>(dict: &'a PyObject, vm: &VirtualMachine) -> PyResult<&'a PyAnyDict> { + PyAnyDict::from_object(dict) + .ok_or_else(|| vm.new_system_error("bad argument to internal function")) +} + +fn writable_dict<'a>( + dict: &'a PyObject, + operation: &str, + vm: &VirtualMachine, +) -> PyResult<&'a Py> { + dict.downcast_ref::().ok_or_else(|| { + if dict.downcast_ref::().is_some() { + vm.new_type_error(format!( + "frozendict object does not support item {operation}" + )) + } else { + vm.new_system_error("bad argument to internal function") + } + }) +} + #[unsafe(no_mangle)] pub extern "C" fn PyDict_New() -> *mut PyObject { with_vm(|vm| vm.ctx.new_dict()) @@ -21,10 +41,10 @@ pub extern "C" fn PyDict_New() -> *mut PyObject { #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Clear(dict: *mut PyObject) { - with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; - dict.clear(); - Ok(()) + with_vm(|_vm| { + if let Some(dict) = unsafe { dict.assume_borrowed() }.downcast_ref::() { + dict.clear(); + } }) } @@ -35,7 +55,7 @@ pub unsafe extern "C" fn PyDict_SetItem( val: *mut PyObject, ) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = writable_dict(unsafe { dict.assume_borrowed() }, "assignment", vm)?; let key = unsafe { key.assume_borrowed() }; let value = unsafe { val.assume_borrowed() }.to_owned(); dict.inner_setitem(key, value, vm) @@ -49,8 +69,8 @@ pub unsafe extern "C" fn PyDict_SetItemString( val: *mut PyObject, ) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; let key = unsafe { key.try_as_str(vm) }?; + let dict = writable_dict(unsafe { dict.assume_borrowed() }, "assignment", vm)?; let value = unsafe { val.assume_borrowed() }.to_owned(); dict.inner_setitem(key, value, vm) }) @@ -59,12 +79,14 @@ pub unsafe extern "C" fn PyDict_SetItemString( #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_GetItem(dict: *mut PyObject, key: *mut PyObject) -> *mut PyObject { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let Some(dict) = PyAnyDict::from_object(unsafe { dict.assume_borrowed() }) else { + return core::ptr::null_mut(); + }; let key = unsafe { key.assume_borrowed() }; match dict.inner_getitem_opt(key, vm) { - Ok(Some(value)) => Ok(value.as_object().as_raw().cast_mut()), - Ok(None) | Err(_) => Ok(core::ptr::null_mut()), + Ok(Some(value)) => value.as_object().as_raw().cast_mut(), + Ok(None) | Err(_) => core::ptr::null_mut(), } }) } @@ -75,7 +97,7 @@ pub unsafe extern "C" fn PyDict_GetItemWithError( key: *mut PyObject, ) -> *mut PyObject { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = any_dict(unsafe { dict.assume_borrowed() }, vm)?; let key = unsafe { key.assume_borrowed() }; if let Some(value) = dict.inner_getitem_opt(key, vm)? { @@ -92,12 +114,16 @@ pub unsafe extern "C" fn PyDict_GetItemString( key: *const c_char, ) -> *mut PyObject { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; - let key = unsafe { key.try_as_str(vm) }?; + let Ok(key) = (unsafe { key.try_as_str(vm) }) else { + return core::ptr::null_mut(); + }; + let Some(dict) = PyAnyDict::from_object(unsafe { dict.assume_borrowed() }) else { + return core::ptr::null_mut(); + }; - match dict.inner_getitem_opt(key, vm)? { - Some(value) => Ok(value.as_object().as_raw().cast_mut()), - None => Ok(core::ptr::null_mut()), + match dict.inner_getitem_opt(key, vm) { + Ok(Some(value)) => value.as_object().as_raw().cast_mut(), + Ok(None) | Err(_) => core::ptr::null_mut(), } }) } @@ -112,8 +138,8 @@ pub unsafe extern "C" fn PyDict_GetItemStringRef( unsafe { *result = core::ptr::null_mut(); } - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; let key = unsafe { key.try_as_str(vm) }?; + let dict = any_dict(unsafe { dict.assume_borrowed() }, vm)?; if let Some(value) = dict.inner_getitem_opt(key, vm)? { unsafe { @@ -134,7 +160,7 @@ pub unsafe extern "C" fn PyDict_GetItemRef( ) -> c_int { with_vm(|vm| { unsafe { *result = core::ptr::null_mut() }; - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = any_dict(unsafe { dict.assume_borrowed() }, vm)?; let key = unsafe { key.assume_borrowed() }; if let Some(value) = dict.inner_getitem_opt(key, vm)? { @@ -165,7 +191,7 @@ pub unsafe extern "C" fn PyDict_SetDefaultRef( result.write(core::ptr::null_mut()); } } - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = writable_dict(unsafe { dict.assume_borrowed() }, "assignment", vm)?; let key = unsafe { key.assume_borrowed() }; if let Some(value) = dict.inner_getitem_opt(key, vm)? { @@ -191,7 +217,9 @@ pub unsafe extern "C" fn PyDict_SetDefaultRef( #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Size(dict: *mut PyObject) -> isize { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = unsafe { dict.assume_borrowed_or_opt() } + .ok_or_else(|| vm.new_system_error("bad argument to internal function"))?; + let dict = any_dict(dict, vm)?; Ok(dict.__len__()) }) } @@ -199,7 +227,7 @@ pub unsafe extern "C" fn PyDict_Size(dict: *mut PyObject) -> isize { #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Contains(dict: *mut PyObject, key: *mut PyObject) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = any_dict(unsafe { dict.assume_borrowed() }, vm)?; let key = unsafe { key.assume_borrowed() }; Ok(dict.inner_getitem_opt(key, vm)?.is_some()) }) @@ -208,7 +236,9 @@ pub unsafe extern "C" fn PyDict_Contains(dict: *mut PyObject, key: *mut PyObject #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Copy(dict: *mut PyObject) -> *mut PyObject { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = unsafe { dict.assume_borrowed_or_opt() } + .and_then(|dict| dict.downcast_ref::()) + .ok_or_else(|| vm.new_system_error("bad argument to internal function"))?; Ok(dict.copy().into_ref(&vm.ctx)) }) } @@ -216,8 +246,12 @@ pub unsafe extern "C" fn PyDict_Copy(dict: *mut PyObject) -> *mut PyObject { #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_DelItem(dict: *mut PyObject, key: *mut PyObject) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = unsafe { dict.assume_borrowed() }; let key = unsafe { key.assume_borrowed() }; + if dict.downcast_ref::().is_some() { + PyFrozenDict::key_hash(key, vm)?; + } + let dict = writable_dict(dict, "deletion", vm)?; dict.del_item(key, vm) }) } @@ -225,8 +259,8 @@ pub unsafe extern "C" fn PyDict_DelItem(dict: *mut PyObject, key: *mut PyObject) #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_DelItemString(dict: *mut PyObject, key: *const c_char) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; let key = unsafe { key.try_as_str(vm) }?; + let dict = writable_dict(unsafe { dict.assume_borrowed() }, "deletion", vm)?; dict.del_item(key, vm) }) } @@ -234,7 +268,9 @@ pub unsafe extern "C" fn PyDict_DelItemString(dict: *mut PyObject, key: *const c #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Items(dict: *mut PyObject) -> *mut PyObject { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = unsafe { dict.assume_borrowed_or_opt() } + .ok_or_else(|| vm.new_system_error("bad argument to internal function"))?; + let dict = any_dict(dict, vm)?; let items = dict .items_vec() .into_iter() @@ -247,7 +283,9 @@ pub unsafe extern "C" fn PyDict_Items(dict: *mut PyObject) -> *mut PyObject { #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Keys(dict: *mut PyObject) -> *mut PyObject { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = unsafe { dict.assume_borrowed_or_opt() } + .ok_or_else(|| vm.new_system_error("bad argument to internal function"))?; + let dict = any_dict(dict, vm)?; Ok(vm.ctx.new_list(dict.keys_vec())) }) } @@ -255,7 +293,9 @@ pub unsafe extern "C" fn PyDict_Keys(dict: *mut PyObject) -> *mut PyObject { #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Values(dict: *mut PyObject) -> *mut PyObject { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = unsafe { dict.assume_borrowed_or_opt() } + .ok_or_else(|| vm.new_system_error("bad argument to internal function"))?; + let dict = any_dict(dict, vm)?; Ok(vm.ctx.new_list(dict.values_vec())) }) } @@ -267,7 +307,7 @@ pub unsafe extern "C" fn PyDict_Merge( override_: c_int, ) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = writable_dict(unsafe { dict.assume_borrowed() }, "assignment", vm)?; let other = unsafe { other.assume_borrowed() }.to_owned(); if override_ != 0 { dict.merge_object(other, vm) @@ -280,7 +320,7 @@ pub unsafe extern "C" fn PyDict_Merge( #[unsafe(no_mangle)] pub unsafe extern "C" fn PyDict_Update(dict: *mut PyObject, other: *mut PyObject) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = writable_dict(unsafe { dict.assume_borrowed() }, "assignment", vm)?; let other = unsafe { other.assume_borrowed() }.to_owned(); dict.merge_object(other, vm) }) @@ -293,7 +333,7 @@ pub unsafe extern "C" fn PyDict_MergeFromSeq2( override_: c_int, ) -> c_int { with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + let dict = writable_dict(unsafe { dict.assume_borrowed() }, "assignment", vm)?; let seq2 = unsafe { seq2.assume_borrowed() }; dict.merge_from_seq2(seq2, override_ != 0, vm) }) @@ -306,8 +346,10 @@ pub unsafe extern "C" fn PyDict_Next( key: *mut *mut PyObject, value: *mut *mut PyObject, ) -> c_int { - with_vm(|vm| { - let dict = unsafe { dict.assume_borrowed_and_cast::(vm) }?; + with_vm(|_vm| { + let Some(dict) = PyAnyDict::from_object(unsafe { dict.assume_borrowed() }) else { + return false; + }; let index = unsafe { *pos } as usize; if let Some((next_pos, k, v)) = dict.next_entry(index) { @@ -320,9 +362,9 @@ pub unsafe extern "C" fn PyDict_Next( value.write(v.as_object().as_raw().cast_mut()); } } - Ok(true) + true } else { - Ok(false) + false } }) } diff --git a/crates/capi/src/pyframe.rs b/crates/capi/src/pyframe.rs index 2214ee55820..c3e81c235b3 100644 --- a/crates/capi/src/pyframe.rs +++ b/crates/capi/src/pyframe.rs @@ -2,9 +2,9 @@ use crate::PyObject; use crate::pystate::with_vm; use crate::util::FfiPtrExt; use core::ffi::c_int; -use rustpython_vm::Py; use rustpython_vm::builtins::PyCode; use rustpython_vm::frame::FrameObject; +use rustpython_vm::{Py, PyObjectRef}; pub type PyFrameObject = Py; pub type PyCodeObject = Py; @@ -26,7 +26,7 @@ pub unsafe extern "C" fn PyFrame_GetLocals(frame: *mut PyFrameObject) -> *mut Py #[unsafe(no_mangle)] pub unsafe extern "C" fn PyFrame_GetGlobals(frame: *mut PyFrameObject) -> *mut PyObject { - with_vm(|_vm| Ok(unsafe { frame.assume_borrowed() }.f_globals())) + with_vm(|_vm| PyObjectRef::from(unsafe { frame.assume_borrowed() }.f_globals())) } #[unsafe(no_mangle)] diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index a98ba4a4356..9d876f2f906 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -7,22 +7,26 @@ use crate::common::lock::LazyLock; use crate::object::{Traverse, TraverseFn}; use crate::stdlib::PyOrderedDictItems; use crate::{ - AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, builtins::{PyList, PyTuple, iter::builtins_iter, type_::PyAttributes}, class::{PyClassDef, PyClassImpl}, - common::{ascii, hash::PyHash}, + common::{ + ascii, + hash::{self, PyHash}, + }, dict_inner::{self, DictKey}, function::{ArgIterable, FuncArgs, KwArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, protocol::{PyIter, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, recursion::ReprGuard, types::{ AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, DefaultConstructor, - Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, + Hashable, Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, vm::VirtualMachine, }; use alloc::fmt; +use core::borrow::Borrow; use core::cell::Cell; use core::ptr::NonNull; use rustpython_common::atomic::{Ordering, PyAtomic, Radium}; @@ -31,15 +35,264 @@ use rustpython_common::wtf8::Wtf8Buf; pub(crate) type DictContentType = dict_inner::Dict; +pub(crate) fn key_hash( + key: &K, + mapping_name: &str, + vm: &VirtualMachine, +) -> PyResult { + match key.key_hash(vm) { + // Exact TypeError only: exception subclasses retain their identity. + Err(exc) if exc.class().is(vm.ctx.exceptions.type_error) => { + let name = key.to_pyobject(vm).class().fully_qualified_name(vm)?; + // Match BaseException_str without the diagnostic formatter's + // fallback: argument __str__/__repr__ failures must propagate. + let args = exc.args(); + let message = match args.as_slice() { + [] => vm.ctx.empty_str.to_owned(), + [arg] => arg.str(vm)?, + _ => args.as_object().str(vm)?, + }; + Err(vm.new_type_error(format!( + "cannot use '{name}' as a {mapping_name} key ({message})" + ))) + } + result => result, + } +} + #[pyclass(module = false, name = "dict", unhashable = true, traverse = "manual")] #[derive(Default)] // OrderedDict contains eight-byte-aligned fields on 32-bit targets too. #[repr(align(8))] pub struct PyDict { - entries: DictContentType, + pub(crate) entries: DictContentType, } pub type PyDictRef = PyRef; +/// A borrowed view of a dict or frozendict, including subclasses. +/// +/// This is an erased header view, not a shared payload layout. Every payload +/// access downcasts to the actual alternative, preserving its mutability. +#[repr(transparent)] +pub struct PyAnyDict { + object: PyObject, +} + +// SAFETY: this owner contributes exactly one strong reference to the object. +unsafe impl Traverse for PyRef { + fn traverse(&self, tracer: &mut TraverseFn<'_>) { + tracer(self.as_object()); + } +} + +impl crate::convert::ToPyObject for PyRef { + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.into_object() + } +} + +impl Borrow for PyAnyDict { + fn borrow(&self) -> &PyObject { + &self.object + } +} + +impl Borrow for PyRef { + fn borrow(&self) -> &PyAnyDict { + self + } +} + +impl ToOwned for PyAnyDict { + type Owned = PyRef; + + fn to_owned(&self) -> PyRef { + // SAFETY: this borrowed view already guarantees a dictionary payload. + unsafe { PyRef::::from_object_unchecked(self.object.to_owned()) } + } +} + +impl From for PyRef { + fn from(dict: PyDictRef) -> Self { + // SAFETY: the typed owner guarantees the dict alternative. + unsafe { Self::from_object_unchecked(dict.into()) } + } +} + +impl From> for PyRef { + fn from(dict: PyRef) -> Self { + // SAFETY: the typed owner guarantees the frozendict alternative. + unsafe { Self::from_object_unchecked(dict.into()) } + } +} + +impl From> for PyObjectRef { + fn from(dict: PyRef) -> Self { + dict.into_object() + } +} + +// SAFETY: the transparent PyRef owner has the same layout as PyObjectRef. +// A successful check proves the actual dict or frozendict payload invariant. +unsafe impl crate::convert::TransmuteFromObject for PyRef { + fn check(vm: &VirtualMachine, obj: &PyObject) -> PyResult<()> { + if PyAnyDict::from_object(obj).is_some() { + Ok(()) + } else { + Err(vm.new_type_error(format!( + "expected dict or frozendict, got {}", + obj.class().name() + ))) + } + } +} + +impl TryFromObject for PyRef { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + ::check(vm, &obj)?; + // SAFETY: the shared checker proved the union view's payload invariant. + Ok(unsafe { Self::from_object_unchecked(obj) }) + } +} + +impl PyAnyDict { + pub fn from_object(obj: &PyObject) -> Option<&Self> { + if obj.downcast_ref::().is_some() || obj.downcast_ref::().is_some() { + // SAFETY: PyAnyDict is transparent over PyObject and the payload was checked. + Some(unsafe { &*(obj as *const PyObject).cast::() }) + } else { + None + } + } + + pub(crate) fn as_mutable(&self) -> Option<&Py> { + self.object.downcast_ref() + } + + pub(crate) fn is_frozen(&self) -> bool { + self.object.downcastable::() + } + + // The raw payload is private: callers outside the VM cannot use it to mutate a frozendict. + pub(crate) fn as_dict(&self) -> &PyDict { + if let Some(dict) = self.as_mutable() { + dict + } else { + &self.object.downcast_ref::().unwrap().dict + } + } + + pub(crate) fn uses_builtin_iter(&self, vm: &VirtualMachine) -> bool { + let base = if self.is_frozen() { + vm.ctx.types.frozendict_type + } else { + vm.ctx.types.dict_type + }; + self.as_object() + .class() + .slots + .iter + .load() + .map(crate::types::fn_addr) + == base.slots.iter.load().map(crate::types::fn_addr) + } + + #[must_use] + pub fn __len__(&self) -> usize { + self.as_dict().__len__() + } + #[must_use] + pub fn is_empty(&self) -> bool { + self.as_dict().is_empty() + } + #[must_use] + pub fn keys_vec(&self) -> Vec { + self.as_dict().keys_vec() + } + #[must_use] + pub fn values_vec(&self) -> Vec { + self.as_dict().values_vec() + } + #[must_use] + pub fn items_vec(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + self.as_dict().items_vec() + } + #[must_use] + pub fn next_entry(&self, position: usize) -> Option<(usize, PyObjectRef, PyObjectRef)> { + self.as_dict().next_entry(position) + } + pub(crate) fn size(&self) -> crate::dict_inner::DictSize { + self.as_dict().size() + } + pub fn inner_getitem_opt( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult> { + if let Some(dict) = self.as_mutable() { + dict.inner_getitem_opt(key, vm) + } else { + self.object + .downcast_ref::() + .unwrap() + .inner_getitem_opt(key, vm) + } + } + pub fn get_item_opt( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult> { + if let Some(dict) = self.as_mutable() { + return dict.get_item_opt(key, vm); + } + if self.object.class().is(vm.ctx.types.frozendict_type) { + return self.inner_getitem_opt(key, vm); + } + match self.object.get_item(key, vm) { + Ok(value) => Ok(Some(value)), + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.key_error) => Ok(None), + Err(exc) => Err(exc), + } + } + pub fn get_item(&self, key: &K, vm: &VirtualMachine) -> PyResult { + if let Some(dict) = self.as_mutable() { + dict.get_item(key, vm) + } else { + self.object.get_item(key, vm) + } + } + pub fn contains_key(&self, key: &K, vm: &VirtualMachine) -> bool { + self.inner_getitem_opt(key, vm) + .is_ok_and(|value| value.is_some()) + } + pub fn set_item( + &self, + key: &K, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + match self.as_mutable() { + Some(dict) => dict.set_item(key, value, vm), + None => Err(vm.new_type_error("frozendict object does not support item assignment")), + } + } + pub fn del_item(&self, key: &K, vm: &VirtualMachine) -> PyResult<()> { + match self.as_mutable() { + Some(dict) => dict.del_item(key, vm), + None => Err(vm.new_type_error("frozendict object does not support item deletion")), + } + } +} + +impl<'a> IntoIterator for &'a PyAnyDict { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter<'a>; + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self.as_dict()) + } +} + // SAFETY: Traverse properly visits all owned PyObjectRefs unsafe impl Traverse for PyDict { fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { @@ -140,51 +393,51 @@ impl PyDict { self.entries.items_with_hashes() } - fn merge_object_with_override( + /// `dict_update_arg`. `mapping_name` names the target type in the + /// unhashable key error, as `dict_unhashable_type` does. + pub(crate) fn update_arg( &self, other: PyObjectRef, override_existing: bool, + mapping_name: &str, vm: &VirtualMachine, ) -> PyResult<()> { - let casted: Result, _> = other.downcast_exact(vm); - let other = match casted { - Ok(dict_other) => { - return self.merge_dict(&dict_other, override_existing, vm); - } - Err(other) => other, - }; + if let Some(other_dict) = PyAnyDict::from_object(&other) + && other_dict.uses_builtin_iter(vm) + { + return self.merge_dict(other_dict.as_dict(), override_existing, vm); + } let dict = &self.entries; // Use get_attr to properly invoke __getattribute__ for proxy objects - let keys_result = other.get_attr(vm.ctx.intern_str("keys"), vm); - let has_keys = match keys_result { - Ok(keys_method) => { - let keys = PyIter::try_from_object(vm, keys_method.call((), vm)?)?; - while let PyIterReturn::Return(key) = keys.next(vm)? { - let hash = Self::hash_or_unhashable(&*key, vm)?; - if !override_existing && dict.contains(vm, &*key, hash)? { - continue; - } - let val = other.get_item(&*key, vm)?; - dict.insert(vm, &*key, hash, val)?; - } - true + let keys_method = match other.get_attr(vm.ctx.intern_str("keys"), vm) { + Ok(keys_method) => keys_method, + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => { + return self.merge_seq2(&other, override_existing, mapping_name, vm); } - Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => false, Err(e) => return Err(e), }; - if !has_keys { - return self.merge_from_seq2(&other, override_existing, vm); + let keys = PyIter::try_from_object(vm, keys_method.call((), vm)?)?; + while let PyIterReturn::Return(key) = keys.next(vm)? { + if !override_existing { + let hash = key_hash(&*key, mapping_name, vm)?; + if dict.contains(vm, &*key, hash)? { + continue; + } + } + let value = other.get_item(&*key, vm)?; + let hash = key_hash(&*key, mapping_name, vm)?; + dict.insert(vm, &*key, hash, value)?; } Ok(()) } // Used in update and ior. pub fn merge_object(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.merge_object_with_override(other, true, vm) + self.update_arg(other, true, "dict", vm) } pub fn merge_object_if_missing(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.merge_object_with_override(other, false, vm) + self.update_arg(other, false, "dict", vm) } fn add_update_sequence_note( @@ -262,6 +515,17 @@ impl PyDict { seq2: &PyObject, override_existing: bool, vm: &VirtualMachine, + ) -> PyResult<()> { + self.merge_seq2(seq2, override_existing, "dict", vm) + } + + /// `dict_merge_from_seq2` + fn merge_seq2( + &self, + seq2: &PyObject, + override_existing: bool, + mapping_name: &str, + vm: &VirtualMachine, ) -> PyResult<()> { let iter = seq2.get_iter(vm)?; let dict = &self.entries; @@ -269,7 +533,7 @@ impl PyDict { for (index, element) in iter.iter::(vm)?.enumerate() { let (key, value) = Self::update_sequence_pair(element?, index, vm)?; - let hash = Self::hash_or_unhashable(&*key, vm)?; + let hash = key_hash(&*key, mapping_name, vm)?; if !override_existing && dict.contains(vm, &*key, hash)? { continue; } @@ -280,7 +544,7 @@ impl PyDict { pub(crate) fn merge_dict( &self, - dict_other: &Py, + dict_other: &Self, override_existing: bool, vm: &VirtualMachine, ) -> PyResult<()> { @@ -314,22 +578,7 @@ impl PyDict { /// Hashing up front also means a `__hash__` that fails only intermittently /// is still reported. fn hash_or_unhashable(key: &K, vm: &VirtualMachine) -> PyResult { - match key.key_hash(vm) { - Ok(hash) => Ok(hash), - // Exact `TypeError` only: a `__hash__` raising a *subclass* of - // `TypeError` (e.g. a user `MyTypeError`) must propagate unchanged, - // matching CPython's exact-type check. - Err(cause) if cause.class().is(vm.ctx.exceptions.type_error) => { - let message = cause.as_object().str(vm)?; - let key = key.to_pyobject(vm); - let name = key.class().fully_qualified_name(vm)?; - let err = - vm.new_type_error(format!("cannot use '{name}' as a dict key ({message})")); - err.set_cause(Some(cause)); - Err(err) - } - Err(other) => Err(other), - } + key_hash(key, "dict", vm) } /// Set item variant which can be called with multiple @@ -409,6 +658,8 @@ impl PyDict { ) -> Option> { if let Some(dict) = obj.downcast_ref_if_exact::(vm) { Some(dict.entries.keys_with_hashes()) + } else if let Some(frozen) = obj.downcast_ref_if_exact::(vm) { + Some(frozen.dict.entries.keys_with_hashes()) } else { set::exact_set_keys_with_hashes(obj, vm) } @@ -426,13 +677,48 @@ struct DictGetArgs { #[derive(FromArgs)] struct FromKeysArgs { #[pyarg(positional)] - iterable: ArgIterable, + iterable: PyObjectRef, #[pyarg(positional, optional)] value: Option, } // Python dict methods: impl PyDict { + pub(crate) fn compare_entries( + &self, + other: &Self, + op: PyComparisonOp, + item: bool, + vm: &VirtualMachine, + ) -> PyResult { + if op == PyComparisonOp::Ne { + return Self::compare_entries(self, other, PyComparisonOp::Eq, item, vm) + .map(|x| x.map(|eq| !eq)); + } + if !op.eval_ord(self.__len__().cmp(&other.__len__())) { + return Ok(PyArithmeticValue::Implemented(false)); + } + let mut position = 0; + while let Some((next, k, v1, hash)) = self.entries.next_entry_with_hash(position) { + position = next; + let found = other.entries.get(vm, &*k, hash)?; + match found { + Some(v2) => { + if v1.is(&v2) { + continue; + } + if item && !vm.bool_eq(&v1, &v2)? { + return Ok(PyArithmeticValue::Implemented(false)); + } + } + None => { + return Ok(PyArithmeticValue::Implemented(false)); + } + } + } + Ok(PyArithmeticValue::Implemented(true)) + } + pub fn __len__(&self) -> usize { self.entries.len() } @@ -460,17 +746,6 @@ impl PyDict { self.entries.setdefault(vm, &*key, hash, || default) } - fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // Accept dict subclasses that inherit this implementation. Subclasses with - // their own reflected slot, such as OrderedDict, take precedence in dispatch. - if let Ok(other) = other.downcast::() { - let self_cp = self.copy(); - self_cp.merge_dict(&other, true, vm)?; - return Ok(self_cp.into_pyobject(vm)); - } - Ok(vm.ctx.not_implemented()) - } - pub(crate) fn __sizeof__(&self) -> usize { core::mem::size_of::() + self.entries.sizeof() } @@ -519,88 +794,20 @@ impl PyDict {} #[pyclass] impl Py { - pub(crate) fn inner_cmp( - &self, - other: &Self, - op: PyComparisonOp, - item: bool, - vm: &VirtualMachine, - ) -> PyResult { - if op == PyComparisonOp::Ne { - return Self::inner_cmp(self, other, PyComparisonOp::Eq, item, vm) - .map(|x| x.map(|eq| !eq)); - } - if !op.eval_ord(self.__len__().cmp(&other.__len__())) { - return Ok(PyArithmeticValue::Implemented(false)); - } - let (superset, subset) = if self.__len__() < other.__len__() { - (other, self) - } else { - (self, other) - }; - for (k, v1) in subset { - match superset.get_item_opt(&*k, vm)? { - Some(v2) => { - if v1.is(&v2) { - continue; - } - if item && !vm.bool_eq(&v1, &v2)? { - return Ok(PyArithmeticValue::Implemented(false)); - } - } - None => { - return Ok(PyArithmeticValue::Implemented(false)); - } - } - } - Ok(PyArithmeticValue::Implemented(true)) - } - #[cfg_attr(feature = "flame-it", flame("PyDictRef"))] #[pymethod(coexist)] fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { self.inner_getitem(&*key, vm) } - fn __ror__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let other_dict = other.downcast::(); - if let Ok(other) = other_dict { - let other_cp = other.copy(); - other_cp.merge_dict(self, true, vm)?; - return Ok(other_cp.into_pyobject(vm)); - } - Ok(vm.ctx.not_implemented()) - } - #[pyclassmethod] fn fromkeys(class: PyTypeRef, args: FromKeysArgs, vm: &VirtualMachine) -> PyResult { - let FromKeysArgs { iterable, value } = args; - let value = value.unwrap_or_else(|| vm.ctx.none()); - let d = PyType::call(&class, ().into(), vm)?; - match d.downcast_exact::(vm) { - Ok(pydict) => { - if let Some(keys) = PyDict::fromkeys_known_hashes(iterable.as_object(), vm) { - if class.is(vm.ctx.types.dict_type) { - pydict.entries.reserve_for_empty(keys.len()); - } - for (key, hash) in keys { - pydict.entries.insert(vm, &*key, hash, value.clone())?; - } - } else { - for key in iterable.iter(vm)? { - let key = key?; - pydict.__setitem__(&key, value.clone(), vm)?; - } - } - Ok(pydict.into_pyref().into()) - } - Err(pyobj) => { - for key in iterable.iter(vm)? { - pyobj.set_item(&*key?, value.clone(), vm)?; - } - Ok(pyobj) - } - } + dict_fromkeys( + class, + args.iterable, + args.value.unwrap_or_else(|| vm.ctx.none()), + vm, + ) } #[pymethod] @@ -681,23 +888,23 @@ impl Py { #[pyclass] impl PyRef { #[pymethod] - const fn keys(self) -> PyDictKeys { - PyDictKeys::new(self) + fn keys(self) -> PyDictKeys { + PyDictKeys::new(self.into()) } #[pymethod] - const fn values(self) -> PyDictValues { - PyDictValues::new(self) + fn values(self) -> PyDictValues { + PyDictValues::new(self.into()) } #[pymethod] - const fn items(self) -> PyDictItems { - PyDictItems::new(self) + fn items(self) -> PyDictItems { + PyDictItems::new(self.into()) } #[pymethod] fn __reversed__(self) -> PyDictReverseKeyIterator { - PyDictReverseKeyIterator::new(self) + PyDictReverseKeyIterator::new(self.into()) } fn __ior__(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -758,13 +965,7 @@ impl AsSequence for PyDict { impl AsNumber for PyDict { fn as_number() -> &'static PyNumberMethods { static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::() { - PyDict::__or__(a, b.to_pyobject(vm), vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), + or: Some(dict_or), inplace_or: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::() { a.to_owned() @@ -787,16 +988,19 @@ impl Comparable for PyDict { op: PyComparisonOp, vm: &VirtualMachine, ) -> PyResult { - op.eq_only(|| { - let other = class_or_notimplemented!(Self, other); - zelf.inner_cmp(other, PyComparisonOp::Eq, true, vm) + op.eq_only(|| match PyAnyDict::from_object(other) { + Some(other) => { + zelf.payload + .compare_entries(other.as_dict(), PyComparisonOp::Eq, true, vm) + } + None => Ok(PyComparisonValue::NotImplemented), }) } } impl Iterable for PyDict { fn iter(zelf: PyRef, vm: &VirtualMachine) -> PyResult { - Ok(PyDictKeyIterator::new(zelf).into_pyobject(vm)) + Ok(PyDictKeyIterator::new(zelf.into()).into_pyobject(vm)) } } @@ -1165,6 +1369,292 @@ impl PyExact { } } +#[pyclass(module = false, name = "frozendict", traverse = "manual")] +#[derive(Debug)] +pub struct PyFrozenDict { + // Own the table directly, never a mutable Python dict visible through gc or a view. + pub(crate) dict: PyDict, + hash: PyAtomic, +} + +unsafe impl Traverse for PyFrozenDict { + fn traverse(&self, tracer: &mut TraverseFn<'_>) { + self.dict.traverse(tracer); + } + fn clear(&mut self, out: &mut Vec) { + Traverse::clear(&mut self.dict, out); + } +} + +impl PyPayload for PyFrozenDict { + fn class(ctx: &Context) -> &'static Py { + ctx.types.frozendict_type + } +} + +impl PyFrozenDict { + pub(crate) fn from_dict(dict: PyDict) -> Self { + Self { + dict, + hash: Radium::new(hash::SENTINEL), + } + } + + /// Hash a dictionary key, adding the frozendict context to an exact TypeError. + pub fn key_hash(key: &K, vm: &VirtualMachine) -> PyResult { + key_hash(key, "frozendict", vm) + } + + pub fn inner_getitem_opt( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult> { + let hash = Self::key_hash(key, vm)?; + self.dict.entries.get(vm, key, hash) + } +} + +impl Constructor for PyFrozenDict { + type Args = (OptionalArg, KwArgs); + fn py_new( + _cls: &Py, + (input, kwargs): Self::Args, + vm: &VirtualMachine, + ) -> PyResult { + // The table is built before the frozendict is published, as + // frozendict_new_untracked keeps it away from the GC. + let dict = PyDict::default(); + if let OptionalArg::Present(input) = input { + dict.update_arg(input, true, "frozendict", vm)?; + } + for (key, value) in kwargs { + dict.inner_setitem(&key, value, vm)?; + } + Ok(Self::from_dict(dict)) + } +} + +#[pyclass( + with( + Py, + PyRef, + Constructor, + Hashable, + Comparable, + Iterable, + AsMapping, + AsSequence, + AsNumber, + Representable + ), + flags(BASETYPE, MAPPING, _MATCH_SELF) +)] +impl PyFrozenDict { + fn __len__(&self) -> usize { + self.dict.__len__() + } + + #[pymethod] + fn get( + zelf: &Py, + key: PyObjectRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(zelf + .inner_getitem_opt(&*key, vm)? + .unwrap_or_else(|| default.unwrap_or_else(|| vm.ctx.none()))) + } + + #[pymethod] + fn __sizeof__(zelf: &Py) -> usize { + zelf.dict.__sizeof__() + core::mem::size_of::>() + } + + #[pyclassmethod] + fn fromkeys( + cls: PyTypeRef, + iterable: PyObjectRef, + value: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + dict_fromkeys(cls, iterable, value.unwrap_or_else(|| vm.ctx.none()), vm) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + PyGenericAlias::from_args(cls, object, vm) + } +} + +#[pyclass] +impl Py { + #[pymethod(coexist)] + fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(value) = self.inner_getitem_opt(&*key, vm)? { + return Ok(value); + } + if !self.class().is(vm.ctx.types.frozendict_type) + && let Some(method) = + vm.get_method(self.to_owned().into(), identifier!(vm, __missing__)) + { + return method?.call((key,), vm); + } + Err(vm.new_key_error(key)) + } + + #[pymethod] + fn copy(&self, vm: &VirtualMachine) -> PyResult> { + if self.class().is(vm.ctx.types.frozendict_type) { + Ok(self.to_owned()) + } else { + let dict = PyDict::default(); + if !self.dict.is_empty() { + dict.update_arg(self.to_owned().into(), true, "frozendict", vm)?; + } + Ok(PyFrozenDict::from_dict(dict).into_ref(&vm.ctx)) + } + } + + #[pymethod] + fn __getnewargs__(&self, vm: &VirtualMachine) -> PyResult { + let dict = vm + .ctx + .types + .dict_type + .as_object() + .call((self.to_owned(),), vm)?; + Ok(vm.new_tuple((dict,))) + } +} + +#[pyclass] +impl PyRef { + #[pymethod] + fn keys(self) -> PyDictKeys { + PyDictKeys::new(self.into()) + } + #[pymethod] + fn values(self) -> PyDictValues { + PyDictValues::new(self.into()) + } + #[pymethod] + fn items(self) -> PyDictItems { + PyDictItems::new(self.into()) + } + #[pymethod] + fn __reversed__(self) -> PyDictReverseKeyIterator { + PyDictReverseKeyIterator::new(self.into()) + } +} + +impl AsMapping for PyFrozenDict { + fn as_mapping() -> &'static PyMappingMethods { + static METHODS: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|obj, _vm| Ok(PyFrozenDict::mapping_downcast(obj).__len__())), + subscript: atomic_func!( + |obj, key, vm| PyFrozenDict::mapping_downcast(obj).__getitem__(key.to_owned(), vm) + ), + ass_subscript: None, + }; + &METHODS + } +} +impl AsSequence for PyFrozenDict { + fn as_sequence() -> &'static PySequenceMethods { + static METHODS: LazyLock = LazyLock::new(|| PySequenceMethods { + contains: atomic_func!(|obj, key, vm| Ok(PyFrozenDict::sequence_downcast(obj) + .inner_getitem_opt(key, vm)? + .is_some())), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &METHODS + } +} +impl AsNumber for PyFrozenDict { + fn as_number() -> &'static PyNumberMethods { + static METHODS: PyNumberMethods = PyNumberMethods { + or: Some(frozendict_or), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &METHODS + } +} +impl Iterable for PyFrozenDict { + fn iter(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + Ok(PyDictKeyIterator::new(zelf.into()).into_pyobject(vm)) + } +} +impl Comparable for PyFrozenDict { + fn cmp( + zelf: &Py, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult { + op.eq_only(|| match PyAnyDict::from_object(other) { + Some(other) => zelf + .dict + .compare_entries(other.as_dict(), PyComparisonOp::Eq, true, vm), + None => Ok(PyComparisonValue::NotImplemented), + }) + } +} +impl Hashable for PyFrozenDict { + fn hash(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let cached = zelf.hash.load(Ordering::Relaxed); + if cached != hash::SENTINEL { + return Ok(cached); + } + let mut hasher = hash::FrozenSetHash::new(zelf.__len__()); + let mut position = 0; + while let Some((next, _key, value, key_hash)) = + zelf.dict.entries.next_entry_with_hash(position) + { + position = next; + // next_entry releases the table lock before invoking arbitrary Python __hash__. + let pair_hash = hash::hash_tuple([Ok(key_hash), value.hash(vm)])?; + hasher.add(pair_hash); + } + let result = hasher.finish(); + zelf.hash.store(result, Ordering::Relaxed); + Ok(result) + } +} +impl Representable for PyFrozenDict { + fn repr(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let name = zelf.class().name(); + if zelf.dict.is_empty() { + return Ok(vm.ctx.new_str(format!("{name}()"))); + } + let mut result = Wtf8Buf::from(format!("{name}(")); + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + result.push_str("{"); + for (index, (key, value)) in DictIter::new(&zelf.dict).enumerate() { + if index != 0 { + result.push_str(", "); + } + result.push_wtf8(key.repr(vm)?.as_wtf8()); + result.push_str(": "); + result.push_wtf8(value.repr(vm)?.as_wtf8()); + } + result.push_str("}"); + } else { + result.push_str("{...}"); + } + result.push_str(")"); + Ok(vm.ctx.new_str(result)) + } + fn repr_str(_zelf: &Py, _vm: &VirtualMachine) -> PyResult { + unreachable!("use repr instead") + } +} + // Implement IntoIterator so that we can easily iterate dictionaries from rust code. impl IntoIterator for PyDictRef { type Item = (PyObjectRef, PyObjectRef); @@ -1226,12 +1716,12 @@ impl ExactSizeIterator for DictIntoIter { } pub struct DictIter<'a> { - dict: &'a Py, + dict: &'a PyDict, position: usize, } impl<'a> DictIter<'a> { - pub const fn new(dict: &'a Py) -> Self { + pub const fn new(dict: &'a PyDict) -> Self { DictIter { dict, position: 0 } } } @@ -1261,7 +1751,7 @@ impl ExactSizeIterator for DictIter<'_> { trait DictView: PyPayload + PyClassDef + Iterable + Representable { type ReverseIter: PyPayload + core::fmt::Debug; - fn dict(&self) -> &Py; + fn dict(&self) -> &PyAnyDict; fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef; fn __len__(&self) -> usize { @@ -1287,14 +1777,14 @@ macro_rules! dict_view { $project_fn: expr, $result_fn: expr ) => { - #[pyclass(module = false, name = $class_name, unhashable = $unhashable)] + #[pyclass(module = false, name = $class_name, unhashable = $unhashable, traverse)] #[derive(Debug)] pub(crate) struct $name { - pub(crate) dict: PyDictRef, + pub(crate) dict: PyRef, } impl $name { - pub(crate) const fn new(dict: PyDictRef) -> Self { + pub(crate) fn new(dict: PyRef) -> Self { $name { dict } } } @@ -1302,7 +1792,7 @@ macro_rules! dict_view { impl DictView for $name { type ReverseIter = $reverse_iter_name; - fn dict(&self) -> &Py { + fn dict(&self) -> &PyAnyDict { &self.dict } @@ -1333,7 +1823,7 @@ macro_rules! dict_view { let s = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { let mut result = Wtf8Buf::from(format!("{}([", Self::NAME)); let mut first = true; - for (key, value) in zelf.dict().clone() { + for (key, value) in zelf.dict() { if !first { result.push_str(", "); } @@ -1354,16 +1844,18 @@ macro_rules! dict_view { } } - #[pyclass(module = false, name = $iter_class_name)] + #[pyclass(module = false, name = $iter_class_name, traverse)] #[derive(Debug)] pub(crate) struct $iter_name { + #[pytraverse(skip)] pub(crate) size: dict_inner::DictSize, /// Whether the dict was found to have changed, which /// `dictiter_iternextkey()` records by writing a size no dict can /// have. Sticky: what it makes the iterator answer, it answers /// from then on. + #[pytraverse(skip)] changed: PyAtomic, - pub(crate) internal: PyMutex>, + pub(crate) internal: PyMutex>>, } impl PyPayload for $iter_name { @@ -1374,7 +1866,7 @@ macro_rules! dict_view { } impl $iter_name { - fn new(dict: PyDictRef) -> Self { + pub(crate) fn new(dict: PyRef) -> Self { $iter_name { size: dict.size(), changed: Radium::new(false), @@ -1411,7 +1903,7 @@ macro_rules! dict_view { let mut position = internal.position; let mut entries = Vec::new(); while let Some((next_position, key, value)) = - dict.entries.next_entry(position) + dict.as_dict().entries.next_entry(position) { entries.push(($result_fn)(vm, ($project_fn)(&key, &value))); position = next_position; @@ -1440,9 +1932,11 @@ macro_rules! dict_view { // raising. return (Err(mutated()), None); } - let entry = - dict.entries - .next_entry_checked(internal.position, &zelf.size, $project_fn); + let entry = dict.as_dict().entries.next_entry_checked( + internal.position, + &zelf.size, + $project_fn, + ); match entry { Err(dict_inner::DictChanged) => { zelf.changed.store(true, Ordering::Relaxed); @@ -1458,14 +1952,17 @@ macro_rules! dict_view { } } - #[pyclass(module = false, name = $reverse_iter_class_name)] + #[pyclass(module = false, name = $reverse_iter_class_name, traverse)] #[derive(Debug)] pub(crate) struct $reverse_iter_name { + #[pytraverse(skip)] pub(crate) size: dict_inner::DictSize, /// As in `$iter_name`. + #[pytraverse(skip)] changed: PyAtomic, + #[pytraverse(skip)] remaining: PyAtomic, - internal: PyMutex>, + internal: PyMutex>>, } impl PyPayload for $reverse_iter_name { @@ -1476,7 +1973,7 @@ macro_rules! dict_view { } impl $reverse_iter_name { - fn new(dict: PyDictRef) -> Self { + pub(crate) fn new(dict: PyRef) -> Self { let size = dict.size(); // usize::MAX represents the position before the first entry. let position = size.entries_size.wrapping_sub(1); @@ -1500,7 +1997,7 @@ macro_rules! dict_view { let mut position = internal.position; let mut entries = Vec::new(); while let Some((found_index, key, value)) = - dict.entries.prev_entry(position) + dict.as_dict().entries.prev_entry(position) { entries.push(($result_fn)(vm, ($project_fn)(&key, &value))); if found_index == 0 { @@ -1549,9 +2046,11 @@ macro_rules! dict_view { // raising. return (Err(mutated()), None); } - let entry = - dict.entries - .prev_entry_checked(internal.position, &zelf.size, $project_fn); + let entry = dict.as_dict().entries.prev_entry_checked( + internal.position, + &zelf.size, + $project_fn, + ); match entry { Err(dict_inner::DictChanged) => { zelf.changed.store(true, Ordering::Relaxed); @@ -1637,14 +2136,6 @@ trait ViewSetOps: DictView { op: PyComparisonOp, vm: &VirtualMachine, ) -> PyResult { - if let Some(dictview) = other.downcast_ref::() { - return zelf.dict().inner_cmp( - dictview.dict(), - op, - !zelf.class().is(vm.ctx.types.dict_keys_type), - vm, - ); - } if !is_set_or_dict_view(other) { return Ok(PyComparisonValue::NotImplemented); } @@ -1718,7 +2209,7 @@ impl PyDictKeys { #[pygetset] fn mapping(zelf: PyRef) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().to_owned()) + PyMappingProxy::from_any_dict(zelf.dict().to_owned()) } } @@ -1738,11 +2229,10 @@ impl AsSequence for PyDictKeys { static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyDictKeys::sequence_downcast(seq).__len__())), contains: atomic_func!(|seq, target, vm| { - let hash = PyDict::hash_or_unhashable(target, vm)?; - PyDictKeys::sequence_downcast(seq) + Ok(PyDictKeys::sequence_downcast(seq) .dict - .entries - .contains(vm, target, hash) + .inner_getitem_opt(target, vm)? + .is_some()) }), ..PySequenceMethods::NOT_IMPLEMENTED }); @@ -1782,7 +2272,7 @@ impl PyDictItems { } #[pygetset] fn mapping(zelf: PyRef) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().to_owned()) + PyMappingProxy::from_any_dict(zelf.dict().to_owned()) } } @@ -1844,7 +2334,7 @@ impl AsNumber for PyDictItems { impl PyDictValues { #[pygetset] fn mapping(zelf: PyRef) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().to_owned()) + PyMappingProxy::from_any_dict(zelf.dict().to_owned()) } } @@ -1944,6 +2434,107 @@ pub(crate) fn set_inner_number_or(a: &PyObject, b: &PyObject, vm: &VirtualMachin set_inner_number_op(a, b, |a, b| a.union(b, vm), vm) } +/// `_PyDict_Or` +fn dict_or(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { + let (Some(left), Some(_)) = (PyAnyDict::from_object(a), PyAnyDict::from_object(b)) else { + return Ok(vm.ctx.not_implemented()); + }; + // anydict_copy_untracked converts any frozendict to an exact frozendict. + let frozen = left.is_frozen(); + let mapping_name = if frozen { "frozendict" } else { "dict" }; + let dict = PyDict::default(); + if !left.is_empty() { + dict.update_arg(a.to_owned(), true, mapping_name, vm)?; + } + dict.update_arg(b.to_owned(), true, mapping_name, vm)?; + Ok(if frozen { + PyFrozenDict::from_dict(dict).into_pyobject(vm) + } else { + dict.into_pyobject(vm) + }) +} + +fn frozendict_or(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { + if a.class().is(vm.ctx.types.frozendict_type) + && let Some(left) = PyAnyDict::from_object(a) + && let Some(right) = PyAnyDict::from_object(b) + { + // frozendict() | frozendict(...) => frozendict(...) + if left.is_empty() && b.class().is(vm.ctx.types.frozendict_type) { + return Ok(b.to_owned()); + } + // frozendict(...) | frozendict() => frozendict(...) + if right.is_empty() + && (b.class().is(vm.ctx.types.dict_type) || b.class().is(vm.ctx.types.frozendict_type)) + { + return Ok(a.to_owned()); + } + } + dict_or(a, b, vm) +} + +pub(crate) fn dict_fromkeys( + class: PyTypeRef, + iterable: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + let exact_frozen = class.is(vm.ctx.types.frozendict_type); + let created = if exact_frozen { + None + } else { + Some(PyType::call(&class, ().into(), vm)?) + }; + let frozen_source = created + .as_ref() + .and_then(|obj| obj.downcast_ref::()); + if exact_frozen || frozen_source.is_some() { + let dict = PyDict::default(); + if let Some(source) = frozen_source { + dict.update_arg(source.to_owned().into(), true, "frozendict", vm)?; + } + let iterable = ArgIterable::::try_from_object(vm, iterable)?; + if let Some(keys) = PyDict::fromkeys_known_hashes(iterable.as_object(), vm) { + for (key, hash) in keys { + dict.entries.insert(vm, &*key, hash, value.clone())?; + } + } else { + for key in iterable.iter(vm)? { + let key = key?; + let hash = PyFrozenDict::key_hash(&*key, vm)?; + dict.entries.insert(vm, &*key, hash, value.clone())?; + } + } + let frozen = PyFrozenDict::from_dict(dict).into_ref(&vm.ctx); + return if exact_frozen { + Ok(frozen.into()) + } else { + class.as_object().call((frozen,), vm) + }; + } + let iterable = ArgIterable::::try_from_object(vm, iterable)?; + let created = created.unwrap(); + if let Some(dict) = created.downcast_ref_if_exact::(vm) { + if let Some(keys) = PyDict::fromkeys_known_hashes(iterable.as_object(), vm) { + if class.is(vm.ctx.types.dict_type) { + dict.entries.reserve_for_empty(keys.len()); + } + for (key, hash) in keys { + dict.entries.insert(vm, &*key, hash, value.clone())?; + } + } else { + for key in iterable.iter(vm)? { + dict.inner_setitem(&*key?, value.clone(), vm)?; + } + } + } else { + for key in iterable.iter(vm)? { + created.set_item(&*key?, value.clone(), vm)?; + } + } + Ok(created) +} + fn vectorcall_dict( zelf_obj: &PyObject, args: Vec, @@ -1958,6 +2549,28 @@ fn vectorcall_dict( Ok(obj.into()) } +fn vectorcall_frozendict( + zelf: &PyObject, + args: Vec, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + // frozendict(frozendict) returns the same object unmodified + if nargs == 1 + && kwnames.is_none_or(<[_]>::is_empty) + && args[0].class().is(vm.ctx.types.frozendict_type) + && zelf.is(vm.ctx.types.frozendict_type) + { + return Ok(args[0].clone()); + } + PyFrozenDict::slot_new( + zelf.downcast_ref::().unwrap().to_owned(), + FuncArgs::from_vectorcall_owned(args, nargs, kwnames), + vm, + ) +} + pub(crate) fn init(context: &'static Context) { PyDict::extend_class(context, context.types.dict_type); context @@ -1975,4 +2588,86 @@ pub(crate) fn init(context: &'static Context) { PyDictItems::extend_class(context, context.types.dict_items_type); PyDictItemIterator::extend_class(context, context.types.dict_itemiterator_type); PyDictReverseItemIterator::extend_class(context, context.types.dict_reverseitemiterator_type); + PyFrozenDict::extend_class(context, context.types.frozendict_type); + context + .types + .frozendict_type + .slots + .vectorcall + .store(Some(vectorcall_frozendict)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn anydict_reference_layout_and_ownership() { + static_assertions::assert_not_impl_any!(PyAnyDict: PyPayload); + static_assertions::assert_type_eq_all!( + ::Target, + PyAnyDict + ); + static_assertions::assert_eq_size!(PyRef, PyObjectRef, Option>); + static_assertions::assert_eq_align!(PyRef, PyObjectRef); + static_assertions::assert_eq_size!(PyAnyDict, PyObject); + static_assertions::assert_eq_align!(PyAnyDict, PyObject); + #[cfg(feature = "threading")] + static_assertions::assert_impl_all!(PyRef: Send, Sync); + #[cfg(not(feature = "threading"))] + static_assertions::assert_not_impl_any!(PyRef: Send, Sync); + + let ctx = Context::genesis(); + let dict: PyRef = PyDict::default().into_ref(ctx).into(); + let frozen: PyRef = PyFrozenDict::from_dict(PyDict::default()) + .into_ref(ctx) + .into(); + for owner in [dict, frozen] { + let object = owner.as_object() as *const PyObject; + assert_eq!(owner.as_object().strong_count(), 1); + let borrowed = PyAnyDict::from_object(owner.as_object()).unwrap(); + assert!(core::ptr::eq(borrowed.as_object(), object)); + assert_eq!(owner.as_object().strong_count(), 1); + assert_eq!(borrowed.as_mutable().is_none(), borrowed.is_frozen()); + let copy = borrowed.to_owned(); + assert_eq!(owner.as_object().strong_count(), 2); + let clone = owner.clone(); + assert_eq!(owner.as_object().strong_count(), 3); + let erased = clone.into_object(); + assert_eq!(erased.strong_count(), 3); + assert!(core::ptr::eq(&*erased, object)); + let mut visited = 0; + owner.traverse(&mut |edge| { + assert!(core::ptr::eq(edge, object)); + visited += 1; + }); + assert_eq!(visited, 1); + drop(copy); + drop(owner); + assert_eq!(erased.strong_count(), 1); + } + } + + #[test] + fn anydict_checked_reference_conversion() { + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let dict = PyDict::default().into_pyobject(vm); + let frozen = PyFrozenDict::from_dict(PyDict::default()).into_pyobject(vm); + for object in [dict, frozen] { + let owner = PyRef::::try_from_object(vm, object.clone()).unwrap(); + assert_eq!(object.strong_count(), 2); + assert!(core::ptr::eq(owner.as_object(), &*object)); + let erased: PyObjectRef = owner.into(); + assert_eq!(object.strong_count(), 2); + assert!(core::ptr::eq(&*erased, &*object)); + drop(erased); + assert_eq!(object.strong_count(), 1); + } + let unrelated: PyObjectRef = vm.ctx.new_int(1000).into(); + assert!(PyAnyDict::from_object(&unrelated).is_none()); + let error = PyRef::::try_from_object(vm, unrelated.clone()).unwrap_err(); + assert!(error.fast_isinstance(vm.ctx.exceptions.type_error)); + assert_eq!(unrelated.strong_count(), 1); + }); + } } diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 905cbb582a4..be8ddd9f391 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -2,7 +2,7 @@ */ -use super::{PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyIntRef}; +use super::{PyAnyDict, PyAsyncGen, PyCode, PyCoroutine, PyGenerator, PyIntRef}; use crate::{ AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, @@ -668,7 +668,7 @@ impl Py { } #[pygetset] - pub fn f_globals(&self) -> PyDictRef { + pub fn f_globals(&self) -> PyRef { self.iframe().globals().to_owned() } diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index be4994a6204..40e51c6b31b 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -2,8 +2,8 @@ mod jit; use super::{ - PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyList, PyModule, PyStr, PyStrRef, - PyTuple, PyTupleRef, PyType, object, + PyAnyDict, PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyList, PyModule, PyStr, + PyStrRef, PyTuple, PyTupleRef, PyType, object, }; use crate::common::hash::PyHash; use crate::common::lock::PyMutex; @@ -66,7 +66,7 @@ fn format_missing_args( pub struct PyFunction { pub(crate) code: PyAtomicRef, #[pymember(name = "__globals__")] - pub(crate) globals: PyDictRef, + pub(crate) globals: PyRef, #[pymember(name = "__builtins__")] pub(crate) builtins: PyObjectRef, #[pymember(name = "__closure__")] @@ -179,15 +179,17 @@ impl PyFunction { #[inline] pub(crate) fn new( code: PyRef, - globals: PyDictRef, + globals: PyRef, vm: &VirtualMachine, ) -> PyResult { let name = PyMutex::new(code.obj_name.to_owned()); - let module = vm.unwrap_or_none(globals.get_item_opt(identifier!(vm, __name__), vm)?); - let builtins = globals.get_item("__builtins__", vm).unwrap_or_else(|_| { - // If not in globals, inherit from current execution context - crate::frame::current_builtins().unwrap_or_else(|| vm.builtins.dict().into()) - }); + let module = vm.unwrap_or_none(globals.inner_getitem_opt(identifier!(vm, __name__), vm)?); + let builtins = globals + .inner_getitem_opt(identifier!(vm, __builtins__), vm)? + .unwrap_or_else(|| { + // If not in globals, inherit from current execution context. + crate::frame::current_builtins().unwrap_or_else(|| vm.builtins.dict().into()) + }); // If builtins is a module, use its __dict__ instead let builtins = if let Some(module) = builtins.downcast_ref::() { module.dict().into() @@ -621,7 +623,7 @@ impl Py { } else if let Some(locals) = locals { Some(locals) } else { - Some(ArgMapping::from_dict_exact(self.globals.clone())) + Some(ArgMapping::from_anydict_exact(self.globals.clone())) }; let use_datastack = !is_gen && !is_coro && !is_async_gen; let frame = FrameObject::new_ref( @@ -654,7 +656,7 @@ impl Py { } else if let Some(locals) = locals { crate::frame::FrameLocals::with_locals(locals) } else { - crate::frame::FrameLocals::with_locals(crate::function::ArgMapping::from_dict_exact( + crate::frame::FrameLocals::with_locals(crate::function::ArgMapping::from_anydict_exact( self.globals.clone(), )) }; @@ -805,7 +807,7 @@ impl Py { let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { None } else { - Some(ArgMapping::from_dict_exact(self.globals.clone())) + Some(ArgMapping::from_anydict_exact(self.globals.clone())) }; let frame = FrameObject::new_ref( @@ -862,7 +864,7 @@ impl Py { let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { None } else { - Some(ArgMapping::from_dict_exact(self.globals.clone())) + Some(ArgMapping::from_anydict_exact(self.globals.clone())) }; // Heap-backed: the frame outlives the call that made it. @@ -897,7 +899,7 @@ impl Py { let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { crate::frame::FrameLocals::lazy() } else { - crate::frame::FrameLocals::with_locals(ArgMapping::from_dict_exact( + crate::frame::FrameLocals::with_locals(ArgMapping::from_anydict_exact( self.globals.clone(), )) }; @@ -1318,7 +1320,7 @@ impl Constructor for PyFunction { None }; - let mut func = Self::new(args.code.clone(), args.globals.clone(), vm)?; + let mut func = Self::new(args.code.clone(), args.globals.clone().into(), vm)?; // Set function name if provided if let Some(name) = args.name.into_option() { *func.name.lock() = name; diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs index 3e53f1729a5..e1e0283d189 100644 --- a/crates/vm/src/builtins/mappingproxy.rs +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -68,6 +68,12 @@ impl Constructor for PyMappingProxy { } impl PyMappingProxy { + pub(crate) fn from_any_dict(dict: PyRef) -> Self { + Self { + mapping: MappingProxyInner::Mapping(ArgMapping::new(dict.into())), + } + } + pub fn from_object(mapping: PyObjectRef, vm: &VirtualMachine) -> PyResult { if mapping.mapping_unchecked().check() && !mapping.downcastable::() diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs index a938b9b3dd1..438cd9e5092 100644 --- a/crates/vm/src/builtins/mod.rs +++ b/crates/vm/src/builtins/mod.rs @@ -20,7 +20,7 @@ pub use complex::PyComplex; pub(crate) mod coroutine; pub use coroutine::PyCoroutine; pub(crate) mod dict; -pub use dict::{PyDict, PyDictRef}; +pub use dict::{PyAnyDict, PyDict, PyDictRef, PyFrozenDict}; pub(crate) mod enumerate; pub use enumerate::PyEnumerate; pub(crate) mod filter; diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index a2449f136ef..153842056c1 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -1,6 +1,6 @@ use super::{ - PyClassMethod, PyDict, PyDictRef, PyList, PyStaticMethod, PyStr, PyStrInterned, PyStrRef, - PyTupleRef, PyUtf8StrRef, PyWeak, mappingproxy::PyMappingProxy, object, union_, + PyAnyDict, PyClassMethod, PyDict, PyDictRef, PyList, PyStaticMethod, PyStr, PyStrInterned, + PyStrRef, PyTupleRef, PyUtf8StrRef, PyWeak, mappingproxy::PyMappingProxy, object, union_, }; use crate::{ AsObject, Context, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, @@ -1798,25 +1798,15 @@ impl Constructor for PyType { })); } - let (name, bases, dict, kwargs): (PyStrRef, PyTupleRef, PyDictRef, KwArgs) = + let (name, bases, dict, kwargs): (PyStrRef, PyTupleRef, PyObjectRef, KwArgs) = args.clone().bind_for(vm, Self::NAME)?; - - // A mapping that is not an exact dict (e.g. OrderedDict) keeps its - // own iteration order; copy via the mapping protocol so that order - // lands in the type dict. - let dict = if args.args[2].class().is(vm.ctx.types.dict_type) { - dict - } else { - let copied = vm.ctx.new_dict(); - copied.merge_object(args.args[2].clone(), vm)?; - copied + let Some(dict) = PyAnyDict::from_object(&dict) else { + return Err(vm.new_type_error(format!( + "type.__new__() argument 3 must be dict or frozendict, not {}", + dict.class().fully_qualified_name(vm)? + ))); }; - if name.as_bytes().contains(&0) { - return Err(vm.new_value_error("type name must not contain null characters")); - } - let name = name.try_into_utf8(vm)?; - let (metatype, base, bases, base_is_type) = if bases.as_slice().is_empty() { let base = vm.ctx.types.object_type.to_owned(); let bases = PyTuple::new_ref_typed(vec![base.clone()], &vm.ctx); @@ -1856,6 +1846,25 @@ impl Constructor for PyType { (metatype, base.to_owned(), bases, base_is_type) }; + // Copy only after resolving the metaclass: its __new__ receives the + // original namespace. A subclass with a custom iterator (e.g. + // OrderedDict) keeps its own mapping order in the mutable type dict. + let dict = if let Some(dict) = dict.as_object().downcast_ref_if_exact::(vm) { + dict.to_owned() + } else { + let copied = vm.ctx.new_dict(); + // Like dict.copy(), an empty table bypasses subclass hooks. + if !dict.is_empty() { + copied.merge_object(dict.to_owned().into(), vm)?; + } + copied + }; + + if name.as_bytes().contains(&0) { + return Err(vm.new_value_error("type name must not contain null characters")); + } + let name = name.try_into_utf8(vm)?; + let qualname = dict .get_item_opt(identifier!(vm, __qualname__), vm)? .map(|obj| downcast_qualname(obj, vm)) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 2c9d6075106..a043838d49b 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -6,10 +6,10 @@ use crate::{ AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, PyStackRef, TryFromObject, VirtualMachine, builtins::{ - PyBaseException, PyBaseExceptionRef, PyBaseObject, PyBytes, PyCode, PyCoroutine, PyDict, - PyDictRef, PyFloat, PyFrozenSet, PyGenerator, PyInt, PyInterpolation, PyList, PyModule, - PyProperty, PySet, PySlice, PyStr, PyStrInterned, PyTemplate, PyTraceback, PyType, - PyUtf8Str, + PyAnyDict, PyBaseException, PyBaseExceptionRef, PyBaseObject, PyBytes, PyCode, PyCoroutine, + PyDict, PyDictRef, PyFloat, PyFrozenSet, PyGenerator, PyInt, PyInterpolation, PyList, + PyModule, PyProperty, PySet, PySlice, PyStr, PyStrInterned, PyTemplate, PyTraceback, + PyType, PyUtf8Str, builtin_func::PyNativeFunction, descriptor::{PyMemberDescriptor, PyMethodDescriptor}, frame::stack_analysis, @@ -108,7 +108,7 @@ pub fn current_thread_frame_materialize(vm: &VirtualMachine) -> Option Option { +pub fn current_globals() -> Option> { let ptr = crate::vm::thread::get_current_frame(); if ptr.is_null() { return None; @@ -923,7 +923,7 @@ pub struct InterpreterFrame { // Borrowed pointers — owned by FrameObject or by PyFunction on caller's stack. pub(crate) code: *const Py, pub(crate) func_obj: *const PyObject, // nullable - pub(crate) globals: *const Py, + pub(crate) globals: *const PyAnyDict, pub(crate) builtins: *const PyObject, /// Unified storage for local variables and evaluation stack. @@ -983,7 +983,7 @@ impl InterpreterFrame { #[inline(always)] pub(crate) unsafe fn new( code: &Py, - globals: &Py, + globals: &PyAnyDict, builtins: &PyObject, func_obj: Option<&PyObject>, localsplus: LocalsPlus, @@ -1027,7 +1027,7 @@ impl InterpreterFrame { Some(obj) => obj as *const PyObject, None => core::ptr::null(), }, - globals: globals as *const Py, + globals: globals as *const PyAnyDict, builtins: builtins as *const PyObject, localsplus, locals, @@ -1060,7 +1060,7 @@ impl InterpreterFrame { #[inline(always)] pub(crate) unsafe fn new_on_datastack<'a>( code: &Py, - globals: &Py, + globals: &PyAnyDict, builtins: &PyObject, func_obj: Option<&PyObject>, locals: FrameLocals, @@ -1266,7 +1266,7 @@ impl InterpreterFrame { // Create a full FrameObject with its own InterpreterFrame copy. // The FrameObject owns references to the same objects (code, globals, etc). let code: PyRef = self.code().to_owned(); - let globals: PyDictRef = self.globals().to_owned(); + let globals = self.globals().to_owned(); let builtins: PyObjectRef = self.builtins().to_owned(); let func_obj: Option = self.func_obj().map(|o| o.to_owned()); @@ -1362,7 +1362,7 @@ impl InterpreterFrame { #[cold] fn materialize_slow_chain(&self, vm: &VirtualMachine) -> FrameObjectRef { let code: PyRef = self.code().to_owned(); - let globals: PyDictRef = self.globals().to_owned(); + let globals = self.globals().to_owned(); let builtins: PyObjectRef = self.builtins().to_owned(); let func_obj: Option = self.func_obj().map(|o| o.to_owned()); @@ -1419,7 +1419,7 @@ impl InterpreterFrame { /// Borrowed globals dict. #[inline(always)] - pub fn globals(&self) -> &Py { + pub fn globals(&self) -> &PyAnyDict { // SAFETY: established by `new` / `init_iframe_ptrs`. unsafe { &*self.globals } } @@ -1534,7 +1534,7 @@ pub struct FrameObject { // raw pointers. Wrapped in Option so Traverse::clear can release them, // allowing GC cycle collection to reclaim referenced objects. pub(crate) owned_code: Option>, - pub(crate) owned_globals: Option, + pub(crate) owned_globals: Option>, pub(crate) owned_builtins: Option, pub(crate) owned_func_obj: Option, @@ -1817,7 +1817,7 @@ impl FrameObject { let locals = match scope.locals { Some(locals) => FrameLocals::with_locals(locals), None if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) => FrameLocals::lazy(), - None => FrameLocals::with_locals(ArgMapping::from_dict_exact(scope.globals.clone())), + None => FrameLocals::with_locals(ArgMapping::from_anydict_exact(scope.globals.clone())), }; // Pointers are initially set from owned fields' references but will be @@ -1853,7 +1853,7 @@ impl FrameObject { fn init_iframe_ptrs(self_: &Py) { let iframe = unsafe { self_.iframe_mut() }; iframe.code = &**self_.owned_code.as_ref().unwrap() as *const Py; - iframe.globals = &**self_.owned_globals.as_ref().unwrap() as *const Py; + iframe.globals = &**self_.owned_globals.as_ref().unwrap() as *const PyAnyDict; iframe.builtins = &**self_.owned_builtins.as_ref().unwrap() as *const PyObject; iframe.func_obj = match &self_.owned_func_obj { Some(obj) => &**obj as *const PyObject, @@ -2497,7 +2497,7 @@ impl Py { }, ) }; - let builtins_dict = if globals.class().is(vm.ctx.types.dict_type) { + let builtins_dict = if globals.as_object().class().is(vm.ctx.types.dict_type) { builtins .downcast_ref_if_exact::(vm) // SAFETY: downcast_ref_if_exact already verified exact type @@ -2739,7 +2739,7 @@ fn exec_iframe<'a>( }, ) }; - let builtins_dict = if globals.class().is(vm.ctx.types.dict_type) { + let builtins_dict = if globals.as_object().class().is(vm.ctx.types.dict_type) { builtins .downcast_ref_if_exact::(vm) .map(|d| unsafe { PyExact::ref_unchecked(d) }) @@ -2931,7 +2931,7 @@ pub(crate) struct ExecutingFrame<'a> { code: &'a Py, localsplus: &'a mut LocalsPlus, locals: &'a FrameLocals, - globals: &'a Py, + globals: &'a PyAnyDict, builtins: &'a PyObject, /// Cached downcast of builtins to PyDict for fast LOAD_GLOBAL. /// Only set when both globals and builtins are exact dict types (not @@ -4651,15 +4651,10 @@ impl ExecutingFrame<'_> { } Instruction::DeleteName { namei: idx } => { let name = self.code.names[idx.get(arg) as usize]; - let res = self.locals.mapping(vm).ass_subscript(name, None, vm); - - match res { - Ok(()) => {} - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - return Err(name_error(name, vm)); - } - Err(e) => return Err(e), - } + self.locals + .mapping(vm) + .ass_subscript(name, None, vm) + .map_err(|_| name_error(name, vm))?; Ok(None) } Instruction::DeleteSubscr => self.execute_delete_subscript(vm), @@ -5254,7 +5249,7 @@ impl ExecutingFrame<'_> { match result { Ok(x) => self.push_value(x), Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - self.push_value(self.load_global_or_builtin(name, vm)?); + self.push_value(self.load_name_global_or_builtin(name, vm)?); } Err(e) => return Err(e), } @@ -8043,9 +8038,9 @@ impl ExecutingFrame<'_> { let cached_version = self.code.instructions.read_cache_u16(cache_base + 1); let cached_index = self.code.instructions.read_cache_u16(cache_base + 3); if cached_version != 0 - && let Some(x) = self - .globals - .get_item_by_index_and_keys_version(cached_version, cached_index) + && let Some(globals) = self.globals.as_mutable() + && let Some(x) = + globals.get_item_by_index_and_keys_version(cached_version, cached_index) { self.push_value(x); if (oparg & 1) != 0 { @@ -8069,7 +8064,8 @@ impl ExecutingFrame<'_> { let cached_index = self.code.instructions.read_cache_u16(cache_base + 3); if cached_globals_ver != 0 && cached_builtins_ver != 0 - && let Ok(current_globals_ver) = u16::try_from(self.globals.keys_version()) + && let Some(globals) = self.globals.as_mutable() + && let Ok(current_globals_ver) = u16::try_from(globals.keys_version()) && cached_globals_ver == current_globals_ver && let Some(builtins_dict) = self.builtins.downcast_ref_if_exact::(vm) && let Some(x) = builtins_dict @@ -8541,7 +8537,12 @@ impl ExecutingFrame<'_> { if let Some(builtins_dict) = self.builtins_dict { // Fast path: both globals and builtins are exact dicts // SAFETY: builtins_dict is only set when globals is also exact dict - let globals_exact = unsafe { PyExact::ref_unchecked(self.globals) }; + let globals = self + .globals + .as_object() + .downcast_ref::() + .expect("exact dict globals"); + let globals_exact = unsafe { PyExact::ref_unchecked(globals) }; globals_exact .get_chain_exact(builtins_dict, name, vm)? .ok_or_else(|| { @@ -8552,16 +8553,29 @@ impl ExecutingFrame<'_> { if let Some(value) = self.globals.get_item_opt(name, vm)? { return Ok(value); } - self.builtins.get_item(name, vm).map_err(|e| { - if e.fast_isinstance(vm.ctx.exceptions.key_error) { - vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) - } else { - e - } - }) + self.load_builtin(name, vm) } } + fn load_name_global_or_builtin(&self, name: &Py, vm: &VirtualMachine) -> PyResult { + // LOAD_NAME reads the stored global entries even for dict subclasses; + // LOAD_GLOBAL instead honors their __getitem__ implementation. + if let Some(value) = self.globals.inner_getitem_opt(name, vm)? { + return Ok(value); + } + self.load_builtin(name, vm) + } + + fn load_builtin(&self, name: &Py, vm: &VirtualMachine) -> PyResult { + self.builtins.get_item(name, vm).map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.key_error) { + vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + } else { + e + } + }) + } + #[cfg_attr(feature = "flame-it", flame("FrameObject"))] fn import(&mut self, vm: &VirtualMachine, module_name: Option<&Py>) -> PyResult<()> { let module_name = module_name.unwrap_or(vm.ctx.empty_str); @@ -11786,7 +11800,7 @@ impl ExecutingFrame<'_> { let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { FrameLocals::lazy() } else { - FrameLocals::with_locals(crate::function::ArgMapping::from_dict_exact( + FrameLocals::with_locals(crate::function::ArgMapping::from_anydict_exact( func.globals.clone(), )) }; @@ -11852,7 +11866,7 @@ impl ExecutingFrame<'_> { let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { FrameLocals::lazy() } else { - FrameLocals::with_locals(crate::function::ArgMapping::from_dict_exact( + FrameLocals::with_locals(crate::function::ArgMapping::from_anydict_exact( func.globals.clone(), )) }; @@ -11938,8 +11952,11 @@ impl ExecutingFrame<'_> { ) { return; } + let Some(globals) = self.globals.as_mutable() else { + return; + }; let name = self.code.names[(oparg >> 1) as usize]; - let Ok(globals_version @ 1..) = u16::try_from(self.globals.assign_keys_version(vm)) else { + let Ok(globals_version @ 1..) = u16::try_from(globals.assign_keys_version(vm)) else { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -11951,7 +11968,7 @@ impl ExecutingFrame<'_> { return; }; - if let Ok(Some(globals_hint)) = self.globals.hint_for_key(name, vm) { + if let Ok(Some(globals_hint)) = globals.hint_for_key(name, vm) { unsafe { self.code .instructions diff --git a/crates/vm/src/function/protocol.rs b/crates/vm/src/function/protocol.rs index 5883a2a4691..c9d8df5b186 100644 --- a/crates/vm/src/function/protocol.rs +++ b/crates/vm/src/function/protocol.rs @@ -1,7 +1,7 @@ use super::IntoFuncArgs; use crate::{ - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, - builtins::{PyDictRef, iter::PySequenceIterator}, + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyAnyDict, PyDictRef, iter::PySequenceIterator}, convert::ToPyObject, object::{Traverse, TraverseFn}, protocol::{PyIter, PyIterIter, PyMapping}, @@ -155,6 +155,12 @@ impl ArgMapping { Self { obj: dict.into() } } + #[inline(always)] + #[must_use] + pub fn from_anydict_exact(dict: PyRef) -> Self { + Self { obj: dict.into() } + } + #[inline(always)] #[must_use] pub fn obj(&self) -> &PyObject { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 8dacc13d303..fa2e7ae648a 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -2666,6 +2666,7 @@ mod ref_target_sealed { impl Sealed for T {} impl Sealed for super::PyObject {} impl Sealed for crate::builtins::set::PyAnySet {} + impl Sealed for crate::builtins::PyAnyDict {} impl Sealed for super::PyTuple> {} } @@ -2709,6 +2710,11 @@ pub trait PyTypedRefTarget: PyRefTarget {} impl>> PyTypedRefTarget for T {} impl PyTypedRefTarget for crate::builtins::set::PyAnySet {} +impl PyTypedRefTarget for crate::builtins::PyAnyDict {} + +impl PyRefTarget for crate::builtins::PyAnyDict { + type Target = Self; +} impl PyRefTarget for crate::builtins::set::PyAnySet { type Target = Self; diff --git a/crates/vm/src/protocol/callable.rs b/crates/vm/src/protocol/callable.rs index 49172a15222..67a5354b131 100644 --- a/crates/vm/src/protocol/callable.rs +++ b/crates/vm/src/protocol/callable.rs @@ -78,6 +78,11 @@ impl<'a> PyCallable<'a> { pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { let args = args.into_args(vm); + // _PyVectorcall_Call + if self.vectorcall.is_some() && args.kwargs.is_empty() { + let nargs = args.args.len(); + return self.invoke_vectorcall(args.args, nargs, None, vm); + } if !vm.use_tracing.get() { return (self.call)(self.obj, args, vm); } diff --git a/crates/vm/src/protocol/mapping.rs b/crates/vm/src/protocol/mapping.rs index e791757901e..997df878ff0 100644 --- a/crates/vm/src/protocol/mapping.rs +++ b/crates/vm/src/protocol/mapping.rs @@ -184,7 +184,7 @@ impl PyMapping<'_> { pub fn keys(self, vm: &VirtualMachine) -> PyResult { if let Some(dict) = self.obj.downcast_ref_if_exact::(vm) { - PyDictKeys::new(dict.to_owned()).to_pyresult(vm) + PyDictKeys::new(dict.to_owned().into()).to_pyresult(vm) } else { self.method_output_as_list(identifier!(vm, keys), vm) } @@ -192,7 +192,7 @@ impl PyMapping<'_> { pub fn values(self, vm: &VirtualMachine) -> PyResult { if let Some(dict) = self.obj.downcast_ref_if_exact::(vm) { - PyDictValues::new(dict.to_owned()).to_pyresult(vm) + PyDictValues::new(dict.to_owned().into()).to_pyresult(vm) } else { self.method_output_as_list(identifier!(vm, values), vm) } @@ -200,7 +200,7 @@ impl PyMapping<'_> { pub fn items(self, vm: &VirtualMachine) -> PyResult { if let Some(dict) = self.obj.downcast_ref_if_exact::(vm) { - PyDictItems::new(dict.to_owned()).to_pyresult(vm) + PyDictItems::new(dict.to_owned().into()).to_pyresult(vm) } else { self.method_output_as_list(identifier!(vm, items), vm) } diff --git a/crates/vm/src/scope.rs b/crates/vm/src/scope.rs index 772d5c176c8..976cf750772 100644 --- a/crates/vm/src/scope.rs +++ b/crates/vm/src/scope.rs @@ -1,10 +1,14 @@ -use crate::{VirtualMachine, builtins::PyDictRef, function::ArgMapping}; +use crate::{ + PyRef, VirtualMachine, + builtins::{PyAnyDict, PyDictRef}, + function::ArgMapping, +}; use alloc::fmt; #[derive(Clone)] pub struct Scope { pub locals: Option, - pub globals: PyDictRef, + pub globals: PyRef, } impl fmt::Debug for Scope { @@ -17,8 +21,11 @@ impl fmt::Debug for Scope { impl Scope { #[inline] #[must_use] - pub const fn new(locals: Option, globals: PyDictRef) -> Self { - Self { locals, globals } + pub fn new(locals: Option, globals: impl Into>) -> Self { + Self { + locals, + globals: globals.into(), + } } pub fn with_builtins( diff --git a/crates/vm/src/stdlib/_testinternalcapi.rs b/crates/vm/src/stdlib/_testinternalcapi.rs index caafcf5ab57..f378eed11c4 100644 --- a/crates/vm/src/stdlib/_testinternalcapi.rs +++ b/crates/vm/src/stdlib/_testinternalcapi.rs @@ -1125,11 +1125,16 @@ fn expandtabs(input: &str, tab_size: usize) -> String { expanded } -type CodeNamespaces = (PyRef, Option>, Option>); +type CodeNamespaces = ( + PyRef, + Option>, + Option>, +); fn code_or_function(obj: &PyObject, vm: &VirtualMachine) -> PyResult { if let Ok(func) = obj.to_owned().downcast::() { - let builtins = func.builtins.clone().downcast::().ok(); + let builtins = + crate::builtins::PyAnyDict::from_object(&func.builtins).map(ToOwned::to_owned); return Ok(( (*func.code).to_owned(), Some(func.globals.clone()), @@ -1142,19 +1147,22 @@ fn code_or_function(obj: &PyObject, vm: &VirtualMachine) -> PyResult>>( override_ns: OptionalArg, - default: Option>, + default: Option, name: &str, vm: &VirtualMachine, -) -> PyResult>> { +) -> PyResult> { match override_ns.into_option() { - Some(obj) => obj.downcast::().map(Some).map_err(|obj| { - vm.new_type_error(format!( - "expected a dict for \"{name}\", got {}", - obj.class().name() - )) - }), + Some(obj) => obj + .downcast::() + .map(|dict| Some(dict.into())) + .map_err(|obj| { + vm.new_type_error(format!( + "expected a dict for \"{name}\", got {}", + obj.class().name() + )) + }), None => Ok(default), } } @@ -1280,8 +1288,8 @@ fn set_unbound_var_counts( counts: &mut VarCounts, globalnames: Option, attrnames: Option, - globalsns: Option<&Py>, - builtinsns: Option<&Py>, + globalsns: Option<&crate::builtins::PyAnyDict>, + builtinsns: Option<&crate::builtins::PyAnyDict>, vm: &VirtualMachine, ) -> PyResult<()> { let globalnames = optional_set(globalnames, "globalnames", vm)?; @@ -1317,8 +1325,8 @@ fn identify_unbound_names( code: &Py, globalnames: Option>, attrnames: Option>, - globalsns: Option<&Py>, - builtinsns: Option<&Py>, + globalsns: Option<&crate::builtins::PyAnyDict>, + builtinsns: Option<&crate::builtins::PyAnyDict>, vm: &VirtualMachine, ) -> PyResult<(UnboundCounts, i32)> { let mut seen_globals = HashSet::new(); diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 655f30f87d8..e4c21007a0b 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -11,7 +11,7 @@ mod builtins { AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{ - PyByteArray, PyBytes, PyDictRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, + PyAnyDict, PyByteArray, PyBytes, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyUtf8StrRef, enumerate::PyReverseSequenceIterator, function::{PyCell, PyCellRef, PyFunction}, @@ -425,6 +425,26 @@ mod builtins { locals: Option, } + /// `_PyEval_EnsureBuiltins` + fn ensure_builtins(globals: &PyAnyDict, vm: &VirtualMachine) -> PyResult<()> { + if globals + .inner_getitem_opt(identifier!(vm, __builtins__), vm)? + .is_some() + { + return Ok(()); + } + // PyEval_GetBuiltins + let builtins = + crate::frame::current_builtins().unwrap_or_else(|| vm.builtins.dict().into()); + match globals.as_mutable() { + Some(dict) => dict.inner_setitem(identifier!(vm, __builtins__), builtins, vm), + None => globals + .as_object() + .set_item(identifier!(vm, __builtins__), builtins, vm) + .map_err(|_| vm.new_type_error("cannot assign __builtins__ to frozendict globals")), + } + } + impl ScopeArgs { fn make_scope( self, @@ -436,21 +456,23 @@ mod builtins { vm: &VirtualMachine, func_name: &'static str, ) -> PyResult<()> { - if !globals.fast_isinstance(vm.ctx.types.dict_type) { + if !globals.fast_isinstance(vm.ctx.types.dict_type) + && !globals.fast_isinstance(vm.ctx.types.frozendict_type) + { return Err(match func_name { "eval" => { let is_mapping = globals.mapping_unchecked().check(); vm.new_type_error(if is_mapping { - "globals must be a real dict; try eval(expr, {}, mapping)" + "globals must be a real dict or a frozendict; try eval(expr, {}, mapping)" } else { - "globals must be a dict" + "globals must be a dict or a frozendict" }) } "exec" => vm.new_type_error(format!( - "exec() globals must be a dict, not {}", + "exec() globals must be a dict or a frozendict, not {}", globals.class().name() )), - _ => vm.new_type_error("globals must be a dict"), + _ => vm.new_type_error("globals must be a dict or a frozendict"), }); } Ok(()) @@ -460,15 +482,11 @@ mod builtins { Some(globals) => { validate_globals_dict(&globals, vm, func_name)?; - let globals = PyDictRef::try_from_object(vm, globals)?; - if !globals.contains_key(identifier!(vm, __builtins__), vm) { - let builtins_dict = vm.builtins.dict().into(); - globals.set_item(identifier!(vm, __builtins__), builtins_dict, vm)?; - } + let globals = PyRef::::try_from_object(vm, globals)?; ( globals.clone(), self.locals - .unwrap_or_else(|| ArgMapping::from_dict_exact(globals.clone())), + .unwrap_or_else(|| ArgMapping::from_anydict_exact(globals.clone())), ) } None => ( @@ -481,7 +499,9 @@ mod builtins { ), }; - let scope = crate::scope::Scope::with_builtins(Some(locals), globals, vm); + ensure_builtins(&globals, vm)?; + + let scope = crate::scope::Scope::new(Some(locals), globals); Ok(scope) } } @@ -650,6 +670,7 @@ mod builtins { func: &str, closure: Option>>, ) -> PyResult { + let from_string = matches!(&source, Either::B(_)); // Determine code object: let code_obj = match source { #[cfg(feature = "rustpython-compiler")] @@ -675,6 +696,18 @@ mod builtins { ))); } + // run_eval_code_obj: set globals['__builtins__'] if it doesn't exist + if from_string + && scope + .globals + .inner_getitem_opt(identifier!(vm, __builtins__), vm)? + .is_none() + { + scope + .globals + .set_item(identifier!(vm, __builtins__), vm.builtins.dict().into(), vm)?; + } + // Run the code: vm.run_code_obj_with_closure(code_obj, scope, closure) } @@ -721,7 +754,7 @@ mod builtins { } #[pyfunction] - fn globals(vm: &VirtualMachine) -> PyDictRef { + fn globals(vm: &VirtualMachine) -> PyRef { vm.current_globals() } @@ -1490,6 +1523,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { "dict" => ctx.types.dict_type.to_owned(), "enumerate" => ctx.types.enumerate_type.to_owned(), "float" => ctx.types.float_type.to_owned(), + "frozendict" => ctx.types.frozendict_type.to_owned(), "frozenset" => ctx.types.frozenset_type.to_owned(), "filter" => ctx.types.filter_type.to_owned(), "int" => ctx.types.int_type.to_owned(), diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index fce1a1e9eb6..5291e677add 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -40,6 +40,7 @@ pub struct TypeZoo { pub float_type: &'static Py, pub frame_type: &'static Py, pub frame_locals_proxy_type: &'static Py, + pub frozendict_type: &'static Py, pub frozenset_type: &'static Py, pub generator_type: &'static Py, pub int_type: &'static Py, @@ -139,6 +140,7 @@ impl TypeZoo { dict_type: dict::PyDict::init_builtin_type(), enumerate_type: enumerate::PyEnumerate::init_builtin_type(), float_type: float::PyFloat::init_builtin_type(), + frozendict_type: dict::PyFrozenDict::init_builtin_type(), frozenset_type: set::PyFrozenSet::init_builtin_type(), filter_type: filter::PyFilter::init_builtin_type(), list_type: list::PyList::init_builtin_type(), diff --git a/crates/vm/src/vm/crossinterp.rs b/crates/vm/src/vm/crossinterp.rs index fbab68f153d..a72d34ea92a 100644 --- a/crates/vm/src/vm/crossinterp.rs +++ b/crates/vm/src/vm/crossinterp.rs @@ -340,7 +340,7 @@ impl SharedValue { // Stateless functions have no globals, so `__main__` is used, // just like for builtins such as exec(). let globals = vm.main_namespace()?; - Ok(PyFunction::new(code, globals, vm)?.into_pyobject(vm)) + Ok(PyFunction::new(code, globals.into(), vm)?.into_pyobject(vm)) } Self::Pickled(data) => pickle_loads(&data, vm), } @@ -763,7 +763,12 @@ pub(crate) fn walk_instructions( /// `_PyFunction_VerifyStateless`. fn verify_stateless_function(func: &Py, vm: &VirtualMachine) -> PyResult<()> { - // `__globals__` is a dict by construction, so only the builtins are checked. + let globals = func.globals.as_mutable().ok_or_else(|| { + vm.new_type_error(format!( + "unsupported globals {}", + render_repr(func.globals.as_object(), vm) + )) + })?; let builtins = func.builtins.downcast_ref::().ok_or_else(|| { vm.new_type_error(format!( "unsupported builtins {}", @@ -786,7 +791,7 @@ fn verify_stateless_function(func: &Py, vm: &VirtualMachine) -> PyRe { return Err(vm.new_value_error("closures not supported")); } - verify_stateless(&func.code, Some((&func.globals, builtins)), vm) + verify_stateless(&func.code, Some((globals, builtins)), vm) } /// `verify_script`: a script takes no arguments and returns only None. diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 28d50958a6a..c3625828cc4 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -24,7 +24,7 @@ mod vm_ops; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ - self, PyBaseExceptionRef, PyBaseObject, PyDict, PyDictRef, PyFrozenSet, PyInt, PyList, + self, PyAnyDict, PyBaseExceptionRef, PyBaseObject, PyDict, PyFrozenSet, PyInt, PyList, PyModule, PySet, PyStr, PyStrInterned, PyStrRef, PyTypeRef, PyUtf8Str, PyUtf8StrInterned, PyWeak, code::PyCode, @@ -3157,7 +3157,7 @@ impl VirtualMachine { .locals(self) } - pub fn current_globals(&self) -> PyDictRef { + pub fn current_globals(&self) -> PyRef { let ptr = crate::vm::thread::get_current_frame(); if !ptr.is_null() { return unsafe { (*ptr).globals().to_owned() }; @@ -3270,7 +3270,7 @@ impl VirtualMachine { // Locals fallback: use the heavy frame if available, otherwise // use globals as locals (light frame locals are on the data stack). let locals_mapping = self.current_frame().map_or_else( - || ArgMapping::from_dict_exact(globals.clone()), + || ArgMapping::from_anydict_exact(globals.clone()), |f| f.iframe().locals.clone_mapping(self), ); (Some(locals_mapping), Some(globals)) diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 1a1696b7106..7c5d43d1643 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -83,15 +83,16 @@ impl VirtualMachine { } pub fn new_scope_with_main(&self) -> PyResult { - let scope = self.new_scope_with_builtins(); - let main_module = self.new_module("__main__", scope.globals.clone(), None); + let globals = self.ctx.new_dict(); + let scope = Scope::with_builtins(None, globals.clone(), self); + let main_module = self.new_module("__main__", globals.clone(), None); self.sys_module.get_attr("modules", self)?.set_item( "__main__", main_module.into(), self, )?; - self.set_main_builtin_importer(&scope.globals)?; + self.set_main_builtin_importer(&globals)?; Ok(scope) } diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs index 14ae7199a9b..e369606f581 100644 --- a/crates/vm/src/warn.rs +++ b/crates/vm/src/warn.rs @@ -762,12 +762,12 @@ fn setup_context( d.downcast::() .map_err(|_| vm.new_type_error("sys.__dict__ is not a dictionary")) })?; - (globals, vm.ctx.intern_str(""), 0) + (globals.into(), vm.ctx.intern_str(""), 0) }; - let registry = match globals.get_item("__warningregistry__", vm) { - Ok(r) => r, - Err(_) => { + let registry = match globals.inner_getitem_opt("__warningregistry__", vm)? { + Some(r) => r, + None => { let r = vm.ctx.new_dict(); globals.set_item("__warningregistry__", r.clone().into(), vm)?; r.into() @@ -776,7 +776,8 @@ fn setup_context( // Setup module. let module = globals - .get_item("__name__", vm) - .unwrap_or_else(|_| vm.new_pyobj("")); + .inner_getitem_opt("__name__", vm)? + .filter(|module| vm.is_none(module) || module.downcastable::()) + .unwrap_or_else(|| vm.new_pyobj("")); Ok((filename.to_owned(), lineno, Some(module), registry)) } diff --git a/extra_tests/snippets/builtin_dict.py b/extra_tests/snippets/builtin_dict.py index e593b92fd87..3d7347a5041 100644 --- a/extra_tests/snippets/builtin_dict.py +++ b/extra_tests/snippets/builtin_dict.py @@ -873,3 +873,297 @@ def __hash__(self): # entirely when popping from an empty dict). {1: 1}.pop(CountingHash()) assert CountingHash.calls == 1, CountingHash.calls + + +# frozendict as a namespace, beyond what CPython's own tests cover. +import builtins +import gc +import sys +import types +import unittest + + +class FrozenDictDescriptorTests(unittest.TestCase): + def test_unrelated_descriptors_cannot_mutate_or_access(self): + frozen = frozendict(protected=23) + for method, args in ( + (dict.__init__, ({"replacement": 8},)), + (dict.__setitem__, ("protected", 99)), + (dict.__delitem__, ("protected",)), + (dict.update, ({"replacement": 8},)), + (dict.clear, ()), + (dict.get, ("protected",)), + ): + with self.subTest(method=method), self.assertRaises(TypeError): + method(frozen, *args) + with self.assertRaises(TypeError): + frozendict.get({"protected": 23}, "protected") + self.assertEqual(frozen, {"protected": 23}) + for name in ( + "clear", + "update", + "pop", + "popitem", + "setdefault", + "__setitem__", + "__delitem__", + "__ior__", + ): + self.assertFalse(hasattr(frozen, name), name) + + +class FrozenGlobalsTests(unittest.TestCase): + def setUp(self): + self.ns = frozendict( + __builtins__=builtins.__dict__, + __name__="frozen_test", + answer=42, + sys=sys, + ) + + def test_eval_and_implicit_scope(self): + self.assertIs(eval("globals()", self.ns), self.ns) + self.assertIs(eval("locals()", self.ns), self.ns) + self.assertEqual(eval("eval('answer')", self.ns), 42) + self.assertEqual(eval("(lambda: answer)()", self.ns), 42) + + def test_function_and_materialized_frame(self): + local = {} + exec("def f():\n return sys._getframe(), globals(), answer\n", self.ns, local) + f = local["f"] + self.assertIs(f.__globals__, self.ns) + self.assertEqual(f.__module__, "frozen_test") + for _ in range(300): + frame, ns, answer = f() + self.assertIs(frame.f_globals, self.ns) + self.assertIs(ns, self.ns) + self.assertEqual(answer, 42) + gc.collect() + self.assertIs(frame.f_globals, self.ns) + + def test_generator_frames(self): + local = {} + exec( + "def gen():\n yield sys._getframe(), globals(), answer\n yield answer\n", + self.ns, + local, + ) + gen = local["gen"]() + self.assertIs(gen.gi_frame.f_globals, self.ns) + frame, ns, answer = next(gen) + self.assertIs(frame.f_globals, self.ns) + self.assertIs(ns, self.ns) + self.assertEqual(answer, 42) + self.assertEqual(next(gen), 42) + with self.assertRaises(StopIteration): + next(gen) + gc.collect() + self.assertIs(frame.f_globals, self.ns) + + def test_coroutine_frames(self): + local = {} + exec( + "async def coro():\n return sys._getframe(), globals(), answer\n", + self.ns, + local, + ) + coro = local["coro"]() + self.assertIs(coro.cr_frame.f_globals, self.ns) + with self.assertRaises(StopIteration) as stopped: + coro.send(None) + frame, ns, answer = stopped.exception.value + self.assertIs(frame.f_globals, self.ns) + self.assertIs(ns, self.ns) + self.assertEqual(answer, 42) + + def test_async_generator_frames(self): + local = {} + exec( + "async def agen():\n yield sys._getframe(), globals(), answer\n", + self.ns, + local, + ) + agen = local["agen"]() + self.assertIs(agen.ag_frame.f_globals, self.ns) + with self.assertRaises(StopIteration) as stopped: + agen.asend(None).send(None) + frame, ns, answer = stopped.exception.value + self.assertIs(frame.f_globals, self.ns) + self.assertIs(ns, self.ns) + self.assertEqual(answer, 42) + with self.assertRaises(StopIteration): + agen.aclose().send(None) + + def test_local_namespace_and_imports(self): + local = {} + exec( + "import math\nclass C:\n value = answer\ndef f():\n import math\n return math.sqrt(answer)\n", + self.ns, + local, + ) + self.assertEqual(local["C"].value, 42) + self.assertEqual(local["C"].__module__, "frozen_test") + self.assertEqual(local["f"](), local["math"].sqrt(42)) + self.assertNotIn("math", self.ns) + + def test_missing_builtins(self): + for func, source in [(eval, ""), (exec, "")]: + with self.assertRaises(TypeError) as caught: + func(source, frozendict()) + self.assertEqual( + str(caught.exception), + "cannot assign __builtins__ to frozendict globals", + ) + + def test_writes_and_deletes(self): + for source, error in [ + ("answer = 0", "'frozendict' object does not support item assignment"), + ( + "global answer; answer = 0", + "frozendict object does not support item assignment", + ), + ( + "global answer; del answer", + "frozendict object does not support item deletion", + ), + ( + "global missing; del missing", + "frozendict object does not support item deletion", + ), + ]: + with self.assertRaises(TypeError) as caught: + exec(source, self.ns) + self.assertEqual(str(caught.exception), error) + self.assertEqual(self.ns["answer"], 42) + with self.assertRaises(NameError): + exec("del answer", self.ns) + + def test_explicit_function_constructor_stays_dict_only(self): + with self.assertRaises(TypeError): + types.FunctionType((lambda: 42).__code__, self.ns) + + def test_subclass_name_and_global_lookup(self): + class F(frozendict): + def __getitem__(self, key): + if key == "answer": + return 81 + return super().__getitem__(key) + + ns = F(self.ns) + self.assertEqual(eval("answer", ns), 81) + self.assertEqual(eval("(lambda: answer)()", ns), 81) + self.assertEqual(eval("answer", ns, {}), 42) + + def test_subclass_function_metadata_reads_storage(self): + class F(frozendict): + def __getitem__(self, key): + if key == "__name__": + return "wrong_name" + if key == "__builtins__": + return {"len": lambda x: 99} + return super().__getitem__(key) + + ns = F(self.ns) + f = eval("lambda: len([])", ns) + self.assertEqual(f.__module__, "frozen_test") + self.assertIs(f.__globals__, ns) + self.assertEqual(f(), 0) + + def test_subclass_builtins_assignment_hook(self): + calls = [] + + class F(frozendict): + def __setitem__(self, key, value): + calls.append((key, value)) + + for operation, source, mode, expected in ( + (eval, "1", "eval", 1), + (exec, "pass", "exec", None), + ): + namespace = F() + calls.clear() + self.assertEqual( + operation(compile(source, "", mode), namespace), + expected, + ) + self.assertEqual(calls, [("__builtins__", builtins.__dict__)]) + self.assertNotIn("__builtins__", namespace) + calls.clear() + with self.assertRaisesRegex( + TypeError, "frozendict object does not support item assignment" + ): + operation(source, namespace) + self.assertEqual(calls, [("__builtins__", builtins.__dict__)]) + + def test_implicit_frozen_builtins_namespace_counts(self): + import _testinternalcapi + + function = eval( + "lambda: len(())", + frozendict(__builtins__=frozendict(len=len)), + ) + counts = _testinternalcapi.get_code_var_counts(function) + self.assertEqual( + counts["unbound"]["globals"], + {"total": 1, "numglobal": 0, "numbuiltin": 1, "numunknown": 0}, + ) + for argument in ("globalsns", "builtinsns"): + with self.assertRaises(TypeError): + _testinternalcapi.get_code_var_counts( + function, **{argument: frozendict()} + ) + + def test_warning_context_reads_frozen_storage(self): + import warnings + + class F(frozendict): + def __getitem__(self, key): + if key in ("__warningregistry__", "__name__"): + raise AssertionError("warning context must read stored entries") + return super().__getitem__(key) + + for has_registry in (False, True): + entries = dict(self.ns, warn=warnings.warn) + if has_registry: + entries["__warningregistry__"] = {} + function = eval("lambda: warn('probe')", F(entries)) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + if has_registry: + function() + self.assertEqual(len(caught), 1) + self.assertEqual(str(caught[0].message), "probe") + else: + with self.assertRaisesRegex( + TypeError, + "frozendict object does not support item assignment", + ): + function() + self.assertEqual(caught, []) + + def test_frozen_builtins(self): + ns = frozendict(__builtins__=frozendict({"len": len})) + self.assertEqual(eval("(lambda: len([1, 2]))()", ns), 2) + + def test_shared_code_does_not_keep_mutable_global_specialization(self): + code = compile("lambda: answer", "", "eval") + mutable = {"__builtins__": builtins.__dict__, "answer": 19} + f = eval(code, mutable) + for _ in range(300): + self.assertEqual(f(), 19) + frozen_f = eval(code, self.ns) + for _ in range(300): + self.assertEqual(frozen_f(), 42) + self.assertEqual(f(), 19) + mutable["answer"] = 23 + self.assertEqual(f(), 23) + self.assertEqual(frozen_f(), 42) + + +frozen_suite = unittest.TestSuite( + unittest.defaultTestLoader.loadTestsFromTestCase(case) + for case in (FrozenDictDescriptorTests, FrozenGlobalsTests) +) +frozen_result = unittest.TextTestRunner(verbosity=2).run(frozen_suite) +assert frozen_result.wasSuccessful() +assert not frozen_result.skipped diff --git a/scripts/update_lib/__main__.py b/scripts/update_lib/__main__.py index 49399db6f43..7eb91970f2f 100644 --- a/scripts/update_lib/__main__.py +++ b/scripts/update_lib/__main__.py @@ -8,6 +8,7 @@ python scripts/update_lib migrate cpython/Lib/test/test_foo.py python scripts/update_lib patches --from Lib/test/foo.py --to cpython/Lib/test/foo.py python scripts/update_lib auto-mark Lib/test/test_foo.py + python scripts/update_lib copy-test SOURCE DESTINATION --test ClassName.method_name --source-sha256 HASH --destination-sha256 HASH """ import argparse @@ -49,6 +50,11 @@ def main(argv: list[str] | None = None) -> int: help="Copy library file/directory from CPython (delete existing first)", add_help=False, ) + subparsers.add_parser( + "copy-test", + help="Copy one complete canonical test from hash-pinned inputs", + add_help=False, + ) subparsers.add_parser( "deps", help="Show dependency information for a module", @@ -72,6 +78,11 @@ def main(argv: list[str] | None = None) -> int: return copy_lib_main(remaining) + if args.command == "copy-test": + from update_lib.cmd_copy_test import main as copy_test_main + + return copy_test_main(remaining) + if args.command == "migrate": from update_lib.cmd_migrate import main as migrate_main diff --git a/scripts/update_lib/cmd_copy_test.py b/scripts/update_lib/cmd_copy_test.py new file mode 100644 index 00000000000..5e14959bb02 --- /dev/null +++ b/scripts/update_lib/cmd_copy_test.py @@ -0,0 +1,134 @@ +"""Copy one complete canonical test definition without refreshing its module.""" + +import argparse +import ast +import hashlib +import json +from pathlib import Path + +from update_lib.patch_spec import _iter_class_methods, _iter_module_classes + + +def _definition(contents: str, qualified_name: str): + parts = qualified_name.split(".") + if len(parts) != 2 or not all(part.isidentifier() for part in parts): + raise ValueError("test name must be ClassName.method_name") + tree = ast.parse(contents) + classes = [node for node in _iter_module_classes(tree) if node.name == parts[0]] + if len(classes) != 1: + raise ValueError(f"Expected one class named {parts[0]}, found {len(classes)}") + methods = [ + node for node in _iter_class_methods(classes[0]) if node.name == parts[1] + ] + if len(methods) != 1: + raise ValueError( + f"Expected one method named {qualified_name}, found {len(methods)}" + ) + method = methods[0] + parents = { + child: parent + for parent in ast.walk(tree) + for child in ast.iter_child_nodes(parent) + } + guards = [] + child = method + while child in parents: + parent = parents[child] + if isinstance(parent, ast.If): + branch = "body" if child in parent.body else "orelse" + guards.append((ast.dump(parent.test), branch)) + elif not isinstance(parent, (ast.Module, ast.ClassDef)): + raise ValueError("Only module/class scopes and if guards are supported") + child = parent + start = min([method.lineno, *(node.lineno for node in method.decorator_list)]) - 1 + return method, guards, start, method.end_lineno + + +def copy_test(source: str, destination: str, qualified_name: str) -> str: + """Replace an unambiguous whole definition, preserving all surrounding bytes. + + Guard, signature, decorator and indentation differences require a broader + migration rather than silently changing the selected test's execution context. + """ + src, src_guards, src_start, src_end = _definition(source, qualified_name) + dst, dst_guards, dst_start, dst_end = _definition(destination, qualified_name) + if src_guards != dst_guards: + raise ValueError("Source and destination test guards differ") + + def signature(node): + return ( + ast.dump(node.args), + ast.dump(node.returns) if node.returns is not None else None, + [ast.dump(param) for param in getattr(node, "type_params", [])], + ) + + if type(src) is not type(dst) or signature(src) != signature(dst): + raise ValueError("Source and destination test signatures differ") + if [ast.dump(node) for node in src.decorator_list] != [ + ast.dump(node) for node in dst.decorator_list + ]: + raise ValueError( + "Source and destination decorators differ; preserve markers explicitly" + ) + if src.col_offset != dst.col_offset: + raise ValueError("Source and destination indentation differs") + source_lines = source.splitlines(keepends=True) + destination_lines = destination.splitlines(keepends=True) + # AST equality omits comments, including project marker rationales. Require + # the complete decorator block to match before replacing the definition. + if ( + source_lines[src_start : src.lineno - 1] + != destination_lines[dst_start : dst.lineno - 1] + ): + raise ValueError( + "Source and destination decorator text differs; preserve markers explicitly" + ) + definition = source_lines[src_start:src_end] + result = "".join( + destination_lines[:dst_start] + definition + destination_lines[dst_end:] + ) + _, _, result_start, result_end = _definition(result, qualified_name) + if result.splitlines(keepends=True)[result_start:result_end] != definition: + raise ValueError("Copied definition is not byte-identical to the source") + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("destination", type=Path) + parser.add_argument("--test", required=True, help="Exact ClassName.method_name") + parser.add_argument("--source-sha256", required=True) + parser.add_argument("--destination-sha256", required=True) + args = parser.parse_args(argv) + try: + source = args.source.read_bytes() + destination = args.destination.read_bytes() + for label, contents, expected in ( + ("source", source, args.source_sha256), + ("destination", destination, args.destination_sha256), + ): + if hashlib.sha256(contents).hexdigest() != expected: + raise ValueError(f"{label} SHA256 does not match the pinned input") + result = copy_test( + source.decode("utf-8"), destination.decode("utf-8"), args.test + ).encode("utf-8") + # Recheck the destination immediately before writing so stale invocations + # fail instead of overwriting an intervening edit. + if args.destination.read_bytes() != destination: + raise ValueError("Destination changed during test preparation") + args.destination.write_bytes(result) + except (OSError, UnicodeError, SyntaxError, ValueError) as exc: + parser.exit(1, f"Error: {exc}\n") + print( + json.dumps( + { + "test": args.test, + "source_sha256": args.source_sha256, + "destination_before_sha256": args.destination_sha256, + "destination_after_sha256": hashlib.sha256(result).hexdigest(), + }, + sort_keys=True, + ) + ) + return 0 diff --git a/scripts/update_lib/patch_spec.py b/scripts/update_lib/patch_spec.py index 996b78bfa18..994fc53b261 100644 --- a/scripts/update_lib/patch_spec.py +++ b/scripts/update_lib/patch_spec.py @@ -197,7 +197,10 @@ def visit_ClassDef(self, node: ast.ClassDef): patch = patch._replace(test_name="__self__") self.patches.append(patch) - self.generic_visit(node) + # Keep conditional methods in this namespace, but do not enter + # nested classes or the bodies of methods. + for item in _iter_class_methods(node): + self.visit(item) class PatchEntry(typing.NamedTuple): @@ -228,22 +231,35 @@ def iter_patch_entries( yield from visitor.patches -def iter_tests( - tree: ast.Module, -) -> "Iterator[tuple[ast.ClassDef, ast.FunctionDef | ast.AsyncFunctionDef]]": - for key, nodes in ast.iter_fields(tree): - if key != "body": - continue +def _iter_module_classes(node: ast.AST) -> "Iterator[ast.ClassDef]": + """Find module-scope classes, including conditional definitions. - for cls_node in nodes: - if not isinstance(cls_node, ast.ClassDef): - continue + Function-local and class-nested definitions have different namespaces and + cannot be addressed by the patch format's unqualified class names. + """ + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.ClassDef): + yield child + elif not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + yield from _iter_module_classes(child) - for fn_node in cls_node.body: - if not isinstance(fn_node, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue - yield (cls_node, fn_node) +def _iter_class_methods( + node: ast.AST, +) -> "Iterator[ast.FunctionDef | ast.AsyncFunctionDef]": + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + yield child + elif not isinstance(child, (ast.ClassDef, ast.Lambda)): + yield from _iter_class_methods(child) + + +def iter_tests( + tree: ast.Module, +) -> "Iterator[tuple[ast.ClassDef, ast.FunctionDef | ast.AsyncFunctionDef]]": + for cls_node in _iter_module_classes(tree): + for fn_node in _iter_class_methods(cls_node): + yield (cls_node, fn_node) def iter_patches(contents: str) -> "Iterator[PatchEntry]": @@ -286,24 +302,28 @@ def _iter_patch_lines( # Track class bases for inherited async method lookup class_bases: dict[str, list[str]] = {} all_classes = set() - all_class_nodes = [] - for node in tree.body: - if not isinstance(node, ast.ClassDef): - continue - + all_class_nodes = list(_iter_module_classes(tree)) + for node in all_class_nodes: + if node.name in all_classes: + raise ValueError(f"Ambiguous module-scope class name: {node.name}") all_classes.add(node.name) - all_class_nodes.append(node) + method_counts = collections.Counter( + item.name for item in _iter_class_methods(node) + ) + for method in patches.get(node.name, {}): + if method_counts[method] > 1: + raise ValueError(f"Ambiguous method name: {node.name}.{method}") - for node in tree.body: + for node in all_class_nodes: if isinstance(node, ast.ClassDef): - cache[node.name] = node.end_lineno + cache[node.name] = (node.end_lineno, node.col_offset + len(DEFAULT_INDENT)) class_bases[node.name] = [ base.id for base in node.bases if isinstance(base, ast.Name) and base.id in all_classes ] cls_async: set[str] = set() - for item in node.body: + for item in _iter_class_methods(node): if isinstance(item, ast.AsyncFunctionDef): cls_async.add(item.name) if cls_async: @@ -319,10 +339,11 @@ def _iter_patch_lines( # Phase 2: Iterate and mark inherited tests for cls_name, tests in sorted(patches.items()): - lineno = cache.get(cls_name) - if not lineno: + location = cache.get(cls_name) + if not location: print(f"WARNING: {cls_name} does not exist in remote file", file=sys.stderr) continue + lineno, indent = location for test_name, specs in sorted(tests.items()): if test_name == "__self__": @@ -356,7 +377,7 @@ async def {test_name}(self): def {test_name}(self): {DEFAULT_INDENT}return super().{test_name}() """.rstrip() - yield (lineno, textwrap.indent(patch_lines, DEFAULT_INDENT)) + yield (lineno, textwrap.indent(patch_lines, " " * indent)) # Phase 3: Mark the class itself for cls_node in all_class_nodes: diff --git a/scripts/update_lib/tests/test_copy_test.py b/scripts/update_lib/tests/test_copy_test.py new file mode 100644 index 00000000000..5dc61452c90 --- /dev/null +++ b/scripts/update_lib/tests/test_copy_test.py @@ -0,0 +1,206 @@ +"""Tests for bounded, hash-pinned canonical test copying.""" + +import ast +import contextlib +import hashlib +import io +import tempfile +import unittest +from pathlib import Path + +from update_lib.cmd_copy_test import copy_test, main + + +class TestCopyTest(unittest.TestCase): + destination = '''# Destination-only header +class Finder: + if int.__doc__: + def example(): + r"""Count is below 800. π\\n""" + + def untouched(self): + return "destination" +''' + source = '''# Canonical header must not be copied +class Finder: + if int.__doc__: + def example(): + r"""Count is below 850. π\\n""" + + def untouched(self): + return "canonical" +''' + + def test_complete_definition_and_surroundings(self): + result = copy_test(self.source, self.destination, "Finder.example") + self.assertEqual(result, self.destination.replace("below 800", "below 850")) + self.assertIn('return "destination"', result) + source_method = ast.parse(self.source).body[0].body[0].body[0] + result_method = ast.parse(result).body[0].body[0].body[0] + self.assertEqual( + ast.get_source_segment(self.source, source_method), + ast.get_source_segment(result, result_method), + ) + + def test_guard_mismatch_is_rejected(self): + with self.assertRaisesRegex(ValueError, "guards differ"): + copy_test( + self.source.replace("if int.__doc__", "if enabled"), + self.destination, + "Finder.example", + ) + + def test_missing_or_ambiguous_definition_is_rejected(self): + for source in ("class Other: pass\n", self.source + self.source): + with self.subTest(source=source): + with self.assertRaisesRegex(ValueError, "Expected one class"): + copy_test(source, self.destination, "Finder.example") + duplicate = self.source.replace( + " def untouched(self):", + " def example():\n pass\n\n def untouched(self):", + ) + with self.assertRaisesRegex(ValueError, "Expected one method"): + copy_test(duplicate, self.destination, "Finder.example") + + def test_decorator_and_signature_changes_are_rejected(self): + decorated = self.source.replace( + " def example():", " @decorator\n def example():" + ) + with self.assertRaisesRegex(ValueError, "decorators differ"): + copy_test(decorated, self.destination, "Finder.example") + with self.assertRaisesRegex(ValueError, "signatures differ"): + copy_test( + self.source.replace("def example():", "def example(arg):"), + self.destination, + "Finder.example", + ) + + def test_nested_local_scope_is_not_selected(self): + local = "def factory():\n" + "".join( + " " + line for line in self.source.splitlines(True) + ) + with self.assertRaisesRegex(ValueError, "Expected one class"): + copy_test(local, self.destination, "Finder.example") + + def test_decorator_comments_must_match_exactly(self): + for source_marker, destination_marker in ( + ( + "@unittest.expectedFailure", + "@unittest.expectedFailure # TODO: RUSTPYTHON; reason", + ), + ( + "@unittest.expectedFailure # original reason", + "@unittest.expectedFailure # changed reason", + ), + ( + "@unittest.expectedFailure\n @other", + "@unittest.expectedFailure\n # marker rationale\n @other", + ), + ( + "@unittest.expectedFailureIf(\n condition,\n 'reason',\n )", + "@unittest.expectedFailureIf(\n condition, # project limitation\n 'reason',\n )", + ), + ): + with self.subTest(destination_marker=destination_marker): + source = self.source.replace( + " def example():", + f" {source_marker}\n def example():", + ) + destination = self.destination.replace( + " def example():", + f" {destination_marker}\n def example():", + ) + with self.assertRaisesRegex(ValueError, "decorator text differs"): + copy_test(source, destination, "Finder.example") + + def test_identical_decorator_comments_are_preserved(self): + prefix = ( + " @unittest.expectedFailure # TODO: RUSTPYTHON; reason\n" + " # Keep this rationale with the marker.\n" + " def example():" + ) + for declaration in ("def example():", "async def example():"): + with self.subTest(declaration=declaration): + source = self.source.replace(" def example():", prefix) + destination = self.destination.replace(" def example():", prefix) + source = source.replace("def example():", declaration) + destination = destination.replace("def example():", declaration) + self.assertEqual( + copy_test(source, destination, "Finder.example"), + destination.replace("below 800", "below 850"), + ) + + def test_cli_does_not_write_on_decorator_comment_mismatch(self): + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "source.py" + destination = Path(temp) / "destination.py" + source.write_text( + self.source.replace( + " def example():", + " @unittest.expectedFailure\n def example():", + ) + ) + destination.write_text( + self.destination.replace( + " def example():", + " @unittest.expectedFailure # TODO: RUSTPYTHON; reason\n" + " def example():", + ) + ) + before = destination.read_bytes() + with ( + contextlib.redirect_stderr(io.StringIO()) as errors, + self.assertRaises(SystemExit), + ): + main( + [ + str(source), + str(destination), + "--test", + "Finder.example", + "--source-sha256", + hashlib.sha256(source.read_bytes()).hexdigest(), + "--destination-sha256", + hashlib.sha256(before).hexdigest(), + ] + ) + self.assertIn("decorator text differs", errors.getvalue()) + self.assertEqual(destination.read_bytes(), before) + + def test_return_annotation_and_type_parameters_are_rejected(self): + for declaration in ("def example() -> int:", "def example[T]():"): + with self.subTest(declaration=declaration): + source = self.source.replace("def example():", declaration) + with self.assertRaisesRegex(ValueError, "signatures differ"): + copy_test(source, self.destination, "Finder.example") + + def test_cli_hashes_and_no_write_on_stale_input(self): + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "source.py" + destination = Path(temp) / "destination.py" + source.write_bytes(self.source.encode()) + destination.write_bytes(self.destination.encode()) + args = [ + str(source), + str(destination), + "--test", + "Finder.example", + "--source-sha256", + hashlib.sha256(source.read_bytes()).hexdigest(), + "--destination-sha256", + "0" * 64, + ] + with ( + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit), + ): + main(args) + self.assertEqual(destination.read_bytes(), self.destination.encode()) + args[-1] = hashlib.sha256(destination.read_bytes()).hexdigest() + with contextlib.redirect_stdout(io.StringIO()) as output: + self.assertEqual(main(args), 0) + self.assertIn('"source_sha256"', output.getvalue()) + self.assertEqual( + destination.read_text(), + self.destination.replace("below 800", "below 850"), + ) diff --git a/scripts/update_lib/tests/test_patch_spec.py b/scripts/update_lib/tests/test_patch_spec.py index 92a0a35a153..2c6bb99ff84 100644 --- a/scripts/update_lib/tests/test_patch_spec.py +++ b/scripts/update_lib/tests/test_patch_spec.py @@ -67,6 +67,38 @@ async def test_async(self): class TestExtractPatches(unittest.TestCase): """Tests for extract_patches function.""" + def test_class_conditional_method_markers_are_preserved(self): + code = """class Owner: + if enabled: + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_value(self): + return 42 +""" + patches = extract_patches(code) + self.assertIn("test_value", patches["Owner"]) + clean = code.replace( + " @unittest.expectedFailure # TODO: RUSTPYTHON\n", "" + ) + result = apply_patches(clean, patches) + self.assertIn(" @unittest.expectedFailure", result) + method = list(iter_tests(ast.parse(result)))[0][1] + self.assertEqual(method.col_offset, 8) + self.assertEqual(ast.dump(method.body[0]), "Return(value=Constant(value=42))") + + def test_nested_class_markers_do_not_target_module_names(self): + code = """class Container: + class Target: + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_value(self): + pass +def factory(): + class Target: + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_value(self): + pass +""" + self.assertEqual(extract_patches(code), {}) + def test_extract_expected_failure(self): """Test extracting @unittest.expectedFailure decorator.""" code = f""" @@ -177,6 +209,106 @@ def test_one(self): class TestApplyPatches(unittest.TestCase): """Tests for apply_patches function.""" + def test_conditional_class_existing_method(self): + code = """import unittest +if enabled: + class TestConditional: + def test_value(self): + assert 1 + 1 == 2 +""" + result = apply_patches( + code, + { + "TestConditional": { + "test_value": [ + PatchSpec( + UtMethod.ExpectedFailureIf, "condition", "known limitation" + ) + ] + } + }, + ) + original = list(iter_tests(ast.parse(code)))[0][1] + updated = list(iter_tests(ast.parse(result)))[0][1] + self.assertEqual(ast.dump(original.body[0]), ast.dump(updated.body[0])) + self.assertEqual(updated.col_offset, 8) + self.assertIn(" @unittest.expectedFailureIf", result) + + def test_conditional_class_inherited_override(self): + code = """import unittest +class Parent: + async def test_value(self): + return 42 +if enabled: + class Child(Parent): + pass +""" + result = apply_patches( + code, {"Child": {"test_value": [PatchSpec(UtMethod.ExpectedSuccess)]}} + ) + methods = list(iter_tests(ast.parse(result))) + self.assertEqual([cls.name for cls, _ in methods], ["Parent", "Child"]) + self.assertIsInstance(methods[1][1], ast.AsyncFunctionDef) + self.assertIn(" return await super().test_value()", result) + self.assertIn(" @unittest.expectedSuccess", result) + + def test_duplicate_conditional_class_names_are_rejected(self): + code = """if enabled: + class Duplicate: + pass +else: + class Duplicate: + pass +""" + with self.assertRaisesRegex(ValueError, "Ambiguous.*Duplicate"): + apply_patches( + code, + {"Duplicate": {"test_value": [PatchSpec(UtMethod.ExpectedFailure)]}}, + ) + + def test_local_class_names_do_not_shadow_module_class(self): + code = """import unittest +class Target: + def test_value(self): + return 1 +class Container: + class Target: + def test_value(self): + return 2 +def factory(): + class Target: + def test_value(self): + return 3 +""" + result = apply_patches( + code, {"Target": {"test_value": [PatchSpec(UtMethod.ExpectedFailure)]}} + ) + tree = ast.parse(result) + targets = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) and node.name == "Target" + ] + self.assertEqual( + [len(node.body[0].decorator_list) for node in targets], [1, 0, 0] + ) + + def test_duplicate_conditional_method_names_are_rejected(self): + code = """class Owner: + if enabled: + def test_value(self): + return 1 + else: + def test_value(self): + return 2 +""" + with self.assertRaisesRegex( + ValueError, "Ambiguous method name: Owner.test_value" + ): + apply_patches( + code, {"Owner": {"test_value": [PatchSpec(UtMethod.ExpectedFailure)]}} + ) + def test_apply_expected_failure(self): """Test applying @unittest.expectedFailure.""" code = """import unittest diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 15fc75563dc..dfd3571bfe4 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,14 +1,14 @@ #![cfg_attr(target_arch = "wasm32", allow(dead_code))] use rustpython_vm::{ - AsObject, PyResult, TryFromObject, VirtualMachine, - builtins::{PyDictRef, PyStrRef}, + AsObject, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyAnyDict, PyStrRef}, function::ArgIterable, identifier, }; pub(super) struct ShellHelper<'vm> { vm: &'vm VirtualMachine, - globals: PyDictRef, + globals: PyRef, } fn reverse_string(s: &mut String) { @@ -54,7 +54,7 @@ fn split_idents_on_dot(line: &str) -> Option<(usize, Vec)> { } impl<'vm> ShellHelper<'vm> { - pub(super) const fn new(vm: &'vm VirtualMachine, globals: PyDictRef) -> Self { + pub(super) const fn new(vm: &'vm VirtualMachine, globals: PyRef) -> Self { ShellHelper { vm, globals } } From bc8080a52f3343b092a5b9ff0c5b958073be8f65 Mon Sep 17 00:00:00 2001 From: Leesoo Ahn Date: Wed, 14 Oct 2026 20:40:37 +0900 Subject: [PATCH 141/229] Fix pwd.getpwuid raising OSError for unknown uids (#9106) When a uid does not exist, some NSS backends (systemd-userdb, sssd, LDAP) make getpwuid_r() return a non-zero errno such as ENOENT. POSIX expects success with a NULL result instead. That errno was surfacing as FileNotFoundError and broke test_pwd.test_errors. Match CPython's pwdmodule.c: raise MemoryError only for ENOMEM, and treat any other failure as "not found" (KeyError). Assisted-by: Antigravity:Claude Opus 5.5 --- crates/vm/src/stdlib/pwd.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/vm/src/stdlib/pwd.rs b/crates/vm/src/stdlib/pwd.rs index 90a37d6d36d..c2eb3391b98 100644 --- a/crates/vm/src/stdlib/pwd.rs +++ b/crates/vm/src/stdlib/pwd.rs @@ -5,8 +5,8 @@ pub(crate) use pwd::module_def; #[pymodule] mod pwd { use crate::{ - PyResult, VirtualMachine, builtins::PyUtf8StrRef, convert::IntoPyException, exceptions, - function::ArgIndex, types::PyStructSequence, + PyResult, VirtualMachine, builtins::PyUtf8StrRef, exceptions, function::ArgIndex, + types::PyStructSequence, }; use core::hint::cold_path; use rustpython_host_env::pwd as host_pwd; @@ -67,12 +67,16 @@ mod pwd { #[pyfunction] fn getpwuid(uidobj: ArgIndex, vm: &VirtualMachine) -> PyResult { let uidobj = uidobj.into_int_ref(); - let uid_t = libc::uid_t::try_from(uidobj.as_bigint()).ok(); - let user = uid_t + let user = match libc::uid_t::try_from(uidobj.as_bigint()) + .ok() .map(host_pwd::getpwuid) - .transpose() - .map_err(|err| err.into_pyexception(vm))? - .flatten(); + { + Some(Ok(user)) => user, + Some(Err(err)) if err.raw_os_error() == Some(libc::ENOMEM) => { + return Err(vm.no_memory_error()); + } + _ => None, + }; let user = user.ok_or_else(|| { vm.new_key_error( vm.ctx From 0816af6543d2da59c43649af19f3254b57769263 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:54:27 +0900 Subject: [PATCH 142/229] Update malachite crates to 0.14.0 (#9117) pymath requires malachite-bigint ^0, so the example projects, which have no lock file, resolve it to 0.14.0 while the workspace pinned 0.13.0, and fail to build. Assisted-by: Claude Code:claude-opus-5-5 --- Cargo.lock | 25 +++++++++++++------------ Cargo.toml | 6 +++--- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f541d0c3eeb..e7c3c7f01be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2378,9 +2378,9 @@ dependencies = [ [[package]] name = "malachite-base" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60f9f06976ea22081565c6f4ddc40e699982c1c6b9bd1bd4871b92eb986f2307" +checksum = "d5399d024a5e8fd066c158d208a02fb2dabb737c401ab03954f27d170fd49ffc" dependencies = [ "hashbrown 0.16.1", "itertools 0.14.0", @@ -2390,9 +2390,9 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a0ea1859fa74e642919ea67bb7afe2e663f88f2e59afdeafda4a0b5ec6855fb" +checksum = "2ba82fa23fb2f63fcebd9ca012d885d41d4e95c9f14904df6cdcad9232646b2a" dependencies = [ "malachite-base", "malachite-nz", @@ -2403,10 +2403,11 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5eef5870b4c0072ab10e28ed6e8228ae224f188bdbb65c739fcc189248381f5" +checksum = "7ac55a9e8426447c374668b93c35b7b6ab43a21dc7ba97ad0e852be2305a5900" dependencies = [ + "bytemuck", "itertools 0.14.0", "libm", "malachite-base", @@ -2415,9 +2416,9 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ff775589824d11de83716ee96d70b7a4fbd70794415ba363b6aeed272010f7" +checksum = "070267491ad6c23e504a71368542a2376d62f6622286be47870d7dcd56f24fce" dependencies = [ "itertools 0.14.0", "libm", @@ -4221,9 +4222,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "safe_arch" -version = "1.0.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7caad094bd561859bcd467734a720c3c1f5d1f338995351fefe2190c45efed" +checksum = "42c6efa15875e6ecb39ca61fb0b0c1a40b84fac5a5ffe71eef7d1000c8eb3f5f" dependencies = [ "bytemuck", ] @@ -5138,9 +5139,9 @@ dependencies = [ [[package]] name = "wide" -version = "1.5.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfdfe6a32973f2d1b268b8895845a8a96cac2f0191e72c27cc929036060dbf89" +checksum = "d920ac99c3c8edce110cb8d07dbb324d6d026011dce85b1e9355b70f0adacc4f" dependencies = [ "bytemuck", "safe_arch", diff --git a/Cargo.toml b/Cargo.toml index 0b55f2e8277..4b84f30930a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -245,9 +245,9 @@ log = "0.4.30" lz4_flex = "0.14" nix = { version = "0.31", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } mac_address = "1.1.3" -malachite-bigint = "0.13.0" -malachite-q = "0.13.0" -malachite-base = "0.13.0" +malachite-bigint = "0.14.0" +malachite-q = "0.14.0" +malachite-base = "0.14.0" md-5 = "0.11" memchr = { version = "2.8.1", default-features = false, features = ["alloc"] } memmap2 = "0.9.10" From a85a6cd762eced5ca74f937ad790ada85e7ee132 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Wed, 14 Oct 2026 05:45:57 -0700 Subject: [PATCH 143/229] Sync stdlib utilities from CPython 3.15.0 final (#9111) Regenerate nine utility modules and associated tests with scripts/update_lib from CPython d220409a13b77de7a6c30abe16ea90db01528edc. Preserve existing mimetypes fixes by excluding that module from this split. OpenAI Codex assisted with source audit, update_lib execution, and focused release validation: 64 tests passed, one existing WASI-only skip. Assisted-by: OpenAI Codex:model version unavailable --- Lib/configparser.py | 6 ++- Lib/csv.py | 39 +++++++++++------- Lib/fractions.py | 10 ++--- Lib/graphlib.py | 61 +++++++++++++++++++++++++++-- Lib/heapq.py | 12 +++--- Lib/ipaddress.py | 16 ++++++-- Lib/netrc.py | 58 +++++++++++++++------------ Lib/plistlib.py | 35 +++++++---------- Lib/statistics.py | 19 ++++----- Lib/test/test_configparser.py | 33 ++++++++++++++-- Lib/test/test_csv.py | 74 +++++++++++++++++++++++++++++++++++ Lib/test/test_ipaddress.py | 14 +++++++ Lib/test/test_netrc.py | 38 ++++++++++++++++++ Lib/test/test_plistlib.py | 19 +++++++++ Lib/test/test_statistics.py | 6 +-- 15 files changed, 343 insertions(+), 97 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index 3c452afe8ad..88015ef6086 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -618,7 +618,8 @@ class RawConfigParser(MutableMapping): _OPT_TMPL = r""" (?P