diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 39d85d46274..fb519878cd8 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -128,7 +128,6 @@ def test_indentation_error(self): self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1\n\nnow exiti [truncated]... doesn't start with 'UnicodeEncodeError: ' def test_unicode_error(self): self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] self.console.interact() diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index 8ef6dd2fee8..d4014a784da 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -87,7 +87,6 @@ def test_fields(self): self.assertEqual(t.n_unnamed_fields, 0) self.assertEqual(t.n_fields, time._STRUCT_TM_ITEMS) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument dict def test_constructor(self): t = time.struct_time @@ -111,7 +110,6 @@ def test_constructor(self): s = "123456789" self.assertEqual("".join(t(s)), s) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_constructor_with_duplicate_fields(self): t = time.struct_time @@ -125,7 +123,6 @@ def test_constructor_with_duplicate_fields(self): with self.assertRaisesRegex(TypeError, error_message): t("1234567890", dict={"error": 0, "tm_zone": "some zone", "tm_mon": 1}) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2 def test_constructor_with_duplicate_unnamed_fields(self): assert os.stat_result.n_unnamed_fields > 0 n_visible_fields = os.stat_result.n_sequence_fields @@ -142,7 +139,6 @@ def test_constructor_with_duplicate_unnamed_fields(self): re.escape("got duplicate or unexpected field name(s)")): os.stat_result((*range(n_visible_fields), -1.0), {'st_atime': -1.0}) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_constructor_with_unknown_fields(self): t = time.struct_time @@ -185,7 +181,6 @@ def test_pickling(self): self.assertEqual(t2.tm_year, t.tm_year) self.assertEqual(t2.tm_zone, t.tm_zone) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2 def test_pickling_with_unnamed_fields(self): assert os.stat_result.n_unnamed_fields > 0 @@ -220,7 +215,6 @@ def test_copying(self): self.assertIsNot(t3[0], t[0]) self.assertIsNot(t3.tm_year, t.tm_year) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2 def test_copying_with_unnamed_fields(self): assert os.stat_result.n_unnamed_fields > 0 diff --git a/crates/capi/src/genericaliasobject.rs b/crates/capi/src/genericaliasobject.rs index bcd31308679..1ab443e13ad 100644 --- a/crates/capi/src/genericaliasobject.rs +++ b/crates/capi/src/genericaliasobject.rs @@ -10,6 +10,6 @@ pub unsafe extern "C" fn Py_GenericAlias( with_vm(|vm| { let origin = unsafe { &*origin }.to_owned(); let args = unsafe { &*args }.to_owned(); - PyGenericAlias::from_args(origin, args, vm).into_pyobject(vm) + PyGenericAlias::from_args(origin, args, vm).map(|alias| alias.into_pyobject(vm)) }) } diff --git a/crates/stdlib/src/_asyncio.rs b/crates/stdlib/src/_asyncio.rs index 3146e39b77d..b311db4a315 100644 --- a/crates/stdlib/src/_asyncio.rs +++ b/crates/stdlib/src/_asyncio.rs @@ -12,8 +12,8 @@ pub(crate) mod _asyncio { vm::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{ - PyBaseException, PyBaseExceptionRef, PyDict, PyDictRef, PyGenericAlias, PyList, - PyListRef, PyModule, PySet, PyTuple, PyType, PyTypeRef, + PyBaseException, PyBaseExceptionRef, PyDict, PyGenericAlias, PyList, PyListRef, + PyModule, PySet, PyTuple, PyType, PyTypeRef, }, extend_module, function::{FuncArgs, KwArgs, OptionalArg, OptionalOption, PySetterValue}, @@ -779,7 +779,7 @@ pub(crate) mod _asyncio { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -1036,7 +1036,7 @@ pub(crate) mod _asyncio { ))); } - let exc = if exc_type.fast_isinstance(vm.ctx.types.type_type) { + let exc: PyBaseExceptionRef = if exc_type.fast_isinstance(vm.ctx.types.type_type) { // exc_type is a class let exc_class: PyTypeRef = exc_type.clone().downcast().unwrap(); // Must be a subclass of BaseException @@ -1047,12 +1047,23 @@ pub(crate) mod _asyncio { } let val = exc_val.unwrap_or_none(vm); - if vm.is_none(&val) { + let exc = if vm.is_none(&val) { exc_type.call((), vm)? } else if val.fast_isinstance(&exc_class) { val } else { exc_type.call((val,), vm)? + }; + match exc.downcast() { + Ok(exc) => exc, + Err(obj) => { + let exc_class_repr = exc_class.as_object().repr(vm)?; + vm.new_type_error(format!( + "calling {} should have returned an instance of BaseException, not {}", + exc_class_repr.as_wtf8(), + obj.class() + )) + } } } else if exc_type.fast_isinstance(vm.ctx.exceptions.base_exception_type) { // exc_type is an exception instance @@ -1063,7 +1074,7 @@ pub(crate) mod _asyncio { vm.new_type_error("instance exception may not have a separate value") ); } - exc_type + exc_type.downcast().unwrap() } else { // exc_type is neither a class nor an exception instance return Err(vm.new_type_error(format!( @@ -1075,10 +1086,11 @@ pub(crate) mod _asyncio { if let OptionalArg::Present(tb) = exc_tb && !vm.is_none(&tb) { - exc.set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?; + exc.as_object() + .set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?; } - Err(exc.downcast().unwrap()) + Err(exc) } #[pymethod] @@ -1840,7 +1852,7 @@ pub(crate) mod _asyncio { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -2405,7 +2417,9 @@ pub(crate) mod _asyncio { // Slow path: look up in the module-level dict for cross-thread queries let current_tasks = get_current_tasks_dict(vm)?; - let dict: PyDictRef = current_tasks.downcast().unwrap(); + let Ok(dict) = current_tasks.downcast::() else { + return Ok(vm.ctx.none()); + }; match dict.get_item(&*loop_obj, vm) { Ok(task) => Ok(task), @@ -2485,15 +2499,17 @@ pub(crate) mod _asyncio { #[pyfunction] fn _enter_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { // Per-thread check, matching CPython's ts->asyncio_running_task - { - let running_task = vm.asyncio_running_task.borrow(); - if running_task.is_some() { - return Err(vm.new_runtime_error(format!( - "Cannot enter into task {:?} while another task {:?} is being executed.", - task, - running_task.as_ref().unwrap() - ))); - } + let running_task = vm.asyncio_running_task.borrow().clone(); + if let Some(running_task) = running_task { + let task_repr = task.repr(vm)?; + let running_task_repr = running_task.repr(vm)?; + return Err(vm.new_runtime_error(wtf8_concat!( + "Cannot enter into task ", + task_repr.as_wtf8(), + " while another task ", + running_task_repr.as_wtf8(), + " is being executed." + ))); } *vm.asyncio_running_task.borrow_mut() = Some(task.clone()); diff --git a/crates/stdlib/src/_queue.rs b/crates/stdlib/src/_queue.rs index 6b150e4c68b..1c8a4b0b21b 100644 --- a/crates/stdlib/src/_queue.rs +++ b/crates/stdlib/src/_queue.rs @@ -282,7 +282,7 @@ mod _queue { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index f2a16d72356..094e690665f 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -1234,7 +1234,7 @@ pub mod array { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/stdlib/src/blake2.rs b/crates/stdlib/src/blake2.rs index 382aec826b1..83504435674 100644 --- a/crates/stdlib/src/blake2.rs +++ b/crates/stdlib/src/blake2.rs @@ -5,7 +5,7 @@ pub(crate) use _blake2::module_def; #[pymodule] mod _blake2 { use crate::hashlib::_hashlib::{BlakeHashArgs, local_blake2b, local_blake2s}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyattr(name = "_GIL_MINSIZE")] const GIL_MINSIZE: u16 = 2048; @@ -43,4 +43,11 @@ mod _blake2 { fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_blake2s(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 0a6e0f12314..19fbcb8412f 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -462,7 +462,7 @@ mod _contextvars { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -562,7 +562,7 @@ mod _contextvars { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 4271d9af62c..3471f7a28d8 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -779,11 +779,16 @@ mod _csv { // TODO: Maybe need to update the obj from HashMap } DialectItem::Obj(o) => Ok(self.update_py_dialect(o.clone())), - DialectItem::None => { - let g = GLOBAL_HASHMAP.lock(); - let res = g.get("excel").unwrap().clone(); - Ok(self.update_py_dialect(res)) - } + DialectItem::None => Ok(self.update_py_dialect(PyDialect { + delimiter: b',', + quotechar: Some(b'"'), + escapechar: None, + doublequote: true, + skipinitialspace: false, + lineterminator: "\r\n".to_owned(), + quoting: QuoteStyle::Minimal, + strict: false, + })), } } diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index 0b699baddbb..6e8a913abaa 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -337,40 +337,43 @@ mod _lzma { } fn parse_filter_chain_spec( - filter_specs: Vec, + filter_specs: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { const LZMA_FILTERS_MAX: usize = 4; - if filter_specs.len() > LZMA_FILTERS_MAX { + let filter_specs_len = filter_specs.length(vm)?; + if filter_specs_len > LZMA_FILTERS_MAX { return Err(new_lzma_error( format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"), vm, )); } + let filter_specs = filter_specs.try_sequence(vm)?; let mut filters = Filters::new(); - for spec in &filter_specs { - let filter_id = get_dict_opt_u64(spec, "id", vm)? + for i in 0..filter_specs_len { + let spec = filter_specs.get_item(i as isize, vm)?; + let filter_id = get_dict_opt_u64(&spec, "id", vm)? .ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?; match filter_id { FILTER_LZMA1 => { - let opts = parse_filter_spec_lzma(spec, vm)?; + let opts = parse_filter_spec_lzma(&spec, vm)?; filters.lzma1(&opts); } FILTER_LZMA2 => { - let opts = parse_filter_spec_lzma(spec, vm)?; + let opts = parse_filter_spec_lzma(&spec, vm)?; filters.lzma2(&opts); } FILTER_DELTA => { - let dist = parse_filter_spec_delta(spec, vm)?; + let dist = parse_filter_spec_delta(&spec, vm)?; filters .delta_properties(&[(dist - 1) as u8]) .map_err(|e| catch_lzma_error(e, vm))?; } FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB | FILTER_SPARC => { - let start_offset = parse_filter_spec_bcj(spec, vm)?; + let start_offset = parse_filter_spec_bcj(&spec, vm)?; add_bcj_filter(&mut filters, filter_id, start_offset) .map_err(|e| catch_lzma_error(e, vm))?; } @@ -570,7 +573,7 @@ mod _lzma { #[pyarg(any, optional)] memlimit: Option, #[pyarg(any, optional)] - filters: Option>, + filters: Option, } impl Constructor for LZMADecompressor { @@ -735,7 +738,7 @@ mod _lzma { fn init_xz( check: i32, preset: u32, - filters: Option>, + filters: Option, vm: &VirtualMachine, ) -> PyResult { let real_check = @@ -751,10 +754,11 @@ mod _lzma { fn init_alone( preset: u32, - filter_specs: Option>, + filter_specs: Option, vm: &VirtualMachine, ) -> PyResult { - if let Some(_filter_specs) = filter_specs { + if let Some(filter_specs) = filter_specs { + filter_specs.length(vm)?; // TODO: validate single LZMA1 filter and use its options let options = LzmaOptions::new_preset(preset).map_err(|_| { new_lzma_error(format!("Invalid compression preset: {preset}"), vm) @@ -768,10 +772,7 @@ mod _lzma { } } - fn init_raw( - filter_specs: Option>, - vm: &VirtualMachine, - ) -> PyResult { + fn init_raw(filter_specs: Option, vm: &VirtualMachine) -> PyResult { let filter_specs = filter_specs .ok_or_else(|| vm.new_value_error("Must specify filters for FORMAT_RAW"))?; let filters = parse_filter_chain_spec(filter_specs, vm)?; @@ -788,7 +789,7 @@ mod _lzma { #[pyarg(any, optional)] preset: Option, #[pyarg(any, optional)] - filters: Option>, + filters: Option, } impl Constructor for LZMACompressor { diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index 92c2a66e93e..3fe1ffd3e63 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -727,25 +727,20 @@ mod math { } // Generic Python path - let (p_i, q_i) = (p_i.unwrap(), q_i.unwrap()); - - // Collect current + remaining elements - let p_remaining: Result, _> = - core::iter::once(Ok(p_i)).chain(p_iter).collect(); - let q_remaining: Result, _> = - core::iter::once(Ok(q_i)).chain(q_iter).collect(); - let (p_vec, q_vec) = (p_remaining?, q_remaining?); - - if p_vec.len() != q_vec.len() { - return Err(vm.new_value_error("Inputs are not the same length")); - } - + let (mut p_i, mut q_i) = (p_i.unwrap(), q_i.unwrap()); let mut total = obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into()); - for (p_item, q_item) in p_vec.into_iter().zip(q_vec) { - let prod = vm._mul(&p_item, &q_item)?; + loop { + let prod = vm._mul(&p_i, &q_i)?; total = vm._add(&total, &prod)?; + + let next_p = p_iter.next().transpose()?; + let next_q = q_iter.next().transpose()?; + match (next_p, next_q) { + (Some(next_p), Some(next_q)) => (p_i, q_i) = (next_p, next_q), + (None, None) => return Ok(total), + _ => return Err(vm.new_value_error("Inputs are not the same length")), + } } - return Ok(total); } Ok(obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into())) diff --git a/crates/stdlib/src/md5.rs b/crates/stdlib/src/md5.rs index 2ff6cd24ff7..0339bf8ace7 100644 --- a/crates/stdlib/src/md5.rs +++ b/crates/stdlib/src/md5.rs @@ -3,10 +3,17 @@ pub(crate) use _md5::module_def; #[pymodule] mod _md5 { use crate::hashlib::_hashlib::{HashArgs, local_md5}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_md5(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index 312d35a4ed4..91d4058a706 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -777,7 +777,10 @@ mod mmap { let start = options .start .map_or_else(|| self.pos(), |start| start.saturated_at(size)); - let end = options.end.map_or(size, |end| end.saturated_at(size)); + let end = options + .end + .map_or(size, |end| end.saturated_at(size)) + .max(start); (start, end) } @@ -886,7 +889,7 @@ mod mmap { let dest = dest.try_to_primitive(vm).ok()?; let src = src.try_to_primitive(vm).ok()?; let cnt = cnt.try_to_primitive(vm).ok()?; - if size - dest < cnt || size - src < cnt { + if dest > size || src > size || size - dest < cnt || size - src < cnt { return None; } Some((dest, src, cnt)) diff --git a/crates/stdlib/src/sha1.rs b/crates/stdlib/src/sha1.rs index 3e3d4928c79..71495435e56 100644 --- a/crates/stdlib/src/sha1.rs +++ b/crates/stdlib/src/sha1.rs @@ -3,10 +3,17 @@ pub(crate) use _sha1::module_def; #[pymodule] mod _sha1 { use crate::hashlib::_hashlib::{HashArgs, local_sha1}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_sha1(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/sha3.rs b/crates/stdlib/src/sha3.rs index 0eb2dfa84d5..642ed838a4d 100644 --- a/crates/stdlib/src/sha3.rs +++ b/crates/stdlib/src/sha3.rs @@ -6,7 +6,7 @@ mod _sha3 { HashArgs, local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128, local_shake_256, }; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { @@ -37,4 +37,11 @@ mod _sha3 { fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_shake_256(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/suggestions.rs b/crates/stdlib/src/suggestions.rs index e0667dfb553..bfde00d2bb9 100644 --- a/crates/stdlib/src/suggestions.rs +++ b/crates/stdlib/src/suggestions.rs @@ -2,19 +2,25 @@ pub(crate) use _suggestions::module_def; #[pymodule] mod _suggestions { - use rustpython_vm::VirtualMachine; + use rustpython_vm::{PyResult, VirtualMachine, builtins::PyList}; use crate::vm::PyObjectRef; #[pyfunction] fn _generate_suggestions( - candidates: Vec, + candidates: PyObjectRef, name: PyObjectRef, vm: &VirtualMachine, - ) -> PyObjectRef { - match crate::vm::suggestion::calculate_suggestions(candidates.iter(), &name) { - Some(suggestion) => suggestion.into(), - None => vm.ctx.none(), - } + ) -> PyResult { + let candidates = candidates + .downcast::() + .map_err(|_| vm.new_type_error("candidates must be a list"))?; + let candidates = candidates.borrow_vec(); + Ok( + match crate::vm::suggestion::calculate_suggestions(candidates.iter(), &name) { + Some(suggestion) => suggestion.into(), + None => vm.ctx.none(), + }, + ) } } diff --git a/crates/vm/src/builtins/asyncgenerator.rs b/crates/vm/src/builtins/asyncgenerator.rs index b53e59d58c1..7ea43f389c6 100644 --- a/crates/vm/src/builtins/asyncgenerator.rs +++ b/crates/vm/src/builtins/asyncgenerator.rs @@ -144,7 +144,11 @@ impl PyAsyncGen { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index a649fe9d8d5..793b269d100 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -554,7 +554,11 @@ impl PyByteArray { // TODO: Uncomment when Python adds __class_getitem__ to bytearray // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index d4c30a7e94d..bb514b84ce1 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -544,7 +544,11 @@ impl PyBytes { // TODO: Uncomment when Python adds __class_getitem__ to bytes // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/classmethod.rs b/crates/vm/src/builtins/classmethod.rs index eb0e15ece01..26dcd251251 100644 --- a/crates/vm/src/builtins/classmethod.rs +++ b/crates/vm/src/builtins/classmethod.rs @@ -27,7 +27,7 @@ use crate::{ /// /// Class methods are different than C++ or Java static methods. /// If you want those, see the staticmethod builtin. -#[pyclass(module = false, name = "classmethod")] +#[pyclass(module = false, name = "classmethod", traverse)] #[derive(Debug)] pub struct PyClassMethod { callable: PyMutex, @@ -187,7 +187,11 @@ impl PyClassMethod { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/coroutine.rs b/crates/vm/src/builtins/coroutine.rs index d472f1a0bfa..0fc50fb1356 100644 --- a/crates/vm/src/builtins/coroutine.rs +++ b/crates/vm/src/builtins/coroutine.rs @@ -103,7 +103,11 @@ impl PyCoroutine { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index fbc23a0dde7..1a380d74d02 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -535,7 +535,11 @@ impl PyDict { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/enumerate.rs b/crates/vm/src/builtins/enumerate.rs index 96073ba7667..95e144dad21 100644 --- a/crates/vm/src/builtins/enumerate.rs +++ b/crates/vm/src/builtins/enumerate.rs @@ -57,7 +57,11 @@ impl Constructor for PyEnumerate { #[pyclass(with(Py, IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyEnumerate { #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/generator.rs b/crates/vm/src/builtins/generator.rs index 52db3c9522a..b06a3a45ea7 100644 --- a/crates/vm/src/builtins/generator.rs +++ b/crates/vm/src/builtins/generator.rs @@ -99,7 +99,11 @@ impl PyGenerator { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index b6f6012fd43..8004bd535be 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -68,7 +68,7 @@ impl Constructor for PyGenericAlias { } else { PyTuple::new_ref(vec![arguments], &vm.ctx) }; - Ok(Self::new(origin, args, false, vm)) + Self::new(origin, args, false, vm) } } @@ -92,14 +92,14 @@ impl PyGenericAlias { args: PyTupleRef, starred: bool, vm: &VirtualMachine, - ) -> Self { - let parameters = make_parameters(&args, vm); - Self { + ) -> PyResult { + let parameters = make_parameters(&args, vm)?; + Ok(Self { origin: origin.into(), args, parameters, starred, - } + }) } /// Create a GenericAlias from an origin and PyObjectRef arguments (helper for compatibility) @@ -107,7 +107,7 @@ impl PyGenericAlias { origin: impl Into, args: PyObjectRef, vm: &VirtualMachine, - ) -> Self { + ) -> PyResult { let args = if let Ok(tuple) = args.try_to_ref::(vm) { tuple.to_owned() } else { @@ -228,7 +228,7 @@ impl PyGenericAlias { vm, )?; - Ok(Self::new(zelf.origin.clone(), new_args, false, vm).into_pyobject(vm)) + Ok(Self::new(zelf.origin.clone(), new_args, false, vm)?.into_pyobject(vm)) } #[pymethod] @@ -247,7 +247,7 @@ impl PyGenericAlias { if zelf.starred { // (next, (iter(GenericAlias(origin, args)),)) let next_fn = vm.builtins.get_attr("next", vm)?; - let non_starred = Self::new(zelf.origin.clone(), zelf.args.clone(), false, vm); + let non_starred = Self::new(zelf.origin.clone(), zelf.args.clone(), false, vm)?; let iter_obj = PyGenericAliasIterator { obj: crate::common::lock::PyMutex::new(Some(non_starred.into_pyobject(vm))), } @@ -292,11 +292,11 @@ impl PyGenericAlias { } } -pub(crate) fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyTupleRef { +pub(crate) fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyResult { make_parameters_from_slice(args.as_slice(), vm) } -fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTupleRef { +fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyResult { let mut parameters: Vec = Vec::with_capacity(args.len()); for arg in args { @@ -326,7 +326,9 @@ fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTu let list = arg.downcast_ref::().unwrap(); list.borrow_vec().to_vec() }; - let sub = make_parameters_from_slice(&items, vm); + let sub = vm.with_recursion("while computing __parameters__", || { + make_parameters_from_slice(&items, vm) + })?; for sub_param in sub.iter() { if tuple_index(¶meters, sub_param).is_none() { parameters.push(sub_param.clone()); @@ -335,7 +337,7 @@ fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTu } } - PyTuple::new_ref(parameters, &vm.ctx) + Ok(PyTuple::new_ref(parameters, &vm.ctx)) } #[inline] @@ -716,7 +718,7 @@ impl crate::types::IterNext for PyGenericAliasIterator { let alias = obj .downcast_ref::() .ok_or_else(|| vm.new_type_error("generic_alias_iterator expected GenericAlias"))?; - let starred = PyGenericAlias::new(alias.origin.clone(), alias.args.clone(), true, vm); + let starred = PyGenericAlias::new(alias.origin.clone(), alias.args.clone(), true, vm)?; Ok(PyIterReturn::Return(starred.into_pyobject(vm))) } } diff --git a/crates/vm/src/builtins/interpolation.rs b/crates/vm/src/builtins/interpolation.rs index 0ae1b33120b..5d5f3774640 100644 --- a/crates/vm/src/builtins/interpolation.rs +++ b/crates/vm/src/builtins/interpolation.rs @@ -144,7 +144,11 @@ impl PyInterpolation { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index c2059e28806..fe674a45821 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -421,7 +421,11 @@ impl PyList { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs index c8b891f7972..dd8c689facb 100644 --- a/crates/vm/src/builtins/mappingproxy.rs +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -177,7 +177,11 @@ impl PyMappingProxy { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index ee5a071287b..9f8312a0704 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -554,7 +554,11 @@ impl Py { )] impl PyMemoryView { #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index 415d34fdb05..5962f90e521 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -364,7 +364,11 @@ impl PyRange { // TODO: Uncomment when Python adds __class_getitem__ to range // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index 6961040c792..860f86f4319 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -386,7 +386,8 @@ impl PySetInner { } fn repr(&self, class_name: Option<&str>, vm: &VirtualMachine) -> PyResult { - collection_repr(class_name, "{", "}", self.elements().iter(), vm) + let empty = format!("{}()", class_name.unwrap_or("set")); + collection_repr(class_name, "{", "}", &empty, self.elements().iter(), vm) } fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { @@ -881,7 +882,11 @@ impl PySet { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -1286,7 +1291,11 @@ impl PyFrozenSet { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index 3c5f13b382d..026b976b65e 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -260,7 +260,11 @@ impl PySlice { // TODO: Uncomment when Python adds __class_getitem__ to slice // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/staticmethod.rs b/crates/vm/src/builtins/staticmethod.rs index addfe8a4e2b..8ae31b67b5c 100644 --- a/crates/vm/src/builtins/staticmethod.rs +++ b/crates/vm/src/builtins/staticmethod.rs @@ -163,7 +163,11 @@ impl PyStaticMethod { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/template.rs b/crates/vm/src/builtins/template.rs index 30812b4f171..94c4d653df3 100644 --- a/crates/vm/src/builtins/template.rs +++ b/crates/vm/src/builtins/template.rs @@ -186,7 +186,11 @@ impl PyTemplate { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index 06fa2519205..7af176840b7 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -504,7 +504,11 @@ impl PyTuple { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -609,7 +613,7 @@ impl Representable for PyTuple { let s = if zelf.len() == 1 { wtf8_concat!("(", zelf.elements[0].repr(vm)?.as_wtf8(), ",)") } else { - collection_repr(None, "(", ")", zelf.elements.iter(), vm)? + collection_repr(None, "(", ")", "()", zelf.elements.iter(), vm)? }; vm.ctx.new_str(s) } else { diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs index cb6dd0d6559..c1be5c8ec9a 100644 --- a/crates/vm/src/builtins/union.rs +++ b/crates/vm/src/builtins/union.rs @@ -234,7 +234,7 @@ pub(crate) fn or_op(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) } fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyResult { - let parameters = genericalias::make_parameters(args, vm); + let parameters = genericalias::make_parameters(args, vm)?; let result = dedup_and_flatten_args(¶meters, vm)?; Ok(result.args) } diff --git a/crates/vm/src/builtins/weakref.rs b/crates/vm/src/builtins/weakref.rs index 9e88ffaa2e6..e0f012f169c 100644 --- a/crates/vm/src/builtins/weakref.rs +++ b/crates/vm/src/builtins/weakref.rs @@ -92,7 +92,11 @@ impl PyWeak { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index c6d18cc6594..11c13912b76 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -60,7 +60,7 @@ pub(super) mod types { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } @@ -255,20 +255,11 @@ pub(super) mod types { ))); } - // Validate exceptions is a sequence (not set or None) + // Validate exceptions is a sequence let exceptions_arg = &args[1]; - - // Check for set/frozenset (not a sequence - unordered) - if exceptions_arg.fast_isinstance(vm.ctx.types.set_type) - || exceptions_arg.fast_isinstance(vm.ctx.types.frozenset_type) - { - return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); - } - - // Check for None - if exceptions_arg.is(&vm.ctx.none) { - return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); - } + exceptions_arg.try_sequence(vm).map_err(|_| { + vm.new_type_error("second argument (exceptions) must be a sequence") + })?; let exceptions: Vec = exceptions_arg.try_to_value(vm).map_err(|_| { vm.new_type_error("second argument (exceptions) must be a sequence") diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 9c42df966ea..0a1c2cb75ee 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -988,6 +988,9 @@ impl ExceptionZoo { extend_exception!(PyImportError, ctx, excs.import_error, { "msg" => ctx.new_readonly_getset("msg", excs.import_error, make_arg_getter(0)), + "name" => ctx.none(), + "path" => ctx.none(), + "name_from" => ctx.none(), }); extend_exception!(PyModuleNotFoundError, ctx, excs.module_not_found_error); @@ -1908,10 +1911,11 @@ pub(super) mod types { #[pymethod] fn __reduce__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { let obj = exc.as_object().to_owned(); - let mut result: Vec = vec![ - obj.class().to_owned().into(), - vm.new_tuple((exc.get_arg(0).unwrap(),)).into(), - ]; + let args: PyObjectRef = match exc.get_arg(0) { + Some(arg) => vm.new_tuple((arg,)).into(), + None => exc.args().into(), + }; + let mut result: Vec = vec![obj.class().to_owned().into(), args]; if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { result.push(dict.into()); @@ -1938,10 +1942,21 @@ pub(super) mod types { ))); } - let dict = crate::builtins::object::object_get_dict(zelf.clone(), vm)?; - dict.set_item("name", vm.unwrap_or_none(name), vm)?; - dict.set_item("path", vm.unwrap_or_none(path), vm)?; - dict.set_item("name_from", vm.unwrap_or_none(name_from), vm)?; + if let Some(name) = name { + zelf.set_attr("name", name, vm)?; + } else if let Some(dict) = zelf.dict() { + dict.del_item("name", vm).ok(); + } + if let Some(path) = path { + zelf.set_attr("path", path, vm)?; + } else if let Some(dict) = zelf.dict() { + dict.del_item("path", vm).ok(); + } + if let Some(name_from) = name_from { + zelf.set_attr("name_from", name_from, vm)?; + } else if let Some(dict) = zelf.dict() { + dict.del_item("name_from", vm).ok(); + } PyBaseException::slot_init(zelf, args, vm) } diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index 69ee0e3c510..186fa8e8a84 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -269,13 +269,16 @@ cfg_select! { _ => {} } -impl fmt::Debug for PyAtomicRef { +impl fmt::Debug for PyAtomicRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "PyAtomicRef(")?; + // The stored pointer is a `Py` — the full object, header included — + // as `Deref`, `load_raw` and `swap` all read it. Formatting it as a + // bare payload would skip the header and print misaligned bytes. unsafe { self.inner .load(Ordering::Relaxed) - .cast::() + .cast::() .as_ref() .fmt(f) }?; diff --git a/crates/vm/src/protocol/iter.rs b/crates/vm/src/protocol/iter.rs index 2f51287b181..1aa0bcd5b13 100644 --- a/crates/vm/src/protocol/iter.rs +++ b/crates/vm/src/protocol/iter.rs @@ -16,7 +16,11 @@ where unsafe impl> Traverse for PyIter { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - self.0.borrow().traverse(tracer_fn); + // Report the iterator itself, not its referents: an owner holding a + // `PyIter` owns the iterator object, and reporting what the iterator + // points at instead leaves the iterator's own reference unaccounted + // for, so a cycle running through it is never collected. + tracer_fn(self.0.borrow()); } } diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index 37007422404..4974fca9343 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -7,7 +7,7 @@ use crate::{ PyType, PyTypeRef, PyUtf8Str, int::check_int_to_str_digits, pystr::AsPyStr, }, common::{hash::PyHash, str::to_ascii}, - convert::{ToPyObject, ToPyResult}, + convert::ToPyObject, dict_inner::DictKey, function::{Either, FuncArgs, PyArithmeticValue, PySetterValue}, object::PyPayload, @@ -694,7 +694,7 @@ impl PyObject { pub fn hash(&self, vm: &VirtualMachine) -> PyResult { if let Some(hash) = self.class().slots.hash.load() { - return hash(self, vm); + return vm.with_recursion("while hashing", || hash(self, vm)); } Err(vm.new_type_error(format!("unhashable type: '{}'", self.class().name()))) @@ -741,8 +741,8 @@ impl PyObject { } else { if self.class().fast_issubclass(vm.ctx.types.type_type) { if self.is(vm.ctx.types.type_type) { - return PyGenericAlias::from_args(self.class().to_owned(), needle, vm) - .to_pyresult(vm); + let alias = PyGenericAlias::from_args(self.class().to_owned(), needle, vm)?; + return Ok(alias.to_pyobject(vm)); } if let Some(class_getitem) = diff --git a/crates/vm/src/sequence.rs b/crates/vm/src/sequence.rs index 4e6ed97f21c..1e126d087ea 100644 --- a/crates/vm/src/sequence.rs +++ b/crates/vm/src/sequence.rs @@ -104,7 +104,12 @@ where return Err(vm.new_memory_error("")); } - let mut v = Vec::with_capacity(n * self.as_ref().len()); + let total = n + .checked_mul(self.as_ref().len()) + .ok_or_else(|| vm.new_memory_error(""))?; + let mut v = Vec::new(); + v.try_reserve_exact(total) + .map_err(|_| vm.new_memory_error(""))?; for _ in 0..n { v.extend_from_slice(self.as_ref()); } diff --git a/crates/vm/src/stdlib/_ast/pyast.rs b/crates/vm/src/stdlib/_ast/pyast.rs index eb97eec8024..ebce1a788d2 100644 --- a/crates/vm/src/stdlib/_ast/pyast.rs +++ b/crates/vm/src/stdlib/_ast/pyast.rs @@ -1718,12 +1718,16 @@ fn populate_field_types(vm: &VirtualMachine, module: &Py) { FieldType::ListOf(name) => { let elem = resolve_node(name); let args = PyTuple::new_ref(vec![elem], &vm.ctx); - PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + PyGenericAlias::new(list_type.clone(), args, false, vm) + .expect("static field types are not nested, so no recursion is possible") + .to_pyobject(vm) } FieldType::ListOfBuiltin(name) => { let elem = resolve_builtin(name); let args = PyTuple::new_ref(vec![elem], &vm.ctx); - PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + PyGenericAlias::new(list_type.clone(), args, false, vm) + .expect("static field types are not nested, so no recursion is possible") + .to_pyobject(vm) } FieldType::Optional(name) => { let base = resolve_node(name); diff --git a/crates/vm/src/stdlib/_collections.rs b/crates/vm/src/stdlib/_collections.rs index c7cce5c735a..b48c0e670ac 100644 --- a/crates/vm/src/stdlib/_collections.rs +++ b/crates/vm/src/stdlib/_collections.rs @@ -13,6 +13,7 @@ mod _collections { convert::ToPyObject, function::{FuncArgs, KwArgs, OptionalArg, PyComparisonValue}, iter::PyExactSizeIterator, + object::{Traverse, TraverseFn}, protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, recursion::ReprGuard, sequence::{MutObjectSequenceOp, OptionalRangeArgs}, @@ -22,13 +23,19 @@ mod _collections { Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, utils::collection_repr, + vm::MAX_MEMORY_SIZE, }; use alloc::collections::VecDeque; - use core::cmp::max; + use core::{cmp::max, mem::size_of}; use crossbeam_utils::atomic::AtomicCell; #[pyattr] - #[pyclass(module = "collections", name = "deque", unhashable = true)] + #[pyclass( + module = "collections", + name = "deque", + unhashable = true, + traverse = "manual" + )] #[derive(Debug, Default, PyPayload)] struct PyDeque { deque: PyRwLock>, @@ -36,6 +43,21 @@ mod _collections { state: AtomicCell, // incremented whenever the indices move } + // SAFETY: Traverse visits each owned Python reference at most once. + unsafe impl Traverse for PyDeque { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + if let Some(deque) = self.deque.try_read_recursive() { + for obj in deque.iter() { + obj.traverse(tracer_fn); + } + } + } + + fn clear(&mut self, out: &mut Vec) { + out.extend(self.deque.get_mut().drain(..)); + } + } + type PyDequeRef = PyRef; #[derive(FromArgs)] @@ -318,6 +340,10 @@ mod _collections { let deque = self.borrow_deque(); let n = vm.check_repeat_or_overflow_error(deque.len(), n)?; let mul_len = n * deque.len(); + let result_len = self.maxlen.map_or(mul_len, |maxlen| mul_len.min(maxlen)); + if n > 1 && result_len.saturating_mul(size_of::()) >= MAX_MEMORY_SIZE { + return Err(vm.new_memory_error("")); + } let iter = deque.iter().cycle().take(mul_len); let skipped = self .maxlen @@ -400,7 +426,7 @@ mod _collections { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -576,9 +602,10 @@ mod _collections { let closing_part = zelf .maxlen .map_or_else(|| "]".to_owned(), |maxlen| format!("], maxlen={maxlen}")); + let empty = format!("{class_name}([{closing_part})"); if zelf.__len__() == 0 { - return Ok(vm.ctx.new_str(format!("{class_name}([{closing_part})"))); + return Ok(vm.ctx.new_str(empty)); } if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { @@ -586,6 +613,7 @@ mod _collections { Some(&class_name), "[", &closing_part, + &empty, deque.iter(), vm, )?)) @@ -753,7 +781,8 @@ mod _collections { module = "collections", name = "defaultdict", base = PyDict, - unhashable = true + unhashable = true, + traverse = "manual" )] #[derive(Debug, Default)] struct PyDefaultDict { @@ -761,6 +790,21 @@ mod _collections { default_factory: PyRwLock>, } + // SAFETY: Traverse visits each owned Python reference at most once. + unsafe impl Traverse for PyDefaultDict { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.dict.traverse(tracer_fn); + self.default_factory.traverse(tracer_fn); + } + + fn clear(&mut self, out: &mut Vec) { + Traverse::clear(&mut self.dict, out); + if let Some(factory) = self.default_factory.get_mut().take() { + out.push(factory); + } + } + } + #[pyclass( with(AsMapping, AsNumber, Constructor, Initializer, Representable), flags(BASETYPE, MAPPING, HAS_DICT) diff --git a/crates/vm/src/stdlib/_ctypes.rs b/crates/vm/src/stdlib/_ctypes.rs index adf047ec750..e4857d0ee06 100644 --- a/crates/vm/src/stdlib/_ctypes.rs +++ b/crates/vm/src/stdlib/_ctypes.rs @@ -141,7 +141,6 @@ pub(crate) mod _ctypes { ffi_value_from_type_code(code.encode_utf8(&mut buf), bytes) } super::CArgValue::Int(v) => FfiValue::I32(*v), - super::CArgValue::Double(v) => FfiValue::F64(*v), super::CArgValue::Pointer(v) => FfiValue::Pointer(*v), // 'V' aggregates format via the object-address default arm below. super::CArgValue::Aggregate { .. } => FfiValue::Pointer(0), diff --git a/crates/vm/src/stdlib/_ctypes/array.rs b/crates/vm/src/stdlib/_ctypes/array.rs index a99fabc812d..c65f9748caf 100644 --- a/crates/vm/src/stdlib/_ctypes/array.rs +++ b/crates/vm/src/stdlib/_ctypes/array.rs @@ -511,7 +511,11 @@ impl AsMapping for PyCArray { )] impl PyCArray { #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } @@ -990,13 +994,19 @@ impl PyCArray { let (range, step, slice_len) = sat_slice.adjust_indices(length); // other_len = PySequence_Length(value); - let items: Vec = vm.extract_elements_with(&value, Ok)?; - let other_len = items.len(); + // Size the operand before consuming it so an unbounded iterable is + // rejected without being materialized. + let other_len = value + .sequence_unchecked() + .length(vm) + .map_err(|_| vm.new_value_error("Can only assign sequence of same size"))?; if other_len != slice_len { return Err(vm.new_value_error("Can only assign sequence of same size")); } + let items: Vec = vm.extract_elements_with(&value, Ok)?; + // Use SaturatedSliceIter for correct index iteration (handles negative step) let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); diff --git a/crates/vm/src/stdlib/_ctypes/base.rs b/crates/vm/src/stdlib/_ctypes/base.rs index 1cc84750cb5..e86fdbc7a42 100644 --- a/crates/vm/src/stdlib/_ctypes/base.rs +++ b/crates/vm/src/stdlib/_ctypes/base.rs @@ -1939,7 +1939,7 @@ fn struct_union_paramfunc(obj: &PyObject, stg_info: &StgInfo, _vm: &VirtualMachi /// A foreign-call argument in a form the unified `call` entry point accepts: a /// simple-typed scalar (its ctypes code plus a native-endian bytes snapshot), -/// an untyped int/float, or an address. Any object whose memory an address +/// an untyped int, or an address. Any object whose memory an address /// refers to is kept alive by the enclosing `Argument`/`CArgObject`, not here. #[derive(Debug, Clone)] pub enum CArgValue { @@ -1947,8 +1947,6 @@ pub enum CArgValue { Typed { code: char, bytes: Vec }, /// Untyped Python int (ConvParam default: C int). Int(i32), - /// Untyped Python float (ConvParam default: C double). - Double(f64), /// Address-valued argument (pointer decay, byref, buffer copies, NULL = 0). Pointer(usize), /// By-value aggregate: its call layout plus a snapshot of its bytes. @@ -1985,7 +1983,6 @@ impl CArgValue { buffer: bytes, }, Self::Int(value) => CallArg::Int(*value), - Self::Double(value) => CallArg::Double(*value), Self::Pointer(value) => CallArg::Pointer(*value), Self::Aggregate { layout, bytes } => CallArg::Aggregate { layout, diff --git a/crates/vm/src/stdlib/_ctypes/function.rs b/crates/vm/src/stdlib/_ctypes/function.rs index 676ee5be8eb..90b41a4e66a 100644 --- a/crates/vm/src/stdlib/_ctypes/function.rs +++ b/crates/vm/src/stdlib/_ctypes/function.rs @@ -9,7 +9,7 @@ use super::{ }; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyBytes, PyDict, PyStr, PyTuple, PyType, PyTypeRef}, + builtins::{PyBytes, PyDict, PyInt, PyStr, PyTuple, PyType, PyTypeRef}, class::StaticType, function::FuncArgs, protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}, @@ -171,7 +171,10 @@ fn conv_param(value: &PyObject, vm: &VirtualMachine) -> PyResult { } // 10. Python int -> i32 (default integer type) - if let Ok(int_val) = value.try_int(vm) { + // PyLong_Check: only an int (or a subclass) converts. Going through + // `__int__` would accept a float and pass its truncated value where the + // callee expects a pointer. + if let Some(int_val) = value.downcast_ref::() { let val = int_val.as_bigint().to_i32().unwrap_or(0); return Ok(Argument { keep: None, @@ -179,15 +182,7 @@ fn conv_param(value: &PyObject, vm: &VirtualMachine) -> PyResult { }); } - // 11. Python float -> f64 - if let Ok(float_val) = value.try_float(vm) { - return Ok(Argument { - keep: None, - value: CArgValue::Double(float_val.to_f64()), - }); - } - - // 12. Check _as_parameter_ attribute + // 11. Check _as_parameter_ attribute if let Ok(as_param) = value.get_attr("_as_parameter_", vm) { return conv_param(&as_param, vm); } @@ -939,6 +934,23 @@ struct CallInfo { ret: RetSpec, } +fn extract_arg_types(argtypes: &PyObject, vm: &VirtualMachine) -> PyResult> { + let error = || vm.new_type_error("_argtypes_ must be a sequence of types"); + let sequence = argtypes.try_sequence(vm).map_err(|_| error())?; + let length = sequence.length(vm).map_err(|_| error())?; + let mut types = Vec::new(); + types + .try_reserve(length) + .map_err(|_| vm.new_memory_error(""))?; + + for index in 0..length { + let item = sequence.get_item(index as isize, vm).map_err(|_| error())?; + types.push(item.downcast::().map_err(|_| error())?); + } + + Ok(types) +} + /// Determine how to retrieve the return value from restype, reproducing the /// prior `ffi_return_type` + `is_pointer_return` dispatch. fn compute_ret_spec( @@ -1007,13 +1019,7 @@ fn extract_call_info(zelf: &Py, vm: &VirtualMachine) -> PyResult> = if let Some(argtypes_obj) = zelf.argtypes.read().as_ref() { if !vm.is_none(argtypes_obj) { - Some( - argtypes_obj - .try_to_value::>(vm)? - .into_iter() - .filter_map(|obj| obj.downcast::().ok()) - .collect(), - ) + Some(extract_arg_types(argtypes_obj, vm)?) } else { None // argtypes is None -> use ConvParam } @@ -1023,13 +1029,7 @@ fn extract_call_info(zelf: &Py, vm: &VirtualMachine) -> PyResult>(vm)? - .into_iter() - .filter_map(|obj| obj.downcast::().ok()) - .collect(), - ) + Some(extract_arg_types(&class_argtypes, vm)?) } else { None // No argtypes -> use ConvParam }; @@ -1944,14 +1944,7 @@ impl PyCThunk { vm: &VirtualMachine, ) -> PyResult { let arg_type_vec: Vec = match arg_types { - Some(args) if !vm.is_none(&args) => args - .try_to_value::>(vm)? - .into_iter() - .map(|item| { - item.downcast::() - .map_err(|_| vm.new_type_error("_argtypes_ must be a sequence of types")) - }) - .collect::>>()?, + Some(args) if !vm.is_none(&args) => extract_arg_types(&args, vm)?, _ => Vec::new(), }; diff --git a/crates/vm/src/stdlib/_ctypes/pointer.rs b/crates/vm/src/stdlib/_ctypes/pointer.rs index f522e6dfb7e..a401fde6fc0 100644 --- a/crates/vm/src/stdlib/_ctypes/pointer.rs +++ b/crates/vm/src/stdlib/_ctypes/pointer.rs @@ -668,7 +668,7 @@ impl PyCPointer { let ptr_val = if vm.is_none(value) { 0usize } else if let Ok(int_val) = value.try_index(vm) { - int_val.as_bigint().to_usize().unwrap_or(0) + super::simple::bigint_to_i128_wrapping(int_val.as_bigint()) as usize } else { return Err(vm.new_type_error("bytes/string or integer address expected")); }; @@ -684,12 +684,13 @@ impl PyCPointer { // Use write_unaligned for safety on strict-alignment architectures if let Ok(int_val) = value.try_int(vm) { let i = int_val.as_bigint(); + let wrapped = super::simple::bigint_to_i128_wrapping(i); let bytes; let write_value = match size { - 1 => AddressWriteValue::U8(i.to_u8().expect("int too large")), - 2 => AddressWriteValue::I16(i.to_i16().expect("int too large")), - 4 => AddressWriteValue::I32(i.to_i32().expect("int too large")), - 8 => AddressWriteValue::I64(i.to_i64().expect("int too large")), + 1 => AddressWriteValue::U8(wrapped as u8), + 2 => AddressWriteValue::I16(wrapped as i16), + 4 => AddressWriteValue::I32(wrapped as i32), + 8 => AddressWriteValue::I64(wrapped as i64), _ => { bytes = i.to_signed_bytes_le(); AddressWriteValue::Bytes(&bytes) diff --git a/crates/vm/src/stdlib/_ctypes/simple.rs b/crates/vm/src/stdlib/_ctypes/simple.rs index c947e56010a..5577fb8d25d 100644 --- a/crates/vm/src/stdlib/_ctypes/simple.rs +++ b/crates/vm/src/stdlib/_ctypes/simple.rs @@ -72,6 +72,17 @@ fn new_simple_type( Ok(PyCSimple(PyCData::from_bytes(zeroed_bytes(size), None))) } +pub(super) fn bigint_to_i128_wrapping(value: &malachite_bigint::BigInt) -> i128 { + let bytes = value.to_signed_bytes_le(); + let fill = bytes + .last() + .map_or(0, |byte| if *byte & 0x80 == 0 { 0 } else { u8::MAX }); + let mut wrapped = [fill; 16]; + let len = bytes.len().min(wrapped.len()); + wrapped[..len].copy_from_slice(&bytes[..len]); + i128::from_le_bytes(wrapped) +} + fn set_primitive(_type_: &str, value: &PyObject, vm: &VirtualMachine) -> PyResult { match _type_ { "c" => { @@ -756,7 +767,7 @@ fn value_to_bytes_endian( "b" => { // c_byte - signed char (1 byte) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -764,7 +775,7 @@ fn value_to_bytes_endian( "B" => { // c_ubyte - unsigned char (1 byte) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -772,7 +783,7 @@ fn value_to_bytes_endian( "h" => { // c_short (2 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -780,7 +791,7 @@ fn value_to_bytes_endian( "H" => { // c_ushort (2 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -788,7 +799,7 @@ fn value_to_bytes_endian( "i" => { // c_int (4 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -796,7 +807,7 @@ fn value_to_bytes_endian( "I" => { // c_uint (4 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -804,7 +815,7 @@ fn value_to_bytes_endian( "l" => { // c_long (platform dependent) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -812,7 +823,7 @@ fn value_to_bytes_endian( "L" => { // c_ulong (platform dependent) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -820,7 +831,7 @@ fn value_to_bytes_endian( "q" => { // c_longlong (8 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -828,7 +839,7 @@ fn value_to_bytes_endian( "Q" => { // c_ulonglong (8 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -889,10 +900,7 @@ fn value_to_bytes_endian( "P" => { // c_void_p - pointer type (platform pointer size) if let Ok(int_val) = value.try_index(vm) { - let v = int_val - .as_bigint() - .to_usize() - .expect("int too large for pointer"); + let v = bigint_to_i128_wrapping(int_val.as_bigint()) as usize; SimpleStorageValue::Pointer(v) } else { SimpleStorageValue::Zero @@ -902,10 +910,7 @@ fn value_to_bytes_endian( // c_char_p - pointer to char (stores pointer value from int) // PyBytes case is handled in slot_new/set_value with make_z_buffer() if let Ok(int_val) = value.try_index(vm) { - let v = int_val - .as_bigint() - .to_usize() - .expect("int too large for pointer"); + let v = bigint_to_i128_wrapping(int_val.as_bigint()) as usize; SimpleStorageValue::Pointer(v) } else { SimpleStorageValue::Zero @@ -915,10 +920,7 @@ fn value_to_bytes_endian( // c_wchar_p - pointer to wchar_t (stores pointer value from int) // PyStr case is handled in slot_new/set_value with make_wchar_buffer() if let Ok(int_val) = value.try_index(vm) { - let v = int_val - .as_bigint() - .to_usize() - .expect("int too large for pointer"); + let v = bigint_to_i128_wrapping(int_val.as_bigint()) as usize; SimpleStorageValue::Pointer(v) } else { SimpleStorageValue::Zero diff --git a/crates/vm/src/stdlib/_functools.rs b/crates/vm/src/stdlib/_functools.rs index 94b9565e79d..9b49e564562 100644 --- a/crates/vm/src/stdlib/_functools.rs +++ b/crates/vm/src/stdlib/_functools.rs @@ -302,7 +302,7 @@ mod _functools { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index 322eaedd7d0..fa979fcadbb 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -179,7 +179,7 @@ mod _imp { PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef, PyUtf8StrRef}, convert::TryFromBorrowedObject, - function::OptionalArg, + function::{FuncArgs, OptionalArg}, import, version, }; @@ -320,14 +320,16 @@ mod _imp { #[allow(clippy::type_complexity)] #[pyfunction] fn find_frozen( - name: PyUtf8StrRef, - withdata: OptionalArg, + args: FuncArgs, vm: &VirtualMachine, ) -> PyResult>, bool, Option)>> { - if withdata.into_option().is_some() { - // this is keyword-only argument in CPython - unimplemented!(); + if args.args.len() > 1 { + return Err(vm.new_type_error(format!( + "find_frozen() takes exactly 1 positional argument ({} given)", + args.args.len() + ))); } + let (name,): (PyUtf8StrRef,) = args.bind(vm)?; let name_str = name.as_str(); let info = match super::find_frozen(name_str, vm) { diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs index 03382549b47..18b4ffde818 100644 --- a/crates/vm/src/stdlib/_sre.rs +++ b/crates/vm/src/stdlib/_sre.rs @@ -498,7 +498,7 @@ mod _sre { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -597,7 +597,7 @@ mod _sre { regs: Vec<(isize, isize)>, } - #[pyclass(with(AsMapping, Representable))] + #[pyclass(with(AsMapping, Representable), flags(DISALLOW_INSTANTIATION))] impl Match { pub(crate) fn new(state: &mut State, pattern: PyRef, string: PyObjectRef) -> Self { let string_position = state.cursor.position; @@ -844,7 +844,7 @@ mod _sre { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/stdlib/_typing.rs b/crates/vm/src/stdlib/_typing.rs index 0214c3cc544..0b19d8e3c32 100644 --- a/crates/vm/src/stdlib/_typing.rs +++ b/crates/vm/src/stdlib/_typing.rs @@ -39,8 +39,17 @@ pub(crate) mod decl { }; #[pyfunction] - pub(crate) fn _idfunc(args: FuncArgs, _vm: &VirtualMachine) -> PyObjectRef { - args.args[0].clone() + pub(crate) fn _idfunc(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("_typing._idfunc() takes no keyword arguments")); + } + if args.args.len() != 1 { + return Err(vm.new_type_error(format!( + "_typing._idfunc() takes exactly one argument ({} given)", + args.args.len() + ))); + } + Ok(args.args[0].clone()) } #[pyfunction(name = "override")] @@ -288,7 +297,7 @@ pub(crate) mod decl { PyTuple::new_ref(vec![args], &vm.ctx) }; let origin: PyObjectRef = zelf.as_object().to_owned(); - Ok(PyGenericAlias::new(origin, args_tuple, false, vm).into_pyobject(vm)) + Ok(PyGenericAlias::new(origin, args_tuple, false, vm)?.into_pyobject(vm)) } #[pymethod] diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 35f404f0f3b..95feea65620 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -341,6 +341,7 @@ mod builtins { }; match &source { ArgStrOrBytesLike::Str(source) => { + let source = source.try_as_utf8(vm)?.as_str(); if source.as_bytes().contains(&0) { return Err(vm.new_exception_msg( vm.ctx.exceptions.syntax_error.to_owned(), @@ -548,13 +549,14 @@ mod builtins { Either::A(either) => { let source = match &either { ArgStrOrBytesLike::Str(source) => { + let source = source.try_as_utf8(vm)?.as_str(); if source.as_bytes().contains(&0) { return Err(vm.new_exception_msg( vm.ctx.exceptions.syntax_error.to_owned(), "source code string cannot contain null bytes".into(), )); } - let source = source.expect_str().trim_start_matches([' ', '\t']); + let source = source.trim_start_matches([' ', '\t']); audit_compile_source(vm, source.as_bytes(), "")?; source.to_owned() } @@ -597,6 +599,7 @@ mod builtins { } let source = match &either { ArgStrOrBytesLike::Str(source) => { + let source = source.try_as_utf8(vm)?.as_str(); if source.as_bytes().contains(&0) { return Err(vm.new_exception_msg( vm.ctx.exceptions.syntax_error.to_owned(), @@ -604,7 +607,7 @@ mod builtins { )); } audit_compile_source(vm, source.as_bytes(), "")?; - source.expect_str().to_owned() + source.to_owned() } ArgStrOrBytesLike::Buf(source) => { let source: &[u8] = &source.borrow_buf(); diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 30a4d8773be..e633404e803 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -26,7 +26,7 @@ mod decl { use num_traits::{Signed, ToPrimitive}; #[pyattr] - #[pyclass(name = "chain")] + #[pyclass(name = "chain", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsChain { source: PyRwLock>, @@ -64,7 +64,7 @@ mod decl { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -119,7 +119,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "compress")] + #[pyclass(name = "compress", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCompress { data: PyIter, @@ -166,7 +166,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "count")] + #[pyclass(name = "count", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCount { cur: PyRwLock, @@ -237,11 +237,12 @@ mod decl { } #[pyattr] - #[pyclass(name = "cycle")] + #[pyclass(name = "cycle", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCycle { iter: PyIter, saved: PyRwLock>, + #[pytraverse(skip)] index: AtomicCell, } @@ -273,11 +274,15 @@ mod decl { return Ok(PyIterReturn::StopIteration(None)); } - let last_index = zelf.index.fetch_add(1); - - if last_index >= saved.len() - 1 { - zelf.index.store(0); - } + // Advance and wrap in a single atomic step. A separate + // fetch_add followed by a reset lets a second thread observe + // an index past the end of `saved`. + let last_index = match zelf.index.fetch_update(|index| { + let next = index + 1; + Some(if next < saved.len() { next } else { 0 }) + }) { + Ok(index) | Err(index) => index, + }; saved[last_index].clone() }; @@ -287,10 +292,11 @@ mod decl { } #[pyattr] - #[pyclass(name = "repeat")] + #[pyclass(name = "repeat", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsRepeat { object: PyObjectRef, + #[pytraverse(skip)] times: Option>, } @@ -365,7 +371,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "starmap")] + #[pyclass(name = "starmap", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsStarmap { function: PyObjectRef, @@ -412,11 +418,12 @@ mod decl { } #[pyattr] - #[pyclass(name = "takewhile")] + #[pyclass(name = "takewhile", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsTakewhile { predicate: PyObjectRef, iterable: PyIter, + #[pytraverse(skip)] stop_flag: AtomicCell, } @@ -474,11 +481,12 @@ mod decl { } #[pyattr] - #[pyclass(name = "dropwhile")] + #[pyclass(name = "dropwhile", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsDropwhile { predicate: PyObjectRef, iterable: PyIter, + #[pytraverse(skip)] start_flag: AtomicCell, } @@ -533,11 +541,13 @@ mod decl { } } - #[derive(Default)] + #[derive(Default, Traverse)] struct GroupByState { current_value: Option, current_key: Option, + #[pytraverse(skip)] next_group: bool, + #[pytraverse(skip)] grouper: Option>, } @@ -561,7 +571,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "groupby")] + #[pyclass(name = "groupby", traverse)] #[derive(PyPayload)] struct PyItertoolsGroupBy { iterable: PyIter, @@ -661,7 +671,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "_grouper")] + #[pyclass(name = "_grouper", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsGrouper { groupby: PyRef, @@ -703,13 +713,17 @@ mod decl { } #[pyattr] - #[pyclass(name = "islice")] + #[pyclass(name = "islice", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsIslice { iterable: PyIter, + #[pytraverse(skip)] cur: AtomicCell, + #[pytraverse(skip)] next: AtomicCell, + #[pytraverse(skip)] stop: Option, + #[pytraverse(skip)] step: usize, } @@ -828,7 +842,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "filterfalse")] + #[pyclass(name = "filterfalse", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsFilterFalse { predicate: PyObjectRef, @@ -887,7 +901,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "accumulate")] + #[pyclass(name = "accumulate", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsAccumulate { iterable: PyIter, @@ -1053,7 +1067,7 @@ mod decl { #[pymethod] fn __copy__(&self) -> Self { Self { - tee_data: PyRc::clone(&self.tee_data), + tee_data: self.tee_data.clone(), index: AtomicCell::new(self.index.load()), } } @@ -1068,12 +1082,15 @@ mod decl { } #[pyattr] - #[pyclass(name = "product")] + #[pyclass(name = "product", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsProduct { pools: Vec>, + #[pytraverse(skip)] idxs: PyRwLock>, + #[pytraverse(skip)] cur: AtomicCell, + #[pytraverse(skip)] stop: AtomicCell, } @@ -1169,13 +1186,16 @@ mod decl { } #[pyattr] - #[pyclass(name = "combinations")] + #[pyclass(name = "combinations", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCombinations { pool: Vec, + #[pytraverse(skip)] indices: PyRwLock>, result: PyRwLock>>, + #[pytraverse(skip)] r: AtomicCell, + #[pytraverse(skip)] exhausted: AtomicCell, } @@ -1201,13 +1221,21 @@ mod decl { if r.is_negative() { return Err(vm.new_value_error("r must be non-negative")); } - let r = r.to_usize().unwrap(); + let r = r.to_isize().ok_or_else(|| { + vm.new_overflow_error("Python int too large to convert to C ssize_t") + })? as usize; let n = pool.len(); + let mut indices = Vec::new(); + indices + .try_reserve_exact(r) + .map_err(|_| vm.new_memory_error(""))?; + indices.extend(0..r); + Ok(Self { pool, - indices: PyRwLock::new((0..r).collect()), + indices: PyRwLock::new(indices), result: PyRwLock::new(None), r: AtomicCell::new(r), exhausted: AtomicCell::new(r > n), @@ -1280,12 +1308,15 @@ mod decl { } #[pyattr] - #[pyclass(name = "combinations_with_replacement")] + #[pyclass(name = "combinations_with_replacement", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCombinationsWithReplacement { pool: Vec, + #[pytraverse(skip)] indices: PyRwLock>, + #[pytraverse(skip)] r: AtomicCell, + #[pytraverse(skip)] exhausted: AtomicCell, } @@ -1302,13 +1333,21 @@ mod decl { if r.is_negative() { return Err(vm.new_value_error("r must be non-negative")); } - let r = r.to_usize().unwrap(); + let r = r.to_isize().ok_or_else(|| { + vm.new_overflow_error("Python int too large to convert to C ssize_t") + })? as usize; let n = pool.len(); + let mut indices = Vec::new(); + indices + .try_reserve_exact(r) + .map_err(|_| vm.new_memory_error(""))?; + indices.resize(r, 0); + Ok(Self { pool, - indices: PyRwLock::new(vec![0; r]), + indices: PyRwLock::new(indices), r: AtomicCell::new(r), exhausted: AtomicCell::new(n == 0 && r > 0), }) @@ -1366,15 +1405,20 @@ mod decl { } #[pyattr] - #[pyclass(name = "permutations")] + #[pyclass(name = "permutations", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsPermutations { - pool: Vec, // Collected input iterable - indices: PyRwLock>, // One index per element in pool - cycles: PyRwLock>, // One rollover counter per element in the result + pool: Vec, // Collected input iterable + #[pytraverse(skip)] + indices: PyRwLock>, // One index per element in pool + #[pytraverse(skip)] + cycles: PyRwLock>, // One rollover counter per element in the result + #[pytraverse(skip)] result: PyRwLock>>, // Indexes of the most recently returned result - r: AtomicCell, // Size of result tuple - exhausted: AtomicCell, // Set when the iterator is exhausted + #[pytraverse(skip)] + r: AtomicCell, // Size of result tuple + #[pytraverse(skip)] + exhausted: AtomicCell, // Set when the iterator is exhausted } #[derive(FromArgs)] @@ -1408,7 +1452,9 @@ mod decl { if val.is_negative() { return Err(vm.new_value_error("r must be non-negative")); } - val.to_usize().unwrap() + val.to_isize().ok_or_else(|| { + vm.new_overflow_error("Python int too large to convert to C ssize_t") + })? as usize } None => n, }; @@ -1524,7 +1570,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "zip_longest")] + #[pyclass(name = "zip_longest", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsZipLongest { iterators: Vec, @@ -1562,7 +1608,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "pairwise")] + #[pyclass(name = "pairwise", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsPairwise { iterator: PyIter, @@ -1611,12 +1657,15 @@ mod decl { } #[pyattr] - #[pyclass(name = "batched")] + #[pyclass(name = "batched", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsBatched { + #[pytraverse(skip)] exhausted: AtomicCell, iterable: PyIter, + #[pytraverse(skip)] n: AtomicCell, + #[pytraverse(skip)] strict: AtomicCell, } diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index a934c6d812f..9156c9fc0bf 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -206,7 +206,10 @@ pub(super) mod _os { ospath::{OsPath, OsPathOrFd, OutputMode, PathConverter}, protocol::PyIterReturn, recursion::ReprGuard, - types::{Destructor, IterNext, Iterable, PyStructSequence, Representable, SelfIter}, + types::{ + Destructor, IterNext, Iterable, PyStructSequence, PyStructSequenceData, Representable, + SelfIter, + }, vm::VirtualMachine, }; #[cfg(not(windows))] @@ -883,7 +886,7 @@ pub(super) mod _os { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } @@ -1314,8 +1317,12 @@ pub(super) mod _os { impl PyStatResult { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let seq: PyObjectRef = args.bind(vm)?; - let result = crate::types::struct_sequence_new(cls.clone(), seq, vm)?; + let result = crate::types::struct_sequence_new( + cls.clone(), + args.bind(vm)?, + StatResultData::OPTIONAL_FIELD_NAMES, + vm, + )?; let tuple = result.downcast_ref::().unwrap(); let mut items: Vec = tuple.to_vec(); @@ -1964,8 +1971,12 @@ pub(super) mod _os { impl PyStatvfsResult { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let seq: PyObjectRef = args.bind(vm)?; - crate::types::struct_sequence_new(cls, seq, vm) + crate::types::struct_sequence_new( + cls, + args.bind(vm)?, + StatvfsResultData::OPTIONAL_FIELD_NAMES, + vm, + ) } } diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index c16da1ee703..6b91950e907 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1333,14 +1333,13 @@ pub mod module { // cfg from nix #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "redox")))] #[pyfunction] - fn setgroups( - group_ids: crate::function::ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<()> { - let gids = group_ids - .iter(vm)? - .map(|gid| gid.map(|gid| gid.0)) - .collect::, _>>()?; + fn setgroups(group_ids: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + group_ids + .try_sequence(vm) + .map_err(|_| vm.new_type_error("setgroups argument must be a sequence"))?; + let gids = vm.extract_elements_with(&group_ids, |gid| { + RawGid::try_from_object(vm, gid).map(|gid| gid.0) + })?; rustpython_host_env::posix::setgroups_raw(&gids).map_err(|err| err.into_pyexception(vm)) } @@ -1400,7 +1399,7 @@ pub mod module { #[pyarg(positional)] path: OsPath, #[pyarg(positional)] - args: crate::function::ArgIterable, + args: PyObjectRef, #[pyarg(positional)] env: Option, #[pyarg(named, default)] @@ -1439,6 +1438,19 @@ pub mod module { .into_cstring(vm) .map_err(|_| vm.new_value_error("path should not have nul bytes"))?; + let function_name = if spawnp { + "posix_spawnp" + } else { + "posix_spawn" + }; + if !self.args.fast_isinstance(vm.ctx.types.list_type) + && !self.args.fast_isinstance(vm.ctx.types.tuple_type) + { + return Err( + vm.new_type_error(format!("{function_name}: argv must be a tuple or list")) + ); + } + let mut file_actions = Vec::new(); if let Some(it) = self.file_actions { for action in it.iter(vm)? { @@ -1478,20 +1490,21 @@ pub mod module { } } - let setsigdef = self - .setsigdef - .map(|sigs| { - let sigs = sigs.iter(vm)?.collect::>>()?; - for &sig in &sigs { - if !rustpython_host_env::posix::validate_posix_spawn_signal(sig) { - return Err( - vm.new_value_error(format!("signal number {sig} out of range")) - ); - } + let collect_signals = |sigs: crate::function::ArgIterable| { + let mut collected = Vec::new(); + for sig in sigs.iter(vm)? { + let sig = sig?; + if !rustpython_host_env::posix::validate_posix_spawn_signal(sig) { + return Err(vm.new_value_error(format!("signal number {sig} out of range"))); } - Ok(sigs) - }) - .transpose()?; + if !collected.contains(&sig) { + collected.push(sig); + } + } + Ok(collected) + }; + + let setsigdef = self.setsigdef.map(&collect_signals).transpose()?; if let Some(_scheduler) = self.scheduler { // TODO: Implement scheduler parameter handling @@ -1507,29 +1520,12 @@ pub mod module { )); } - let setsigmask = self - .setsigmask - .map(|sigs| { - let sigs = sigs.iter(vm)?.collect::>>()?; - for &sig in &sigs { - if !rustpython_host_env::posix::validate_posix_spawn_signal(sig) { - return Err( - vm.new_value_error(format!("signal number {sig} out of range")) - ); - } - } - Ok(sigs) - }) - .transpose()?; + let setsigmask = self.setsigmask.map(collect_signals).transpose()?; - let args: Vec = self - .args - .iter(vm)? - .map(|res| { - CString::new(res?.into_bytes()) - .map_err(|_| vm.new_value_error("path should not have nul bytes")) - }) - .collect::>()?; + let args = vm.extract_elements_with(&self.args, |arg| { + CString::new(OsPath::try_from_object(vm, arg)?.into_bytes()) + .map_err(|_| vm.new_value_error("path should not have nul bytes")) + })?; let env = if let Some(env_dict) = self.env { envp_from_dict(env_dict, vm)? } else { diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index c7cc2fd298a..66257806e22 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -888,8 +888,7 @@ pub mod sys { format!("Ignoring unimportable $PYTHONBREAKPOINT: \"{env_var}\"",), 0, vm, - ) - .unwrap(); + )?; Ok(vm.ctx.none()) }; diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index 3d777c24b89..a5daa9cd2ff 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -18,7 +18,7 @@ mod decl { AsObject, Py, PyObjectRef, PyResult, VirtualMachine, builtins::{PyStrRef, PyTypeRef}, function::{Either, FuncArgs, OptionalArg}, - types::{PyStructSequence, struct_sequence_new}, + types::{PyStructSequence, PyStructSequenceData, struct_sequence_new}, }; #[cfg(any(unix, windows))] use crate::{ @@ -811,8 +811,12 @@ mod decl { impl PyStructTime { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let (seq, _dict): (PyObjectRef, OptionalArg) = args.bind(vm)?; - struct_sequence_new(cls, seq, vm) + struct_sequence_new( + cls, + args.bind(vm)?, + StructTimeData::OPTIONAL_FIELD_NAMES, + vm, + ) } } diff --git a/crates/vm/src/types/mod.rs b/crates/vm/src/types/mod.rs index b17a737545f..11c3a4dc51e 100644 --- a/crates/vm/src/types/mod.rs +++ b/crates/vm/src/types/mod.rs @@ -5,5 +5,7 @@ mod zoo; pub use slot::*; pub use slot_defs::{SLOT_DEFS, SlotAccessor, SlotDef}; -pub use structseq::{PyStructSequence, PyStructSequenceData, struct_sequence_new}; +pub use structseq::{ + PyStructSequence, PyStructSequenceData, StructSequenceNewArgs, struct_sequence_new, +}; pub(crate) use zoo::TypeZoo; diff --git a/crates/vm/src/types/structseq.rs b/crates/vm/src/types/structseq.rs index 703cc79c193..7f8099e7efb 100644 --- a/crates/vm/src/types/structseq.rs +++ b/crates/vm/src/types/structseq.rs @@ -2,9 +2,11 @@ use crate::common::lock::LazyLock; use crate::common::wtf8::Wtf8; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, - builtins::{PyBaseExceptionRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef}, + builtins::{ + PyBaseExceptionRef, PyDict, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, + }, class::{PyClassImpl, StaticType}, - function::{Either, FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags}, + function::{Either, FuncArgs, OptionalArg, PyComparisonValue, PyMethodDef, PyMethodFlags}, iter::PyExactSizeIterator, protocol::{PyMappingMethods, PySequenceMethods}, sliceable::{SequenceIndex, SliceableSequenceOp}, @@ -21,12 +23,35 @@ const DEFAULT_STRUCTSEQ_REDUCE: PyMethodDef = PyMethodDef::new_const( None, ); +/// The arguments every struct sequence constructor takes. +#[derive(FromArgs)] +pub struct StructSequenceNewArgs { + #[pyarg(any)] + pub sequence: PyObjectRef, + #[pyarg(any, optional)] + pub dict: OptionalArg, +} + /// Create a new struct sequence instance from a sequence. /// +/// `dict` supplies the hidden fields — the ones past `n_sequence_fields`, named +/// by `hidden_field_names` in order — that the sequence itself did not cover. It +/// may not name a field the sequence already supplied, nor one that does not +/// exist. +/// /// The class must have `n_sequence_fields` and `n_fields` attributes set /// (done automatically by `PyStructSequence::extend_pyclass`). -pub fn struct_sequence_new(cls: PyTypeRef, seq: PyObjectRef, vm: &VirtualMachine) -> PyResult { +pub fn struct_sequence_new( + cls: PyTypeRef, + args: StructSequenceNewArgs, + hidden_field_names: &[&str], + vm: &VirtualMachine, +) -> PyResult { // = structseq_new + let StructSequenceNewArgs { + sequence: seq, + dict, + } = args; #[cold] fn length_error( @@ -60,6 +85,16 @@ pub fn struct_sequence_new(cls: PyTypeRef, seq: PyObjectRef, vm: &VirtualMachine .ok_or_else(|| vm.new_type_error("missing n_fields attribute"))? .try_into_value(vm)?; + let dict = match dict { + OptionalArg::Missing => None, + OptionalArg::Present(dict) => Some(dict.downcast::().map_err(|_| { + vm.new_type_error(format!( + "{}() takes a dict as second arg, if any", + cls.slot_name() + )) + })?), + }; + let seq: Vec = seq.try_into_value(vm)?; let len = seq.len(); @@ -67,10 +102,30 @@ pub fn struct_sequence_new(cls: PyTypeRef, seq: PyObjectRef, vm: &VirtualMachine return Err(length_error(&cls.slot_name(), min_len, max_len, len, vm)); } - // Copy items and pad with None + // Copy items and pad the hidden fields the sequence did not cover with None. let mut items = seq; items.resize_with(max_len, || vm.ctx.none()); + // Fill those padded slots from `dict`. Every key has to land in one of them: + // a key naming a field the sequence already supplied, or no field at all, + // would otherwise be silently dropped. + if let Some(dict) = dict.filter(|dict| !dict.is_empty()) { + let mut found = 0; + let names = hidden_field_names.get(len - min_len..).unwrap_or(&[]); + for (item, name) in items[len..].iter_mut().zip(names) { + if let Some(value) = dict.get_item_opt(*name, vm)? { + *item = value; + found += 1; + } + } + if found != dict.__len__() { + return Err(vm.new_type_error(format!( + "{}() got duplicate or unexpected field name(s)", + cls.slot_name() + ))); + } + } + PyTuple::new_unchecked(items.into_boxed_slice()) .into_ref_with_type(vm, cls) .map(Into::into) @@ -193,6 +248,11 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { /// The Data struct that provides field definitions. type Data: PyStructSequenceData; + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + struct_sequence_new(cls, args.bind(vm)?, Self::Data::OPTIONAL_FIELD_NAMES, vm) + } + /// Convert a Data struct into a PyStructSequence instance. fn from_data(data: Self::Data, vm: &VirtualMachine) -> PyTupleRef { let tuple = diff --git a/crates/vm/src/utils.rs b/crates/vm/src/utils.rs index 80402480cfd..8a28a32f663 100644 --- a/crates/vm/src/utils.rs +++ b/crates/vm/src/utils.rs @@ -33,6 +33,7 @@ pub(crate) fn collection_repr<'a, I>( class_name: Option<&str>, prefix: &str, suffix: &str, + empty: &str, iter: I, vm: &VirtualMachine, ) -> PyResult @@ -47,10 +48,9 @@ where repr.push_str(prefix); { let mut parts_iter = iter.map(|o| o.repr(vm)); - let first = parts_iter - .next() - .transpose()? - .expect("this is not called for empty collection"); + let Some(first) = parts_iter.next().transpose()? else { + return Ok(Wtf8Buf::from(empty)); + }; repr.push_wtf8(first.as_wtf8()); for part in parts_iter { repr.push_str(", "); diff --git a/extra_tests/snippets/builtin_compile.py b/extra_tests/snippets/builtin_compile.py index 49295bf26d2..73247e50df1 100644 --- a/extra_tests/snippets/builtin_compile.py +++ b/extra_tests/snippets/builtin_compile.py @@ -145,3 +145,8 @@ def _check_flags_error(flags): assert exc.args[0] == "incomplete input", repr(exc) else: raise AssertionError("expected _IncompleteInputError") + +# The source is encoded before it is parsed, so a lone surrogate has to be +# reported rather than assumed away. +with assert_raises(UnicodeEncodeError): + compile(chr(0xD800), "", "eval") diff --git a/extra_tests/snippets/builtin_eval.py b/extra_tests/snippets/builtin_eval.py index 2f2405c8d9e..1648a1a271d 100644 --- a/extra_tests/snippets/builtin_eval.py +++ b/extra_tests/snippets/builtin_eval.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + assert 3 == eval("1+2") code = compile("5+3", "x.py", "eval") @@ -75,3 +77,8 @@ def make_closure(): assert False, "eval with code containing free variables should fail" except NameError as e: pass + +# The source is encoded before it is parsed, so a lone surrogate has to be +# reported rather than assumed away. +with assert_raises(UnicodeEncodeError): + eval(chr(0xD800)) diff --git a/extra_tests/snippets/builtin_exceptions.py b/extra_tests/snippets/builtin_exceptions.py index 8879e130bc2..080294a3c8a 100644 --- a/extra_tests/snippets/builtin_exceptions.py +++ b/extra_tests/snippets/builtin_exceptions.py @@ -1,4 +1,5 @@ import builtins +import itertools import pickle import platform import sys @@ -393,3 +394,19 @@ class SubError(MyError): assert err.exceptions[0].args == ("x",) else: assert False, "except* handler did not run" + +# The exceptions argument is a sequence, so an arbitrary iterable must be +# rejected rather than drained. +try: + ExceptionGroup("m", itertools.count()) +except TypeError: + pass +else: + assert False, "ExceptionGroup accepted an unbounded iterable" + +# ImportError.__reduce__ has to cope with the exception carrying no args. +assert pickle.loads(pickle.dumps(ImportError())).args == () +restored = pickle.loads(pickle.dumps(ImportError("m", name="n", path="p"))) +assert restored.args == ("m",) +assert restored.name == "n" +assert restored.path == "p" diff --git a/extra_tests/snippets/builtin_exec.py b/extra_tests/snippets/builtin_exec.py index 2eae90e91c5..cfb88c15dc1 100644 --- a/extra_tests/snippets/builtin_exec.py +++ b/extra_tests/snippets/builtin_exec.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + exec("def square(x):\n return x * x\n") assert 16 == square(4) # noqa: F821 @@ -71,3 +73,8 @@ def f(): f() + +# The source is encoded before it is parsed, so a lone surrogate has to be +# reported rather than assumed away. +with assert_raises(UnicodeEncodeError): + exec(chr(0xD800)) diff --git a/extra_tests/snippets/builtin_hash.py b/extra_tests/snippets/builtin_hash.py index 9b2c8388790..b3128cecc5a 100644 --- a/extra_tests/snippets/builtin_hash.py +++ b/extra_tests/snippets/builtin_hash.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises @@ -28,3 +30,19 @@ def __hash__(self): with assert_raises(TypeError): hash([]) + +# Hashing a deeply nested tuple must not run off the native stack: the hash +# slot dispatch is what recurses, so that is where the depth is checked. + +if sys.implementation.name == "rustpython": + # CPython, which also runs this snippet, survives this depth unguarded. + deep_tuple = () + for _ in range(sys.getrecursionlimit() * 2): + deep_tuple = (deep_tuple,) + with assert_raises(RecursionError): + hash(deep_tuple) + # a dict key and a set member are hashed on insertion, same dispatch + with assert_raises(RecursionError): + {deep_tuple: 1} + with assert_raises(RecursionError): + {deep_tuple} diff --git a/extra_tests/snippets/builtin_list.py b/extra_tests/snippets/builtin_list.py index d62cae03b50..44492092bad 100644 --- a/extra_tests/snippets/builtin_list.py +++ b/extra_tests/snippets/builtin_list.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises x = [1, 2, 3] @@ -923,3 +925,8 @@ def __eq__(self, other): list1 = rewrite_list_eq([poc()]) list1.remove(list1) assert list1 == [] + +# The repeat count is multiplied by the element size; a count that overflows +# that product must raise instead of wrapping into a short allocation. +with assert_raises(MemoryError): + [1] * sys.maxsize diff --git a/extra_tests/snippets/builtin_tuple.py b/extra_tests/snippets/builtin_tuple.py index fc2f8d5bb75..a679d2a99a8 100644 --- a/extra_tests/snippets/builtin_tuple.py +++ b/extra_tests/snippets/builtin_tuple.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises assert (1, 2) == (1, 2) @@ -93,3 +95,8 @@ def __eq__(self, x): assert (float("inf"), float("inf")) >= (float("inf"), float("inf")) assert not (float("inf"), float("inf")) < (float("inf"), float("inf")) assert not (float("inf"), float("inf")) > (float("inf"), float("inf")) + +# The repeat count is multiplied by the element size; a count that overflows +# that product must raise instead of wrapping into a short allocation. +with assert_raises(MemoryError): + (1,) * sys.maxsize diff --git a/extra_tests/snippets/forbidden_instantiation.py b/extra_tests/snippets/forbidden_instantiation.py index 50b6f58f07f..50a0e2cf635 100644 --- a/extra_tests/snippets/forbidden_instantiation.py +++ b/extra_tests/snippets/forbidden_instantiation.py @@ -1,3 +1,4 @@ +import re from types import ( AsyncGeneratorType, BuiltinFunctionType, @@ -62,3 +63,9 @@ def check_forbidden_instantiation(typ, reverse=False): for typ in internal_types: with assert_raises(TypeError): typ() + +# a match object carries state that only the matcher can fill in +with assert_raises(TypeError): + re.Match() +with assert_raises(TypeError): + re.Match.__new__(re.Match) diff --git a/extra_tests/snippets/stdlib_asyncio.py b/extra_tests/snippets/stdlib_asyncio.py new file mode 100644 index 00000000000..7f03aeb436b --- /dev/null +++ b/extra_tests/snippets/stdlib_asyncio.py @@ -0,0 +1,52 @@ +"""The private _asyncio accessors, reached directly instead of through a loop. + +CPython's _asyncio rejects every call below with "loop ... is not the running +loop" before it gets anywhere, and does not expose _current_tasks at all, so +these only run where they are reachable. +""" + +import sys + +from testutils import assert_raises + +if sys.implementation.name != "rustpython": + sys.exit(0) + +import _asyncio + + +def _task(): + pass + + +# The "already entered" message formats both tasks; a plain function used to be +# formatted as the wrong type there. +_asyncio._enter_task(0, _task) +with assert_raises(RuntimeError) as cm: + _asyncio._enter_task(0, _task) +assert "Cannot enter into task" in str(cm.exception), cm.exception +assert " str: # print(get_win_folder_via_ctypes("CSIDL_DOWNLOADS")) +# A value wider than the C type is masked down to it instead of failing an +# unchecked conversion. +assert ctypes.c_char_p(2**64).value is None +assert ctypes.c_int(2**64 + 7).value == 7 +buf = (ctypes.c_int * 1)() +int_ptr = ctypes.cast(buf, ctypes.POINTER(ctypes.c_int)) +int_ptr[0] = 2**64 + 5 +assert int_ptr[0] == 5 + +# A slice assignment is length-checked against the slice, so the right-hand +# side must not be drained first. +array3 = (ctypes.c_int * 3)() +try: + array3[0:3] = itertools.count() +except ValueError: + pass +else: + assert False, "slice assignment accepted an unbounded iterable" +array3[0:3] = [7, 8, 9] +assert list(array3) == [7, 8, 9] + print("done") diff --git a/extra_tests/snippets/stdlib_ctypes_calls.py b/extra_tests/snippets/stdlib_ctypes_calls.py index 1de29931429..cc4e8020511 100644 --- a/extra_tests/snippets/stdlib_ctypes_calls.py +++ b/extra_tests/snippets/stdlib_ctypes_calls.py @@ -1,6 +1,7 @@ # Exercises the migrated _ctypes foreign-call path (routed through the unified # host_env `call` entry point): scalar int/double arguments and returns, -# pointer (c_char_p / c_void_p) returns, and a use_errno round-trip. +# pointer (c_char_p / c_void_p) returns, a use_errno round-trip, and the +# argument conversion an untyped call performs. # # Prints "OK" and exits 0; any failed assertion aborts. Output is identical # under CPython and RustPython on the same platform. @@ -61,4 +62,17 @@ libc.strtol(b"9" * 40, None, 10) assert get_errno() == errno.ERANGE, (get_errno(), errno.ERANGE) +# 7. A float has no implicit conversion to an integer argument: converting it +# would pass a truncated value where the callee expects an int or a pointer. +libc.abs.argtypes = None +for bad in (1.5, 0.0, 1e300): + try: + libc.abs(bad) + except (TypeError, ctypes.ArgumentError): + pass + else: + assert False, f"{bad!r} was accepted as an integer argument" +assert libc.abs(-3) == 3 +assert libc.abs(True) == 1 + print("OK") diff --git a/extra_tests/snippets/stdlib_gc.py b/extra_tests/snippets/stdlib_gc.py new file mode 100644 index 00000000000..134b1b9f458 --- /dev/null +++ b/extra_tests/snippets/stdlib_gc.py @@ -0,0 +1,66 @@ +"""The cycle collector has to walk the internal fields of containers and +iterators. + +Every type below is built into the cycle + + node -> node.__dict__ -> wrapper -> container -> node + +so the only path back to `node` runs through a field of the wrapper. A type +that reports nothing while being traversed, or reports the objects it iterates +instead of the iterator it holds, leaves its own reference unaccounted for: the +cycle is then classified as reachable and `node` is never freed. +""" + +import gc +import itertools +import weakref +from collections import defaultdict, deque + + +class Node: + pass + + +def collects(wrap): + """Report whether the collector breaks the cycle built around wrap().""" + + def build(): + container = [] + node = Node() + container.append(node) + node.held = wrap(container) + return weakref.ref(node) + + gc.collect() + ref = build() + gc.collect() + return ref() is None + + +# containers keeping their items in a field of their own +assert collects(deque) +assert collects(lambda c: defaultdict(int, {"k": c})) +assert collects(lambda c: classmethod(lambda cls: c)) + +# iterators: the wrapper holds an iterator, and that iterator holds the +# container +assert collects(iter) +assert collects(lambda c: map(str, c)) +assert collects(lambda c: filter(None, c)) +assert collects(lambda c: zip(c)) +assert collects(enumerate) +assert collects(reversed) +assert collects(itertools.chain) +assert collects(itertools.cycle) +assert collects(lambda c: itertools.islice(c, 5)) +assert collects(itertools.groupby) +assert collects(itertools.accumulate) +assert collects(lambda c: itertools.starmap(str, c)) +assert collects(lambda c: itertools.takewhile(bool, c)) +assert collects(lambda c: itertools.dropwhile(bool, c)) +assert collects(lambda c: itertools.filterfalse(None, c)) +assert collects(lambda c: itertools.compress(c, [1])) +assert collects(lambda c: itertools.product(c)) +assert collects(lambda c: itertools.combinations(c, 1)) + +print("ok") diff --git a/extra_tests/snippets/stdlib_hashlib.py b/extra_tests/snippets/stdlib_hashlib.py index c5feb709e17..a463941b29a 100644 --- a/extra_tests/snippets/stdlib_hashlib.py +++ b/extra_tests/snippets/stdlib_hashlib.py @@ -1,3 +1,5 @@ +import _md5 +import _sha1 import hashlib # print(hashlib.md5) @@ -48,3 +50,9 @@ assert ( h.hexdigest() == "25738bfe4cc104131e1b45bece4dfd4e7e1d6f0dffda1211e996e9d5d3b66e81" ) + +# The single-algorithm modules set up their own types rather than relying on +# hashlib having done it. + +assert _md5.md5(b"").hexdigest() == "d41d8cd98f00b204e9800998ecf8427e" +assert _sha1.sha1(b"").hexdigest() == "da39a3ee5e6b4b0d3255bfef95601890afd80709" diff --git a/extra_tests/snippets/stdlib_imp.py b/extra_tests/snippets/stdlib_imp.py index 835b50d6171..64f1a0ad67e 100644 --- a/extra_tests/snippets/stdlib_imp.py +++ b/extra_tests/snippets/stdlib_imp.py @@ -1,6 +1,8 @@ import _imp import time as import_time +from testutils import assert_raises + assert _imp.is_builtin("time") == True assert _imp.is_builtin("os") == False assert _imp.is_builtin("not existing module") == False @@ -29,3 +31,8 @@ def __init__(self, name): hello = _imp.init_frozen("__hello__") assert hello.initialized == True + +# withdata is keyword-only +with assert_raises(TypeError): + _imp.find_frozen("x", True) +assert _imp.find_frozen("_this_module_does_not_exist_") is None diff --git a/extra_tests/snippets/stdlib_itertools.py b/extra_tests/snippets/stdlib_itertools.py index ce7a494713a..029d0d4229a 100644 --- a/extra_tests/snippets/stdlib_itertools.py +++ b/extra_tests/snippets/stdlib_itertools.py @@ -524,3 +524,19 @@ def __iter__(self): assert next(it) == (2, None) with assert_raises(StopIteration): next(it) + +# r is an arbitrary Python int: one too large for an index must raise +# OverflowError, and a representable one that cannot be allocated must raise +# MemoryError. +for factory in ( + itertools.combinations, + itertools.combinations_with_replacement, + itertools.permutations, +): + with assert_raises(OverflowError): + factory(range(5), 2**64) + +with assert_raises(MemoryError): + itertools.combinations(range(5), 2**44) +with assert_raises(MemoryError): + itertools.combinations_with_replacement(range(5), 2**44) diff --git a/extra_tests/snippets/stdlib_lzma.py b/extra_tests/snippets/stdlib_lzma.py new file mode 100644 index 00000000000..5ebce3c7fb1 --- /dev/null +++ b/extra_tests/snippets/stdlib_lzma.py @@ -0,0 +1,22 @@ +import itertools +import lzma + +from testutils import assert_raises + +# A raw-format compressor needs the filter chain's length before it can build +# it, so a filter argument that is not a sequence has to be rejected instead of +# being drained. +with assert_raises(TypeError): + lzma.LZMACompressor( + format=lzma.FORMAT_RAW, + filters=({"id": lzma.FILTER_LZMA2} for _ in itertools.count()), + ) + +compressor = lzma.LZMACompressor( + format=lzma.FORMAT_RAW, filters=[{"id": lzma.FILTER_LZMA2}] +) +compressed = compressor.compress(b"data") + compressor.flush() +decompressor = lzma.LZMADecompressor( + format=lzma.FORMAT_RAW, filters=[{"id": lzma.FILTER_LZMA2}] +) +assert decompressor.decompress(compressed) == b"data" diff --git a/extra_tests/snippets/stdlib_math.py b/extra_tests/snippets/stdlib_math.py index a6bb0099c05..bc8673797c0 100644 --- a/extra_tests/snippets/stdlib_math.py +++ b/extra_tests/snippets/stdlib_math.py @@ -1,3 +1,4 @@ +import itertools import math from testutils import assert_raises, skip_if_unsupported @@ -311,3 +312,9 @@ def assertAllNotClose(examples, *args, **kwargs): assert math.fmod(0.0, NINF) == 0.0 assert math.gamma(1) == 1.0 + +# sumprod compares the two lengths as it goes; it must not drain either +# argument first. +assert_raises(ValueError, lambda: math.sumprod(itertools.count(), [1, 2, 3])) +assert_raises(ValueError, lambda: math.sumprod([1, 2, 3], itertools.count())) +assert math.sumprod(iter([1, 2, 3]), iter([4, 5, 6])) == 32 diff --git a/extra_tests/snippets/stdlib_mmap.py b/extra_tests/snippets/stdlib_mmap.py index 3a2b139a333..2dee29e6bab 100644 --- a/extra_tests/snippets/stdlib_mmap.py +++ b/extra_tests/snippets/stdlib_mmap.py @@ -1,6 +1,19 @@ import mmap +from testutils import assert_raises + mapped = mmap.mmap(-1, 1) assert mapped.seekable() mapped.close() assert mapped.seekable() + +mapped = mmap.mmap(-1, 10) +# an inverted range finds nothing rather than being subtracted into a huge one +assert mapped.find(b"x", 5, 2) == -1 +assert mapped.rfind(b"x", 5, 2) == -1 +# both offsets are bounds-checked before anything is copied +with assert_raises(ValueError): + mapped.move(20, 0, 1) +with assert_raises(ValueError): + mapped.move(0, 20, 1) +mapped.close() diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index d00924e10f2..a1f40ef4c45 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -1,3 +1,4 @@ +import itertools import os import stat import sys @@ -528,3 +529,19 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert os.access("nonexistent_file_12345", os.W_OK) is False assert os.access("README.md", os.F_OK) is True assert os.access("README.md", os.R_OK) is True + +# argv and the group list are sequences; an arbitrary iterable must be rejected +# rather than drained. +if hasattr(os, "posix_spawn"): + with assert_raises(TypeError): + os.posix_spawn("/bin/true", map(str, itertools.count()), os.environ) +if hasattr(os, "setgroups"): + with assert_raises(TypeError): + os.setgroups(itertools.count()) + +# The optional second argument fills the fields past the visible ones, and the +# getters must not index past what __new__ stored. +assert os.stat_result(tuple(range(10))).st_atime == 7 +assert os.stat_result(tuple(range(10)), {"st_atime": 1.5}).st_atime == 1.5 +with assert_raises(TypeError): + os.stat_result(tuple(range(10)), ["st_atime"]) diff --git a/extra_tests/snippets/stdlib_pwd.py b/extra_tests/snippets/stdlib_pwd.py index c3aeb7c8703..6229f631c91 100644 --- a/extra_tests/snippets/stdlib_pwd.py +++ b/extra_tests/snippets/stdlib_pwd.py @@ -12,3 +12,7 @@ fake_name = "fake_user" while pwd.getpwnam(fake_name): fake_name += "1" + +# The field getters must not index a struct sequence that __new__ never filled. +with assert_raises(TypeError): + pwd.struct_passwd() diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index 155fc905a73..9dba301fb01 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -1,6 +1,7 @@ import os import subprocess import sys +import warnings from testutils import assert_raises @@ -158,3 +159,18 @@ def test_getframemodulename(): test_getframemodulename.__module__ = "awesome_module" assert test_getframemodulename() == "awesome_module" + +# An unimportable $PYTHONBREAKPOINT warns, and the hook has to survive that +# warning being turned into an exception. +saved_breakpoint_env = os.environ.get("PYTHONBREAKPOINT") +os.environ["PYTHONBREAKPOINT"] = "nonexistent_xyz.foo" +try: + with warnings.catch_warnings(): + warnings.simplefilter("error") + with assert_raises(RuntimeWarning): + sys.breakpointhook() +finally: + if saved_breakpoint_env is None: + del os.environ["PYTHONBREAKPOINT"] + else: + os.environ["PYTHONBREAKPOINT"] = saved_breakpoint_env diff --git a/extra_tests/snippets/stdlib_threading_itertools_cycle.py b/extra_tests/snippets/stdlib_threading_itertools_cycle.py new file mode 100644 index 00000000000..b50a31b2443 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_itertools_cycle.py @@ -0,0 +1,26 @@ +"""Stress itertools.cycle from several threads at once. + +cycle() advances its index and wraps it back to zero when it reaches the end of +the saved items. Doing that in two separate steps lets another thread observe +the index past the end and read out of bounds, so the update has to be a single +atomic step. +""" + +import itertools +import threading + +shared_cycle = itertools.cycle([1, 2, 3]) + + +def spin(): + for _ in range(20000): + next(shared_cycle) + + +threads = [threading.Thread(target=spin) for _ in range(4)] +for t in threads: + t.start() +for t in threads: + t.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_set_repr.py b/extra_tests/snippets/stdlib_threading_set_repr.py new file mode 100644 index 00000000000..e2ce2d94357 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_set_repr.py @@ -0,0 +1,44 @@ +"""Stress set repr against concurrent mutation. + +repr() checks that the set is non-empty and then reads its first element. The +two steps are separate, so another thread can empty the set in between; the +read has to cope with that rather than trusting the earlier check. + +Threads that observe a mutation mid-iteration raise RuntimeError, which is a +legitimate outcome here; a regression shows up as a crash instead. +""" + +import threading + +shared_set = {1, 2, 3, 4, 5} +stop = False + + +def mutate(): + while not stop: + try: + shared_set.clear() + shared_set.update({1, 2, 3}) + except RuntimeError: # changed size during iteration + pass + + +def read(): + for _ in range(20000): + try: + repr(shared_set) + except RuntimeError: # changed size during iteration + pass + + +mutators = [threading.Thread(target=mutate) for _ in range(2)] +readers = [threading.Thread(target=read) for _ in range(2)] +for t in mutators + readers: + t.start() +for t in readers: + t.join() +stop = True +for t in mutators: + t.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index 68ceab89521..b74d5bbc638 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -82,3 +82,16 @@ assert monotonic_elapsed >= 0.01 assert perf_elapsed >= 0.01 + +# The optional second argument fills the fields that are not part of the +# sequence. +fields = (2024, 1, 2, 3, 4, 5, 6, 7, 0) +assert time.struct_time(fields).tm_zone is None +assert time.struct_time(fields, {"tm_zone": "UTC"}).tm_zone == "UTC" +assert time.struct_time(fields, {"tm_gmtoff": 60}).tm_gmtoff == 60 +try: + time.struct_time(fields, ["tm_zone", "UTC"]) +except TypeError: + pass +else: + assert False, "struct_time accepted a non-dict second argument" diff --git a/extra_tests/snippets/stdlib_traceback.py b/extra_tests/snippets/stdlib_traceback.py index c2cc5773dbc..b1b11a75503 100644 --- a/extra_tests/snippets/stdlib_traceback.py +++ b/extra_tests/snippets/stdlib_traceback.py @@ -1,5 +1,9 @@ +import itertools import traceback +import _suggestions +from testutils import assert_raises + try: 1 / 0 except ZeroDivisionError as ex: @@ -25,3 +29,10 @@ except ZeroDivisionError as ex2: tb = traceback.extract_tb(ex2.__traceback__) assert len(tb) == 1 + +# The candidate list backing "Did you mean" suggestions is a list; an arbitrary +# iterable must be rejected rather than drained. + +with assert_raises(TypeError): + _suggestions._generate_suggestions(itertools.count(), "x") +assert _suggestions._generate_suggestions(["value"], "valu") == "value" diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index cdecf12dd2b..335069811a8 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -1,5 +1,6 @@ import _ast import platform +import sys import types from testutils import assert_raises @@ -34,3 +35,26 @@ def _run_missing_type_params_regression(): _run_missing_type_params_regression() + +if sys.implementation.name == "rustpython": + # __parameters__ is computed when the alias is built, and the walk descends + # into every list and tuple argument, so a self-referential or deeply + # nested argument must be caught. CPython, which also runs this snippet, + # does not walk into plain lists at all. + self_referential = [] + self_referential.append(self_referential) + with assert_raises(RecursionError): + list[self_referential] + + nested = [0] + for _ in range(sys.getrecursionlimit() * 2): + nested = [nested] + with assert_raises(RecursionError): + list[nested] + + # hashing an alias walks the same shape + deep_alias = int + for _ in range(sys.getrecursionlimit() * 2): + deep_alias = list[deep_alias] + with assert_raises(RecursionError): + hash(deep_alias) diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py index 07348945842..98d368c02cd 100644 --- a/extra_tests/snippets/stdlib_typing.py +++ b/extra_tests/snippets/stdlib_typing.py @@ -1,6 +1,9 @@ from collections.abc import Awaitable, Callable from typing import TypeVar +import _typing +from testutils import assert_raises + T = TypeVar("T") @@ -35,3 +38,10 @@ def __init__( def method(self, value: Union[int, float]) -> Union[str, bytes]: return str(value) + + +# _idfunc takes exactly one argument, checked before the argument is read. + +assert _typing._idfunc(1) == 1 +with assert_raises(TypeError): + _typing._idfunc()