Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Lib/test/test_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,6 @@ def test_has_strftime_extensions(self):
else:
self.assertTrue(support.has_strftime_extensions)

@unittest.expectedFailure # TODO: RUSTPYTHON; - _testinternalcapi module not available
def test_get_recursion_depth(self):
# test support.get_recursion_depth()
code = textwrap.dedent("""
Expand Down
25 changes: 13 additions & 12 deletions crates/common/src/str.rs
Original file line number Diff line number Diff line change
Expand Up @@ -416,20 +416,21 @@ pub fn codepoint_range_end(s: &Wtf8, n_chars: usize) -> Option<usize> {
}

#[must_use]
pub fn zfill(bytes: &[u8], width: usize) -> Vec<u8> {
/// Returns `None` for a width whose result cannot be allocated.
pub fn zfill(bytes: &[u8], width: usize) -> Option<Vec<u8>> {
if width <= bytes.len() {
bytes.to_vec()
} else {
let (sign, s) = match bytes.first() {
Some(_sign @ (b'+' | b'-')) => (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]),
_ => (&b""[..], bytes),
};
let mut filled = Vec::new();
filled.extend_from_slice(sign);
filled.extend(core::iter::repeat_n(b'0', width - bytes.len()));
filled.extend_from_slice(s);
filled
return Some(bytes.to_vec());
}
let (sign, s) = match bytes.first() {
Some(_sign @ (b'+' | b'-')) => (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]),
_ => (&b""[..], bytes),
};
let mut filled = Vec::new();
filled.try_reserve_exact(width).ok()?;
filled.extend_from_slice(sign);
filled.extend(core::iter::repeat_n(b'0', width - bytes.len()));
filled.extend_from_slice(s);
Some(filled)
}

/// Convert a string to ascii compatible, escaping unicode-s into escape
Expand Down
40 changes: 25 additions & 15 deletions crates/compiler-core/src/marshal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ pub enum MarshalError {
InvalidLocation,
/// Bad type marker
BadType,
/// A container length that is negative or does not fit, named by what it counts
BadSize(&'static str),
}

impl core::fmt::Display for MarshalError {
Expand All @@ -29,6 +31,7 @@ impl core::fmt::Display for MarshalError {
Self::InvalidUtf8 => f.write_str("invalid utf8"),
Self::InvalidLocation => f.write_str("invalid source location"),
Self::BadType => f.write_str("bad type marker"),
Self::BadSize(what) => write!(f, "{what} size out of range"),
}
}
}
Expand Down Expand Up @@ -146,6 +149,13 @@ pub trait Read {
fn read_u64(&mut self) -> Result<u64> {
Ok(u64::from_le_bytes(*self.read_array()?))
}

/// A length, read the way `r_long` reads one: it is signed, so a value
/// with the top bit set is out of range rather than four billion items.
fn read_len(&mut self, what: &'static str) -> Result<usize> {
let len = self.read_u32()? as i32;
usize::try_from(len).map_err(|_| MarshalError::BadSize(what))
}
}

pub(crate) trait ReadBorrowed<'a>: Read {
Expand Down Expand Up @@ -553,7 +563,7 @@ pub trait MarshalBag: Copy {
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value;
) -> Result<Self::Value>;

/// Construct a runtime code object while retaining the exact values read
/// from ``co_consts``. Compiler bags ignore this second channel; runtime
Expand All @@ -563,7 +573,7 @@ pub trait MarshalBag: Copy {
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
_constants: Vec<Self::Value>,
) -> Self::Value {
) -> Result<Self::Value> {
self.make_code(code)
}

Expand Down Expand Up @@ -725,8 +735,8 @@ impl<Bag: ConstantBag> MarshalBag for Bag {
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value {
self.make_code(code)
) -> Result<Self::Value> {
Ok(self.make_code(code))
}

fn make_stop_iter(&self) -> Result<Self::Value> {
Expand Down Expand Up @@ -986,7 +996,7 @@ fn deserialize_code_value_inner<R: Read, Bag: MarshalBag>(
linetable,
exceptiontable,
};
Ok(bag.make_code_with_constants(code, constant_values))
bag.make_code_with_constants(code, constant_values)
}

fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
Expand Down Expand Up @@ -1033,13 +1043,13 @@ fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
bag.make_complex(value)
}
Type::Ascii | Type::Unicode => {
let len = rdr.read_u32()?;
let value = rdr.read_wtf8(len)?;
let len = rdr.read_len("string")?;
let value = rdr.read_wtf8(len as u32)?;
bag.make_str(value)
}
Type::AsciiInterned | Type::Interned => {
let len = rdr.read_u32()?;
let value = rdr.read_wtf8(len)?;
let len = rdr.read_len("string")?;
let value = rdr.read_wtf8(len as u32)?;
bag.make_interned_str(value)
}
Type::ShortAscii => {
Expand Down Expand Up @@ -1077,7 +1087,7 @@ fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
return Err(MarshalError::BadType);
}
Type::Tuple => {
let len = rdr.read_u32()? as usize;
let len = rdr.read_len("tuple")?;
let d = depth - 1;
if let Some(index) = slot
&& let Some(tuple) = bag.make_tuple_placeholder(len)
Expand All @@ -1094,7 +1104,7 @@ fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
}
}
Type::List => {
let len = rdr.read_u32()? as usize;
let len = rdr.read_len("list")?;
let d = depth - 1;
if let Some(index) = slot
&& let Some(list) = bag.make_list_placeholder(len)
Expand All @@ -1111,7 +1121,7 @@ fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
}
}
Type::Set => {
let len = rdr.read_u32()? as usize;
let len = rdr.read_len("set")?;
let d = depth - 1;
if let Some(index) = slot
&& let Some(set) = bag.make_set_placeholder()
Expand All @@ -1128,7 +1138,7 @@ fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
}
}
Type::FrozenSet => {
let len = rdr.read_u32()?;
let len = rdr.read_len("set")?;
let d = depth - 1;
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_frozenset(it))??
Expand Down Expand Up @@ -1165,8 +1175,8 @@ fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
}
Type::Bytes => {
// After marshaling, byte arrays are converted into bytes.
let len = rdr.read_u32()?;
let value = rdr.read_slice(len)?;
let len = rdr.read_len("bytes object")?;
let value = rdr.read_slice(len as u32)?;
bag.make_bytes(value)
}
Type::Code => return Err(MarshalError::BadType),
Expand Down
71 changes: 40 additions & 31 deletions crates/stdlib/src/_asyncio.rs
Original file line number Diff line number Diff line change
Expand Up @@ -724,47 +724,56 @@ pub(crate) mod _asyncio {

/// Add waiter to fut_awaited_by with single-object optimization
fn awaited_by_add(&self, waiter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
let mut awaited_by = self.fut_awaited_by.write();
if awaited_by.is_none() {
// First waiter - store directly
*awaited_by = Some(waiter);
return Ok(());
}
// Storing a waiter in the set runs its __hash__ and __eq__, which can
// come back to this future, so the field is locked only while it is
// read or written.
let existing = {
let mut awaited_by = self.fut_awaited_by.write();
match awaited_by.as_ref() {
// First waiter - store directly
None => {
*awaited_by = Some(waiter);
return Ok(());
}
Some(existing) => existing.clone(),
}
};

if self.fut_awaited_by_is_set.load(Ordering::Relaxed) {
// Already a Set - add to it
let set = awaited_by.as_ref().unwrap();
vm.call_method(set, "add", (waiter,))?;
} else {
// Single object - convert to Set
let existing = awaited_by.take().unwrap();
let new_set = PySet::default().into_ref(&vm.ctx);
new_set.add(existing, vm)?;
new_set.add(waiter, vm)?;
*awaited_by = Some(new_set.into());
self.fut_awaited_by_is_set.store(true, Ordering::Relaxed);
return vm.call_method(&existing, "add", (waiter,)).map(drop);
}

// Single object - convert to Set
let new_set = PySet::default().into_ref(&vm.ctx);
new_set.add(existing, vm)?;
new_set.add(waiter, vm)?;
*self.fut_awaited_by.write() = Some(new_set.into());
self.fut_awaited_by_is_set.store(true, Ordering::Relaxed);
Ok(())
}

/// Discard waiter from fut_awaited_by with single-object optimization
fn awaited_by_discard(&self, waiter: &PyObject, vm: &VirtualMachine) -> PyResult<()> {
let mut awaited_by = self.fut_awaited_by.write();
if awaited_by.is_none() {
return Ok(());
}

let obj = awaited_by.as_ref().unwrap();
if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) {
// Single object - check if it matches
if obj.is(waiter) {
*awaited_by = None;
// As in awaited_by_add, discarding from the set runs Python.
let set = {
let mut awaited_by = self.fut_awaited_by.write();
let Some(obj) = awaited_by.as_ref() else {
return Ok(());
};
if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) {
// Single object - check if it matches
if obj.is(waiter) {
*awaited_by = None;
}
return Ok(());
}
} else {
// It's a Set - use discard
vm.call_method(obj, "discard", (waiter.to_owned(),))?;
}
Ok(())
obj.clone()
};

// It's a Set - use discard
vm.call_method(&set, "discard", (waiter.to_owned(),))
.map(drop)
}

#[pymethod]
Expand Down
48 changes: 37 additions & 11 deletions crates/stdlib/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ pub mod array {
$($n(Vec<$t>),)*
}

/// One item, already converted to the array's element type.
enum ArrayItem {
$($n($t),)*
}

impl ArrayContentType {
fn from_char(c: char) -> Result<Self, String> {
match c {
Expand Down Expand Up @@ -303,17 +308,31 @@ pub mod array {
}
}

fn setitem_by_index(
/// Convert an object to the element type of the array with
/// this typecode. This runs the object's conversion methods,
/// which can reach the array, so it takes the typecode by
/// value and holds no lock on it.
fn item_from_object(
typecode: char,
value: PyObjectRef,
vm: &VirtualMachine
) -> PyResult<ArrayItem> {
match typecode {
$($c => Ok(ArrayItem::$n(<$t>::try_into_from_object(vm, value)?)),)*
_ => unreachable!("array has a typecode"),
}
}

fn setitem_by_item(
&mut self,
i: isize,
value: PyObjectRef,
item: ArrayItem,
vm: &VirtualMachine
) -> PyResult<()> {
match self {
$(ArrayContentType::$n(v) => {
let value = <$t>::try_into_from_object(vm, value)?;
v.setitem_by_index(vm, i, value)
})*
match (self, item) {
$((ArrayContentType::$n(v), ArrayItem::$n(value)) =>
v.setitem_by_index(vm, i, value),)*
_ => unreachable!("item was converted for this array"),
}
}

Expand Down Expand Up @@ -1047,7 +1066,11 @@ pub mod array {
vm: &VirtualMachine,
) -> PyResult<()> {
match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? {
SequenceIndex::Int(i) => zelf.write().setitem_by_index(i, value, vm),
SequenceIndex::Int(i) => {
let typecode = zelf.read().typecode();
let item = ArrayContentType::item_from_object(typecode, value, vm)?;
zelf.write().setitem_by_item(i, item, vm)
}
SequenceIndex::Slice(slice) => {
let cloned;
let guard;
Expand Down Expand Up @@ -1386,7 +1409,9 @@ pub mod array {
ass_item: atomic_func!(|seq, i, value, vm| {
let zelf = PyArray::sequence_downcast(seq);
if let Some(value) = value {
zelf.write().setitem_by_index(i, value, vm)
let typecode = zelf.read().typecode();
let item = ArrayContentType::item_from_object(typecode, value, vm)?;
zelf.write().setitem_by_item(i, item, vm)
} else {
zelf.write().delitem_by_index(i, vm)
}
Expand Down Expand Up @@ -1421,8 +1446,9 @@ pub mod array {
type Resizable<'a> = PyRwLockWriteGuard<'a, ArrayContentType>;

fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> {
let w = self.write();
(self.exports.load(atomic::Ordering::SeqCst) == 0).then_some(w)
// An export is a borrow someone else still holds, so it is
// answered before the lock rather than by waiting on it.
(self.exports.load(atomic::Ordering::SeqCst) == 0).then(|| self.write())
}
}

Expand Down
4 changes: 2 additions & 2 deletions crates/stdlib/src/hashlib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -847,8 +847,8 @@ pub(crate) mod _hashlib {
if len < 1 {
return Err(vm.new_value_error("key length must be greater than 0."));
}
usize::try_from(len)
.map_err(|_| vm.new_overflow_error("key length is too great."))?
i32::try_from(len).map_err(|_| vm.new_overflow_error("key length is too great."))?
as usize
}
None => hash_digest_size(&name).ok_or_else(|| unsupported_hash(&name, vm))?,
};
Expand Down
Loading
Loading