Skip to content

Commit 7623743

Browse files
youknowoneyouknowone
committed
Implement PEP 688 __buffer__ and __release_buffer__
A Python class could not export a buffer: the slot machinery had no bf_getbuffer or bf_releasebuffer, and every consumer acquired buffers as PyBUF_FULL_RO through a module of PyBUF_* constants. Add both slots. PyBuffer::release now runs a Python __release_buffer__ before the exporter's own release, once per acquisition, which PyBuffer tracks with an `acquired` flag that clones do not inherit. An export made by a Python __buffer__ is held by a _buffer_wrapper payload that counts its exports and drops the returned memoryview with the last one, and the view handed to __release_buffer__ is a _buffer_window that owns no export, so releasing it inside the hook is inert instead of re-entering it. Replace the PyBUF_* constants with a BufferFlags bitflags type whose composite requests are supersets of the simpler ones, so `contains` answers the REQ_* questions, and pass the request to PyBuffer::from_object. Each consumer now asks for what its counterpart asks for: y* arguments for SIMPLE, w* for WRITABLE, BytesIO.write for CONTIG_RO, bytes(), bytearray() and memoryview() for FULL_RO. memoryview checks the request in memory_getbuf, and array.array and mmap.mmap expose __release_buffer__. Test buffer support with PyObject::check_buffer (PyObject_CheckBuffer) instead of attempting an acquisition, so an exception raised by __buffer__ is no longer reported as the object not being bytes-like, and a __buffer__ with side effects runs once. PyBytesInner becomes a y* conversion as a result: bytes and bytearray methods no longer accept iterables of ints, and find, index, count and __contains__ take the arguments parse_args_finds_byte and bytes_contains describe. A view exports its start offset in the descriptor rather than in its window, which fixes a panic when collecting from a negative-stride view. BytesIO.write rechecks closed after acquiring its buffer, which __buffer__ can close in between. Assisted-by: Claude Code:claude-opus-5
1 parent c9a6244 commit 7623743

32 files changed

Lines changed: 976 additions & 214 deletions

‎.cspell.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
"alnum",
6060
"csock",
6161
"coro",
62+
"contig",
6263
"Crnl",
6364
"dedentations",
6465
"dedents",

‎Lib/test/test_buffer.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4471,7 +4471,6 @@ def test_flags_overflow(self):
44714471

44724472

44734473
class TestPythonBufferProtocol(unittest.TestCase):
4474-
@unittest.expectedFailure # TODO: RUSTPYTHON
44754474
def test_basic(self):
44764475
class MyBuffer:
44774476
def __buffer__(self, flags):
@@ -4500,7 +4499,6 @@ def __buffer__(self):
45004499

45014500
self.assertRaises(TypeError, memoryview, WrongArity())
45024501

4503-
@unittest.expectedFailure # TODO: RUSTPYTHON
45044502
def test_release_buffer(self):
45054503
class WhatToRelease:
45064504
def __init__(self):
@@ -4523,7 +4521,6 @@ def __release_buffer__(self, buffer):
45234521
self.assertEqual(mv.tobytes(), b"hello")
45244522
self.assertFalse(wr.held)
45254523

4526-
@unittest.expectedFailure # TODO: RUSTPYTHON
45274524
def test_same_buffer_returned(self):
45284525
class WhatToRelease:
45294526
def __init__(self):
@@ -4549,7 +4546,6 @@ def __release_buffer__(self, buffer):
45494546
self.assertEqual(mv.tobytes(), b"hello")
45504547
self.assertFalse(wr.held)
45514548

4552-
@unittest.expectedFailure # TODO: RUSTPYTHON
45534549
def test_buffer_flags(self):
45544550
class PossiblyMutable:
45554551
def __init__(self, data, mutable) -> None:
@@ -4589,7 +4585,6 @@ def __buffer__(self, flags):
45894585
mv[0] = ord(b'x')
45904586
self.assertEqual(mv.tobytes(), b"hello")
45914587

4592-
@unittest.expectedFailure # TODO: RUSTPYTHON
45934588
def test_call_builtins(self):
45944589
ba = bytearray(b"hello")
45954590
mv = ba.__buffer__(0)
@@ -4651,7 +4646,6 @@ def __buffer__(self, flags):
46514646
mv = memoryview(a)
46524647
self.assertEqual(mv.tobytes(), b"hello")
46534648

4654-
@unittest.expectedFailure # TODO: RUSTPYTHON
46554649
def test_inheritance_releasebuffer(self):
46564650
rb_call_count = 0
46574651
class B(bytearray):
@@ -4668,7 +4662,6 @@ def __release_buffer__(self, view):
46684662
self.assertEqual(rb_call_count, 0)
46694663
self.assertEqual(rb_call_count, 1)
46704664

4671-
@unittest.expectedFailure # TODO: RUSTPYTHON
46724665
def test_inherit_but_return_something_else(self):
46734666
class A(bytearray):
46744667
def __buffer__(self, flags):
@@ -4708,7 +4701,6 @@ def __release_buffer__(self, buffer):
47084701
with memoryview(c) as mv:
47094702
self.assertEqual(mv.tobytes(), b"hello")
47104703

4711-
@unittest.expectedFailure # TODO: RUSTPYTHON
47124704
def test_release_saves_reference(self):
47134705
smuggled_buffer = None
47144706

@@ -4736,7 +4728,6 @@ def __release_buffer__(s, buffer: memoryview):
47364728
with self.assertRaises(ValueError):
47374729
smuggled_buffer.tobytes()
47384730

4739-
@unittest.expectedFailure # TODO: RUSTPYTHON
47404731
def test_release_saves_reference_no_subclassing(self):
47414732
ba = bytearray(b"hello")
47424733

@@ -4757,7 +4748,6 @@ def __release_buffer__(self, buffer):
47574748
c.buffer.release()
47584749
ba.clear()
47594750

4760-
@unittest.expectedFailure # TODO: RUSTPYTHON
47614751
def test_multiple_inheritance_buffer_last(self):
47624752
class A:
47634753
def __buffer__(self, flags):
@@ -4817,7 +4807,6 @@ def __buffer__(self, flags):
48174807
c.clear()
48184808
self.assertIs(c.buffer, None)
48194809

4820-
@unittest.expectedFailure # TODO: RUSTPYTHON
48214810
def test_release_buffer_with_exception_set(self):
48224811
class A:
48234812
def __buffer__(self, flags):

‎Lib/test/test_collections.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1956,7 +1956,6 @@ class X(ByteString): pass
19561956
# No metaclass conflict
19571957
class Z(ByteString, Awaitable): pass
19581958

1959-
@unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types)
19601959
def test_Buffer(self):
19611960
for sample in [bytes, bytearray, memoryview]:
19621961
self.assertIsInstance(sample(b"x"), Buffer)

‎Lib/test/test_memoryio.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -587,7 +587,6 @@ def test_issue5449(self):
587587
self.ioclass(initial_bytes=buf)
588588
self.assertRaises(TypeError, self.ioclass, buf, foo=None)
589589

590-
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
591590
def test_write_concurrent_close(self):
592591
class B:
593592
def __buffer__(self, flags):
@@ -601,7 +600,6 @@ def __buffer__(self, flags):
601600
# concurrently mutates (e.g., closes or exports) 'memio'.
602601
# See: https://github.com/python/cpython/issues/143378.
603602

604-
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
605603
def test_writelines_concurrent_close(self):
606604
class B:
607605
def __buffer__(self, flags):
@@ -611,7 +609,6 @@ def __buffer__(self, flags):
611609
memio = self.ioclass()
612610
self.assertRaises(ValueError, memio.writelines, [B()])
613611

614-
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
615612
def test_write_concurrent_export(self):
616613
class B:
617614
buf = None
@@ -622,7 +619,6 @@ def __buffer__(self, flags):
622619
memio = self.ioclass()
623620
self.assertRaises(BufferError, memio.write, B())
624621

625-
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
626622
def test_writelines_concurrent_export(self):
627623
class B:
628624
buf = None
@@ -633,7 +629,6 @@ def __buffer__(self, flags):
633629
memio = self.ioclass()
634630
self.assertRaises(BufferError, memio.writelines, [B()])
635631

636-
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
637632
def test_write_mutating_buffer(self):
638633
# Test that buffer is exported only once during write().
639634
# See: https://github.com/python/cpython/issues/143602.
@@ -930,9 +925,6 @@ def test_cow_mutable(self):
930925
def test_flags(self):
931926
return super().test_flags()
932927

933-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by write
934-
def test_write(self):
935-
return super().test_write()
936928

937929
class CStringIOTest(PyStringIOTest):
938930
ioclass = io.StringIO

‎Lib/test/test_memoryview.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -797,7 +797,6 @@ def __bool__(self):
797797
m[0] = MyBool()
798798
self.assertEqual(ba[:8], b'\0'*8)
799799

800-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'memoryview' object has no attribute '__buffer__'
801800
def test_buffer_reference_loop(self):
802801
m = memoryview(b'abc').__buffer__(0)
803802
o = MyObject()

‎Lib/test/test_struct.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -498,12 +498,10 @@ def _test_pack_into(self, pack_into):
498498
with self.assertRaises((IndexError, OverflowError)):
499499
pack_into(writable_buf, -2**1000, test_string)
500500

501-
@unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object
502501
def test_pack_into(self):
503502
s = struct.Struct('21s')
504503
self._test_pack_into(s.pack_into)
505504

506-
@unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object
507505
def test_pack_into_fn(self):
508506
pack_into = lambda *args: struct.pack_into('21s', *args)
509507
self._test_pack_into(pack_into)

‎crates/derive-impl/src/pyclass.rs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1162,13 +1162,16 @@ where
11621162
let slot_ident = Ident::new(&slot_ident.to_string().to_lowercase(), slot_ident.span());
11631163
let slot_name = slot_ident.to_string();
11641164
let tokens = {
1165-
const NON_ATOMIC_SLOTS: &[&str] = &["as_buffer"];
11661165
const POINTER_SLOTS: &[&str] = &["as_sequence", "as_mapping"];
11671166
const STATIC_GEN_SLOTS: &[&str] = &["as_number"];
11681167

1169-
if NON_ATOMIC_SLOTS.contains(&slot_name.as_str()) {
1168+
if slot_name == "as_buffer" {
1169+
// bf_releasebuffer is not a separate function in RustPython; the
1170+
// exporter's BufferMethods already release. Only its presence is
1171+
// observable, and AsBuffer declares that.
11701172
quote_spanned! { span =>
1171-
slots.#slot_ident = Some(Self::#ident as _);
1173+
slots.#slot_ident.store(Some(Self::#ident as _));
1174+
slots.has_release_buffer.store(Self::RELEASE_BUFFER);
11721175
}
11731176
} else if POINTER_SLOTS.contains(&slot_name.as_str()) {
11741177
quote_spanned! { span =>

‎crates/stdlib/src/array.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -732,12 +732,12 @@ pub mod array {
732732
}
733733
} else if init.downcastable::<PyBytes>() || init.downcastable::<PyByteArray>() {
734734
init.try_bytes_like(vm, |x| array.frombytes(x))?;
735-
} else if let Ok(iter) = ArgIterable::try_from_object(vm, init.clone()) {
735+
} else {
736+
// Everything else is taken item by item, buffer or not.
737+
let iter = ArgIterable::try_from_object(vm, init)?;
736738
for obj in iter.iter(vm)? {
737739
array.push(obj?, vm)?;
738740
}
739-
} else {
740-
init.try_bytes_like(vm, |x| array.frombytes(x))?;
741741
}
742742
}
743743

@@ -1292,6 +1292,8 @@ pub mod array {
12921292
}
12931293

12941294
impl AsBuffer for PyArray {
1295+
const RELEASE_BUFFER: bool = true;
1296+
12951297
fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
12961298
let array = zelf.read();
12971299
let buf = PyBuffer::new(

‎crates/stdlib/src/mmap.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -611,6 +611,8 @@ mod mmap {
611611
};
612612

613613
impl AsBuffer for PyMmap {
614+
const RELEASE_BUFFER: bool = true;
615+
614616
fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
615617
let readonly = matches!(zelf.access, AccessMode::Read);
616618
let buf = PyBuffer::new(

‎crates/stdlib/src/ssl.rs‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1158,19 +1158,19 @@ mod _ssl {
11581158
let pwd_result = callable.call((), vm)?;
11591159

11601160
// Convert callable result to string
1161-
let password_from_callable = if let Ok(pwd_str) =
1162-
PyUtf8StrRef::try_from_object(vm, pwd_result.clone())
1163-
{
1164-
pwd_str.as_str().to_owned()
1165-
} else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, pwd_result) {
1166-
String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()).map_err(|_| {
1167-
vm.new_type_error("password callback returned invalid UTF-8 bytes")
1168-
})?
1169-
} else {
1170-
return Err(
1171-
vm.new_type_error("password callback must return a string or bytes")
1172-
);
1173-
};
1161+
let password_from_callable =
1162+
if let Ok(pwd_str) = PyUtf8StrRef::try_from_object(vm, pwd_result.clone()) {
1163+
pwd_str.as_str().to_owned()
1164+
} else if pwd_result.check_buffer() {
1165+
let pwd_bytes_like = ArgBytesLike::try_from_object(vm, pwd_result)?;
1166+
String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()).map_err(|_| {
1167+
vm.new_type_error("password callback returned invalid UTF-8 bytes")
1168+
})?
1169+
} else {
1170+
return Err(
1171+
vm.new_type_error("password callback must return a string or bytes")
1172+
);
1173+
};
11741174

11751175
// Validate callable password length
11761176
if password_from_callable.len() > PEM_BUFSIZE {
@@ -1808,7 +1808,8 @@ mod _ssl {
18081808
// Validate filepath is str or bytes
18091809
let path_str = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, filepath.clone()) {
18101810
s.as_str().to_owned()
1811-
} else if let Ok(b) = ArgBytesLike::try_from_object(vm, filepath) {
1811+
} else if filepath.check_buffer() {
1812+
let b = ArgBytesLike::try_from_object(vm, filepath)?;
18121813
String::from_utf8(b.borrow_buf().to_vec())
18131814
.map_err(|_| vm.new_value_error("Invalid path encoding"))?
18141815
} else {
@@ -1863,7 +1864,8 @@ mod _ssl {
18631864
// Validate name is str or bytes
18641865
let curve_name = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, name.clone()) {
18651866
s.as_str().to_owned()
1866-
} else if let Ok(b) = ArgBytesLike::try_from_object(vm, name) {
1867+
} else if name.check_buffer() {
1868+
let b = ArgBytesLike::try_from_object(vm, name)?;
18671869
String::from_utf8(b.borrow_buf().to_vec())
18681870
.map_err(|_| vm.new_value_error("Invalid curve name encoding"))?
18691871
} else {
@@ -2106,8 +2108,8 @@ mod _ssl {
21062108
Ok((Some(pwd_str.as_str().to_owned()), None))
21072109
}
21082110
// Try bytes-like
2109-
else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, p.clone())
2110-
{
2111+
else if p.check_buffer() {
2112+
let pwd_bytes_like = ArgBytesLike::try_from_object(vm, p.clone())?;
21112113
let pwd = String::from_utf8(pwd_bytes_like.borrow_buf().to_vec())
21122114
.map_err(|_| vm.new_type_error("password bytes must be valid UTF-8"))?;
21132115
Ok((Some(pwd), None))

0 commit comments

Comments
 (0)