Skip to content

Implement PEP 688 and rework the buffer protocol around managed exports #8523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
youknowone wants to merge 2 commits into
base: main
Choose a base branch
from
+1,702 −453
Draft

Implement PEP 688 and rework the buffer protocol around managed exports #8523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a985ae4
Implement PEP 688 __buffer__ and __release_buffer__
youknowone Aug 13, 2026
a63e3ac
Rework the buffer protocol around managed exports and view offsets
youknowone Aug 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .cspell.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"alnum",
"csock",
"coro",
"contig",
"Crnl",
"dedentations",
"dedents",
Expand Down
11 changes: 0 additions & 11 deletions Lib/test/test_buffer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4471,7 +4471,6 @@ def test_flags_overflow(self):


class TestPythonBufferProtocol(unittest.TestCase):
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_basic(self):
class MyBuffer:
def __buffer__(self, flags):
Expand Down Expand Up @@ -4500,7 +4499,6 @@ def __buffer__(self):

self.assertRaises(TypeError, memoryview, WrongArity())

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_buffer(self):
class WhatToRelease:
def __init__(self):
Expand All @@ -4523,7 +4521,6 @@ def __release_buffer__(self, buffer):
self.assertEqual(mv.tobytes(), b"hello")
self.assertFalse(wr.held)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_same_buffer_returned(self):
class WhatToRelease:
def __init__(self):
Expand All @@ -4549,7 +4546,6 @@ def __release_buffer__(self, buffer):
self.assertEqual(mv.tobytes(), b"hello")
self.assertFalse(wr.held)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_buffer_flags(self):
class PossiblyMutable:
def __init__(self, data, mutable) -> None:
Expand Down Expand Up @@ -4589,7 +4585,6 @@ def __buffer__(self, flags):
mv[0] = ord(b'x')
self.assertEqual(mv.tobytes(), b"hello")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_call_builtins(self):
ba = bytearray(b"hello")
mv = ba.__buffer__(0)
Expand Down Expand Up @@ -4651,7 +4646,6 @@ def __buffer__(self, flags):
mv = memoryview(a)
self.assertEqual(mv.tobytes(), b"hello")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_inheritance_releasebuffer(self):
rb_call_count = 0
class B(bytearray):
Expand All @@ -4668,7 +4662,6 @@ def __release_buffer__(self, view):
self.assertEqual(rb_call_count, 0)
self.assertEqual(rb_call_count, 1)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_inherit_but_return_something_else(self):
class A(bytearray):
def __buffer__(self, flags):
Expand Down Expand Up @@ -4708,7 +4701,6 @@ def __release_buffer__(self, buffer):
with memoryview(c) as mv:
self.assertEqual(mv.tobytes(), b"hello")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_saves_reference(self):
smuggled_buffer = None

Expand Down Expand Up @@ -4736,7 +4728,6 @@ def __release_buffer__(s, buffer: memoryview):
with self.assertRaises(ValueError):
smuggled_buffer.tobytes()

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_saves_reference_no_subclassing(self):
ba = bytearray(b"hello")

Expand All @@ -4757,7 +4748,6 @@ def __release_buffer__(self, buffer):
c.buffer.release()
ba.clear()

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_multiple_inheritance_buffer_last(self):
class A:
def __buffer__(self, flags):
Expand Down Expand Up @@ -4817,7 +4807,6 @@ def __buffer__(self, flags):
c.clear()
self.assertIs(c.buffer, None)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_buffer_with_exception_set(self):
class A:
def __buffer__(self, flags):
Expand Down
1 change: 0 additions & 1 deletion Lib/test/test_collections.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1956,7 +1956,6 @@ class X(ByteString): pass
# No metaclass conflict
class Z(ByteString, Awaitable): pass

@unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types)
def test_Buffer(self):
for sample in [bytes, bytearray, memoryview]:
self.assertIsInstance(sample(b"x"), Buffer)
Expand Down
8 changes: 0 additions & 8 deletions Lib/test/test_memoryio.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,6 @@ def test_issue5449(self):
self.ioclass(initial_bytes=buf)
self.assertRaises(TypeError, self.ioclass, buf, foo=None)

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_write_concurrent_close(self):
class B:
def __buffer__(self, flags):
Expand All @@ -601,7 +600,6 @@ def __buffer__(self, flags):
# concurrently mutates (e.g., closes or exports) 'memio'.
# See: https://github.com/python/cpython/issues/143378.

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_writelines_concurrent_close(self):
class B:
def __buffer__(self, flags):
Expand All @@ -611,7 +609,6 @@ def __buffer__(self, flags):
memio = self.ioclass()
self.assertRaises(ValueError, memio.writelines, [B()])

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_write_concurrent_export(self):
class B:
buf = None
Expand All @@ -622,7 +619,6 @@ def __buffer__(self, flags):
memio = self.ioclass()
self.assertRaises(BufferError, memio.write, B())

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_writelines_concurrent_export(self):
class B:
buf = None
Expand All @@ -633,7 +629,6 @@ def __buffer__(self, flags):
memio = self.ioclass()
self.assertRaises(BufferError, memio.writelines, [B()])

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_write_mutating_buffer(self):
# Test that buffer is exported only once during write().
# See: https://github.com/python/cpython/issues/143602.
Expand Down Expand Up @@ -930,9 +925,6 @@ def test_cow_mutable(self):
def test_flags(self):
return super().test_flags()

@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by write
def test_write(self):
return super().test_write()

class CStringIOTest(PyStringIOTest):
ioclass = io.StringIO
Expand Down
1 change: 0 additions & 1 deletion Lib/test/test_memoryview.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -797,7 +797,6 @@ def __bool__(self):
m[0] = MyBool()
self.assertEqual(ba[:8], b'\0'*8)

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'memoryview' object has no attribute '__buffer__'
def test_buffer_reference_loop(self):
m = memoryview(b'abc').__buffer__(0)
o = MyObject()
Expand Down
2 changes: 0 additions & 2 deletions Lib/test/test_struct.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,12 +498,10 @@ def _test_pack_into(self, pack_into):
with self.assertRaises((IndexError, OverflowError)):
pack_into(writable_buf, -2**1000, test_string)

@unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object
def test_pack_into(self):
s = struct.Struct('21s')
self._test_pack_into(s.pack_into)

@unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object
def test_pack_into_fn(self):
pack_into = lambda *args: struct.pack_into('21s', *args)
self._test_pack_into(pack_into)
Expand Down
9 changes: 6 additions & 3 deletions crates/derive-impl/src/pyclass.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1162,13 +1162,16 @@ where
let slot_ident = Ident::new(&slot_ident.to_string().to_lowercase(), slot_ident.span());
let slot_name = slot_ident.to_string();
let tokens = {
const NON_ATOMIC_SLOTS: &[&str] = &["as_buffer"];
const POINTER_SLOTS: &[&str] = &["as_sequence", "as_mapping"];
const STATIC_GEN_SLOTS: &[&str] = &["as_number"];

if NON_ATOMIC_SLOTS.contains(&slot_name.as_str()) {
if slot_name == "as_buffer" {
// bf_releasebuffer is not a separate function in RustPython; the
// exporter's BufferMethods already release. Only its presence is
// observable, and AsBuffer declares that.
quote_spanned! { span =>
slots.#slot_ident = Some(Self::#ident as _);
slots.#slot_ident.store(Some(Self::#ident as _));
slots.has_release_buffer.store(Self::RELEASE_BUFFER);
}
} else if POINTER_SLOTS.contains(&slot_name.as_str()) {
quote_spanned! { span =>
Expand Down
52 changes: 37 additions & 15 deletions crates/stdlib/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ pub mod array {
ArgBytesLike, ArgIntoFloat, ArgIterable, KwArgs, OptionalArg, PyComparisonValue,
},
protocol::{
BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn,
PyMappingMethods, PySequenceMethods,
BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer,
PyIterReturn, PyMappingMethods, PySequenceMethods,
},
sequence::{OptionalRangeArgs, SequenceExt, SequenceMutExt},
sliceable::{
Expand Down Expand Up @@ -732,12 +732,12 @@ pub mod array {
}
} else if init.downcastable::<PyBytes>() || init.downcastable::<PyByteArray>() {
init.try_bytes_like(vm, |x| array.frombytes(x))?;
} else if let Ok(iter) = ArgIterable::try_from_object(vm, init.clone()) {
} else {
// Everything else is taken item by item, buffer or not.
let iter = ArgIterable::try_from_object(vm, init)?;
for obj in iter.iter(vm)? {
array.push(obj?, vm)?;
}
} else {
init.try_bytes_like(vm, |x| array.frombytes(x))?;
}
}

Expand Down Expand Up @@ -1291,20 +1291,42 @@ pub mod array {
}
}

impl PyArray {
fn buffer_desc(&self) -> BufferDescriptor {
let array = self.read();
BufferDescriptor::format(
array.len() * array.itemsize(),
false,
array.itemsize(),
array.typecode_str().into(),
)
}
}

impl AsBuffer for PyArray {
const RELEASE_BUFFER: bool = true;

// array_buffer_getbuf, which reports the type code only when the request
// asked for a format.
fn slot_as_buffer(
zelf: &PyObject,
flags: BufferFlags,
vm: &VirtualMachine,
) -> PyResult<PyBuffer> {
let zelf = zelf
.downcast_ref::<Self>()
.ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?;
let desc = zelf.buffer_desc().projected(flags);
flags.check_writable(desc.readonly, "Object is not writable.", vm)?;
Ok(PyBuffer::new(zelf.to_owned().into(), desc, &BUFFER_METHODS))
}

fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let array = zelf.read();
let buf = PyBuffer::new(
Ok(PyBuffer::new(
zelf.to_owned().into(),
BufferDescriptor::format(
array.len() * array.itemsize(),
false,
array.itemsize(),
array.typecode_str().into(),
),
zelf.buffer_desc(),
&BUFFER_METHODS,
);
Ok(buf)
))
}
}

Expand Down
2 changes: 2 additions & 0 deletions crates/stdlib/src/mmap.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,6 +611,8 @@ mod mmap {
};

impl AsBuffer for PyMmap {
const RELEASE_BUFFER: bool = true;

fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let readonly = matches!(zelf.access, AccessMode::Read);
let buf = PyBuffer::new(
Expand Down
26 changes: 19 additions & 7 deletions crates/stdlib/src/overlapped.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ mod _overlapped {
builtins::{PyBaseExceptionRef, PyBytesRef, PyModule, PyStrRef, PyTupleRef, PyType},
common::lock::PyMutex,
convert::{ToPyException, ToPyObject},
function::OptionalArg,
function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg},
object::{Traverse, TraverseFn},
protocol::PyBuffer,
types::{Constructor, Destructor},
Expand Down Expand Up @@ -428,12 +428,14 @@ mod _overlapped {
fn ReadFileInto(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// w*, as _overlapped.Overlapped.ReadFileInto takes
buf: ArgMemoryBuffer,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{
ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS,
};
let buf: PyBuffer = buf.into();

let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
Expand Down Expand Up @@ -530,13 +532,15 @@ mod _overlapped {
fn WSARecvInto(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// w*, as _overlapped.Overlapped.WSARecvInto takes
buf: ArgMemoryBuffer,
flags: u32,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{
ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS,
};
let buf: PyBuffer = buf.into();

let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
Expand Down Expand Up @@ -583,10 +587,12 @@ mod _overlapped {
fn WriteFile(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// y*, as _overlapped.Overlapped.WriteFile takes
buf: ArgBytesLike,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS};
let buf: PyBuffer = buf.into();

let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
Expand Down Expand Up @@ -629,11 +635,13 @@ mod _overlapped {
fn WSASend(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// y*, as _overlapped.Overlapped.WSASend takes
buf: ArgBytesLike,
flags: u32,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS};
let buf: PyBuffer = buf.into();

let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
Expand Down Expand Up @@ -870,12 +878,14 @@ mod _overlapped {
fn WSASendTo(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// y*, as _overlapped.Overlapped.WSASendTo takes
buf: ArgBytesLike,
flags: u32,
address: PyTupleRef,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS};
let buf: PyBuffer = buf.into();

let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
Expand Down Expand Up @@ -1001,14 +1011,16 @@ mod _overlapped {
fn WSARecvFromInto(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// w*, as _overlapped.Overlapped.WSARecvFromInto takes
buf: ArgMemoryBuffer,
size: u32,
flags: OptionalArg<u32>,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{
ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS,
};
let buf: PyBuffer = buf.into();

let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
Expand Down
Loading
Loading