Skip to content

gh-112014: correct buffer protocol support in ctypes (use native formats) #155472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
skirpichev wants to merge 1 commit into
base: main
Choose a base branch
from
+74 −78
Open

gh-112014: correct buffer protocol support in ctypes (use native formats) #155472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions Lib/test/test_ctypes/test_numbers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,10 @@ def test_complex(self):
self.assertEqual(t(FloatLike()).value, 2+0j)
self.assertEqual(t(ComplexLike()).value, 1+1j)

prefix = '>' if sys.byteorder == 'big' else '<'
num = t(1.0)
self.assertEqual(memoryview(num).format, prefix + format)
self.assertEqual(memoryview(num).format, format)
array = (t * 3)()
self.assertEqual(memoryview(array).format, prefix + format)
self.assertEqual(memoryview(array).format, format)

@unittest.skipUnless(hasattr(ctypes, "c_double_complex"),
"requires C11 complex type")
Expand Down
126 changes: 62 additions & 64 deletions Lib/test/test_ctypes/test_pep3118.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import re
import sys
import unittest
from ctypes import (CFUNCTYPE, POINTER, sizeof, Union,
Expand All @@ -9,29 +8,12 @@
c_bool, c_float, c_double, c_longdouble, py_object)


if sys.byteorder == "little":
THIS_ENDIAN = "<"
OTHER_ENDIAN = ">"
else:
THIS_ENDIAN = ">"
OTHER_ENDIAN = "<"


def normalize(format):
# Remove current endian specifier and white space from a format
# string
if format is None:
return ""
format = format.replace(OTHER_ENDIAN, THIS_ENDIAN)
return re.sub(r"\s", "", format)


class Test(unittest.TestCase):
def test_native_types(self):
for tp, fmt, shape, itemtp in native_types:
ob = tp()
v = memoryview(ob)
self.assertEqual(normalize(v.format), normalize(fmt))
self.assertEqual(v.format, fmt)
if shape:
self.assertEqual(len(v), shape[0])
else:
Expand Down Expand Up @@ -73,6 +55,15 @@ def test_endian_types(self):
n = n * dim
self.assertEqual(n * v.itemsize, len(v.tobytes()))

def test_memoryview_supports_ctypes_arrays(self):
ArrayType = c_int * 5
a = ArrayType(123, 42, 1, 2, 3)
m = memoryview(a)
self.assertEqual(m.shape, (5,))
self.assertEqual(m.format, c_int._type_)
self.assertEqual(list(m), [123, 42, 1, 2, 3])
self.assertEqual(m[1], 42)


# define some structure classes

Expand Down Expand Up @@ -124,8 +115,7 @@ class Complete(Structure):

################################################################
#
# This table contains format strings as they look on little endian
# machines. The test replaces '<' with '>' on big endian machines.
# This table contains format strings with native endianness.
#

# Platform-specific type codes
Expand Down Expand Up @@ -160,67 +150,67 @@ class Complete(Structure):

## simple types

(c_char, "<c", (), c_char),
(c_byte, "<b", (), c_byte),
(c_ubyte, "<B", (), c_ubyte),
(c_short, "<" + s_short, (), c_short),
(c_ushort, "<" + s_ushort, (), c_ushort),
(c_char, "c", (), c_char),
(c_byte, "b", (), c_byte),
(c_ubyte, "B", (), c_ubyte),
(c_short, s_short, (), c_short),
(c_ushort, s_ushort, (), c_ushort),

(c_int, "<" + s_int, (), c_int),
(c_uint, "<" + s_uint, (), c_uint),
(c_int, s_int, (), c_int),
(c_uint, s_uint, (), c_uint),

(c_long, "<" + s_long, (), c_long),
(c_ulong, "<" + s_ulong, (), c_ulong),
(c_long, s_long, (), c_long),
(c_ulong, s_ulong, (), c_ulong),

(c_longlong, "<" + s_longlong, (), c_longlong),
(c_ulonglong, "<" + s_ulonglong, (), c_ulonglong),
(c_longlong, s_longlong, (), c_longlong),
(c_ulonglong, s_ulonglong, (), c_ulonglong),

(c_float, "<f", (), c_float),
(c_double, "<d", (), c_double),
(c_float, "f", (), c_float),
(c_double, "d", (), c_double),

(c_longdouble, "<" + s_longdouble, (), c_longdouble),
(c_longdouble, s_longdouble, (), c_longdouble),

(c_bool, "<" + s_bool, (), c_bool),
(py_object, "<O", (), py_object),
(c_bool, s_bool, (), c_bool),
(py_object, "O", (), py_object),

## pointers

(POINTER(c_byte), "&<b", (), POINTER(c_byte)),
(POINTER(POINTER(c_long)), "&&<" + s_long, (), POINTER(POINTER(c_long))),
(POINTER(c_byte), "&b", (), POINTER(c_byte)),
(POINTER(POINTER(c_long)), "&&" + s_long, (), POINTER(POINTER(c_long))),

## arrays and pointers

(c_double * 4, "<d", (4,), c_double),
(c_double * 0, "<d", (0,), c_double),
(c_float * 4 * 3 * 2, "<f", (2,3,4), c_float),
(c_float * 4 * 0 * 2, "<f", (2,0,4), c_float),
(POINTER(c_short) * 2, "&<" + s_short, (2,), POINTER(c_short)),
(POINTER(c_short) * 2 * 3, "&<" + s_short, (3,2,), POINTER(c_short)),
(POINTER(c_short * 2), "&(2)<" + s_short, (), POINTER(c_short)),
(c_double * 4, "d", (4,), c_double),
(c_double * 0, "d", (0,), c_double),
(c_float * 4 * 3 * 2, "f", (2,3,4), c_float),
(c_float * 4 * 0 * 2, "f", (2,0,4), c_float),
(POINTER(c_short) * 2, "&" + s_short, (2,), POINTER(c_short)),
(POINTER(c_short) * 2 * 3, "&" + s_short, (3,2,), POINTER(c_short)),
(POINTER(c_short * 2), "&(2)" + s_short, (), POINTER(c_short)),

## structures and unions

(Point2, "T{<l:x:<l:y:}".replace('l', s_long), (), Point2),
(Point, "T{<l:x:<l:y:}".replace('l', s_long), (), Point),
(PackedPoint, "T{<l:x:<l:y:}".replace('l', s_long), (), PackedPoint),
(PointMidPad, "T{<b:x:3x<I:y:}".replace('I', s_uint), (), PointMidPad),
(PackedPointMidPad, "T{<b:x:x<Q:y:}", (), PackedPointMidPad),
(PointEndPad, "T{<I:x:<b:y:3x}".replace('I', s_uint), (), PointEndPad),
(PackedPointEndPad, "T{<Q:x:<b:y:x}", (), PackedPointEndPad),
(EmptyStruct, "T{}", (), EmptyStruct),
(Point2, "T{l:x:l:y:}".replace('l', s_long), (), Point2),
(Point, "T{l:x:l:y:}".replace('l', s_long), (), Point),
(PackedPoint, "T{l:x:l:y:}".replace('l', s_long), (), PackedPoint),
(PointMidPad, "T{b:x:3xI:y:}".replace('I', s_uint), (), PointMidPad),
(PackedPointMidPad, "T{b:x:xQ:y:}", (), PackedPointMidPad),
(PointEndPad, "T{I:x:b:y:3x}".replace('I', s_uint), (), PointEndPad),
(PackedPointEndPad, "T{Q:x:b:y:x}", (), PackedPointEndPad),
(EmptyStruct, "T{}", (), EmptyStruct),
# the pep doesn't support unions
(aUnion, "B", (), aUnion),
(aUnion, "B", (), aUnion),
# structure with sub-arrays
(StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (), StructWithArrays),
(StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
(StructWithArrays, "T{(2,3)l:x:(4)T{l:x:l:y:}:y:}".replace('l', s_long), (), StructWithArrays),
(StructWithArrays * 3, "T{(2,3)l:x:(4)T{l:x:l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),

## pointer to incomplete structure
(Incomplete, "B", (), Incomplete),
(POINTER(Incomplete), "&B", (), POINTER(Incomplete)),

# 'Complete' is a structure that starts incomplete, but is completed after the
# pointer type to it has been created.
(Complete, "T{<l:a:}".replace('l', s_long), (), Complete),
(Complete, "T{l:a:}".replace('l', s_long), (), Complete),
# Unfortunately the pointer format string is not fixed...
(POINTER(Complete), "&B", (), POINTER(Complete)),

Expand All @@ -241,12 +231,20 @@ class LEPoint(LittleEndianStructure):

# This table contains format strings as they really look, on both big
# and little endian machines.
endian_types = [
(BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), (), BEPoint),
(LEPoint * 1, "T{<l:x:<l:y:}".replace('l', s_long), (1,), LEPoint),
(POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), (), POINTER(LEPoint)),
]
if sys.byteorder == "little":
endian_types = [
(BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), (), BEPoint),
(LEPoint * 1, "T{l:x:l:y:}".replace('l', s_long), (1,), LEPoint),
(POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{l:x:l:y:}".replace('l', s_long), (), POINTER(LEPoint)),
]
else:
endian_types = [
(BEPoint * 1, "T{l:x:l:y:}".replace('l', s_long), (1,), BEPoint),
(LEPoint, "T{<l:x:<l:y:}".replace('l', s_long), (), LEPoint),
(POINTER(BEPoint), "&T{l:x:l:y:}".replace('l', s_long), (), POINTER(BEPoint)),
(POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), (), POINTER(LEPoint)),
]


if __name__ == "__main__":
Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/Library/2026-08-10-10-35-19.gh-issue-112014.nLEBCv.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Correct the :ref:`buffer protocol <bufferobjects>` support in the
:mod:`ctypes` module to use the machine’s native format and byte order,
rather than explicitly specify endianness (by ``'<'`` or ``'>'``). The
later kept for byte-swapped types. Patch by Sergey B Kirpichev.
17 changes: 6 additions & 11 deletions Modules/_ctypes/_ctypes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ _PyDict_GetItemProxy(PyObject *dict, PyObject *key, PyObject **presult)
later on.
*/
static char *
_ctypes_alloc_format_string_for_type(const char *code, int big_endian)
_ctypes_alloc_format_string_for_type(const char *code)
{
const char *pep_code = NULL;

Expand Down Expand Up @@ -310,14 +310,13 @@ _ctypes_alloc_format_string_for_type(const char *code, int big_endian)
break;
}

char *result = PyMem_Malloc(1 + strlen(pep_code) + 1);
char *result = PyMem_Malloc(1 + strlen(pep_code));
if (result == NULL) {
PyErr_NoMemory();
return NULL;
}

result[0] = big_endian ? '>' : '<';
strcpy(result + 1, pep_code);
strcpy(result, pep_code);
return result;
}

Expand Down Expand Up @@ -2405,11 +2404,7 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds)
stginfo->size = fmt->pffi_type->size;
stginfo->setfunc = fmt->setfunc;
stginfo->getfunc = fmt->getfunc;
#ifdef WORDS_BIGENDIAN
stginfo->format = _ctypes_alloc_format_string_for_type(proto_str, 1);
#else
stginfo->format = _ctypes_alloc_format_string_for_type(proto_str, 0);
#endif
stginfo->format = _ctypes_alloc_format_string_for_type(proto_str);
if (stginfo->format == NULL) {
Py_DECREF(proto);
return -1;
Expand Down Expand Up @@ -2504,14 +2499,14 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds)
PyObject_SetAttrString(swapped, "__ctype_be__", self);
PyObject_SetAttrString(swapped, "__ctype_le__", swapped);
/* We are creating the type for the OTHER endian */
sw_info->format = _ctypes_alloc_format_string("<", stginfo->format+1);
sw_info->format = _ctypes_alloc_format_string("<", stginfo->format);
#else
PyObject_SetAttrString(self, "__ctype_be__", swapped);
PyObject_SetAttrString(self, "__ctype_le__", self);
PyObject_SetAttrString(swapped, "__ctype_le__", self);
PyObject_SetAttrString(swapped, "__ctype_be__", swapped);
/* We are creating the type for the OTHER endian */
sw_info->format = _ctypes_alloc_format_string(">", stginfo->format+1);
sw_info->format = _ctypes_alloc_format_string(">", stginfo->format);
#endif
Py_DECREF(swapped);
if (PyErr_Occurred()) {
Expand Down
Loading