Skip to content

gh-112433: Add optional _align_ attribute to ctypes.Structure #113790

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
serhiy-storchaka merged 9 commits into from
Feb 15, 2024
Merged

gh-112433: Add optional _align_ attribute to ctypes.Structure #113790

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a6ef3ac
Add optional _align_ attribute to ctypes.Structure
monkeyman192 Jan 6, 2024
e2cceec
📜🤖 Added by blurb_it.
blurb-it[bot] Jan 28, 2024
2329fa3
Fix issue with aligned subclasses
monkeyman192 Jan 28, 2024
ebaed7e
Improve tests to pass on big endian machines
monkeyman192 Jan 28, 2024
6b183b5
Merge branch 'main' into add_structure_alignment
serhiy-storchaka Jan 28, 2024
c8b37d4
Further improvements to tests
monkeyman192 Jan 29, 2024
68e2d61
Test cleanup and improvements
monkeyman192 Jan 29, 2024
f6a93dd
More test improvements
monkeyman192 Jan 31, 2024
a7bc0fe
Add extra test and fix documentation typo
monkeyman192 Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Improve tests to pass on big endian machines
  • Loading branch information
@monkeyman192
monkeyman192 committed Jan 28, 2024
commit ebaed7ebba6dc79879537f90db8dea44445e4821
238 changes: 127 additions & 111 deletions Lib/test/test_ctypes/test_aligned_structures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,116 +1,134 @@
from ctypes import Structure, c_char, c_uint32, c_ubyte, alignment, sizeof, Union
from ctypes import (
c_char, c_uint32, c_ubyte, alignment, sizeof,
Structure, BigEndianStructure, LittleEndianStructure,
Union, BigEndianUnion, LittleEndianUnion,
)
import inspect
import unittest



class TestAlignedStructures(unittest.TestCase):
def test_aligned_string(self):
data = bytearray(
b'\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64\x21\x00\x00\x00\x00'
)

class Aligned(Structure):
_align_ = 16
_fields_ = [
('value', c_char * 16),
]

class Main(Structure):
_fields_ = [
('first', c_uint32),
('string', Aligned),
]

main = Main.from_buffer(data)
self.assertEqual(main.first, 7)
self.assertEqual(main.string.value, b'hello world!')
self.assertEqual(
bytes(main.string.__buffer__(inspect.BufferFlags.SIMPLE)),
b'\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64\x21\x00\x00\x00\x00'
)

def test_aligned_structures(self):
data = bytearray(
b'\x01\x00\x01\x00\x07\x00\x00\x00'
)

class SomeBools(Structure):
_align_ = 4
_fields_ = [
("bool1", c_ubyte),
("bool2", c_ubyte),
("bool3", c_ubyte),
]
class Main(Structure):
_fields_ = [
("x", SomeBools),
("y", c_uint32),
]

class SomeBoolsTooBig(Structure):
_align_ = 8
_fields_ = [
("bool1", c_ubyte),
("bool2", c_ubyte),
("bool3", c_ubyte),
]
class MainTooBig(Structure):
_fields_ = [
("x", SomeBoolsTooBig),
("y", c_uint32),
]
main = Main.from_buffer(data)
self.assertEqual(main.y, 7)
self.assertEqual(alignment(SomeBools), 4)
self.assertEqual(main.x.bool1, True)
self.assertEqual(main.x.bool2, False)
self.assertEqual(main.x.bool3, True)

with self.assertRaises(ValueError) as ctx:
MainTooBig.from_buffer(data)
for base, data in (
(LittleEndianStructure, bytearray(
Copy link
Copy Markdown
Member

@serhiy-storchaka serhiy-storchaka Jan 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can also use struct.pack() to generate data. It may be more compact and more readable.

b'\x07\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x68\x65\x6c\x6c\x6f\x20\x77\x6f'
b'\x72\x6c\x64\x21\x00\x00\x00\x00'
)),
(BigEndianStructure, bytearray(
b'\x00\x00\x00\x07\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x68\x65\x6c\x6c\x6f\x20\x77\x6f'
b'\x72\x6c\x64\x21\x00\x00\x00\x00'
)),
):

class Aligned(base):
_align_ = 16
_fields_ = [
('value', c_char * 16),
]

class Main(base):
_fields_ = [
('first', c_uint32),
('string', Aligned),
]

main = Main.from_buffer(data)
self.assertEqual(main.first, 7)
self.assertEqual(main.string.value, b'hello world!')
self.assertEqual(
ctx.exception.args[0],
'Buffer size too small (4 instead of at least 8 bytes)'
bytes(main.string.__buffer__(inspect.BufferFlags.SIMPLE)),
Copy link
Copy Markdown
Member

@serhiy-storchaka serhiy-storchaka Jan 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe simply bytes(main.string)?

b'\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64\x21\x00\x00\x00\x00'
Copy link
Copy Markdown
Member

@serhiy-storchaka serhiy-storchaka Jan 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

b'hello world!\0\0\0\0' could be more readable.

)
self.assertEqual(Main.string.offset, 16)
Comment thread
serhiy-storchaka marked this conversation as resolved.

def test_aligned_subclasses(self):
data = bytearray(
b"\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00"
)

class UnalignedSub(Structure):
x: c_uint32
_fields_ = [
("x", c_uint32),
]

class AlignedStruct(UnalignedSub):
_align_ = 8
y: c_uint32
_fields_ = [
("y", c_uint32),
]

class Main(Structure):
_fields_ = [
("a", c_uint32),
("b", AlignedStruct)
]
def test_aligned_structures(self):
Comment thread
serhiy-storchaka marked this conversation as resolved.
for base, data in (
(LittleEndianStructure, bytearray(b"\1\0\1\0\7\0\0\0")),
(BigEndianStructure, bytearray(b"\1\0\1\0\0\0\0\7")),
):
class SomeBools(base):
_align_ = 4
_fields_ = [
("bool1", c_ubyte),
("bool2", c_ubyte),
("bool3", c_ubyte),
]
class Main(base):
_fields_ = [
("x", SomeBools),
("y", c_uint32),
]

class SomeBoolsTooBig(base):
_align_ = 8
_fields_ = [
("bool1", c_ubyte),
("bool2", c_ubyte),
("bool3", c_ubyte),
]
class MainTooBig(base):
_fields_ = [
("x", SomeBoolsTooBig),
("y", c_uint32),
]
main = Main.from_buffer(data)
self.assertEqual(main.y, 7)
self.assertEqual(alignment(SomeBools), 4)
self.assertEqual(main.x.bool1, True)
self.assertEqual(main.x.bool2, False)
self.assertEqual(main.x.bool3, True)

with self.assertRaises(ValueError) as ctx:
Copy link
Copy Markdown
Member

@serhiy-storchaka serhiy-storchaka Jan 30, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is still not very informative test. If you want to test that the _align_ setting affects the size of Main, you can do this more directly by checking sizeof(Main). If you want to check also content, use the data of size 12.

Copy link
Copy Markdown
Contributor Author

@monkeyman192 monkeyman192 Jan 31, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess with this test I was just wanting to ensure that the appropriate error is raised when you have a Structure/Union with an alignment which will cause the size it expects to be bigger than what it is given. Rather than checking anything about the content itself.

MainTooBig.from_buffer(data)
self.assertEqual(
ctx.exception.args[0],
'Buffer size too small (4 instead of at least 8 bytes)'
)

main = Main.from_buffer(data)
self.assertEqual(alignment(main.b), 8)
self.assertEqual(alignment(main), 8)
self.assertEqual(sizeof(main.b), 8)
self.assertEqual(sizeof(main), 16)
self.assertEqual(main.a, 1)
self.assertEqual(main.b.x, 3)
self.assertEqual(main.b.y, 4)
def test_aligned_subclasses(self):
for base, data in (
(LittleEndianStructure, bytearray(
b"\1\0\0\0\2\0\0\0\3\0\0\0\4\0\0\0"
)),
(BigEndianStructure, bytearray(
b"\0\0\0\1\0\0\0\2\0\0\0\3\0\0\0\4"
)),
):
class UnalignedSub(base):
x: c_uint32
_fields_ = [
("x", c_uint32),
]

class AlignedStruct(UnalignedSub):
_align_ = 8
y: c_uint32
_fields_ = [
("y", c_uint32),
]

class Main(base):
_fields_ = [
("a", c_uint32),
("b", AlignedStruct)
]

main = Main.from_buffer(data)
self.assertEqual(alignment(main.b), 8)
self.assertEqual(alignment(main), 8)
self.assertEqual(sizeof(main.b), 8)
self.assertEqual(sizeof(main), 16)
self.assertEqual(main.a, 1)
self.assertEqual(main.b.x, 3)
self.assertEqual(main.b.y, 4)

def test_aligned_union(self):
data = bytearray(
b"\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00"
)
data = bytearray(b"\1\0\0\0\2\0\0\0\3\0\0\0\4\0\0\0")

class AlignedUnion(Union):
_align_ = 8
Expand All @@ -126,14 +144,11 @@ class Main(Structure):
]

main = Main.from_buffer(data)
self.assertEqual(main.first, 1)
self.assertEqual(main.union.a, 3)
self.assertEqual(bytes(main.union.b), b"\x03\x00\x00\x00\x04\x00\x00\x00")
self.assertEqual(Main.union.offset, 8)
self.assertEqual(len(bytes(main.union.b)), 8)

def test_aligned_struct_in_union(self):
data = bytearray(
b"\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00"
)
data = bytearray(b"\1\0\0\0\2\0\0\0\3\0\0\3\4\0\0\4")

class Sub(Structure):
_align_ = 8
Expand All @@ -155,10 +170,11 @@ class Main(Structure):
]

main = Main.from_buffer(data)
self.assertEqual(main.first, 1)
self.assertEqual(main.union.a, 3)
self.assertEqual(main.union.b.x, 3)
self.assertEqual(main.union.b.y, 4)
self.assertEqual(Main.first.size, 4)
self.assertEqual(Main.union.offset, 8)
self.assertEqual(main.union.a, 0x03000003)
self.assertEqual(main.union.b.x, 0x03000003)
self.assertEqual(main.union.b.y, 0x04000004)


if __name__ == '__main__':
Expand Down
3 changes: 2 additions & 1 deletion Modules/_ctypes/stgdict.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,8 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
size = offset = basedict->size;
align = basedict->align;
union_size = 0;
total_align = max(align ? align : 1, forced_alignment);
total_align = align ? align : 1;
total_align = max(total_align, forced_alignment);
stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT;
stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, basedict->length + len + 1);
if (stgdict->ffi_type_pointer.elements == NULL) {
Expand Down