Skip to content

Commit 1e9e19b

Browse files
youknowdotyouknowdot
committed
Align bytes and bytearray operations with Python 3.15
Correct quote escaping, replacement count keyword binding, signed-minimum hex grouping, and clear storage/export handling. Migrate canonical bytes coverage through fail-closed preservation of the existing RustPython customizations, and retire only two freshly verified formatting guards. Assisted-by: OpenAI Codex:model version unavailable
1 parent 8720306 commit 1e9e19b

6 files changed

Lines changed: 698 additions & 30 deletions

File tree

‎Lib/test/test_bytes.py‎

Lines changed: 274 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -553,10 +553,10 @@ def test_hex_separator_basics(self):
553553
self.assertEqual(three_bytes.hex('*', -2), 'b901*ef')
554554
self.assertEqual(three_bytes.hex(sep=':', bytes_per_sep=2), 'b9:01ef')
555555
self.assertEqual(three_bytes.hex(sep='*', bytes_per_sep=-2), 'b901*ef')
556-
for bytes_per_sep in 3, -3, 2**31-1, -(2**31-1):
556+
for bytes_per_sep in 3, -3, sys.maxsize, -sys.maxsize:
557557
with self.subTest(bytes_per_sep=bytes_per_sep):
558558
self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef')
559-
for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000:
559+
for bytes_per_sep in sys.maxsize+1, -sys.maxsize-1, 2**1000, -2**1000:
560560
with self.subTest(bytes_per_sep=bytes_per_sep):
561561
try:
562562
self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef')
@@ -586,6 +586,37 @@ def test_hex_separator_six_bytes(self):
586586
self.assertEqual(six_bytes.hex(':', -6), '0306090c0f12')
587587
self.assertEqual(six_bytes.hex(' ', -95), '0306090c0f12')
588588

589+
def test_hex_simd_boundaries(self):
590+
# Test lengths around the SIMD threshold (16 bytes).
591+
# SIMD processes 16 bytes at a time; smaller inputs use scalar code.
592+
for length in (14, 15, 16, 17, 31, 32, 33, 64, 65):
593+
data = self.type2test(bytes(range(length)))
594+
expected = ''.join(f'{b:02x}' for b in range(length))
595+
with self.subTest(length=length):
596+
self.assertEqual(data.hex(), expected)
597+
598+
def test_hex_nibble_boundaries(self):
599+
# Test the nibble value boundary at 9/10 (where '9' becomes 'a').
600+
# SIMD uses signed comparison for efficiency; verify correctness
601+
# at this boundary for various nibble combinations.
602+
boundary_bytes = self.type2test(bytes([
603+
0x09, # both nibbles: 0, 9
604+
0x0a, # both nibbles: 0, 10
605+
0x90, # both nibbles: 9, 0
606+
0x99, # both nibbles: 9, 9 (max all-digit)
607+
0x9a, # both nibbles: 9, 10
608+
0xa0, # both nibbles: 10, 0
609+
0xa9, # both nibbles: 10, 9
610+
0xaa, # both nibbles: 10, 10 (min all-letter)
611+
0x00, # min value
612+
0xff, # max value
613+
]))
614+
self.assertEqual(boundary_bytes.hex(), '090a90999aa0a9aa00ff')
615+
616+
# Repeat with 16+ bytes to exercise SIMD path
617+
simd_boundary = self.type2test(boundary_bytes * 2)
618+
self.assertEqual(simd_boundary.hex(), '090a90999aa0a9aa00ff' * 2)
619+
589620
def test_join(self):
590621
self.assertEqual(self.type2test(b"").join([]), b"")
591622
self.assertEqual(self.type2test(b"").join([b""]), b"")
@@ -616,6 +647,32 @@ def test_join(self):
616647
with self.assertRaises(TypeError):
617648
dot_join([memoryview(b"ab"), "cd", b"ef"])
618649

650+
def test_join_concurrent_buffer_mutation(self):
651+
# __buffer__() can release the GIL, letting another thread concurrently
652+
# mutate the joined sequence (simulated here by mutating in __buffer__).
653+
# See: https://github.com/python/cpython/issues/151295
654+
def make_seq(mutate):
655+
# Item is only referenced from the list slot, so mutate() frees it.
656+
class Item:
657+
def __buffer__(self, flags):
658+
mutate(seq)
659+
return memoryview(b'x')
660+
seq = [b'a', Item(), b'c']
661+
return seq
662+
663+
for sep in (self.type2test(b''), self.type2test(b'::')):
664+
with self.subTest(sep=sep):
665+
# Changing the list length is reported as a RuntimeError.
666+
seq = make_seq(lambda seq: seq.clear())
667+
self.assertRaises(RuntimeError, sep.join, seq)
668+
669+
# The list length is unchanged, so the size-change recheck
670+
# cannot fire: only keeping the item alive avoids the crash.
671+
def replace(seq):
672+
seq[1] = b'z'
673+
seq = make_seq(replace)
674+
self.assertEqual(sep.join(seq), sep.join([b'a', b'x', b'c']))
675+
619676
def test_count(self):
620677
b = self.type2test(b'mississippi')
621678
i = 105
@@ -794,16 +851,16 @@ def __int__(self):
794851
pi = PseudoFloat(3.1415)
795852

796853
exceptions_params = [
797-
('%x format: an integer is required, not float', b'%x', 3.14),
798-
('%X format: an integer is required, not float', b'%X', 2.11),
799-
('%o format: an integer is required, not float', b'%o', 1.79),
800-
('%x format: an integer is required, not PseudoFloat', b'%x', pi),
801-
('%x format: an integer is required, not complex', b'%x', 3j),
802-
('%X format: an integer is required, not complex', b'%X', 2j),
803-
('%o format: an integer is required, not complex', b'%o', 1j),
804-
('%u format: a real number is required, not complex', b'%u', 3j),
805-
('%i format: a real number is required, not complex', b'%i', 2j),
806-
('%d format: a real number is required, not complex', b'%d', 2j),
854+
('%x requires an integer, not float', b'%x', 3.14),
855+
('%X requires an integer, not float', b'%X', 2.11),
856+
('%o requires an integer, not float', b'%o', 1.79),
857+
(r'%x requires an integer, not .*\.PseudoFloat', b'%x', pi),
858+
('%x requires an integer, not complex', b'%x', 3j),
859+
('%X requires an integer, not complex', b'%X', 2j),
860+
('%o requires an integer, not complex', b'%o', 1j),
861+
('%u requires a real number, not complex', b'%u', 3j),
862+
('%i requires a real number, not complex', b'%i', 2j),
863+
('%d requires a real number, not complex', b'%d', 2j),
807864
(
808865
r'%c requires an integer in range\(256\)'
809866
r' or a single byte, not .*\.PseudoFloat',
@@ -812,7 +869,7 @@ def __int__(self):
812869
]
813870

814871
for msg, format_bytes, value in exceptions_params:
815-
with self.assertRaisesRegex(TypeError, msg):
872+
with self.assertRaisesRegex(TypeError, 'format argument: ' + msg):
816873
operator.mod(format_bytes, value)
817874

818875
def test_memory_leak_gh_140939(self):
@@ -849,6 +906,13 @@ def test_replace(self):
849906
self.assertEqual(b.replace(b'i', b'a'), b'massassappa')
850907
self.assertEqual(b.replace(b'ss', b'x'), b'mixixippi')
851908

909+
def test_replace_count_keyword(self):
910+
b = self.type2test(b'aa')
911+
self.assertEqual(b.replace(b'a', b'b', count=0), b'aa')
912+
self.assertEqual(b.replace(b'a', b'b', count=1), b'ba')
913+
self.assertEqual(b.replace(b'a', b'b', count=2), b'bb')
914+
self.assertEqual(b.replace(b'a', b'b', count=3), b'bb')
915+
852916
def test_replace_int_error(self):
853917
self.assertRaises(TypeError, self.type2test(b'a b').replace, 32, b'')
854918

@@ -1337,10 +1401,6 @@ class SubBytes(bytes):
13371401
self.assertNotEqual(id(s), id(1 * s))
13381402
self.assertNotEqual(id(s), id(s * 2))
13391403

1340-
@unittest.expectedFailure # TODO: RUSTPYTHON
1341-
def test_mod(self):
1342-
return super().test_mod()
1343-
13441404

13451405
class ByteArrayTest(BaseBytesTest, unittest.TestCase):
13461406
type2test = bytearray
@@ -1431,6 +1491,16 @@ def test_clear(self):
14311491
b.append(ord('p'))
14321492
self.assertEqual(b, b'p')
14331493

1494+
# Cleared object should be empty.
1495+
b = bytearray(b'abc')
1496+
b.clear()
1497+
self.assertEqual(b.__alloc__(), 0)
1498+
base_size = sys.getsizeof(bytearray())
1499+
self.assertEqual(sys.getsizeof(b), base_size)
1500+
c = b.copy()
1501+
self.assertEqual(c.__alloc__(), 0)
1502+
self.assertEqual(sys.getsizeof(c), base_size)
1503+
14341504
def test_copy(self):
14351505
b = bytearray(b'abc')
14361506
bb = b.copy()
@@ -1493,6 +1563,124 @@ def test_resize(self):
14931563
self.assertRaises(MemoryError, bytearray().resize, sys.maxsize)
14941564
self.assertRaises(MemoryError, bytearray(1000).resize, sys.maxsize)
14951565

1566+
def test_take_bytes(self):
1567+
ba = bytearray(b'ab')
1568+
self.assertEqual(ba.take_bytes(), b'ab')
1569+
self.assertEqual(len(ba), 0)
1570+
self.assertEqual(ba, bytearray(b''))
1571+
self.assertEqual(ba.__alloc__(), 0)
1572+
base_size = sys.getsizeof(bytearray())
1573+
self.assertEqual(sys.getsizeof(ba), base_size)
1574+
1575+
# Positive and negative slicing.
1576+
ba = bytearray(b'abcdef')
1577+
self.assertEqual(ba.take_bytes(1), b'a')
1578+
self.assertEqual(ba, bytearray(b'bcdef'))
1579+
self.assertEqual(len(ba), 5)
1580+
self.assertEqual(ba.take_bytes(-5), b'')
1581+
self.assertEqual(ba, bytearray(b'bcdef'))
1582+
self.assertEqual(len(ba), 5)
1583+
self.assertEqual(ba.take_bytes(-3), b'bc')
1584+
self.assertEqual(ba, bytearray(b'def'))
1585+
self.assertEqual(len(ba), 3)
1586+
self.assertEqual(ba.take_bytes(3), b'def')
1587+
self.assertEqual(ba, bytearray(b''))
1588+
self.assertEqual(len(ba), 0)
1589+
1590+
# Take nothing from emptiness.
1591+
self.assertEqual(ba.take_bytes(0), b'')
1592+
self.assertEqual(ba.take_bytes(), b'')
1593+
self.assertEqual(ba.take_bytes(None), b'')
1594+
1595+
# Out of bounds, bad take value.
1596+
self.assertRaises(IndexError, ba.take_bytes, -1)
1597+
self.assertRaises(TypeError, ba.take_bytes, 3.14)
1598+
ba = bytearray(b'abcdef')
1599+
self.assertRaises(IndexError, ba.take_bytes, 7)
1600+
1601+
# Offset between physical and logical start (ob_bytes != ob_start).
1602+
ba = bytearray(b'abcde')
1603+
del ba[:2]
1604+
self.assertEqual(ba, bytearray(b'cde'))
1605+
self.assertEqual(ba.take_bytes(), b'cde')
1606+
1607+
# Overallocation at end.
1608+
ba = bytearray(b'abcde')
1609+
del ba[-2:]
1610+
self.assertEqual(ba, bytearray(b'abc'))
1611+
self.assertEqual(ba.take_bytes(), b'abc')
1612+
ba = bytearray(b'abcde')
1613+
ba.resize(4)
1614+
self.assertEqual(ba.take_bytes(), b'abcd')
1615+
1616+
# Take of a bytearray with references should fail.
1617+
ba = bytearray(b'abc')
1618+
with memoryview(ba) as mv:
1619+
self.assertRaises(BufferError, ba.take_bytes)
1620+
self.assertEqual(ba.take_bytes(), b'abc')
1621+
1622+
# Leaving one byte must not adopt the shared single-byte bytes object
1623+
# as the buffer.
1624+
ba = bytearray(b'abc')
1625+
self.assertEqual(ba.take_bytes(2), b'ab')
1626+
ba[0] = ord('A')
1627+
self.assertEqual(ba, bytearray(b'A'))
1628+
self.assertEqual(ord(b'c'), ord('c'))
1629+
1630+
@support.cpython_only # tests an implementation detail
1631+
def test_take_bytes_optimization(self):
1632+
# Validate optimization around taking lots of little chunks out of a
1633+
# much bigger buffer. Save work by only copying a little rather than
1634+
# moving a lot.
1635+
ba = bytearray(b'abcdef' + b'0' * 1000)
1636+
start_alloc = ba.__alloc__()
1637+
1638+
# Take two bytes at a time, checking alloc doesn't change.
1639+
self.assertEqual(ba.take_bytes(2), b'ab')
1640+
self.assertEqual(ba.__alloc__(), start_alloc)
1641+
self.assertEqual(len(ba), 4 + 1000)
1642+
self.assertEqual(ba.take_bytes(2), b'cd')
1643+
self.assertEqual(ba.__alloc__(), start_alloc)
1644+
self.assertEqual(len(ba), 2 + 1000)
1645+
self.assertEqual(ba.take_bytes(2), b'ef')
1646+
self.assertEqual(ba.__alloc__(), start_alloc)
1647+
self.assertEqual(len(ba), 0 + 1000)
1648+
self.assertEqual(ba.__alloc__(), start_alloc)
1649+
1650+
# Take over half, alloc shrinks to exact size.
1651+
self.assertEqual(ba.take_bytes(501), b'0' * 501)
1652+
self.assertEqual(len(ba), 499)
1653+
bytes_header_size = sys.getsizeof(b'')
1654+
self.assertEqual(ba.__alloc__(), 499 + bytes_header_size)
1655+
1656+
def test_take_bytes_reentrant_resize(self):
1657+
# gh-153570: n.__index__() can resize the bytearray, so take_bytes()
1658+
# must re-read the size afterwards. It cached the size before the
1659+
# call and used it for the bounds check and the buffer reads, so a
1660+
# reentrant clear() returned freed memory (a use-after-free read).
1661+
def take(target, resize, n):
1662+
class Evil:
1663+
def __index__(self):
1664+
resize(target)
1665+
return n
1666+
return target.take_bytes(Evil())
1667+
1668+
# clear() during __index__: nothing is left to take.
1669+
ba = bytearray(b'abcdefgh')
1670+
with self.assertRaises(IndexError):
1671+
take(ba, lambda b: b.clear(), 8)
1672+
self.assertEqual(ba, b'')
1673+
1674+
# shrink during __index__: n past the new size is out of range.
1675+
ba = bytearray(b'abcdefgh')
1676+
with self.assertRaises(IndexError):
1677+
take(ba, lambda b: b.__delitem__(slice(4, None)), 8)
1678+
self.assertEqual(ba, b'abcd')
1679+
1680+
# grow during __index__: the take runs against the new, larger size.
1681+
ba = bytearray(b'abcd')
1682+
self.assertEqual(take(ba, lambda b: b.extend(b'efgh'), 8), b'abcdefgh')
1683+
self.assertEqual(ba, b'')
14961684

14971685
@unittest.expectedFailure # TODO: RUSTPYTHON
14981686
def test_setitem(self):
@@ -2073,9 +2261,45 @@ def __len__(self):
20732261

20742262
self.assertRaises(BufferError, ba.hex, S(b':'))
20752263

2076-
@unittest.expectedFailure # TODO: RUSTPYTHON
2077-
def test_mod(self):
2078-
return super().test_mod()
2264+
def test_no_init_called(self):
2265+
# A bytearray created without calling bytearray.__init__
2266+
# should not crash the interpreter (see gh-153419).
2267+
def bytearray_new():
2268+
return bytearray.__new__(bytearray)
2269+
2270+
bytearray_new().insert(0, 1)
2271+
bytearray_new().extend(b"x")
2272+
bytearray_new().extend([1, 2, 3])
2273+
bytearray_new().resize(4)
2274+
bytearray_new().__init__(5)
2275+
bytearray_new().__init__(b"xyz")
2276+
bytearray_new().take_bytes()
2277+
bytearray_new().take_bytes(0)
2278+
2279+
a = bytearray_new()
2280+
a.append(1)
2281+
2282+
a = bytearray_new()
2283+
a += b"x"
2284+
2285+
a = bytearray_new()
2286+
a[:] = b"xyz"
2287+
2288+
def test_reinit_length(self):
2289+
# There is a shortcut taken when resizing, where alloc/2 < newsize.
2290+
# In this case, the existing buffer is reused, rather than reset.
2291+
# If this happens when newsize == 0 and alloc == 1, then various
2292+
# code assumptions can be violated. This test should catch those
2293+
# in debug builds. (see gh-153419)
2294+
a = bytearray(1)
2295+
a.__init__()
2296+
self.assertEqual(a, b"")
2297+
2298+
def test_reinit_with_view(self):
2299+
a = bytearray()
2300+
with memoryview(a):
2301+
self.assertRaises(BufferError, a.__init__, "x", "ascii")
2302+
self.assertEqual(a, b"")
20792303

20802304

20812305
class AssortedBytesTest(unittest.TestCase):
@@ -2105,7 +2329,7 @@ def test_bytearray_repr(self, f=repr):
21052329
self.assertEqual(f(bytearray([7, 8, 9, 10, 11, 12, 13])),
21062330
r"bytearray(b'\x07\x08\t\n\x0b\x0c\r')")
21072331
self.assertEqual(f(bytearray(b'"')), """bytearray(b'"')""") # '"'
2108-
self.assertEqual(f(bytearray(b"'")), r'''bytearray(b"\'")''') # "\'"
2332+
self.assertEqual(f(bytearray(b"'")), '''bytearray(b"'")''') # "'"
21092333
self.assertEqual(f(bytearray(b"'\"")), r"""bytearray(b'\'"')""") # '\'"'
21102334
self.assertEqual(f(bytearray(b"\"'\"")), r"""bytearray(b'"\'"')""") # '"\'"'
21112335
self.assertEqual(f(bytearray(b'\'"\'')), r"""bytearray(b'\'"\'')""") # '\'"\''
@@ -2698,6 +2922,18 @@ def resize(b, a): # MODIFIES!
26982922
b.wait()
26992923
a.resize(10)
27002924

2925+
def take_bytes(b, a): # MODIFIES!
2926+
b.wait()
2927+
c = a.take_bytes()
2928+
assert not c or c[0] == 48 # '0'
2929+
2930+
def take_bytes_n(b, a): # MODIFIES!
2931+
b.wait()
2932+
try:
2933+
c = a.take_bytes(10)
2934+
assert c == b'0123456789'
2935+
except IndexError: pass
2936+
27012937
def check(funcs, a=None, *args):
27022938
if a is None:
27032939
a = bytearray(b'0' * 0x400000)
@@ -2762,6 +2998,10 @@ def check(funcs, a=None, *args):
27622998

27632999
check([clear] + [resize] * 10)
27643000

3001+
check([clear] + [take_bytes] * 10)
3002+
check([take_bytes_n] * 10, bytearray(b'0123456789' * 0x400))
3003+
check([take_bytes_n] * 10, bytearray(b'0123456789' * 5))
3004+
27653005
check([clear] + [contains] * 10)
27663006
check([clear] + [subscript] * 10)
27673007
check([clear2] + [ass_subscript2] * 10, None, bytearray(b'0' * 0x400000))
@@ -2834,5 +3074,18 @@ def resize_stress(ba):
28343074
with threading_helper.start_threads(threads):
28353075
pass
28363076

3077+
@threading_helper.reap_threads
3078+
@threading_helper.requires_working_threading()
3079+
def test_free_threading_bytearray_resize_other_thread(self):
3080+
# Shrinking a bytearray whose buffer another thread owns must not
3081+
# adopt the immortal single-byte bytes object a the buffer.
3082+
ba = bytearray(b'abc')
3083+
thread = threading.Thread(target=ba.resize, args=(1,))
3084+
with threading_helper.start_threads([thread]):
3085+
pass
3086+
ba[0] = ord('X')
3087+
self.assertEqual(ba, bytearray(b'X'))
3088+
self.assertEqual(ord(b'a'), ord('a'))
3089+
28373090
if __name__ == "__main__":
28383091
unittest.main()

0 commit comments

Comments
 (0)