Skip to content

Commit 60d241f

Browse files
gpsheadgpshead
committed
For PEP3137: Adds missing methods to the mutable PyBytes object (soon
to be called a buffer). Shares code with stringobject when possible. Adds unit tests with common code that should be usable to test the PEPs mutable buffer() and immutable bytes() types. http://bugs.python.org/issue1261
1 parent 3d2fd7f commit 60d241f

File tree

12 files changed

+1595
-1079
lines changed

12 files changed

+1595
-1079
lines changed

‎Include/bytes_methods.h‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#ifndef Py_BYTES_CTYPE_H
2+
#define Py_BYTES_CTYPE_H
3+
4+
/*
5+
* The internal implementation behind PyString (bytes) and PyBytes (buffer)
6+
* methods of the given names, they operate on ASCII byte strings.
7+
*/
8+
extern PyObject* _Py_bytes_isspace(const char *cptr, Py_ssize_t len);
9+
extern PyObject* _Py_bytes_isalpha(const char *cptr, Py_ssize_t len);
10+
extern PyObject* _Py_bytes_isalnum(const char *cptr, Py_ssize_t len);
11+
extern PyObject* _Py_bytes_isdigit(const char *cptr, Py_ssize_t len);
12+
extern PyObject* _Py_bytes_islower(const char *cptr, Py_ssize_t len);
13+
extern PyObject* _Py_bytes_isupper(const char *cptr, Py_ssize_t len);
14+
extern PyObject* _Py_bytes_istitle(const char *cptr, Py_ssize_t len);
15+
16+
/* These store their len sized answer in the given preallocated *result arg. */
17+
extern void _Py_bytes_lower(char *result, const char *cptr, Py_ssize_t len);
18+
extern void _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len);
19+
extern void _Py_bytes_title(char *result, char *s, Py_ssize_t len);
20+
extern void _Py_bytes_capitalize(char *result, char *s, Py_ssize_t len);
21+
extern void _Py_bytes_swapcase(char *result, char *s, Py_ssize_t len);
22+
23+
/* Shared __doc__ strings. */
24+
extern const char _Py_isspace__doc__[];
25+
extern const char _Py_isalpha__doc__[];
26+
extern const char _Py_isalnum__doc__[];
27+
extern const char _Py_isdigit__doc__[];
28+
extern const char _Py_islower__doc__[];
29+
extern const char _Py_isupper__doc__[];
30+
extern const char _Py_istitle__doc__[];
31+
extern const char _Py_lower__doc__[];
32+
extern const char _Py_upper__doc__[];
33+
extern const char _Py_title__doc__[];
34+
extern const char _Py_capitalize__doc__[];
35+
extern const char _Py_swapcase__doc__[];
36+
37+
#define FLAG_LOWER 0x01
38+
#define FLAG_UPPER 0x02
39+
#define FLAG_ALPHA (FLAG_LOWER|FLAG_UPPER)
40+
#define FLAG_DIGIT 0x04
41+
#define FLAG_ALNUM (FLAG_ALPHA|FLAG_DIGIT)
42+
#define FLAG_SPACE 0x08
43+
#define FLAG_XDIGIT 0x10
44+
45+
extern const unsigned int _Py_ctype_table[256];
46+
47+
#define ISLOWER(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_LOWER)
48+
#define ISUPPER(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_UPPER)
49+
#define ISALPHA(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_ALPHA)
50+
#define ISDIGIT(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_DIGIT)
51+
#define ISXDIGIT(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_XDIGIT)
52+
#define ISALNUM(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_ALNUM)
53+
#define ISSPACE(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_SPACE)
54+
55+
#undef islower
56+
#define islower(c) undefined_islower(c)
57+
#undef isupper
58+
#define isupper(c) undefined_isupper(c)
59+
#undef isalpha
60+
#define isalpha(c) undefined_isalpha(c)
61+
#undef isdigit
62+
#define isdigit(c) undefined_isdigit(c)
63+
#undef isxdigit
64+
#define isxdigit(c) undefined_isxdigit(c)
65+
#undef isalnum
66+
#define isalnum(c) undefined_isalnum(c)
67+
#undef isspace
68+
#define isspace(c) undefined_isspace(c)
69+
70+
extern const unsigned char _Py_ctype_tolower[256];
71+
extern const unsigned char _Py_ctype_toupper[256];
72+
73+
#define TOLOWER(c) (_Py_ctype_tolower[Py_CHARMASK(c)])
74+
#define TOUPPER(c) (_Py_ctype_toupper[Py_CHARMASK(c)])
75+
76+
#undef tolower
77+
#define tolower(c) undefined_tolower(c)
78+
#undef toupper
79+
#define toupper(c) undefined_toupper(c)
80+
81+
/* this is needed because some docs are shared from the .o, not static */
82+
#define PyDoc_STRVAR_shared(name,str) const char name[] = PyDoc_STR(str)
83+
84+
#endif /* !Py_BYTES_CTYPE_H */

‎Lib/test/buffer_tests.py‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
# Tests that work for both str8 (bytes) and bytes (buffer) objects.
2+
# See PEP 3137.
3+
4+
import struct
5+
import sys
6+
7+
class MixinBytesBufferCommonTests(object):
8+
"""Tests that work for both str8 (bytes) and bytes (buffer) objects.
9+
See PEP 3137.
10+
"""
11+
12+
def marshal(self, x):
13+
"""Convert x into the appropriate type for these tests."""
14+
raise RuntimeError('test class must provide a marshal method')
15+
16+
def test_islower(self):
17+
self.assertFalse(self.marshal(b'').islower())
18+
self.assert_(self.marshal(b'a').islower())
19+
self.assertFalse(self.marshal(b'A').islower())
20+
self.assertFalse(self.marshal(b'\n').islower())
21+
self.assert_(self.marshal(b'abc').islower())
22+
self.assertFalse(self.marshal(b'aBc').islower())
23+
self.assert_(self.marshal(b'abc\n').islower())
24+
self.assertRaises(TypeError, self.marshal(b'abc').islower, 42)
25+
26+
def test_isupper(self):
27+
self.assertFalse(self.marshal(b'').isupper())
28+
self.assertFalse(self.marshal(b'a').isupper())
29+
self.assert_(self.marshal(b'A').isupper())
30+
self.assertFalse(self.marshal(b'\n').isupper())
31+
self.assert_(self.marshal(b'ABC').isupper())
32+
self.assertFalse(self.marshal(b'AbC').isupper())
33+
self.assert_(self.marshal(b'ABC\n').isupper())
34+
self.assertRaises(TypeError, self.marshal(b'abc').isupper, 42)
35+
36+
def test_istitle(self):
37+
self.assertFalse(self.marshal(b'').istitle())
38+
self.assertFalse(self.marshal(b'a').istitle())
39+
self.assert_(self.marshal(b'A').istitle())
40+
self.assertFalse(self.marshal(b'\n').istitle())
41+
self.assert_(self.marshal(b'A Titlecased Line').istitle())
42+
self.assert_(self.marshal(b'A\nTitlecased Line').istitle())
43+
self.assert_(self.marshal(b'A Titlecased, Line').istitle())
44+
self.assertFalse(self.marshal(b'Not a capitalized String').istitle())
45+
self.assertFalse(self.marshal(b'Not\ta Titlecase String').istitle())
46+
self.assertFalse(self.marshal(b'Not--a Titlecase String').istitle())
47+
self.assertFalse(self.marshal(b'NOT').istitle())
48+
self.assertRaises(TypeError, self.marshal(b'abc').istitle, 42)
49+
50+
def test_isspace(self):
51+
self.assertFalse(self.marshal(b'').isspace())
52+
self.assertFalse(self.marshal(b'a').isspace())
53+
self.assert_(self.marshal(b' ').isspace())
54+
self.assert_(self.marshal(b'\t').isspace())
55+
self.assert_(self.marshal(b'\r').isspace())
56+
self.assert_(self.marshal(b'\n').isspace())
57+
self.assert_(self.marshal(b' \t\r\n').isspace())
58+
self.assertFalse(self.marshal(b' \t\r\na').isspace())
59+
self.assertRaises(TypeError, self.marshal(b'abc').isspace, 42)
60+
61+
def test_isalpha(self):
62+
self.assertFalse(self.marshal(b'').isalpha())
63+
self.assert_(self.marshal(b'a').isalpha())
64+
self.assert_(self.marshal(b'A').isalpha())
65+
self.assertFalse(self.marshal(b'\n').isalpha())
66+
self.assert_(self.marshal(b'abc').isalpha())
67+
self.assertFalse(self.marshal(b'aBc123').isalpha())
68+
self.assertFalse(self.marshal(b'abc\n').isalpha())
69+
self.assertRaises(TypeError, self.marshal(b'abc').isalpha, 42)
70+
71+
def test_isalnum(self):
72+
self.assertFalse(self.marshal(b'').isalnum())
73+
self.assert_(self.marshal(b'a').isalnum())
74+
self.assert_(self.marshal(b'A').isalnum())
75+
self.assertFalse(self.marshal(b'\n').isalnum())
76+
self.assert_(self.marshal(b'123abc456').isalnum())
77+
self.assert_(self.marshal(b'a1b3c').isalnum())
78+
self.assertFalse(self.marshal(b'aBc000 ').isalnum())
79+
self.assertFalse(self.marshal(b'abc\n').isalnum())
80+
self.assertRaises(TypeError, self.marshal(b'abc').isalnum, 42)
81+
82+
def test_isdigit(self):
83+
self.assertFalse(self.marshal(b'').isdigit())
84+
self.assertFalse(self.marshal(b'a').isdigit())
85+
self.assert_(self.marshal(b'0').isdigit())
86+
self.assert_(self.marshal(b'0123456789').isdigit())
87+
self.assertFalse(self.marshal(b'0123456789a').isdigit())
88+
89+
self.assertRaises(TypeError, self.marshal(b'abc').isdigit, 42)
90+
91+
def test_lower(self):
92+
self.assertEqual(b'hello', self.marshal(b'HeLLo').lower())
93+
self.assertEqual(b'hello', self.marshal(b'hello').lower())
94+
self.assertRaises(TypeError, self.marshal(b'hello').lower, 42)
95+
96+
def test_upper(self):
97+
self.assertEqual(b'HELLO', self.marshal(b'HeLLo').upper())
98+
self.assertEqual(b'HELLO', self.marshal(b'HELLO').upper())
99+
self.assertRaises(TypeError, self.marshal(b'hello').upper, 42)
100+
101+
def test_capitalize(self):
102+
self.assertEqual(b' hello ', self.marshal(b' hello ').capitalize())
103+
self.assertEqual(b'Hello ', self.marshal(b'Hello ').capitalize())
104+
self.assertEqual(b'Hello ', self.marshal(b'hello ').capitalize())
105+
self.assertEqual(b'Aaaa', self.marshal(b'aaaa').capitalize())
106+
self.assertEqual(b'Aaaa', self.marshal(b'AaAa').capitalize())
107+
108+
self.assertRaises(TypeError, self.marshal(b'hello').capitalize, 42)
109+
110+
def test_ljust(self):
111+
self.assertEqual(b'abc ', self.marshal(b'abc').ljust(10))
112+
self.assertEqual(b'abc ', self.marshal(b'abc').ljust(6))
113+
self.assertEqual(b'abc', self.marshal(b'abc').ljust(3))
114+
self.assertEqual(b'abc', self.marshal(b'abc').ljust(2))
115+
self.assertEqual(b'abc*******', self.marshal(b'abc').ljust(10, '*'))
116+
self.assertRaises(TypeError, self.marshal(b'abc').ljust)
117+
118+
def test_rjust(self):
119+
self.assertEqual(b' abc', self.marshal(b'abc').rjust(10))
120+
self.assertEqual(b' abc', self.marshal(b'abc').rjust(6))
121+
self.assertEqual(b'abc', self.marshal(b'abc').rjust(3))
122+
self.assertEqual(b'abc', self.marshal(b'abc').rjust(2))
123+
self.assertEqual(b'*******abc', self.marshal(b'abc').rjust(10, '*'))
124+
self.assertRaises(TypeError, self.marshal(b'abc').rjust)
125+
126+
def test_center(self):
127+
self.assertEqual(b' abc ', self.marshal(b'abc').center(10))
128+
self.assertEqual(b' abc ', self.marshal(b'abc').center(6))
129+
self.assertEqual(b'abc', self.marshal(b'abc').center(3))
130+
self.assertEqual(b'abc', self.marshal(b'abc').center(2))
131+
self.assertEqual(b'***abc****', self.marshal(b'abc').center(10, '*'))
132+
self.assertRaises(TypeError, self.marshal(b'abc').center)
133+
134+
def test_swapcase(self):
135+
self.assertEqual(b'hEllO CoMPuTErS',
136+
self.marshal(b'HeLLo cOmpUteRs').swapcase())
137+
138+
self.assertRaises(TypeError, self.marshal(b'hello').swapcase, 42)
139+
140+
def test_zfill(self):
141+
self.assertEqual(b'123', self.marshal(b'123').zfill(2))
142+
self.assertEqual(b'123', self.marshal(b'123').zfill(3))
143+
self.assertEqual(b'0123', self.marshal(b'123').zfill(4))
144+
self.assertEqual(b'+123', self.marshal(b'+123').zfill(3))
145+
self.assertEqual(b'+123', self.marshal(b'+123').zfill(4))
146+
self.assertEqual(b'+0123', self.marshal(b'+123').zfill(5))
147+
self.assertEqual(b'-123', self.marshal(b'-123').zfill(3))
148+
self.assertEqual(b'-123', self.marshal(b'-123').zfill(4))
149+
self.assertEqual(b'-0123', self.marshal(b'-123').zfill(5))
150+
self.assertEqual(b'000', self.marshal(b'').zfill(3))
151+
self.assertEqual(b'34', self.marshal(b'34').zfill(1))
152+
self.assertEqual(b'0034', self.marshal(b'34').zfill(4))
153+
154+
self.assertRaises(TypeError, self.marshal(b'123').zfill)
155+
156+
def test_expandtabs(self):
157+
self.assertEqual(b'abc\rab def\ng hi',
158+
self.marshal(b'abc\rab\tdef\ng\thi').expandtabs())
159+
self.assertEqual(b'abc\rab def\ng hi',
160+
self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(8))
161+
self.assertEqual(b'abc\rab def\ng hi',
162+
self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(4))
163+
self.assertEqual(b'abc\r\nab def\ng hi',
164+
self.marshal(b'abc\r\nab\tdef\ng\thi').expandtabs(4))
165+
self.assertEqual(b'abc\rab def\ng hi',
166+
self.marshal(b'abc\rab\tdef\ng\thi').expandtabs())
167+
self.assertEqual(b'abc\rab def\ng hi',
168+
self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(8))
169+
self.assertEqual(b'abc\r\nab\r\ndef\ng\r\nhi',
170+
self.marshal(b'abc\r\nab\r\ndef\ng\r\nhi').expandtabs(4))
171+
self.assertEqual(b' a\n b', self.marshal(b' \ta\n\tb').expandtabs(1))
172+
173+
self.assertRaises(TypeError, self.marshal(b'hello').expandtabs, 42, 42)
174+
# This test is only valid when sizeof(int) == sizeof(void*) == 4.
175+
if sys.maxint < (1 << 32) and struct.calcsize('P') == 4:
176+
self.assertRaises(OverflowError,
177+
self.marshal(b'\ta\n\tb').expandtabs, sys.maxint)
178+
179+
def test_title(self):
180+
self.assertEqual(b' Hello ', self.marshal(b' hello ').title())
181+
self.assertEqual(b'Hello ', self.marshal(b'hello ').title())
182+
self.assertEqual(b'Hello ', self.marshal(b'Hello ').title())
183+
self.assertEqual(b'Format This As Title String',
184+
self.marshal(b'fOrMaT thIs aS titLe String').title())
185+
self.assertEqual(b'Format,This-As*Title;String',
186+
self.marshal(b'fOrMaT,thIs-aS*titLe;String').title())
187+
self.assertEqual(b'Getint', self.marshal(b'getInt').title())
188+
self.assertRaises(TypeError, self.marshal(b'hello').title, 42)
189+
190+
def test_splitlines(self):
191+
self.assertEqual([b'abc', b'def', b'', b'ghi'],
192+
self.marshal(b'abc\ndef\n\rghi').splitlines())
193+
self.assertEqual([b'abc', b'def', b'', b'ghi'],
194+
self.marshal(b'abc\ndef\n\r\nghi').splitlines())
195+
self.assertEqual([b'abc', b'def', b'ghi'],
196+
self.marshal(b'abc\ndef\r\nghi').splitlines())
197+
self.assertEqual([b'abc', b'def', b'ghi'],
198+
self.marshal(b'abc\ndef\r\nghi\n').splitlines())
199+
self.assertEqual([b'abc', b'def', b'ghi', b''],
200+
self.marshal(b'abc\ndef\r\nghi\n\r').splitlines())
201+
self.assertEqual([b'', b'abc', b'def', b'ghi', b''],
202+
self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines())
203+
self.assertEqual([b'\n', b'abc\n', b'def\r\n', b'ghi\n', b'\r'],
204+
self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(1))
205+
206+
self.assertRaises(TypeError, self.marshal(b'abc').splitlines, 42, 42)

‎Lib/test/test_bytes.py‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import unittest
99
import test.test_support
1010
import test.string_tests
11+
import test.buffer_tests
1112

1213

1314
class BytesTest(unittest.TestCase):
@@ -454,17 +455,18 @@ def test_alloc(self):
454455
def test_fromhex(self):
455456
self.assertRaises(TypeError, bytes.fromhex)
456457
self.assertRaises(TypeError, bytes.fromhex, 1)
457-
self.assertEquals(bytes.fromhex(''), bytes())
458+
self.assertEquals(bytes.fromhex(b''), bytes())
458459
b = bytes([0x1a, 0x2b, 0x30])
459-
self.assertEquals(bytes.fromhex('1a2B30'), b)
460-
self.assertEquals(bytes.fromhex(' 1A 2B 30 '), b)
460+
self.assertEquals(bytes.fromhex(b'1a2B30'), b)
461+
self.assertEquals(bytes.fromhex(b' 1A 2B 30 '), b)
461462
self.assertEquals(bytes.fromhex(memoryview(b'')), bytes())
462463
self.assertEquals(bytes.fromhex(memoryview(b'0000')), bytes([0, 0]))
463-
self.assertRaises(ValueError, bytes.fromhex, 'a')
464-
self.assertRaises(ValueError, bytes.fromhex, 'rt')
465-
self.assertRaises(ValueError, bytes.fromhex, '1a b cd')
466-
self.assertRaises(ValueError, bytes.fromhex, '\x00')
467-
self.assertRaises(ValueError, bytes.fromhex, '12 \x00 34')
464+
self.assertRaises(TypeError, bytes.fromhex, '1B')
465+
self.assertRaises(ValueError, bytes.fromhex, b'a')
466+
self.assertRaises(ValueError, bytes.fromhex, b'rt')
467+
self.assertRaises(ValueError, bytes.fromhex, b'1a b cd')
468+
self.assertRaises(ValueError, bytes.fromhex, b'\x00')
469+
self.assertRaises(ValueError, bytes.fromhex, b'12 \x00 34')
468470

469471
def test_join(self):
470472
self.assertEqual(b"".join([]), bytes())
@@ -504,11 +506,12 @@ def test_remove(self):
504506
self.assertEqual(b, b'heo')
505507
self.assertRaises(ValueError, lambda: b.remove(ord('l')))
506508
self.assertRaises(ValueError, lambda: b.remove(400))
507-
self.assertRaises(ValueError, lambda: b.remove('e'))
509+
self.assertRaises(TypeError, lambda: b.remove('e'))
508510
# remove first and last
509511
b.remove(ord('o'))
510512
b.remove(ord('h'))
511513
self.assertEqual(b, b'e')
514+
self.assertRaises(TypeError, lambda: b.remove(b'e'))
512515

513516
def test_pop(self):
514517
b = b'world'
@@ -542,6 +545,7 @@ def test_append(self):
542545
b = bytes()
543546
b.append(ord('A'))
544547
self.assertEqual(len(b), 1)
548+
self.assertRaises(TypeError, lambda: b.append(b'o'))
545549

546550
def test_insert(self):
547551
b = b'msssspp'
@@ -550,6 +554,7 @@ def test_insert(self):
550554
b.insert(-2, ord('i'))
551555
b.insert(1000, ord('i'))
552556
self.assertEqual(b, b'mississippi')
557+
self.assertRaises(TypeError, lambda: b.insert(0, b'1'))
553558

554559
def test_startswith(self):
555560
b = b'hello'
@@ -734,6 +739,29 @@ def test_ord(self):
734739
# Unfortunately they are all bundled with tests that
735740
# are not appropriate for bytes
736741

742+
# I've started porting some of those into buffer_tests.py, we should port
743+
# the rest that make sense (the code can be cleaned up to use modern
744+
# unittest methods at the same time).
745+
746+
class BufferPEP3137Test(unittest.TestCase,
747+
test.buffer_tests.MixinBytesBufferCommonTests):
748+
def marshal(self, x):
749+
return bytes(x)
750+
# TODO this should become:
751+
#return buffer(x)
752+
# once the bytes -> buffer and str8 -> bytes rename happens
753+
754+
def test_returns_new_copy(self):
755+
val = self.marshal(b'1234')
756+
# On immutable types these MAY return a reference to themselves
757+
# but on mutable types like buffer they MUST return a new copy.
758+
for methname in ('zfill', 'rjust', 'ljust', 'center'):
759+
method = getattr(val, methname)
760+
newval = method(3)
761+
self.assertEqual(val, newval)
762+
self.assertTrue(val is not newval,
763+
methname+' returned self on a mutable object')
764+
737765

738766
class BytesAsStringTest(test.string_tests.BaseTest):
739767
type2test = bytes
@@ -759,7 +787,7 @@ def test_lower(self):
759787
def test_main():
760788
test.test_support.run_unittest(BytesTest)
761789
test.test_support.run_unittest(BytesAsStringTest)
762-
790+
test.test_support.run_unittest(BufferPEP3137Test)
763791

764792
if __name__ == "__main__":
765793
##test_main()

0 commit comments

Comments
 (0)