Skip to content

Commit 61d6e4a

Browse files
vadmiumvadmium
committed
Issue #24802: Merge null termination fixes from 3.4 into 3.5
2 parents 9b566c3 + eeb896c commit 61d6e4a

File tree

8 files changed

+162
-37
lines changed

8 files changed

+162
-37
lines changed

‎Lib/test/test_compile.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,27 @@ def check_limit(prefix, repeated):
542542
check_limit("a", "[0]")
543543
check_limit("a", "*a")
544544

545+
def test_null_terminated(self):
546+
# The source code is null-terminated internally, but bytes-like
547+
# objects are accepted, which could be not terminated.
548+
# Exception changed from TypeError to ValueError in 3.5
549+
with self.assertRaisesRegex(Exception, "cannot contain null"):
550+
compile("123\x00", "<dummy>", "eval")
551+
with self.assertRaisesRegex(Exception, "cannot contain null"):
552+
compile(memoryview(b"123\x00"), "<dummy>", "eval")
553+
code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval")
554+
self.assertEqual(eval(code), 23)
555+
code = compile(memoryview(b"1234")[1:-1], "<dummy>", "eval")
556+
self.assertEqual(eval(code), 23)
557+
code = compile(memoryview(b"$23$")[1:-1], "<dummy>", "eval")
558+
self.assertEqual(eval(code), 23)
559+
560+
# Also test when eval() and exec() do the compilation step
561+
self.assertEqual(eval(memoryview(b"1234")[1:-1]), 23)
562+
namespace = dict()
563+
exec(memoryview(b"ax = 123")[1:-1], namespace)
564+
self.assertEqual(namespace['x'], 12)
565+
545566

546567
class TestStackSize(unittest.TestCase):
547568
# These tests check that the computed stack size for a code object

‎Lib/test/test_float.py‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ def test_float(self):
3131
self.assertEqual(float(3.14), 3.14)
3232
self.assertEqual(float(314), 314.0)
3333
self.assertEqual(float(" 3.14 "), 3.14)
34-
self.assertEqual(float(b" 3.14 "), 3.14)
3534
self.assertRaises(ValueError, float, " 0x3.1 ")
3635
self.assertRaises(ValueError, float, " -0x3.p-1 ")
3736
self.assertRaises(ValueError, float, " +0x3.p-1 ")
@@ -43,7 +42,6 @@ def test_float(self):
4342
self.assertRaises(ValueError, float, "+.inf")
4443
self.assertRaises(ValueError, float, ".")
4544
self.assertRaises(ValueError, float, "-.")
46-
self.assertRaises(ValueError, float, b"-")
4745
self.assertRaises(TypeError, float, {})
4846
self.assertRaisesRegex(TypeError, "not 'dict'", float, {})
4947
# Lone surrogate
@@ -57,6 +55,42 @@ def test_float(self):
5755
float(b'.' + b'1'*1000)
5856
float('.' + '1'*1000)
5957

58+
def test_non_numeric_input_types(self):
59+
# Test possible non-numeric types for the argument x, including
60+
# subclasses of the explicitly documented accepted types.
61+
class CustomStr(str): pass
62+
class CustomBytes(bytes): pass
63+
class CustomByteArray(bytearray): pass
64+
65+
factories = [
66+
bytes,
67+
bytearray,
68+
lambda b: CustomStr(b.decode()),
69+
CustomBytes,
70+
CustomByteArray,
71+
memoryview,
72+
]
73+
try:
74+
from array import array
75+
except ImportError:
76+
pass
77+
else:
78+
factories.append(lambda b: array('B', b))
79+
80+
for f in factories:
81+
x = f(b" 3.14 ")
82+
with self.subTest(type(x)):
83+
self.assertEqual(float(x), 3.14)
84+
with self.assertRaisesRegex(ValueError, "could not convert"):
85+
float(f(b'A' * 0x10))
86+
87+
def test_float_memoryview(self):
88+
self.assertEqual(float(memoryview(b'12.3')[1:4]), 2.3)
89+
self.assertEqual(float(memoryview(b'12.3\x00')[1:4]), 2.3)
90+
self.assertEqual(float(memoryview(b'12.3 ')[1:4]), 2.3)
91+
self.assertEqual(float(memoryview(b'12.3A')[1:4]), 2.3)
92+
self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3)
93+
6094
def test_error_message(self):
6195
testlist = ('\xbd', '123\xbd', ' 123 456 ')
6296
for s in testlist:

‎Lib/test/test_int.py‎

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -276,16 +276,40 @@ class CustomStr(str): pass
276276
class CustomBytes(bytes): pass
277277
class CustomByteArray(bytearray): pass
278278

279-
values = [b'100',
280-
bytearray(b'100'),
281-
CustomStr('100'),
282-
CustomBytes(b'100'),
283-
CustomByteArray(b'100')]
284-
285-
for x in values:
286-
msg = 'x has type %s' % type(x).__name__
287-
self.assertEqual(int(x), 100, msg=msg)
288-
self.assertEqual(int(x, 2), 4, msg=msg)
279+
factories = [
280+
bytes,
281+
bytearray,
282+
lambda b: CustomStr(b.decode()),
283+
CustomBytes,
284+
CustomByteArray,
285+
memoryview,
286+
]
287+
try:
288+
from array import array
289+
except ImportError:
290+
pass
291+
else:
292+
factories.append(lambda b: array('B', b))
293+
294+
for f in factories:
295+
x = f(b'100')
296+
with self.subTest(type(x)):
297+
self.assertEqual(int(x), 100)
298+
if isinstance(x, (str, bytes, bytearray)):
299+
self.assertEqual(int(x, 2), 4)
300+
else:
301+
msg = "can't convert non-string"
302+
with self.assertRaisesRegex(TypeError, msg):
303+
int(x, 2)
304+
with self.assertRaisesRegex(ValueError, 'invalid literal'):
305+
int(f(b'A' * 0x10))
306+
307+
def test_int_memoryview(self):
308+
self.assertEqual(int(memoryview(b'123')[1:3]), 23)
309+
self.assertEqual(int(memoryview(b'123\x00')[1:3]), 23)
310+
self.assertEqual(int(memoryview(b'123 ')[1:3]), 23)
311+
self.assertEqual(int(memoryview(b'123A')[1:3]), 23)
312+
self.assertEqual(int(memoryview(b'1234')[1:3]), 23)
289313

290314
def test_string_float(self):
291315
self.assertRaises(ValueError, int, '1.2')

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ Release date: TBA
1111
Core and Builtins
1212
-----------------
1313

14+
- Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec()
15+
and eval() are passed bytes-like objects. These objects are not
16+
necessarily terminated by a null byte, but the functions assumed they were.
17+
1418
- Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that
1519
was mutated by direct calls of dict methods.
1620

‎Objects/abstract.c‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1309,12 +1309,30 @@ PyNumber_Long(PyObject *o)
13091309
/* The below check is done in PyLong_FromUnicode(). */
13101310
return PyLong_FromUnicodeObject(o, 10);
13111311

1312-
if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
1312+
if (PyBytes_Check(o))
13131313
/* need to do extra error checking that PyLong_FromString()
13141314
* doesn't do. In particular int('9\x005') must raise an
13151315
* exception, not truncate at the null.
13161316
*/
1317-
PyObject *result = _PyLong_FromBytes(view.buf, view.len, 10);
1317+
return _PyLong_FromBytes(PyBytes_AS_STRING(o),
1318+
PyBytes_GET_SIZE(o), 10);
1319+
1320+
if (PyByteArray_Check(o))
1321+
return _PyLong_FromBytes(PyByteArray_AS_STRING(o),
1322+
PyByteArray_GET_SIZE(o), 10);
1323+
1324+
if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
1325+
PyObject *result, *bytes;
1326+
1327+
/* Copy to NUL-terminated buffer. */
1328+
bytes = PyBytes_FromStringAndSize((const char *)view.buf, view.len);
1329+
if (bytes == NULL) {
1330+
PyBuffer_Release(&view);
1331+
return NULL;
1332+
}
1333+
result = _PyLong_FromBytes(PyBytes_AS_STRING(bytes),
1334+
PyBytes_GET_SIZE(bytes), 10);
1335+
Py_DECREF(bytes);
13181336
PyBuffer_Release(&view);
13191337
return result;
13201338
}

‎Objects/complexobject.c‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -767,7 +767,6 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
767767
int got_bracket=0;
768768
PyObject *s_buffer = NULL;
769769
Py_ssize_t len;
770-
Py_buffer view = {NULL, NULL};
771770

772771
if (PyUnicode_Check(v)) {
773772
s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v);
@@ -777,10 +776,6 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
777776
if (s == NULL)
778777
goto error;
779778
}
780-
else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
781-
s = (const char *)view.buf;
782-
len = view.len;
783-
}
784779
else {
785780
PyErr_Format(PyExc_TypeError,
786781
"complex() argument must be a string or a number, not '%.200s'",
@@ -895,15 +890,13 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
895890
if (s-start != len)
896891
goto parse_error;
897892

898-
PyBuffer_Release(&view);
899893
Py_XDECREF(s_buffer);
900894
return complex_subtype_from_doubles(type, x, y);
901895

902896
parse_error:
903897
PyErr_SetString(PyExc_ValueError,
904898
"complex() arg is a malformed string");
905899
error:
906-
PyBuffer_Release(&view);
907900
Py_XDECREF(s_buffer);
908901
return NULL;
909902
}

‎Objects/floatobject.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,9 +144,24 @@ PyFloat_FromString(PyObject *v)
144144
return NULL;
145145
}
146146
}
147+
else if (PyBytes_Check(v)) {
148+
s = PyBytes_AS_STRING(v);
149+
len = PyBytes_GET_SIZE(v);
150+
}
151+
else if (PyByteArray_Check(v)) {
152+
s = PyByteArray_AS_STRING(v);
153+
len = PyByteArray_GET_SIZE(v);
154+
}
147155
else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
148156
s = (const char *)view.buf;
149157
len = view.len;
158+
/* Copy to NUL-terminated buffer. */
159+
s_buffer = PyBytes_FromStringAndSize(s, len);
160+
if (s_buffer == NULL) {
161+
PyBuffer_Release(&view);
162+
return NULL;
163+
}
164+
s = PyBytes_AS_STRING(s_buffer);
150165
}
151166
else {
152167
PyErr_Format(PyExc_TypeError,

‎Python/bltinmodule.c‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -599,20 +599,37 @@ builtin_chr_impl(PyModuleDef *module, int i)
599599

600600

601601
static const char *
602-
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, Py_buffer *view)
602+
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
603603
{
604604
const char *str;
605605
Py_ssize_t size;
606+
Py_buffer view;
606607

608+
*cmd_copy = NULL;
607609
if (PyUnicode_Check(cmd)) {
608610
cf->cf_flags |= PyCF_IGNORE_COOKIE;
609611
str = PyUnicode_AsUTF8AndSize(cmd, &size);
610612
if (str == NULL)
611613
return NULL;
612614
}
613-
else if (PyObject_GetBuffer(cmd, view, PyBUF_SIMPLE) == 0) {
614-
str = (const char *)view->buf;
615-
size = view->len;
615+
else if (PyBytes_Check(cmd)) {
616+
str = PyBytes_AS_STRING(cmd);
617+
size = PyBytes_GET_SIZE(cmd);
618+
}
619+
else if (PyByteArray_Check(cmd)) {
620+
str = PyByteArray_AS_STRING(cmd);
621+
size = PyByteArray_GET_SIZE(cmd);
622+
}
623+
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
624+
/* Copy to NUL-terminated buffer. */
625+
*cmd_copy = PyBytes_FromStringAndSize(
626+
(const char *)view.buf, view.len);
627+
PyBuffer_Release(&view);
628+
if (*cmd_copy == NULL) {
629+
return NULL;
630+
}
631+
str = PyBytes_AS_STRING(*cmd_copy);
632+
size = PyBytes_GET_SIZE(*cmd_copy);
616633
}
617634
else {
618635
PyErr_Format(PyExc_TypeError,
@@ -624,7 +641,7 @@ source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompil
624641
if (strlen(str) != (size_t)size) {
625642
PyErr_SetString(PyExc_ValueError,
626643
"source code string cannot contain null bytes");
627-
PyBuffer_Release(view);
644+
Py_CLEAR(*cmd_copy);
628645
return NULL;
629646
}
630647
return str;
@@ -660,7 +677,7 @@ builtin_compile_impl(PyModuleDef *module, PyObject *source,
660677
int dont_inherit, int optimize)
661678
/*[clinic end generated code: output=31881762c1bb90c4 input=9d53e8cfb3c86414]*/
662679
{
663-
Py_buffer view = {NULL, NULL};
680+
PyObject *source_copy;
664681
const char *str;
665682
int compile_mode = -1;
666683
int is_ast;
@@ -732,12 +749,12 @@ builtin_compile_impl(PyModuleDef *module, PyObject *source,
732749
goto finally;
733750
}
734751

735-
str = source_as_string(source, "compile", "string, bytes or AST", &cf, &view);
752+
str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy);
736753
if (str == NULL)
737754
goto error;
738755

739756
result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);
740-
PyBuffer_Release(&view);
757+
Py_XDECREF(source_copy);
741758
goto finally;
742759

743760
error:
@@ -812,8 +829,7 @@ builtin_eval_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
812829
PyObject *locals)
813830
/*[clinic end generated code: output=7284501fb7b4d666 input=11ee718a8640e527]*/
814831
{
815-
PyObject *result, *tmp = NULL;
816-
Py_buffer view = {NULL, NULL};
832+
PyObject *result, *source_copy;
817833
const char *str;
818834
PyCompilerFlags cf;
819835

@@ -861,7 +877,7 @@ builtin_eval_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
861877
}
862878

863879
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
864-
str = source_as_string(source, "eval", "string, bytes or code", &cf, &view);
880+
str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
865881
if (str == NULL)
866882
return NULL;
867883

@@ -870,8 +886,7 @@ builtin_eval_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
870886

871887
(void)PyEval_MergeCompilerFlags(&cf);
872888
result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
873-
PyBuffer_Release(&view);
874-
Py_XDECREF(tmp);
889+
Py_XDECREF(source_copy);
875890
return result;
876891
}
877892

@@ -942,20 +957,21 @@ builtin_exec_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
942957
v = PyEval_EvalCode(source, globals, locals);
943958
}
944959
else {
945-
Py_buffer view = {NULL, NULL};
960+
PyObject *source_copy;
946961
const char *str;
947962
PyCompilerFlags cf;
948963
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
949964
str = source_as_string(source, "exec",
950-
"string, bytes or code", &cf, &view);
965+
"string, bytes or code", &cf,
966+
&source_copy);
951967
if (str == NULL)
952968
return NULL;
953969
if (PyEval_MergeCompilerFlags(&cf))
954970
v = PyRun_StringFlags(str, Py_file_input, globals,
955971
locals, &cf);
956972
else
957973
v = PyRun_String(str, Py_file_input, globals, locals);
958-
PyBuffer_Release(&view);
974+
Py_XDECREF(source_copy);
959975
}
960976
if (v == NULL)
961977
return NULL;

0 commit comments

Comments
 (0)