Skip to content

Commit eeb896c

Browse files
vadmiumvadmium
committed
Issue #24802: Copy bytes-like objects to null-terminated buffers if necessary
This avoids possible buffer overreads when int(), float(), compile(), exec() and eval() are passed bytes-like objects. Similar code is removed from the complex() constructor, where it was not reachable. Patch by John Leitch, Serhiy Storchaka and Martin Panter.
1 parent 9ad0aae commit eeb896c

File tree

8 files changed

+162
-37
lines changed

8 files changed

+162
-37
lines changed

‎Lib/test/test_compile.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,27 @@ def check_limit(prefix, repeated):
530530
check_limit("a", "[0]")
531531
check_limit("a", "*a")
532532

533+
def test_null_terminated(self):
534+
# The source code is null-terminated internally, but bytes-like
535+
# objects are accepted, which could be not terminated.
536+
# Exception changed from TypeError to ValueError in 3.5
537+
with self.assertRaisesRegex(Exception, "cannot contain null"):
538+
compile("123\x00", "<dummy>", "eval")
539+
with self.assertRaisesRegex(Exception, "cannot contain null"):
540+
compile(memoryview(b"123\x00"), "<dummy>", "eval")
541+
code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval")
542+
self.assertEqual(eval(code), 23)
543+
code = compile(memoryview(b"1234")[1:-1], "<dummy>", "eval")
544+
self.assertEqual(eval(code), 23)
545+
code = compile(memoryview(b"$23$")[1:-1], "<dummy>", "eval")
546+
self.assertEqual(eval(code), 23)
547+
548+
# Also test when eval() and exec() do the compilation step
549+
self.assertEqual(eval(memoryview(b"1234")[1:-1]), 23)
550+
namespace = dict()
551+
exec(memoryview(b"ax = 123")[1:-1], namespace)
552+
self.assertEqual(namespace['x'], 12)
553+
533554

534555
class TestStackSize(unittest.TestCase):
535556
# These tests check that the computed stack size for a code object

‎Lib/test/test_float.py‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ def test_float(self):
3131
self.assertEqual(float(3.14), 3.14)
3232
self.assertEqual(float(314), 314.0)
3333
self.assertEqual(float(" 3.14 "), 3.14)
34-
self.assertEqual(float(b" 3.14 "), 3.14)
3534
self.assertRaises(ValueError, float, " 0x3.1 ")
3635
self.assertRaises(ValueError, float, " -0x3.p-1 ")
3736
self.assertRaises(ValueError, float, " +0x3.p-1 ")
@@ -43,7 +42,6 @@ def test_float(self):
4342
self.assertRaises(ValueError, float, "+.inf")
4443
self.assertRaises(ValueError, float, ".")
4544
self.assertRaises(ValueError, float, "-.")
46-
self.assertRaises(ValueError, float, b"-")
4745
self.assertRaises(TypeError, float, {})
4846
self.assertRaisesRegex(TypeError, "not 'dict'", float, {})
4947
# Lone surrogate
@@ -57,6 +55,42 @@ def test_float(self):
5755
float(b'.' + b'1'*1000)
5856
float('.' + '1'*1000)
5957

58+
def test_non_numeric_input_types(self):
59+
# Test possible non-numeric types for the argument x, including
60+
# subclasses of the explicitly documented accepted types.
61+
class CustomStr(str): pass
62+
class CustomBytes(bytes): pass
63+
class CustomByteArray(bytearray): pass
64+
65+
factories = [
66+
bytes,
67+
bytearray,
68+
lambda b: CustomStr(b.decode()),
69+
CustomBytes,
70+
CustomByteArray,
71+
memoryview,
72+
]
73+
try:
74+
from array import array
75+
except ImportError:
76+
pass
77+
else:
78+
factories.append(lambda b: array('B', b))
79+
80+
for f in factories:
81+
x = f(b" 3.14 ")
82+
with self.subTest(type(x)):
83+
self.assertEqual(float(x), 3.14)
84+
with self.assertRaisesRegex(ValueError, "could not convert"):
85+
float(f(b'A' * 0x10))
86+
87+
def test_float_memoryview(self):
88+
self.assertEqual(float(memoryview(b'12.3')[1:4]), 2.3)
89+
self.assertEqual(float(memoryview(b'12.3\x00')[1:4]), 2.3)
90+
self.assertEqual(float(memoryview(b'12.3 ')[1:4]), 2.3)
91+
self.assertEqual(float(memoryview(b'12.3A')[1:4]), 2.3)
92+
self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3)
93+
6094
def test_error_message(self):
6195
testlist = ('\xbd', '123\xbd', ' 123 456 ')
6296
for s in testlist:

‎Lib/test/test_int.py‎

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -276,16 +276,40 @@ class CustomStr(str): pass
276276
class CustomBytes(bytes): pass
277277
class CustomByteArray(bytearray): pass
278278

279-
values = [b'100',
280-
bytearray(b'100'),
281-
CustomStr('100'),
282-
CustomBytes(b'100'),
283-
CustomByteArray(b'100')]
284-
285-
for x in values:
286-
msg = 'x has type %s' % type(x).__name__
287-
self.assertEqual(int(x), 100, msg=msg)
288-
self.assertEqual(int(x, 2), 4, msg=msg)
279+
factories = [
280+
bytes,
281+
bytearray,
282+
lambda b: CustomStr(b.decode()),
283+
CustomBytes,
284+
CustomByteArray,
285+
memoryview,
286+
]
287+
try:
288+
from array import array
289+
except ImportError:
290+
pass
291+
else:
292+
factories.append(lambda b: array('B', b))
293+
294+
for f in factories:
295+
x = f(b'100')
296+
with self.subTest(type(x)):
297+
self.assertEqual(int(x), 100)
298+
if isinstance(x, (str, bytes, bytearray)):
299+
self.assertEqual(int(x, 2), 4)
300+
else:
301+
msg = "can't convert non-string"
302+
with self.assertRaisesRegex(TypeError, msg):
303+
int(x, 2)
304+
with self.assertRaisesRegex(ValueError, 'invalid literal'):
305+
int(f(b'A' * 0x10))
306+
307+
def test_int_memoryview(self):
308+
self.assertEqual(int(memoryview(b'123')[1:3]), 23)
309+
self.assertEqual(int(memoryview(b'123\x00')[1:3]), 23)
310+
self.assertEqual(int(memoryview(b'123 ')[1:3]), 23)
311+
self.assertEqual(int(memoryview(b'123A')[1:3]), 23)
312+
self.assertEqual(int(memoryview(b'1234')[1:3]), 23)
289313

290314
def test_string_float(self):
291315
self.assertRaises(ValueError, int, '1.2')

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ Release date: tba
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec()
14+
and eval() are passed bytes-like objects. These objects are not
15+
necessarily terminated by a null byte, but the functions assumed they were.
16+
1317
- Issue #24402: Fix input() to prompt to the redirected stdout when
1418
sys.stdout.fileno() fails.
1519

‎Objects/abstract.c‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,12 +1264,30 @@ PyNumber_Long(PyObject *o)
12641264
/* The below check is done in PyLong_FromUnicode(). */
12651265
return PyLong_FromUnicodeObject(o, 10);
12661266

1267-
if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
1267+
if (PyBytes_Check(o))
12681268
/* need to do extra error checking that PyLong_FromString()
12691269
* doesn't do. In particular int('9\x005') must raise an
12701270
* exception, not truncate at the null.
12711271
*/
1272-
PyObject *result = _PyLong_FromBytes(view.buf, view.len, 10);
1272+
return _PyLong_FromBytes(PyBytes_AS_STRING(o),
1273+
PyBytes_GET_SIZE(o), 10);
1274+
1275+
if (PyByteArray_Check(o))
1276+
return _PyLong_FromBytes(PyByteArray_AS_STRING(o),
1277+
PyByteArray_GET_SIZE(o), 10);
1278+
1279+
if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
1280+
PyObject *result, *bytes;
1281+
1282+
/* Copy to NUL-terminated buffer. */
1283+
bytes = PyBytes_FromStringAndSize((const char *)view.buf, view.len);
1284+
if (bytes == NULL) {
1285+
PyBuffer_Release(&view);
1286+
return NULL;
1287+
}
1288+
result = _PyLong_FromBytes(PyBytes_AS_STRING(bytes),
1289+
PyBytes_GET_SIZE(bytes), 10);
1290+
Py_DECREF(bytes);
12731291
PyBuffer_Release(&view);
12741292
return result;
12751293
}

‎Objects/complexobject.c‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -767,7 +767,6 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
767767
int got_bracket=0;
768768
PyObject *s_buffer = NULL;
769769
Py_ssize_t len;
770-
Py_buffer view = {NULL, NULL};
771770

772771
if (PyUnicode_Check(v)) {
773772
s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v);
@@ -777,10 +776,6 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
777776
if (s == NULL)
778777
goto error;
779778
}
780-
else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
781-
s = (const char *)view.buf;
782-
len = view.len;
783-
}
784779
else {
785780
PyErr_Format(PyExc_TypeError,
786781
"complex() argument must be a string or a number, not '%.200s'",
@@ -895,15 +890,13 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
895890
if (s-start != len)
896891
goto parse_error;
897892

898-
PyBuffer_Release(&view);
899893
Py_XDECREF(s_buffer);
900894
return complex_subtype_from_doubles(type, x, y);
901895

902896
parse_error:
903897
PyErr_SetString(PyExc_ValueError,
904898
"complex() arg is a malformed string");
905899
error:
906-
PyBuffer_Release(&view);
907900
Py_XDECREF(s_buffer);
908901
return NULL;
909902
}

‎Objects/floatobject.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,9 +144,24 @@ PyFloat_FromString(PyObject *v)
144144
return NULL;
145145
}
146146
}
147+
else if (PyBytes_Check(v)) {
148+
s = PyBytes_AS_STRING(v);
149+
len = PyBytes_GET_SIZE(v);
150+
}
151+
else if (PyByteArray_Check(v)) {
152+
s = PyByteArray_AS_STRING(v);
153+
len = PyByteArray_GET_SIZE(v);
154+
}
147155
else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
148156
s = (const char *)view.buf;
149157
len = view.len;
158+
/* Copy to NUL-terminated buffer. */
159+
s_buffer = PyBytes_FromStringAndSize(s, len);
160+
if (s_buffer == NULL) {
161+
PyBuffer_Release(&view);
162+
return NULL;
163+
}
164+
s = PyBytes_AS_STRING(s_buffer);
150165
}
151166
else {
152167
PyErr_Format(PyExc_TypeError,

‎Python/bltinmodule.c‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -560,20 +560,37 @@ Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff.");
560560

561561

562562
static const char *
563-
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, Py_buffer *view)
563+
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
564564
{
565565
const char *str;
566566
Py_ssize_t size;
567+
Py_buffer view;
567568

569+
*cmd_copy = NULL;
568570
if (PyUnicode_Check(cmd)) {
569571
cf->cf_flags |= PyCF_IGNORE_COOKIE;
570572
str = PyUnicode_AsUTF8AndSize(cmd, &size);
571573
if (str == NULL)
572574
return NULL;
573575
}
574-
else if (PyObject_GetBuffer(cmd, view, PyBUF_SIMPLE) == 0) {
575-
str = (const char *)view->buf;
576-
size = view->len;
576+
else if (PyBytes_Check(cmd)) {
577+
str = PyBytes_AS_STRING(cmd);
578+
size = PyBytes_GET_SIZE(cmd);
579+
}
580+
else if (PyByteArray_Check(cmd)) {
581+
str = PyByteArray_AS_STRING(cmd);
582+
size = PyByteArray_GET_SIZE(cmd);
583+
}
584+
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
585+
/* Copy to NUL-terminated buffer. */
586+
*cmd_copy = PyBytes_FromStringAndSize(
587+
(const char *)view.buf, view.len);
588+
PyBuffer_Release(&view);
589+
if (*cmd_copy == NULL) {
590+
return NULL;
591+
}
592+
str = PyBytes_AS_STRING(*cmd_copy);
593+
size = PyBytes_GET_SIZE(*cmd_copy);
577594
}
578595
else {
579596
PyErr_Format(PyExc_TypeError,
@@ -585,7 +602,7 @@ source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompil
585602
if (strlen(str) != size) {
586603
PyErr_SetString(PyExc_TypeError,
587604
"source code string cannot contain null bytes");
588-
PyBuffer_Release(view);
605+
Py_CLEAR(*cmd_copy);
589606
return NULL;
590607
}
591608
return str;
@@ -594,7 +611,7 @@ source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompil
594611
static PyObject *
595612
builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
596613
{
597-
Py_buffer view = {NULL, NULL};
614+
PyObject *cmd_copy;
598615
const char *str;
599616
PyObject *filename;
600617
char *startstr;
@@ -681,12 +698,12 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
681698
goto finally;
682699
}
683700

684-
str = source_as_string(cmd, "compile", "string, bytes or AST", &cf, &view);
701+
str = source_as_string(cmd, "compile", "string, bytes or AST", &cf, &cmd_copy);
685702
if (str == NULL)
686703
goto error;
687704

688705
result = Py_CompileStringObject(str, filename, start[mode], &cf, optimize);
689-
PyBuffer_Release(&view);
706+
Py_XDECREF(cmd_copy);
690707
goto finally;
691708

692709
error:
@@ -754,9 +771,8 @@ Return the tuple ((x-x%y)/y, x%y). Invariant: div*y + mod == x.");
754771
static PyObject *
755772
builtin_eval(PyObject *self, PyObject *args)
756773
{
757-
PyObject *cmd, *result, *tmp = NULL;
774+
PyObject *cmd, *result, *cmd_copy;
758775
PyObject *globals = Py_None, *locals = Py_None;
759-
Py_buffer view = {NULL, NULL};
760776
const char *str;
761777
PyCompilerFlags cf;
762778

@@ -806,7 +822,7 @@ builtin_eval(PyObject *self, PyObject *args)
806822
}
807823

808824
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
809-
str = source_as_string(cmd, "eval", "string, bytes or code", &cf, &view);
825+
str = source_as_string(cmd, "eval", "string, bytes or code", &cf, &cmd_copy);
810826
if (str == NULL)
811827
return NULL;
812828

@@ -815,8 +831,7 @@ builtin_eval(PyObject *self, PyObject *args)
815831

816832
(void)PyEval_MergeCompilerFlags(&cf);
817833
result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
818-
PyBuffer_Release(&view);
819-
Py_XDECREF(tmp);
834+
Py_XDECREF(cmd_copy);
820835
return result;
821836
}
822837

@@ -882,20 +897,21 @@ builtin_exec(PyObject *self, PyObject *args)
882897
v = PyEval_EvalCode(prog, globals, locals);
883898
}
884899
else {
885-
Py_buffer view = {NULL, NULL};
900+
PyObject *prog_copy;
886901
const char *str;
887902
PyCompilerFlags cf;
888903
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
889904
str = source_as_string(prog, "exec",
890-
"string, bytes or code", &cf, &view);
905+
"string, bytes or code", &cf,
906+
&prog_copy);
891907
if (str == NULL)
892908
return NULL;
893909
if (PyEval_MergeCompilerFlags(&cf))
894910
v = PyRun_StringFlags(str, Py_file_input, globals,
895911
locals, &cf);
896912
else
897913
v = PyRun_String(str, Py_file_input, globals, locals);
898-
PyBuffer_Release(&view);
914+
Py_XDECREF(prog_copy);
899915
}
900916
if (v == NULL)
901917
return NULL;

0 commit comments

Comments
 (0)