Skip to content

Commit 39837b6

Browse files
mdickinsongpshead
mdickinson
and
gpshead
committed
pythongh-95778: Correctly pre-check for int-to-str conversion (python#96537)
Converting a large enough `int` to a decimal string raises `ValueError` as expected. However, the raise comes _after_ the quadratic-time base-conversion algorithm has run to completion. For effective DOS prevention, we need some kind of check before entering the quadratic-time loop. Oops! =) The quick fix: essentially we catch _most_ values that exceed the threshold up front. Those that slip through will still be on the small side (read: sufficiently fast), and will get caught by the existing check so that the limit remains exact. The justification for the current check. The C code check is: ```c max_str_digits / (3 * PyLong_SHIFT) <= (size_a - 11) / 10 ``` In GitHub markdown math-speak, writing $M$ for `max_str_digits`, $L$ for `PyLong_SHIFT` and $s$ for `size_a`, that check is: $$\left\lfloor\frac{M}{3L}\right\rfloor \le \left\lfloor\frac{s - 11}{10}\right\rfloor$$ From this it follows that $$\frac{M}{3L} < \frac{s-1}{10}$$ hence that $$\frac{L(s-1)}{M} > \frac{10}{3} > \log_2(10).$$ So $$2^{L(s-1)} > 10^M.$$ But our input integer $a$ satisfies $|a| \ge 2^{L(s-1)}$, so $|a|$ is larger than $10^M$. This shows that we don't accidentally capture anything _below_ the intended limit in the check. <!-- gh-issue-number: pythongh-95778 --> * Issue: pythongh-95778 <!-- /gh-issue-number --> Co-authored-by: Gregory P. Smith [Google LLC] <greg@krypto.org>
1 parent f69b587 commit 39837b6

3 files changed

Lines changed: 105 additions & 5 deletions

File tree

‎Lib/test/test_int.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import sys
2+
import time
23

34
import unittest
45
from test import support
@@ -571,6 +572,87 @@ def test_max_str_digits(self):
571572
with self.assertRaises(ValueError):
572573
str(i)
573574

575+
def test_denial_of_service_prevented_int_to_str(self):
576+
"""Regression test: ensure we fail before performing O(N**2) work."""
577+
maxdigits = sys.get_int_max_str_digits()
578+
assert maxdigits < 50_000, maxdigits # A test prerequisite.
579+
get_time = time.process_time
580+
if get_time() <= 0: # some platforms like WASM lack process_time()
581+
get_time = time.monotonic
582+
583+
huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits.
584+
digits = 78_268
585+
with support.adjust_int_max_str_digits(digits):
586+
start = get_time()
587+
huge_decimal = str(huge_int)
588+
seconds_to_convert = get_time() - start
589+
self.assertEqual(len(huge_decimal), digits)
590+
# Ensuring that we chose a slow enough conversion to measure.
591+
# It takes 0.1 seconds on a Zen based cloud VM in an opt build.
592+
if seconds_to_convert < 0.005:
593+
raise unittest.SkipTest('"slow" conversion took only '
594+
f'{seconds_to_convert} seconds.')
595+
596+
# We test with the limit almost at the size needed to check performance.
597+
# The performant limit check is slightly fuzzy, give it a some room.
598+
with support.adjust_int_max_str_digits(int(.995 * digits)):
599+
with self.assertRaises(ValueError) as err:
600+
start = get_time()
601+
str(huge_int)
602+
seconds_to_fail_huge = get_time() - start
603+
self.assertIn('conversion', str(err.exception))
604+
self.assertLess(seconds_to_fail_huge, seconds_to_convert/8)
605+
606+
# Now we test that a conversion that would take 30x as long also fails
607+
# in a similarly fast fashion.
608+
extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits.
609+
with self.assertRaises(ValueError) as err:
610+
start = get_time()
611+
# If not limited, 8 seconds said Zen based cloud VM.
612+
str(extra_huge_int)
613+
seconds_to_fail_extra_huge = get_time() - start
614+
self.assertIn('conversion', str(err.exception))
615+
self.assertLess(seconds_to_fail_extra_huge, seconds_to_convert/8)
616+
617+
def test_denial_of_service_prevented_str_to_int(self):
618+
"""Regression test: ensure we fail before performing O(N**2) work."""
619+
maxdigits = sys.get_int_max_str_digits()
620+
assert maxdigits < 100_000, maxdigits # A test prerequisite.
621+
get_time = time.process_time
622+
if get_time() <= 0: # some platforms like WASM lack process_time()
623+
get_time = time.monotonic
624+
625+
digits = 133700
626+
huge = '8'*digits
627+
with support.adjust_int_max_str_digits(digits):
628+
start = get_time()
629+
int(huge)
630+
seconds_to_convert = get_time() - start
631+
# Ensuring that we chose a slow enough conversion to measure.
632+
# It takes 0.1 seconds on a Zen based cloud VM in an opt build.
633+
if seconds_to_convert < 0.005:
634+
raise unittest.SkipTest('"slow" conversion took only '
635+
f'{seconds_to_convert} seconds.')
636+
637+
with support.adjust_int_max_str_digits(digits - 1):
638+
with self.assertRaises(ValueError) as err:
639+
start = get_time()
640+
int(huge)
641+
seconds_to_fail_huge = get_time() - start
642+
self.assertIn('conversion', str(err.exception))
643+
self.assertLess(seconds_to_fail_huge, seconds_to_convert/8)
644+
645+
# Now we test that a conversion that would take 30x as long also fails
646+
# in a similarly fast fashion.
647+
extra_huge = '7'*1_200_000
648+
with self.assertRaises(ValueError) as err:
649+
start = get_time()
650+
# If not limited, 8 seconds in the Zen based cloud VM.
651+
int(extra_huge)
652+
seconds_to_fail_extra_huge = get_time() - start
653+
self.assertIn('conversion', str(err.exception))
654+
self.assertLess(seconds_to_fail_extra_huge, seconds_to_convert/8)
655+
574656
def test_power_of_two_bases_unlimited(self):
575657
"""The limit does not apply to power of 2 bases."""
576658
maxdigits = sys.get_int_max_str_digits()

‎Misc/NEWS.d/next/Security/2022-08-07-16-53-38.gh-issue-95778.ch010gps.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@ limitation <int_max_str_digits>` documentation. The default limit is 4300
1111
digits in string form.
1212

1313
Patch by Gregory P. Smith [Google] and Christian Heimes [Red Hat] with feedback
14-
from Victor Stinner, Thomas Wouters, Steve Dower, and Ned Deily.
14+
from Victor Stinner, Thomas Wouters, Steve Dower, Ned Deily, and Mark Dickinson.

‎Objects/longobject.c‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ static PyLongObject small_ints[NSMALLNEGINTS + NSMALLPOSINTS];
4747
Py_ssize_t quick_int_allocs, quick_neg_int_allocs;
4848
#endif
4949

50-
#define _MAX_STR_DIGITS_ERROR_FMT "Exceeds the limit (%d) for integer string conversion: value has %zd digits"
50+
#define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d) for integer string conversion: value has %zd digits"
51+
#define _MAX_STR_DIGITS_ERROR_FMT_TO_STR "Exceeds the limit (%d) for integer string conversion"
5152

5253
static PyObject *
5354
get_small_int(sdigit ival)
@@ -1606,6 +1607,23 @@ long_to_decimal_string_internal(PyObject *aa,
16061607
size_a = Py_ABS(Py_SIZE(a));
16071608
negative = Py_SIZE(a) < 0;
16081609

1610+
/* quick and dirty pre-check for overflowing the decimal digit limit,
1611+
based on the inequality 10/3 >= log2(10)
1612+
1613+
explanation in https://github.com/python/cpython/pull/96537
1614+
*/
1615+
if (size_a >= 10 * _PY_LONG_MAX_STR_DIGITS_THRESHOLD
1616+
/ (3 * PyLong_SHIFT) + 2) {
1617+
PyInterpreterState *interp = _PyInterpreterState_GET();
1618+
int max_str_digits = interp->int_max_str_digits;
1619+
if ((max_str_digits > 0) &&
1620+
(max_str_digits / (3 * PyLong_SHIFT) <= (size_a - 11) / 10)) {
1621+
PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT_TO_STR,
1622+
max_str_digits);
1623+
return -1;
1624+
}
1625+
}
1626+
16091627
/* quick and dirty upper bound for the number of digits
16101628
required to express a in base _PyLong_DECIMAL_BASE:
16111629
@@ -1670,8 +1688,8 @@ long_to_decimal_string_internal(PyObject *aa,
16701688
Py_ssize_t strlen_nosign = strlen - negative;
16711689
if ((max_str_digits > 0) && (strlen_nosign > max_str_digits)) {
16721690
Py_DECREF(scratch);
1673-
PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT,
1674-
max_str_digits, strlen_nosign);
1691+
PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT_TO_STR,
1692+
max_str_digits);
16751693
return -1;
16761694
}
16771695
}
@@ -2344,7 +2362,7 @@ digit beyond the first.
23442362
if (digits > _PY_LONG_MAX_STR_DIGITS_THRESHOLD) {
23452363
int max_str_digits = _PyRuntime.int_max_str_digits;
23462364
if ((max_str_digits > 0) && (digits > max_str_digits)) {
2347-
PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT,
2365+
PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT_TO_INT,
23482366
max_str_digits, digits);
23492367
return NULL;
23502368
}

0 commit comments

Comments
 (0)