Skip to content

bpo-5438: Update memory requirements and optimize test_bigmem.py. #11123

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
serhiy-storchaka merged 1 commit into from
Dec 20, 2018
Merged

bpo-5438: Update memory requirements and optimize test_bigmem.py. #11123

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 69 additions & 68 deletions Lib/test/test_bigmem.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
ascii_char_size = 1
ucs2_char_size = 2
ucs4_char_size = 4
pointer_size = 4 if sys.maxsize < 2**32 else 8


class BaseStrTest:
Expand Down Expand Up @@ -372,7 +373,7 @@ def test_split_small(self, size):
# suffer for the list size. (Otherwise, it'd cost another 48 times
# size in bytes!) Nevertheless, a list of size takes
# 8*size bytes.
@bigmemtest(size=_2G + 5, memuse=2 * ascii_char_size + 8)
@bigmemtest(size=_2G + 5, memuse=ascii_char_size * 2 + pointer_size)
def test_split_large(self, size):
_ = self.from_latin1
s = _(' a') * size + _(' ')
Expand Down Expand Up @@ -604,15 +605,15 @@ def tearDown(self):
for name, memuse in self._adjusted.items():
getattr(type(self), name).memuse = memuse

@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
def test_capitalize(self, size):
self._test_capitalize(size)

@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
def test_title(self, size):
self._test_title(size)

@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
def test_swapcase(self, size):
self._test_swapcase(size)

Expand All @@ -630,7 +631,7 @@ def test_encode_raw_unicode_escape(self, size):
except MemoryError:
pass # acceptable on 32-bit

@bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + ucs4_char_size + 1)
@bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + 8 + 1)
def test_encode_utf7(self, size):
try:
return self.basic_encode_test(size, 'utf7')
Expand Down Expand Up @@ -820,7 +821,7 @@ class TupleTest(unittest.TestCase):
# having more than 2<<31 references to any given object. Hence the
# use of different types of objects as contents in different tests.

@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2)
def test_compare(self, size):
t1 = ('',) * size
t2 = ('',) * size
Expand All @@ -843,31 +844,31 @@ def basic_concat_test(self, size):
t = t + t
self.assertEqual(len(t), size * 2)

@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_concat_small(self, size):
return self.basic_concat_test(size)

@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_concat_large(self, size):
return self.basic_concat_test(size)

@bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 10, memuse=pointer_size * 5)
def test_contains(self, size):
t = (1, 2, 3, 4, 5) * size
self.assertEqual(len(t), size * 5)
self.assertTrue(5 in t)
self.assertFalse((1, 2, 3, 4, 5) in t)
self.assertFalse(0 in t)

@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_hash(self, size):
t1 = (0,) * size
h1 = hash(t1)
del t1
t2 = (0,) * (size + 1)
self.assertFalse(h1 == hash(t2))

@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_index_and_slice(self, size):
t = (None,) * size
self.assertEqual(len(t), size)
Expand All @@ -892,60 +893,54 @@ def basic_test_repeat(self, size):
t = t * 2
self.assertEqual(len(t), size * 2)

@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_repeat_small(self, size):
return self.basic_test_repeat(size)

@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_repeat_large(self, size):
return self.basic_test_repeat(size)

@bigmemtest(size=_1G - 1, memuse=12)
def test_repeat_large_2(self, size):
return self.basic_test_repeat(size)

@bigmemtest(size=_1G - 1, memuse=9)
@bigmemtest(size=_1G - 1, memuse=pointer_size * 2)
def test_from_2G_generator(self, size):
self.skipTest("test needs much more memory than advertised, see issue5438")
try:
t = tuple(range(size))
t = tuple(iter([42]*size))
except MemoryError:
pass # acceptable on 32-bit
else:
count = 0
for item in t:
self.assertEqual(item, count)
count += 1
self.assertEqual(count, size)
self.assertEqual(len(t), size)
self.assertEqual(t[:10], (42,) * 10)
self.assertEqual(t[-10:], (42,) * 10)

@bigmemtest(size=_1G - 25, memuse=9)
@bigmemtest(size=_1G - 25, memuse=pointer_size * 2)
def test_from_almost_2G_generator(self, size):
self.skipTest("test needs much more memory than advertised, see issue5438")
try:
t = tuple(range(size))
count = 0
for item in t:
self.assertEqual(item, count)
count += 1
self.assertEqual(count, size)
t = tuple(iter([42]*size))
except MemoryError:
pass # acceptable, expected on 32-bit
pass # acceptable on 32-bit
else:
self.assertEqual(len(t), size)
self.assertEqual(t[:10], (42,) * 10)
self.assertEqual(t[-10:], (42,) * 10)

# Like test_concat, split in two.
def basic_test_repr(self, size):
t = (0,) * size
t = (False,) * size
s = repr(t)
# The repr of a tuple of 0's is exactly three times the tuple length.
self.assertEqual(len(s), size * 3)
self.assertEqual(s[:5], '(0, 0')
self.assertEqual(s[-5:], '0, 0)')
self.assertEqual(s.count('0'), size)
# The repr of a tuple of Falses is exactly 7 times the tuple length.
self.assertEqual(len(s), size * 7)
self.assertEqual(s[:10], '(False, Fa')
self.assertEqual(s[-10:], 'se, False)')

@bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_small(self, size):
return self.basic_test_repr(size)

@bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_large(self, size):
return self.basic_test_repr(size)

Expand All @@ -956,7 +951,7 @@ class ListTest(unittest.TestCase):
# lists hold references to various objects to test their refcount
# limits.

@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2)
def test_compare(self, size):
l1 = [''] * size
l2 = [''] * size
Expand All @@ -979,43 +974,45 @@ def basic_test_concat(self, size):
l = l + l
self.assertEqual(len(l), size * 2)

@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_concat_small(self, size):
return self.basic_test_concat(size)

@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_concat_large(self, size):
return self.basic_test_concat(size)

# XXX This tests suffers from overallocation, just like test_append.
# This should be fixed in future.
def basic_test_inplace_concat(self, size):
l = [sys.stdout] * size
l += l
self.assertEqual(len(l), size * 2)
self.assertTrue(l[0] is l[-1])
self.assertTrue(l[size - 1] is l[size + 1])

@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_concat_small(self, size):
return self.basic_test_inplace_concat(size)

@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_concat_large(self, size):
return self.basic_test_inplace_concat(size)

@bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 10, memuse=pointer_size * 5)
def test_contains(self, size):
l = [1, 2, 3, 4, 5] * size
self.assertEqual(len(l), size * 5)
self.assertTrue(5 in l)
self.assertFalse([1, 2, 3, 4, 5] in l)
self.assertFalse(0 in l)

@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_hash(self, size):
l = [0] * size
self.assertRaises(TypeError, hash, l)

@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_index_and_slice(self, size):
l = [None] * size
self.assertEqual(len(l), size)
Expand Down Expand Up @@ -1079,14 +1076,16 @@ def basic_test_repeat(self, size):
l = l * 2
self.assertEqual(len(l), size * 2)

@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_repeat_small(self, size):
return self.basic_test_repeat(size)

@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_repeat_large(self, size):
return self.basic_test_repeat(size)

# XXX This tests suffers from overallocation, just like test_append.
# This should be fixed in future.
def basic_test_inplace_repeat(self, size):
l = ['']
l *= size
Expand All @@ -1099,63 +1098,65 @@ def basic_test_inplace_repeat(self, size):
self.assertEqual(len(l), size * 2)
self.assertTrue(l[size - 1] is l[-1])

@bigmemtest(size=_2G // 2 + 2, memuse=16)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_repeat_small(self, size):
return self.basic_test_inplace_repeat(size)

@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_repeat_large(self, size):
return self.basic_test_inplace_repeat(size)

def basic_test_repr(self, size):
l = [0] * size
l = [False] * size
s = repr(l)
# The repr of a list of 0's is exactly three times the list length.
self.assertEqual(len(s), size * 3)
self.assertEqual(s[:5], '[0, 0')
self.assertEqual(s[-5:], '0, 0]')
self.assertEqual(s.count('0'), size)
# The repr of a list of Falses is exactly 7 times the list length.
self.assertEqual(len(s), size * 7)
self.assertEqual(s[:10], '[False, Fa')
self.assertEqual(s[-10:], 'se, False]')
self.assertEqual(s.count('F'), size)

@bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_small(self, size):
return self.basic_test_repr(size)

@bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_large(self, size):
return self.basic_test_repr(size)

# list overallocates ~1/8th of the total size (on first expansion) so
# the single list.append call puts memuse at 9 bytes per size.
@bigmemtest(size=_2G, memuse=9)
@bigmemtest(size=_2G, memuse=pointer_size * 9/8)
def test_append(self, size):
l = [object()] * size
l.append(object())
self.assertEqual(len(l), size+1)
self.assertTrue(l[-3] is l[-2])
self.assertFalse(l[-2] is l[-1])

@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
def test_count(self, size):
l = [1, 2, 3, 4, 5] * size
self.assertEqual(l.count(1), size)
self.assertEqual(l.count("1"), 0)

# XXX This tests suffers from overallocation, just like test_append.
# This should be fixed in future.
def basic_test_extend(self, size):
l = [object] * size
l.extend(l)
self.assertEqual(len(l), size * 2)
self.assertTrue(l[0] is l[-1])
self.assertTrue(l[size - 1] is l[size + 1])

@bigmemtest(size=_2G // 2 + 2, memuse=16)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
def test_extend_small(self, size):
return self.basic_test_extend(size)

@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
def test_extend_large(self, size):
return self.basic_test_extend(size)

@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
def test_index(self, size):
l = [1, 2, 3, 4, 5] * size
size *= 5
Expand All @@ -1166,7 +1167,7 @@ def test_index(self, size):
self.assertRaises(ValueError, l.index, 6)

# This tests suffers from overallocation, just like test_append.
@bigmemtest(size=_2G + 10, memuse=9)
@bigmemtest(size=_2G + 10, memuse=pointer_size * 9/8)
def test_insert(self, size):
l = [1.0] * size
l.insert(size - 1, "A")
Expand All @@ -1185,7 +1186,7 @@ def test_insert(self, size):
self.assertEqual(l[:3], [1.0, "C", 1.0])
self.assertEqual(l[size - 3:], ["A", 1.0, "B"])

@bigmemtest(size=_2G // 5 + 4, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 4, memuse=pointer_size * 5)
def test_pop(self, size):
l = ["a", "b", "c", "d", "e"] * size
size *= 5
Expand All @@ -1209,7 +1210,7 @@ def test_pop(self, size):
self.assertEqual(item, "c")
self.assertEqual(l[-2:], ["b", "d"])

@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_remove(self, size):
l = [10] * size
self.assertEqual(len(l), size)
Expand All @@ -1229,15 +1230,15 @@ def test_remove(self, size):
self.assertEqual(len(l), size)
self.assertEqual(l[-2:], [10, 10])

@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
def test_reverse(self, size):
l = [1, 2, 3, 4, 5] * size
l.reverse()
self.assertEqual(len(l), size * 5)
self.assertEqual(l[-5:], [5, 4, 3, 2, 1])
self.assertEqual(l[:5], [5, 4, 3, 2, 1])

@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5 * 1.5)
def test_sort(self, size):
l = [1, 2, 3, 4, 5] * size
l.sort()
Expand Down