Skip to content

Commit b13a20f

Browse files
serhiy-storchakaserhiy-storchaka
authored
bpo-5438: Update memory requirements and optimize test_bigmem.py. (pythonGH-11123)
1 parent 83dd4e8 commit b13a20f

1 file changed

Lines changed: 69 additions & 68 deletions

File tree

‎Lib/test/test_bigmem.py‎

Lines changed: 69 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
ascii_char_size = 1
6565
ucs2_char_size = 2
6666
ucs4_char_size = 4
67+
pointer_size = 4 if sys.maxsize < 2**32 else 8
6768

6869

6970
class BaseStrTest:
@@ -372,7 +373,7 @@ def test_split_small(self, size):
372373
# suffer for the list size. (Otherwise, it'd cost another 48 times
373374
# size in bytes!) Nevertheless, a list of size takes
374375
# 8*size bytes.
375-
@bigmemtest(size=_2G + 5, memuse=2 * ascii_char_size + 8)
376+
@bigmemtest(size=_2G + 5, memuse=ascii_char_size * 2 + pointer_size)
376377
def test_split_large(self, size):
377378
_ = self.from_latin1
378379
s = _(' a') * size + _(' ')
@@ -604,15 +605,15 @@ def tearDown(self):
604605
for name, memuse in self._adjusted.items():
605606
getattr(type(self), name).memuse = memuse
606607

607-
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
608+
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
608609
def test_capitalize(self, size):
609610
self._test_capitalize(size)
610611

611-
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
612+
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
612613
def test_title(self, size):
613614
self._test_title(size)
614615

615-
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
616+
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
616617
def test_swapcase(self, size):
617618
self._test_swapcase(size)
618619

@@ -630,7 +631,7 @@ def test_encode_raw_unicode_escape(self, size):
630631
except MemoryError:
631632
pass # acceptable on 32-bit
632633

633-
@bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + ucs4_char_size + 1)
634+
@bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + 8 + 1)
634635
def test_encode_utf7(self, size):
635636
try:
636637
return self.basic_encode_test(size, 'utf7')
@@ -820,7 +821,7 @@ class TupleTest(unittest.TestCase):
820821
# having more than 2<<31 references to any given object. Hence the
821822
# use of different types of objects as contents in different tests.
822823

823-
@bigmemtest(size=_2G + 2, memuse=16)
824+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2)
824825
def test_compare(self, size):
825826
t1 = ('',) * size
826827
t2 = ('',) * size
@@ -843,31 +844,31 @@ def basic_concat_test(self, size):
843844
t = t + t
844845
self.assertEqual(len(t), size * 2)
845846

846-
@bigmemtest(size=_2G // 2 + 2, memuse=24)
847+
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
847848
def test_concat_small(self, size):
848849
return self.basic_concat_test(size)
849850

850-
@bigmemtest(size=_2G + 2, memuse=24)
851+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
851852
def test_concat_large(self, size):
852853
return self.basic_concat_test(size)
853854

854-
@bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
855+
@bigmemtest(size=_2G // 5 + 10, memuse=pointer_size * 5)
855856
def test_contains(self, size):
856857
t = (1, 2, 3, 4, 5) * size
857858
self.assertEqual(len(t), size * 5)
858859
self.assertTrue(5 in t)
859860
self.assertFalse((1, 2, 3, 4, 5) in t)
860861
self.assertFalse(0 in t)
861862

862-
@bigmemtest(size=_2G + 10, memuse=8)
863+
@bigmemtest(size=_2G + 10, memuse=pointer_size)
863864
def test_hash(self, size):
864865
t1 = (0,) * size
865866
h1 = hash(t1)
866867
del t1
867868
t2 = (0,) * (size + 1)
868869
self.assertFalse(h1 == hash(t2))
869870

870-
@bigmemtest(size=_2G + 10, memuse=8)
871+
@bigmemtest(size=_2G + 10, memuse=pointer_size)
871872
def test_index_and_slice(self, size):
872873
t = (None,) * size
873874
self.assertEqual(len(t), size)
@@ -892,60 +893,54 @@ def basic_test_repeat(self, size):
892893
t = t * 2
893894
self.assertEqual(len(t), size * 2)
894895

895-
@bigmemtest(size=_2G // 2 + 2, memuse=24)
896+
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
896897
def test_repeat_small(self, size):
897898
return self.basic_test_repeat(size)
898899

899-
@bigmemtest(size=_2G + 2, memuse=24)
900+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
900901
def test_repeat_large(self, size):
901902
return self.basic_test_repeat(size)
902903

903904
@bigmemtest(size=_1G - 1, memuse=12)
904905
def test_repeat_large_2(self, size):
905906
return self.basic_test_repeat(size)
906907

907-
@bigmemtest(size=_1G - 1, memuse=9)
908+
@bigmemtest(size=_1G - 1, memuse=pointer_size * 2)
908909
def test_from_2G_generator(self, size):
909-
self.skipTest("test needs much more memory than advertised, see issue5438")
910910
try:
911-
t = tuple(range(size))
911+
t = tuple(iter([42]*size))
912912
except MemoryError:
913913
pass # acceptable on 32-bit
914914
else:
915-
count = 0
916-
for item in t:
917-
self.assertEqual(item, count)
918-
count += 1
919-
self.assertEqual(count, size)
915+
self.assertEqual(len(t), size)
916+
self.assertEqual(t[:10], (42,) * 10)
917+
self.assertEqual(t[-10:], (42,) * 10)
920918

921-
@bigmemtest(size=_1G - 25, memuse=9)
919+
@bigmemtest(size=_1G - 25, memuse=pointer_size * 2)
922920
def test_from_almost_2G_generator(self, size):
923-
self.skipTest("test needs much more memory than advertised, see issue5438")
924921
try:
925-
t = tuple(range(size))
926-
count = 0
927-
for item in t:
928-
self.assertEqual(item, count)
929-
count += 1
930-
self.assertEqual(count, size)
922+
t = tuple(iter([42]*size))
931923
except MemoryError:
932-
pass # acceptable, expected on 32-bit
924+
pass # acceptable on 32-bit
925+
else:
926+
self.assertEqual(len(t), size)
927+
self.assertEqual(t[:10], (42,) * 10)
928+
self.assertEqual(t[-10:], (42,) * 10)
933929

934930
# Like test_concat, split in two.
935931
def basic_test_repr(self, size):
936-
t = (0,) * size
932+
t = (False,) * size
937933
s = repr(t)
938-
# The repr of a tuple of 0's is exactly three times the tuple length.
939-
self.assertEqual(len(s), size * 3)
940-
self.assertEqual(s[:5], '(0, 0')
941-
self.assertEqual(s[-5:], '0, 0)')
942-
self.assertEqual(s.count('0'), size)
934+
# The repr of a tuple of Falses is exactly 7 times the tuple length.
935+
self.assertEqual(len(s), size * 7)
936+
self.assertEqual(s[:10], '(False, Fa')
937+
self.assertEqual(s[-10:], 'se, False)')
943938

944-
@bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
939+
@bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
945940
def test_repr_small(self, size):
946941
return self.basic_test_repr(size)
947942

948-
@bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
943+
@bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
949944
def test_repr_large(self, size):
950945
return self.basic_test_repr(size)
951946

@@ -956,7 +951,7 @@ class ListTest(unittest.TestCase):
956951
# lists hold references to various objects to test their refcount
957952
# limits.
958953

959-
@bigmemtest(size=_2G + 2, memuse=16)
954+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2)
960955
def test_compare(self, size):
961956
l1 = [''] * size
962957
l2 = [''] * size
@@ -979,43 +974,45 @@ def basic_test_concat(self, size):
979974
l = l + l
980975
self.assertEqual(len(l), size * 2)
981976

982-
@bigmemtest(size=_2G // 2 + 2, memuse=24)
977+
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
983978
def test_concat_small(self, size):
984979
return self.basic_test_concat(size)
985980

986-
@bigmemtest(size=_2G + 2, memuse=24)
981+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
987982
def test_concat_large(self, size):
988983
return self.basic_test_concat(size)
989984

985+
# XXX This tests suffers from overallocation, just like test_append.
986+
# This should be fixed in future.
990987
def basic_test_inplace_concat(self, size):
991988
l = [sys.stdout] * size
992989
l += l
993990
self.assertEqual(len(l), size * 2)
994991
self.assertTrue(l[0] is l[-1])
995992
self.assertTrue(l[size - 1] is l[size + 1])
996993

997-
@bigmemtest(size=_2G // 2 + 2, memuse=24)
994+
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
998995
def test_inplace_concat_small(self, size):
999996
return self.basic_test_inplace_concat(size)
1000997

1001-
@bigmemtest(size=_2G + 2, memuse=24)
998+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
1002999
def test_inplace_concat_large(self, size):
10031000
return self.basic_test_inplace_concat(size)
10041001

1005-
@bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
1002+
@bigmemtest(size=_2G // 5 + 10, memuse=pointer_size * 5)
10061003
def test_contains(self, size):
10071004
l = [1, 2, 3, 4, 5] * size
10081005
self.assertEqual(len(l), size * 5)
10091006
self.assertTrue(5 in l)
10101007
self.assertFalse([1, 2, 3, 4, 5] in l)
10111008
self.assertFalse(0 in l)
10121009

1013-
@bigmemtest(size=_2G + 10, memuse=8)
1010+
@bigmemtest(size=_2G + 10, memuse=pointer_size)
10141011
def test_hash(self, size):
10151012
l = [0] * size
10161013
self.assertRaises(TypeError, hash, l)
10171014

1018-
@bigmemtest(size=_2G + 10, memuse=8)
1015+
@bigmemtest(size=_2G + 10, memuse=pointer_size)
10191016
def test_index_and_slice(self, size):
10201017
l = [None] * size
10211018
self.assertEqual(len(l), size)
@@ -1079,14 +1076,16 @@ def basic_test_repeat(self, size):
10791076
l = l * 2
10801077
self.assertEqual(len(l), size * 2)
10811078

1082-
@bigmemtest(size=_2G // 2 + 2, memuse=24)
1079+
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
10831080
def test_repeat_small(self, size):
10841081
return self.basic_test_repeat(size)
10851082

1086-
@bigmemtest(size=_2G + 2, memuse=24)
1083+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
10871084
def test_repeat_large(self, size):
10881085
return self.basic_test_repeat(size)
10891086

1087+
# XXX This tests suffers from overallocation, just like test_append.
1088+
# This should be fixed in future.
10901089
def basic_test_inplace_repeat(self, size):
10911090
l = ['']
10921091
l *= size
@@ -1099,63 +1098,65 @@ def basic_test_inplace_repeat(self, size):
10991098
self.assertEqual(len(l), size * 2)
11001099
self.assertTrue(l[size - 1] is l[-1])
11011100

1102-
@bigmemtest(size=_2G // 2 + 2, memuse=16)
1101+
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
11031102
def test_inplace_repeat_small(self, size):
11041103
return self.basic_test_inplace_repeat(size)
11051104

1106-
@bigmemtest(size=_2G + 2, memuse=16)
1105+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
11071106
def test_inplace_repeat_large(self, size):
11081107
return self.basic_test_inplace_repeat(size)
11091108

11101109
def basic_test_repr(self, size):
1111-
l = [0] * size
1110+
l = [False] * size
11121111
s = repr(l)
1113-
# The repr of a list of 0's is exactly three times the list length.
1114-
self.assertEqual(len(s), size * 3)
1115-
self.assertEqual(s[:5], '[0, 0')
1116-
self.assertEqual(s[-5:], '0, 0]')
1117-
self.assertEqual(s.count('0'), size)
1112+
# The repr of a list of Falses is exactly 7 times the list length.
1113+
self.assertEqual(len(s), size * 7)
1114+
self.assertEqual(s[:10], '[False, Fa')
1115+
self.assertEqual(s[-10:], 'se, False]')
1116+
self.assertEqual(s.count('F'), size)
11181117

1119-
@bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
1118+
@bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
11201119
def test_repr_small(self, size):
11211120
return self.basic_test_repr(size)
11221121

1123-
@bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
1122+
@bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
11241123
def test_repr_large(self, size):
11251124
return self.basic_test_repr(size)
11261125

11271126
# list overallocates ~1/8th of the total size (on first expansion) so
11281127
# the single list.append call puts memuse at 9 bytes per size.
1129-
@bigmemtest(size=_2G, memuse=9)
1128+
@bigmemtest(size=_2G, memuse=pointer_size * 9/8)
11301129
def test_append(self, size):
11311130
l = [object()] * size
11321131
l.append(object())
11331132
self.assertEqual(len(l), size+1)
11341133
self.assertTrue(l[-3] is l[-2])
11351134
self.assertFalse(l[-2] is l[-1])
11361135

1137-
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
1136+
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
11381137
def test_count(self, size):
11391138
l = [1, 2, 3, 4, 5] * size
11401139
self.assertEqual(l.count(1), size)
11411140
self.assertEqual(l.count("1"), 0)
11421141

1142+
# XXX This tests suffers from overallocation, just like test_append.
1143+
# This should be fixed in future.
11431144
def basic_test_extend(self, size):
11441145
l = [object] * size
11451146
l.extend(l)
11461147
self.assertEqual(len(l), size * 2)
11471148
self.assertTrue(l[0] is l[-1])
11481149
self.assertTrue(l[size - 1] is l[size + 1])
11491150

1150-
@bigmemtest(size=_2G // 2 + 2, memuse=16)
1151+
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
11511152
def test_extend_small(self, size):
11521153
return self.basic_test_extend(size)
11531154

1154-
@bigmemtest(size=_2G + 2, memuse=16)
1155+
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
11551156
def test_extend_large(self, size):
11561157
return self.basic_test_extend(size)
11571158

1158-
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
1159+
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
11591160
def test_index(self, size):
11601161
l = [1, 2, 3, 4, 5] * size
11611162
size *= 5
@@ -1166,7 +1167,7 @@ def test_index(self, size):
11661167
self.assertRaises(ValueError, l.index, 6)
11671168

11681169
# This tests suffers from overallocation, just like test_append.
1169-
@bigmemtest(size=_2G + 10, memuse=9)
1170+
@bigmemtest(size=_2G + 10, memuse=pointer_size * 9/8)
11701171
def test_insert(self, size):
11711172
l = [1.0] * size
11721173
l.insert(size - 1, "A")
@@ -1185,7 +1186,7 @@ def test_insert(self, size):
11851186
self.assertEqual(l[:3], [1.0, "C", 1.0])
11861187
self.assertEqual(l[size - 3:], ["A", 1.0, "B"])
11871188

1188-
@bigmemtest(size=_2G // 5 + 4, memuse=8 * 5)
1189+
@bigmemtest(size=_2G // 5 + 4, memuse=pointer_size * 5)
11891190
def test_pop(self, size):
11901191
l = ["a", "b", "c", "d", "e"] * size
11911192
size *= 5
@@ -1209,7 +1210,7 @@ def test_pop(self, size):
12091210
self.assertEqual(item, "c")
12101211
self.assertEqual(l[-2:], ["b", "d"])
12111212

1212-
@bigmemtest(size=_2G + 10, memuse=8)
1213+
@bigmemtest(size=_2G + 10, memuse=pointer_size)
12131214
def test_remove(self, size):
12141215
l = [10] * size
12151216
self.assertEqual(len(l), size)
@@ -1229,15 +1230,15 @@ def test_remove(self, size):
12291230
self.assertEqual(len(l), size)
12301231
self.assertEqual(l[-2:], [10, 10])
12311232

1232-
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
1233+
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
12331234
def test_reverse(self, size):
12341235
l = [1, 2, 3, 4, 5] * size
12351236
l.reverse()
12361237
self.assertEqual(len(l), size * 5)
12371238
self.assertEqual(l[-5:], [5, 4, 3, 2, 1])
12381239
self.assertEqual(l[:5], [5, 4, 3, 2, 1])
12391240

1240-
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
1241+
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5 * 1.5)
12411242
def test_sort(self, size):
12421243
l = [1, 2, 3, 4, 5] * size
12431244
l.sort()

0 commit comments

Comments
 (0)