Skip to content

Commit 41bbdec

Browse files
author
christian.heimes
committed
Patch #1953
I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers. The patch also renames sys._cleartypecache to sys._clear_type_cache git-svn-id: http://svn.python.org/projects/python/trunk@60567 6015fed2-1504-0410-9fe1-9d1591cc4771
1 parent e777e55 commit 41bbdec

File tree

11 files changed

+160
-42
lines changed

11 files changed

+160
-42
lines changed

‎Doc/c-api/float.rst‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,3 +84,12 @@ Floating Point Objects
8484
Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`.
8585

8686
.. versionadded:: 2.6
87+
88+
89+
.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
90+
91+
Compact the float free list. *bc* is the number of allocated blocks before
92+
blocks are freed, *bf* is the number of freed blocks and *sum* is the number
93+
of remaining objects in the blocks.
94+
95+
.. versionadded:: 2.6

‎Doc/c-api/int.rst‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,3 +120,12 @@ Plain Integer Objects
120120

121121
Return the system's idea of the largest integer it can handle
122122
(:const:`LONG_MAX`, as defined in the system header files).
123+
124+
125+
.. cfunction:: void PyInt_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
126+
127+
Compact the integer free list. *bc* is the number of allocated blocks before
128+
blocks are freed, *bf* is the number of freed blocks and *sum* is the number
129+
of remaining objects in the blocks.
130+
131+
.. versionadded:: 2.6

‎Doc/library/sys.rst‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,29 @@ always available.
5858
A string containing the copyright pertaining to the Python interpreter.
5959

6060

61-
.. function:: _cleartypecache()
61+
.. function:: _compact_freelists()
6262

63-
Clear the internal type lookup cache.
63+
Compact the free lists of integers and floats by deallocating unused blocks.
64+
It can reduce the memory usage of the Python process several tenth of
65+
thousands of integers or floats have been allocated at once.
66+
67+
The return value is a tuple of tuples each containing three elements,
68+
amount of used objects, total block count before the blocks are deallocated
69+
and amount of freed blocks. The first tuple refers to ints, the second to
70+
floats.
71+
72+
This function should be used for specialized purposes only.
73+
74+
.. versionadded:: 2.6
75+
76+
77+
.. function:: _clear_type_cache()
78+
79+
Clear the internal type cache. The type cache is used to speed up attribute
80+
and method lookups. Use the function *only* to drop unnecessary references
81+
during reference leak debugging.
82+
83+
This function should be used for internal and specialized purposes only.
6484

6585
.. versionadded:: 2.6
6686

‎Include/floatobject.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void);
101101
PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
102102
PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
103103

104+
/* free list api */
105+
PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *);
104106

105107
#ifdef __cplusplus
106108
}

‎Include/intobject.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ PyAPI_FUNC(long) PyInt_GetMax(void);
5959
PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int);
6060
PyAPI_FUNC(long) PyOS_strtol(char *, char **, int);
6161

62+
/* free list api */
63+
PyAPI_FUNC(void) PyInt_CompactFreeList(size_t *, size_t *, size_t *);
64+
6265
#ifdef __cplusplus
6366
}
6467
#endif

‎Lib/test/regrtest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -710,7 +710,7 @@ def dash_R_cleanup(fs, ps, pic, abcs):
710710
sys.path_importer_cache.update(pic)
711711

712712
# clear type cache
713-
sys._cleartypecache()
713+
sys._clear_type_cache()
714714

715715
# Clear ABC registries, restoring previously saved ABC registries.
716716
for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:

‎Lib/test/test_sys.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,24 @@ def test_sys_flags(self):
363363
self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
364364
self.assert_(repr(sys.flags))
365365

366+
def test_clear_type_cache(self):
367+
sys._clear_type_cache()
368+
369+
def test_compact_freelists(self):
370+
sys._compact_freelists()
371+
r = sys._compact_freelists()
372+
# freed blocks shouldn't change
373+
self.assertEqual(r[0][2], 0)
374+
self.assertEqual(r[1][2], 0)
375+
# fill freelists
376+
ints = list(range(12000))
377+
floats = [float(i) for i in ints]
378+
del ints
379+
del floats
380+
# should free more than 200 blocks each
381+
r = sys._compact_freelists()
382+
self.assert_(r[0][2] > 200, r[0][2])
383+
self.assert_(r[1][2] > 200, r[1][2])
366384

367385
def test_main():
368386
test.test_support.run_unittest(SysModuleTest)

‎Misc/NEWS‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1?
1212
Core and builtins
1313
-----------------
1414

15+
- Patch #1953: Added ´´sys._compact_freelists()´´ and the C API functions
16+
´´PyInt_CompactFreeList´´ and ´´PyFloat_CompactFreeList´´
17+
to compact the internal free lists of pre-allocted ints and floats.
18+
1519
- Bug #1983: Fixed return type of fork(), fork1() and forkpty() calls.
1620
Python expected the return type int but the fork familie returns pi_t.
1721

@@ -21,7 +25,7 @@ Core and builtins
2125
- Patch #1970 by Antoine Pitrou: Speedup unicode whitespace and linebreak
2226
detection
2327

24-
- Added ``PyType_ClearCache()`` and ``sys._cleartypecache`` to clear the
28+
- Added ``PyType_ClearCache()`` and ``sys._clear_type_cache`` to clear the
2529
internal lookup cache for ref leak tests.
2630

2731
- Patch #1473257: generator objects gain a gi_code attribute. This is the

‎Objects/floatobject.c‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1609,17 +1609,15 @@ _PyFloat_Init(void)
16091609
}
16101610

16111611
void
1612-
PyFloat_Fini(void)
1612+
PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
16131613
{
16141614
PyFloatObject *p;
16151615
PyFloatBlock *list, *next;
16161616
unsigned i;
1617-
int bc, bf; /* block count, number of freed blocks */
1618-
int frem, fsum; /* remaining unfreed floats per block, total */
1617+
size_t bc = 0, bf = 0; /* block count, number of freed blocks */
1618+
size_t fsum = 0; /* total unfreed ints */
1619+
int frem; /* remaining unfreed ints per block */
16191620

1620-
bc = 0;
1621-
bf = 0;
1622-
fsum = 0;
16231621
list = block_list;
16241622
block_list = NULL;
16251623
free_list = NULL;
@@ -1654,6 +1652,22 @@ PyFloat_Fini(void)
16541652
fsum += frem;
16551653
list = next;
16561654
}
1655+
*pbc = bc;
1656+
*pbf = bf;
1657+
*bsum = fsum;
1658+
}
1659+
1660+
void
1661+
PyFloat_Fini(void)
1662+
{
1663+
PyFloatObject *p;
1664+
PyFloatBlock *list;
1665+
unsigned i;
1666+
size_t bc, bf; /* block count, number of freed blocks */
1667+
size_t fsum; /* total unfreed floats per block */
1668+
1669+
PyFloat_CompactFreeList(&bc, &bf, &fsum);
1670+
16571671
if (!Py_VerboseFlag)
16581672
return;
16591673
fprintf(stderr, "# cleanup floats");
@@ -1662,7 +1676,9 @@ PyFloat_Fini(void)
16621676
}
16631677
else {
16641678
fprintf(stderr,
1665-
": %d unfreed float%s in %d out of %d block%s\n",
1679+
": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %"
1680+
PY_FORMAT_SIZE_T "d out of %"
1681+
PY_FORMAT_SIZE_T "d block%s\n",
16661682
fsum, fsum == 1 ? "" : "s",
16671683
bc - bf, bc, bc == 1 ? "" : "s");
16681684
}

‎Objects/intobject.c‎

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,28 +1202,15 @@ _PyInt_Init(void)
12021202
}
12031203

12041204
void
1205-
PyInt_Fini(void)
1205+
PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
12061206
{
12071207
PyIntObject *p;
12081208
PyIntBlock *list, *next;
1209-
int i;
12101209
unsigned int ctr;
1211-
int bc, bf; /* block count, number of freed blocks */
1212-
int irem, isum; /* remaining unfreed ints per block, total */
1210+
size_t bc = 0, bf = 0; /* block count, number of freed blocks */
1211+
size_t isum = 0; /* total unfreed ints */
1212+
int irem; /* remaining unfreed ints per block */
12131213

1214-
#if NSMALLNEGINTS + NSMALLPOSINTS > 0
1215-
PyIntObject **q;
1216-
1217-
i = NSMALLNEGINTS + NSMALLPOSINTS;
1218-
q = small_ints;
1219-
while (--i >= 0) {
1220-
Py_XDECREF(*q);
1221-
*q++ = NULL;
1222-
}
1223-
#endif
1224-
bc = 0;
1225-
bf = 0;
1226-
isum = 0;
12271214
list = block_list;
12281215
block_list = NULL;
12291216
free_list = NULL;
@@ -1268,6 +1255,33 @@ PyInt_Fini(void)
12681255
isum += irem;
12691256
list = next;
12701257
}
1258+
1259+
*pbc = bc;
1260+
*pbf = bf;
1261+
*bsum = isum;
1262+
}
1263+
1264+
void
1265+
PyInt_Fini(void)
1266+
{
1267+
PyIntObject *p;
1268+
PyIntBlock *list;
1269+
unsigned int ctr;
1270+
size_t bc, bf; /* block count, number of freed blocks */
1271+
size_t isum; /* total unfreed ints per block */
1272+
1273+
#if NSMALLNEGINTS + NSMALLPOSINTS > 0
1274+
int i;
1275+
PyIntObject **q;
1276+
1277+
i = NSMALLNEGINTS + NSMALLPOSINTS;
1278+
q = small_ints;
1279+
while (--i >= 0) {
1280+
Py_XDECREF(*q);
1281+
*q++ = NULL;
1282+
}
1283+
#endif
1284+
PyInt_CompactFreeList(&bc, &bf, &isum);
12711285
if (!Py_VerboseFlag)
12721286
return;
12731287
fprintf(stderr, "# cleanup ints");
@@ -1276,7 +1290,9 @@ PyInt_Fini(void)
12761290
}
12771291
else {
12781292
fprintf(stderr,
1279-
": %d unfreed int%s in %d out of %d block%s\n",
1293+
": %" PY_FORMAT_SIZE_T "d unfreed ints%s in %"
1294+
PY_FORMAT_SIZE_T "d out of %"
1295+
PY_FORMAT_SIZE_T "d block%s\n",
12801296
isum, isum == 1 ? "" : "s",
12811297
bc - bf, bc, bc == 1 ? "" : "s");
12821298
}

0 commit comments

Comments
 (0)