Skip to content

Commit db067af

Browse files
vstinnervstinner
committed
Issue python#21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(),
PyObject_Calloc(), _PyObject_GC_Calloc(). bytes(int) and bytearray(int) are now using ``calloc()`` instead of ``malloc()`` for large objects which is faster and use less memory (until the bytearray buffer is filled with data).
1 parent d50c3f3 commit db067af

11 files changed

Lines changed: 366 additions & 71 deletions

File tree

‎Doc/c-api/memory.rst‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ functions are thread-safe, the :term:`GIL <global interpreter lock>` does not
9292
need to be held.
9393

9494
The default raw memory block allocator uses the following functions:
95-
:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
96-
requesting zero bytes.
95+
:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
96+
``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
9797

9898
.. versionadded:: 3.4
9999

@@ -106,6 +106,17 @@ requesting zero bytes.
106106
been initialized in any way.
107107
108108
109+
.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize)
110+
111+
Allocates *nelem* elements each whose size in bytes is *elsize* and returns
112+
a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
113+
request fails. The memory is initialized to zeros. Requesting zero elements
114+
or elements of size zero bytes returns a distinct non-*NULL* pointer if
115+
possible, as if ``PyMem_RawCalloc(1, 1)`` had been called instead.
116+
117+
.. versionadded:: 3.5
118+
119+
109120
.. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
110121
111122
Resizes the memory block pointed to by *p* to *n* bytes. The contents will
@@ -136,8 +147,8 @@ behavior when requesting zero bytes, are available for allocating and releasing
136147
memory from the Python heap.
137148
138149
The default memory block allocator uses the following functions:
139-
:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
140-
requesting zero bytes.
150+
:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
151+
``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
141152
142153
.. warning::
143154
@@ -152,6 +163,17 @@ requesting zero bytes.
152163
been called instead. The memory will not have been initialized in any way.
153164
154165
166+
.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize)
167+
168+
Allocates *nelem* elements each whose size in bytes is *elsize* and returns
169+
a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
170+
request fails. The memory is initialized to zeros. Requesting zero elements
171+
or elements of size zero bytes returns a distinct non-*NULL* pointer if
172+
possible, as if ``PyMem_Calloc(1, 1)`` had been called instead.
173+
174+
.. versionadded:: 3.5
175+
176+
155177
.. c:function:: void* PyMem_Realloc(void *p, size_t n)
156178
157179
Resizes the memory block pointed to by *p* to *n* bytes. The contents will be
@@ -222,11 +244,17 @@ Customize Memory Allocators
222244
+----------------------------------------------------------+---------------------------------------+
223245
| ``void* malloc(void *ctx, size_t size)`` | allocate a memory block |
224246
+----------------------------------------------------------+---------------------------------------+
247+
| ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized |
248+
| | with zeros |
249+
+----------------------------------------------------------+---------------------------------------+
225250
| ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block |
226251
+----------------------------------------------------------+---------------------------------------+
227252
| ``void free(void *ctx, void *ptr)`` | free a memory block |
228253
+----------------------------------------------------------+---------------------------------------+
229254
255+
.. versionchanged:: 3.5
256+
Add a new field ``calloc``.
257+
230258
.. c:type:: PyMemAllocatorDomain
231259
232260
Enum used to identify an allocator domain. Domains:

‎Doc/whatsnew/3.5.rst‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,15 +164,23 @@ Optimizations
164164

165165
Major performance enhancements have been added:
166166

167-
* None yet.
167+
* Construction of ``bytes(int)`` and ``bytearray(int)`` (filled by zero bytes)
168+
is faster and use less memory (until the bytearray buffer is filled with
169+
data) for large objects. ``calloc()`` is used instead of ``malloc()`` to
170+
allocate memory for these objects.
168171

169172

170173
Build and C API Changes
171174
=======================
172175

173176
Changes to Python's build process and to the C API include:
174177

175-
* None yet.
178+
* New ``calloc`` functions:
179+
180+
* :c:func:`PyMem_RawCalloc`
181+
* :c:func:`PyMem_Calloc`
182+
* :c:func:`PyObject_Calloc`
183+
* :c:func:`_PyObject_GC_Calloc`
176184

177185

178186
Deprecated
@@ -209,6 +217,9 @@ Porting to Python 3.5
209217
This section lists previously described changes and other bugfixes
210218
that may require changes to your code.
211219

220+
Changes in the Python API
221+
-------------------------
222+
212223
* Before Python 3.5, a :class:`datetime.time` object was considered to be false
213224
if it represented midnight in UTC. This behavior was considered obscure and
214225
error-prone and has been removed in Python 3.5. See :issue:`13936` for full
@@ -217,3 +228,8 @@ that may require changes to your code.
217228
* :meth:`ssl.SSLSocket.send()` now raises either :exc:`ssl.SSLWantReadError`
218229
or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation
219230
would block. Previously, it would return 0. See :issue:`20951`.
231+
232+
Changes in the C API
233+
--------------------
234+
235+
* The :c:type:`PyMemAllocator` structure has a new ``calloc`` field.

‎Include/objimpl.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ PyObject_{New, NewVar, Del}.
9595
the raw memory.
9696
*/
9797
PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
98+
PyAPI_FUNC(void *) PyObject_Calloc(size_t nelem, size_t elsize);
9899
PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
99100
PyAPI_FUNC(void) PyObject_Free(void *ptr);
100101

@@ -321,7 +322,8 @@ extern PyGC_Head *_PyGC_generation0;
321322
(!PyTuple_CheckExact(obj) || _PyObject_GC_IS_TRACKED(obj)))
322323
#endif /* Py_LIMITED_API */
323324

324-
PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t);
325+
PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size);
326+
PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size);
325327
PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *);
326328
PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t);
327329
PyAPI_FUNC(void) PyObject_GC_Track(void *);

‎Include/pymem.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ extern "C" {
1313

1414
#ifndef Py_LIMITED_API
1515
PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
16+
PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
1617
PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
1718
PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
1819
#endif
@@ -57,6 +58,7 @@ PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
5758
*/
5859

5960
PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
61+
PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
6062
PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
6163
PyAPI_FUNC(void) PyMem_Free(void *ptr);
6264

@@ -132,6 +134,9 @@ typedef struct {
132134
/* allocate a memory block */
133135
void* (*malloc) (void *ctx, size_t size);
134136

137+
/* allocate a memory block initialized by zeros */
138+
void* (*calloc) (void *ctx, size_t nelem, size_t elsize);
139+
135140
/* allocate or resize a memory block */
136141
void* (*realloc) (void *ctx, void *ptr, size_t new_size);
137142

‎Misc/NEWS‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ Release date: TBA
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(),
14+
PyObject_Calloc(), _PyObject_GC_Calloc(). bytes(int) and bytearray(int)
15+
are now using ``calloc()`` instead of ``malloc()`` for large objects which
16+
is faster and use less memory (until the bytearray buffer is filled with
17+
data).
18+
1319
- Issue #21377: PyBytes_Concat() now tries to concatenate in-place when the
1420
first argument has a reference count of 1. Patch by Nikolaus Rath.
1521

‎Modules/_testcapimodule.c‎

Lines changed: 68 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2710,27 +2710,57 @@ test_pymem_alloc0(PyObject *self)
27102710
{
27112711
void *ptr;
27122712

2713+
ptr = PyMem_RawMalloc(0);
2714+
if (ptr == NULL) {
2715+
PyErr_SetString(PyExc_RuntimeError, "PyMem_RawMalloc(0) returns NULL");
2716+
return NULL;
2717+
}
2718+
PyMem_RawFree(ptr);
2719+
2720+
ptr = PyMem_RawCalloc(0, 0);
2721+
if (ptr == NULL) {
2722+
PyErr_SetString(PyExc_RuntimeError, "PyMem_RawCalloc(0, 0) returns NULL");
2723+
return NULL;
2724+
}
2725+
PyMem_RawFree(ptr);
2726+
27132727
ptr = PyMem_Malloc(0);
27142728
if (ptr == NULL) {
27152729
PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
27162730
return NULL;
27172731
}
27182732
PyMem_Free(ptr);
27192733

2734+
ptr = PyMem_Calloc(0, 0);
2735+
if (ptr == NULL) {
2736+
PyErr_SetString(PyExc_RuntimeError, "PyMem_Calloc(0, 0) returns NULL");
2737+
return NULL;
2738+
}
2739+
PyMem_Free(ptr);
2740+
27202741
ptr = PyObject_Malloc(0);
27212742
if (ptr == NULL) {
27222743
PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
27232744
return NULL;
27242745
}
27252746
PyObject_Free(ptr);
27262747

2748+
ptr = PyObject_Calloc(0, 0);
2749+
if (ptr == NULL) {
2750+
PyErr_SetString(PyExc_RuntimeError, "PyObject_Calloc(0, 0) returns NULL");
2751+
return NULL;
2752+
}
2753+
PyObject_Free(ptr);
2754+
27272755
Py_RETURN_NONE;
27282756
}
27292757

27302758
typedef struct {
27312759
PyMemAllocator alloc;
27322760

27332761
size_t malloc_size;
2762+
size_t calloc_nelem;
2763+
size_t calloc_elsize;
27342764
void *realloc_ptr;
27352765
size_t realloc_new_size;
27362766
void *free_ptr;
@@ -2743,6 +2773,14 @@ static void* hook_malloc (void* ctx, size_t size)
27432773
return hook->alloc.malloc(hook->alloc.ctx, size);
27442774
}
27452775

2776+
static void* hook_calloc (void* ctx, size_t nelem, size_t elsize)
2777+
{
2778+
alloc_hook_t *hook = (alloc_hook_t *)ctx;
2779+
hook->calloc_nelem = nelem;
2780+
hook->calloc_elsize = elsize;
2781+
return hook->alloc.calloc(hook->alloc.ctx, nelem, elsize);
2782+
}
2783+
27462784
static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
27472785
{
27482786
alloc_hook_t *hook = (alloc_hook_t *)ctx;
@@ -2765,16 +2803,14 @@ test_setallocators(PyMemAllocatorDomain domain)
27652803
const char *error_msg;
27662804
alloc_hook_t hook;
27672805
PyMemAllocator alloc;
2768-
size_t size, size2;
2806+
size_t size, size2, nelem, elsize;
27692807
void *ptr, *ptr2;
27702808

2771-
hook.malloc_size = 0;
2772-
hook.realloc_ptr = NULL;
2773-
hook.realloc_new_size = 0;
2774-
hook.free_ptr = NULL;
2809+
memset(&hook, 0, sizeof(hook));
27752810

27762811
alloc.ctx = &hook;
27772812
alloc.malloc = &hook_malloc;
2813+
alloc.calloc = &hook_calloc;
27782814
alloc.realloc = &hook_realloc;
27792815
alloc.free = &hook_free;
27802816
PyMem_GetAllocator(domain, &hook.alloc);
@@ -2831,6 +2867,33 @@ test_setallocators(PyMemAllocatorDomain domain)
28312867
goto fail;
28322868
}
28332869

2870+
nelem = 2;
2871+
elsize = 5;
2872+
switch(domain)
2873+
{
2874+
case PYMEM_DOMAIN_RAW: ptr = PyMem_RawCalloc(nelem, elsize); break;
2875+
case PYMEM_DOMAIN_MEM: ptr = PyMem_Calloc(nelem, elsize); break;
2876+
case PYMEM_DOMAIN_OBJ: ptr = PyObject_Calloc(nelem, elsize); break;
2877+
default: ptr = NULL; break;
2878+
}
2879+
2880+
if (ptr == NULL) {
2881+
error_msg = "calloc failed";
2882+
goto fail;
2883+
}
2884+
2885+
if (hook.calloc_nelem != nelem || hook.calloc_elsize != elsize) {
2886+
error_msg = "calloc invalid nelem or elsize";
2887+
goto fail;
2888+
}
2889+
2890+
switch(domain)
2891+
{
2892+
case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr); break;
2893+
case PYMEM_DOMAIN_MEM: PyMem_Free(ptr); break;
2894+
case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr); break;
2895+
}
2896+
28342897
Py_INCREF(Py_None);
28352898
res = Py_None;
28362899
goto finally;

0 commit comments

Comments
 (0)