/* Python's malloc wrappers (see pymem.h) */

#include "Python.h"

#include "pycore_code.h" // stats

#include "pycore_pystate.h" // _PyInterpreterState_GET

#include "pycore_obmalloc.h"

#include "pycore_pymem.h"

#include <stdlib.h> // malloc()

#include <stdbool.h>

#undef uint

#define uint pymem_uint

/* Defined in tracemalloc.c */

extern void _PyMem_DumpTraceback(int fd, const void *ptr);

static void _PyObject_DebugDumpAddress(const void *p);

static void _PyMem_DebugCheckAddress(const char *func, char api_id, const void *p);

static void set_up_debug_hooks_domain_unlocked(PyMemAllocatorDomain domain);

static void set_up_debug_hooks_unlocked(void);

static void get_allocator_unlocked(PyMemAllocatorDomain, PyMemAllocatorEx *);

static void set_allocator_unlocked(PyMemAllocatorDomain, PyMemAllocatorEx *);

/***************************************/

/* low-level allocator implementations */

/***************************************/

/* the default raw allocator (wraps malloc) */

void *

_PyMem_RawMalloc(void *Py_UNUSED(ctx), size_t size)

{

/* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL

for malloc(0), which would be treated as an error. Some platforms would

return a pointer with no memory behind it, which would break pymalloc.

To solve these problems, allocate an extra byte. */

if (size == 0)

size = 1;

return malloc(size);

}

void *

_PyMem_RawCalloc(void *Py_UNUSED(ctx), size_t nelem, size_t elsize)

{

/* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL

for calloc(0, 0), which would be treated as an error. Some platforms

would return a pointer with no memory behind it, which would break

pymalloc. To solve these problems, allocate an extra byte. */

if (nelem == 0 || elsize == 0) {

nelem = 1;

elsize = 1;

}

return calloc(nelem, elsize);

}

void *

_PyMem_RawRealloc(void *Py_UNUSED(ctx), void *ptr, size_t size)

{

if (size == 0)

size = 1;

return realloc(ptr, size);

}

void

_PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr)

{

free(ptr);

}

#define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}

#define PYRAW_ALLOC MALLOC_ALLOC

/* the default object allocator */

// The actual implementation is further down.

#ifdef WITH_PYMALLOC

void* _PyObject_Malloc(void *ctx, size_t size);

void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);

void _PyObject_Free(void *ctx, void *p);

void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);

# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}

# define PYOBJ_ALLOC PYMALLOC_ALLOC

#else

# define PYOBJ_ALLOC MALLOC_ALLOC

#endif // WITH_PYMALLOC

#define PYMEM_ALLOC PYOBJ_ALLOC

/* the default debug allocators */

// The actual implementation is further down.

void* _PyMem_DebugRawMalloc(void *ctx, size_t size);

void* _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize);

void* _PyMem_DebugRawRealloc(void *ctx, void *ptr, size_t size);

void _PyMem_DebugRawFree(void *ctx, void *ptr);

void* _PyMem_DebugMalloc(void *ctx, size_t size);

void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize);

void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);

void _PyMem_DebugFree(void *ctx, void *p);

#define PYDBGRAW_ALLOC \

{&_PyRuntime.allocators.debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree}

#define PYDBGMEM_ALLOC \

{&_PyRuntime.allocators.debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

#define PYDBGOBJ_ALLOC \

{&_PyRuntime.allocators.debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree}

/* the low-level virtual memory allocator */

#ifdef WITH_PYMALLOC

# ifdef MS_WINDOWS

# include <windows.h>

# elif defined(HAVE_MMAP)

# include <sys/mman.h>

# ifdef MAP_ANONYMOUS

# define ARENAS_USE_MMAP

# endif

# endif

#endif

void *

_PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size)

{

#ifdef MS_WINDOWS

return VirtualAlloc(NULL, size,

MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);

#elif defined(ARENAS_USE_MMAP)

void *ptr;

ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,

MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);

if (ptr == MAP_FAILED)

return NULL;

assert(ptr != NULL);

return ptr;

#else

return malloc(size);

#endif

}

void

_PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr,

#if defined(ARENAS_USE_MMAP)

size_t size

#else

size_t Py_UNUSED(size)

#endif

)

{

#ifdef MS_WINDOWS

VirtualFree(ptr, 0, MEM_RELEASE);

#elif defined(ARENAS_USE_MMAP)

munmap(ptr, size);

#else

free(ptr);

#endif

}

/*******************************************/

/* end low-level allocator implementations */

/*******************************************/

#if defined(__has_feature) /* Clang */

# if __has_feature(address_sanitizer) /* is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize("address")))

# endif

# if __has_feature(thread_sanitizer) /* is TSAN enabled? */

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

# if __has_feature(memory_sanitizer) /* is MSAN enabled? */

# define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))

# endif

#elif defined(__GNUC__)

# if defined(__SANITIZE_ADDRESS__) /* GCC 4.8+, is ASAN enabled? */

# define _Py_NO_SANITIZE_ADDRESS \

__attribute__((no_sanitize_address))

# endif

// TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro

// is provided only since GCC 7.

# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1)

# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))

# endif

#endif

#ifndef _Py_NO_SANITIZE_ADDRESS

# define _Py_NO_SANITIZE_ADDRESS

#endif

#ifndef _Py_NO_SANITIZE_THREAD

# define _Py_NO_SANITIZE_THREAD

#endif

#ifndef _Py_NO_SANITIZE_MEMORY

# define _Py_NO_SANITIZE_MEMORY

#endif

#define ALLOCATORS_MUTEX (_PyRuntime.allocators.mutex)

#define _PyMem_Raw (_PyRuntime.allocators.standard.raw)

#define _PyMem (_PyRuntime.allocators.standard.mem)

#define _PyObject (_PyRuntime.allocators.standard.obj)

#define _PyMem_Debug (_PyRuntime.allocators.debug)

#define _PyObject_Arena (_PyRuntime.allocators.obj_arena)

/***************************/

/* managing the allocators */

/***************************/

static int

set_default_allocator_unlocked(PyMemAllocatorDomain domain, int debug,

PyMemAllocatorEx *old_alloc)

{

if (old_alloc != NULL) {

get_allocator_unlocked(domain, old_alloc);

}

PyMemAllocatorEx new_alloc;

switch(domain)

{

case PYMEM_DOMAIN_RAW:

new_alloc = (PyMemAllocatorEx)PYRAW_ALLOC;

break;

case PYMEM_DOMAIN_MEM:

new_alloc = (PyMemAllocatorEx)PYMEM_ALLOC;

break;

case PYMEM_DOMAIN_OBJ:

new_alloc = (PyMemAllocatorEx)PYOBJ_ALLOC;

break;

default:

/* unknown domain */

return -1;

}

set_allocator_unlocked(domain, &new_alloc);

if (debug) {

set_up_debug_hooks_domain_unlocked(domain);

}

return 0;

}

#ifdef Py_DEBUG

static const int pydebug = 1;

#else

static const int pydebug = 0;

#endif

int

_PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain,

PyMemAllocatorEx *old_alloc)

{

if (ALLOCATORS_MUTEX == NULL) {

/* The runtime must be initializing. */

return set_default_allocator_unlocked(domain, pydebug, old_alloc);

}

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

int res = set_default_allocator_unlocked(domain, pydebug, old_alloc);

PyThread_release_lock(ALLOCATORS_MUTEX);

return res;

}

int

_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)

{

if (name == NULL || *name == '\0') {

/* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line

nameions): use default memory allocators */

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "default") == 0) {

*allocator = PYMEM_ALLOCATOR_DEFAULT;

}

else if (strcmp(name, "debug") == 0) {

*allocator = PYMEM_ALLOCATOR_DEBUG;

}

#ifdef WITH_PYMALLOC

else if (strcmp(name, "pymalloc") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC;

}

else if (strcmp(name, "pymalloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG;

}

#endif

else if (strcmp(name, "malloc") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC;

}

else if (strcmp(name, "malloc_debug") == 0) {

*allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;

}

else {

/* unknown allocator */

return -1;

}

return 0;

}

static int

set_up_allocators_unlocked(PyMemAllocatorName allocator)

{

switch (allocator) {

case PYMEM_ALLOCATOR_NOT_SET:

/* do nothing */

break;

case PYMEM_ALLOCATOR_DEFAULT:

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, pydebug, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, pydebug, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, pydebug, NULL);

break;

case PYMEM_ALLOCATOR_DEBUG:

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_RAW, 1, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_MEM, 1, NULL);

(void)set_default_allocator_unlocked(PYMEM_DOMAIN_OBJ, 1, NULL);

break;

#ifdef WITH_PYMALLOC

case PYMEM_ALLOCATOR_PYMALLOC:

case PYMEM_ALLOCATOR_PYMALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_RAW, &malloc_alloc);

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_MEM, &pymalloc);

set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &pymalloc);

if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) {

set_up_debug_hooks_unlocked();

}

break;

}

#endif

case PYMEM_ALLOCATOR_MALLOC:

case PYMEM_ALLOCATOR_MALLOC_DEBUG:

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

set_allocator_unlocked(PYMEM_DOMAIN_RAW, &malloc_alloc);

set_allocator_unlocked(PYMEM_DOMAIN_MEM, &malloc_alloc);

set_allocator_unlocked(PYMEM_DOMAIN_OBJ, &malloc_alloc);

if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) {

set_up_debug_hooks_unlocked();

}

break;

}

default:

/* unknown allocator */

return -1;

}

return 0;

}

int

_PyMem_SetupAllocators(PyMemAllocatorName allocator)

{

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

int res = set_up_allocators_unlocked(allocator);

PyThread_release_lock(ALLOCATORS_MUTEX);

return res;

}

static int

pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b)

{

return (memcmp(a, b, sizeof(PyMemAllocatorEx)) == 0);

}

static const char*

get_current_allocator_name_unlocked(void)

{

PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC;

#ifdef WITH_PYMALLOC

PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC;

#endif

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &malloc_alloc) &&

pymemallocator_eq(&_PyObject, &malloc_alloc))

{

return "malloc";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) &&

pymemallocator_eq(&_PyMem, &pymalloc) &&

pymemallocator_eq(&_PyObject, &pymalloc))

{

return "pymalloc";

}

#endif

PyMemAllocatorEx dbg_raw = PYDBGRAW_ALLOC;

PyMemAllocatorEx dbg_mem = PYDBGMEM_ALLOC;

PyMemAllocatorEx dbg_obj = PYDBGOBJ_ALLOC;

if (pymemallocator_eq(&_PyMem_Raw, &dbg_raw) &&

pymemallocator_eq(&_PyMem, &dbg_mem) &&

pymemallocator_eq(&_PyObject, &dbg_obj))

{

/* Debug hooks installed */

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &malloc_alloc))

{

return "malloc_debug";

}

#ifdef WITH_PYMALLOC

if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) &&

pymemallocator_eq(&_PyMem_Debug.mem.alloc, &pymalloc) &&

pymemallocator_eq(&_PyMem_Debug.obj.alloc, &pymalloc))

{

return "pymalloc_debug";

}

#endif

}

return NULL;

}

const char*

_PyMem_GetCurrentAllocatorName(void)

{

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

const char *name = get_current_allocator_name_unlocked();

PyThread_release_lock(ALLOCATORS_MUTEX);

return name;

}

#ifdef WITH_PYMALLOC

static int

_PyMem_DebugEnabled(void)

{

return (_PyObject.malloc == _PyMem_DebugMalloc);

}

static int

_PyMem_PymallocEnabled(void)

{

if (_PyMem_DebugEnabled()) {

return (_PyMem_Debug.obj.alloc.malloc == _PyObject_Malloc);

}

else {

return (_PyObject.malloc == _PyObject_Malloc);

}

}

#endif

static void

set_up_debug_hooks_domain_unlocked(PyMemAllocatorDomain domain)

{

PyMemAllocatorEx alloc;

if (domain == PYMEM_DOMAIN_RAW) {

if (_PyMem_Raw.malloc == _PyMem_DebugRawMalloc) {

return;

}

get_allocator_unlocked(domain, &_PyMem_Debug.raw.alloc);

alloc.ctx = &_PyMem_Debug.raw;

alloc.malloc = _PyMem_DebugRawMalloc;

alloc.calloc = _PyMem_DebugRawCalloc;

alloc.realloc = _PyMem_DebugRawRealloc;

alloc.free = _PyMem_DebugRawFree;

set_allocator_unlocked(domain, &alloc);

}

else if (domain == PYMEM_DOMAIN_MEM) {

if (_PyMem.malloc == _PyMem_DebugMalloc) {

return;

}

get_allocator_unlocked(domain, &_PyMem_Debug.mem.alloc);

alloc.ctx = &_PyMem_Debug.mem;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

set_allocator_unlocked(domain, &alloc);

}

else if (domain == PYMEM_DOMAIN_OBJ) {

if (_PyObject.malloc == _PyMem_DebugMalloc) {

return;

}

get_allocator_unlocked(domain, &_PyMem_Debug.obj.alloc);

alloc.ctx = &_PyMem_Debug.obj;

alloc.malloc = _PyMem_DebugMalloc;

alloc.calloc = _PyMem_DebugCalloc;

alloc.realloc = _PyMem_DebugRealloc;

alloc.free = _PyMem_DebugFree;

set_allocator_unlocked(domain, &alloc);

}

}

static void

set_up_debug_hooks_unlocked(void)

{

set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_RAW);

set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_MEM);

set_up_debug_hooks_domain_unlocked(PYMEM_DOMAIN_OBJ);

}

void

PyMem_SetupDebugHooks(void)

{

if (ALLOCATORS_MUTEX == NULL) {

/* The runtime must not be completely initialized yet. */

set_up_debug_hooks_unlocked();

return;

}

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

set_up_debug_hooks_unlocked();

PyThread_release_lock(ALLOCATORS_MUTEX);

}

static void

get_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break;

case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;

case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;

default:

/* unknown domain: set all attributes to NULL */

allocator->ctx = NULL;

allocator->malloc = NULL;

allocator->calloc = NULL;

allocator->realloc = NULL;

allocator->free = NULL;

}

}

static void

set_allocator_unlocked(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

switch(domain)

{

case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break;

case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break;

case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break;

/* ignore unknown domain */

}

}

void

PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

if (ALLOCATORS_MUTEX == NULL) {

/* The runtime must not be completely initialized yet. */

get_allocator_unlocked(domain, allocator);

return;

}

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

get_allocator_unlocked(domain, allocator);

PyThread_release_lock(ALLOCATORS_MUTEX);

}

void

PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)

{

if (ALLOCATORS_MUTEX == NULL) {

/* The runtime must not be completely initialized yet. */

set_allocator_unlocked(domain, allocator);

return;

}

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

set_allocator_unlocked(domain, allocator);

PyThread_release_lock(ALLOCATORS_MUTEX);

}

void

PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)

{

if (ALLOCATORS_MUTEX == NULL) {

/* The runtime must not be completely initialized yet. */

*allocator = _PyObject_Arena;

return;

}

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

*allocator = _PyObject_Arena;

PyThread_release_lock(ALLOCATORS_MUTEX);

}

void

PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)

{

if (ALLOCATORS_MUTEX == NULL) {

/* The runtime must not be completely initialized yet. */

_PyObject_Arena = *allocator;

return;

}

PyThread_acquire_lock(ALLOCATORS_MUTEX, WAIT_LOCK);

_PyObject_Arena = *allocator;

PyThread_release_lock(ALLOCATORS_MUTEX);

}

/* Note that there is a possible, but very unlikely, race in any place

* below where we call one of the allocator functions. We access two

* fields in each case: "malloc", etc. and "ctx".

*

* It is unlikely that the allocator will be changed while one of those

* calls is happening, much less in that very narrow window.

* Furthermore, the likelihood of a race is drastically reduced by the

* fact that the allocator may not be changed after runtime init

* (except with a wrapper).

*

* With the above in mind, we currently don't worry about locking

* around these uses of the runtime-global allocators state. */

/*************************/

/* the "arena" allocator */

/*************************/

void *

_PyObject_VirtualAlloc(size_t size)

{

return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size);

}

void

_PyObject_VirtualFree(void *obj, size_t size)

{

_PyObject_Arena.free(_PyObject_Arena.ctx, obj, size);

}

/***********************/

/* the "raw" allocator */

/***********************/

void *

PyMem_RawMalloc(size_t size)

{

/*

* Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.

* Most python internals blindly use a signed Py_ssize_t to track

* things without checking for overflows or negatives.

* As size_t is unsigned, checking for size < 0 is not required.

*/

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);

}

void *

PyMem_RawCalloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);

}

void*

PyMem_RawRealloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);

}

void PyMem_RawFree(void *ptr)

{

_PyMem_Raw.free(_PyMem_Raw.ctx, ptr);

}

/***********************/

/* the "mem" allocator */

/***********************/

void *

PyMem_Malloc(size_t size)

{

/* see PyMem_RawMalloc() */

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

OBJECT_STAT_INC_COND(allocations512, size < 512);

OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);

OBJECT_STAT_INC_COND(allocations_big, size >= 4094);

OBJECT_STAT_INC(allocations);

return _PyMem.malloc(_PyMem.ctx, size);

}

void *

PyMem_Calloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

OBJECT_STAT_INC_COND(allocations512, elsize < 512);

OBJECT_STAT_INC_COND(allocations4k, elsize >= 512 && elsize < 4094);

OBJECT_STAT_INC_COND(allocations_big, elsize >= 4094);

OBJECT_STAT_INC(allocations);

return _PyMem.calloc(_PyMem.ctx, nelem, elsize);

}

void *

PyMem_Realloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyMem.realloc(_PyMem.ctx, ptr, new_size);

}

void

PyMem_Free(void *ptr)

{

OBJECT_STAT_INC(frees);

_PyMem.free(_PyMem.ctx, ptr);

}

/***************************/

/* pymem utility functions */

/***************************/

wchar_t*

_PyMem_RawWcsdup(const wchar_t *str)

{

assert(str != NULL);

size_t len = wcslen(str);

if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {

return NULL;

}

size_t size = (len + 1) * sizeof(wchar_t);

wchar_t *str2 = PyMem_RawMalloc(size);

if (str2 == NULL) {

return NULL;

}

memcpy(str2, str, size);

return str2;

}

char *

_PyMem_RawStrdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_RawMalloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

char *

_PyMem_Strdup(const char *str)

{

assert(str != NULL);

size_t size = strlen(str) + 1;

char *copy = PyMem_Malloc(size);

if (copy == NULL) {

return NULL;

}

memcpy(copy, str, size);

return copy;

}

/**************************/

/* the "object" allocator */

/**************************/

void *

PyObject_Malloc(size_t size)

{

/* see PyMem_RawMalloc() */

if (size > (size_t)PY_SSIZE_T_MAX)

return NULL;

OBJECT_STAT_INC_COND(allocations512, size < 512);

OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);

OBJECT_STAT_INC_COND(allocations_big, size >= 4094);

OBJECT_STAT_INC(allocations);

return _PyObject.malloc(_PyObject.ctx, size);

}

void *

PyObject_Calloc(size_t nelem, size_t elsize)

{

/* see PyMem_RawMalloc() */

if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)

return NULL;

OBJECT_STAT_INC_COND(allocations512, elsize < 512);

OBJECT_STAT_INC_COND(allocations4k, elsize >= 512 && elsize < 4094);

OBJECT_STAT_INC_COND(allocations_big, elsize >= 4094);

OBJECT_STAT_INC(allocations);

return _PyObject.calloc(_PyObject.ctx, nelem, elsize);

}

void *

PyObject_Realloc(void *ptr, size_t new_size)

{

/* see PyMem_RawMalloc() */

if (new_size > (size_t)PY_SSIZE_T_MAX)

return NULL;

return _PyObject.realloc(_PyObject.ctx, ptr, new_size);

}

void

PyObject_Free(void *ptr)

{

OBJECT_STAT_INC(frees);

_PyObject.free(_PyObject.ctx, ptr);

}

/* If we're using GCC, use __builtin_expect() to reduce overhead of

the valgrind checks */

#if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)

# define UNLIKELY(value) __builtin_expect((value), 0)

# define LIKELY(value) __builtin_expect((value), 1)

#else

# define UNLIKELY(value) (value)

# define LIKELY(value) (value)

#endif

#ifdef WITH_PYMALLOC

#ifdef WITH_VALGRIND

#include <valgrind/valgrind.h>

/* -1 indicates that we haven't checked that we're running on valgrind yet. */

static int running_on_valgrind = -1;

#endif

typedef struct _obmalloc_state OMState;

static inline int

has_own_state(PyInterpreterState *interp)

{

return (_Py_IsMainInterpreter(interp) ||

!(interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) ||

_Py_IsMainInterpreterFinalizing(interp));

}

static inline OMState *

get_state(void)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

if (!has_own_state(interp)) {

interp = _PyInterpreterState_Main();

}

return &interp->obmalloc;

}

// These macros all rely on a local "state" variable.

#define usedpools (state->pools.used)

#define allarenas (state->mgmt.arenas)

#define maxarenas (state->mgmt.maxarenas)

#define unused_arena_objects (state->mgmt.unused_arena_objects)

#define usable_arenas (state->mgmt.usable_arenas)

#define nfp2lasta (state->mgmt.nfp2lasta)

#define narenas_currently_allocated (state->mgmt.narenas_currently_allocated)

#define ntimes_arena_allocated (state->mgmt.ntimes_arena_allocated)

#define narenas_highwater (state->mgmt.narenas_highwater)

#define raw_allocated_blocks (state->mgmt.raw_allocated_blocks)

Py_ssize_t

_PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp)

{

#ifdef Py_DEBUG

assert(has_own_state(interp));

#else

if (!has_own_state(interp)) {

_Py_FatalErrorFunc(__func__,

"the interpreter doesn't have its own allocator");

}

#endif

OMState *state = &interp->obmalloc;

Py_ssize_t n = raw_allocated_blocks;

/* add up allocated blocks for used pools */

for (uint i = 0; i < maxarenas; ++i) {

/* Skip arenas which are not allocated. */

if (allarenas[i].address == 0) {

continue;

}

uintptr_t base = (uintptr_t)_Py_ALIGN_UP(allarenas[i].address, POOL_SIZE);

/* visit every pool in the arena */

assert(base <= (uintptr_t) allarenas[i].pool_address);

for (; base < (uintptr_t) allarenas[i].pool_address; base += POOL_SIZE) {

poolp p = (poolp)base;

n += p->ref.count;

}

}

return n;

}

void

_PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp)

{

if (has_own_state(interp)) {

Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp);

assert(has_own_state(interp) || leaked == 0);

interp->runtime->obmalloc.interpreter_leaks += leaked;

}

}

static Py_ssize_t get_num_global_allocated_blocks(_PyRuntimeState *);

/* We preserve the number of blockss leaked during runtime finalization,

so they can be reported if the runtime is initialized again. */

// XXX We don't lose any information by dropping this,

// so we should consider doing so.

static Py_ssize_t last_final_leaks = 0;

void

_Py_FinalizeAllocatedBlocks(_PyRuntimeState *runtime)

{

last_final_leaks = get_num_global_allocated_blocks(runtime);

runtime->obmalloc.interpreter_leaks = 0;

}

static Py_ssize_t

get_num_global_allocated_blocks(_PyRuntimeState *runtime)

{

Py_ssize_t total = 0;

if (_PyRuntimeState_GetFinalizing(runtime) != NULL) {

PyInterpreterState *interp = _PyInterpreterState_Main();

if (interp == NULL) {

/* We are at the very end of runtime finalization.

We can't rely on finalizing->interp since that thread

state is probably already freed, so we don't worry

about it. */

assert(PyInterpreterState_Head() == NULL);

}

else {

assert(interp != NULL);

/* It is probably the last interpreter but not necessarily. */

assert(PyInterpreterState_Next(interp) == NULL);

total += _PyInterpreterState_GetAllocatedBlocks(interp);

}

}

else {

HEAD_LOCK(runtime);

PyInterpreterState *interp = PyInterpreterState_Head();

assert(interp != NULL);

#ifdef Py_DEBUG

int got_main = 0;

#endif

for (; interp != NULL; interp = PyInterpreterState_Next(interp)) {

#ifdef Py_DEBUG

if (_Py_IsMainInterpreter(interp)) {

assert(!got_main);

got_main = 1;

assert(has_own_state(interp));

}

#endif

if (has_own_state(interp)) {

total += _PyInterpreterState_GetAllocatedBlocks(interp);

}

}

HEAD_UNLOCK(runtime);

#ifdef Py_DEBUG

assert(got_main);

#endif

}

total += runtime->obmalloc.interpreter_leaks;

total += last_final_leaks;

return total;

}

Py_ssize_t

_Py_GetGlobalAllocatedBlocks(void)

{

return get_num_global_allocated_blocks(&_PyRuntime);

}

#if WITH_PYMALLOC_RADIX_TREE

/*==========================================================================*/

/* radix tree for tracking arena usage. */