/* Thread and interpreter state structures and their interfaces */

#include "Python.h"

#include "pycore_ceval.h"

#include "pycore_frame.h"

#include "pycore_initconfig.h"

#include "pycore_object.h" // _PyType_InitCache()

#include "pycore_pyerrors.h"

#include "pycore_pylifecycle.h"

#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator()

#include "pycore_pystate.h" // _PyThreadState_GET()

#include "pycore_sysmodule.h"

/* --------------------------------------------------------------------------

CAUTION

Always use PyMem_RawMalloc() and PyMem_RawFree() directly in this file. A

number of these functions are advertised as safe to call when the GIL isn't

held, and in a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's

debugging obmalloc functions. Those aren't thread-safe (they rely on the GIL

to avoid the expense of doing their own locking).

-------------------------------------------------------------------------- */

#ifdef HAVE_DLOPEN

#ifdef HAVE_DLFCN_H

#include <dlfcn.h>

#endif

#if !HAVE_DECL_RTLD_LAZY

#define RTLD_LAZY 1

#endif

#endif

#ifdef __cplusplus

extern "C" {

#endif

#define _PyRuntimeGILState_GetThreadState(gilstate) \

((PyThreadState*)_Py_atomic_load_relaxed(&(gilstate)->tstate_current))

#define _PyRuntimeGILState_SetThreadState(gilstate, value) \

_Py_atomic_store_relaxed(&(gilstate)->tstate_current, \

(uintptr_t)(value))

/* Forward declarations */

static PyThreadState *_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate);

static void _PyThreadState_Delete(PyThreadState *tstate, int check_current);

static int

alloc_for_runtime(PyThread_type_lock *plock1, PyThread_type_lock *plock2,

PyThread_type_lock *plock3)

{

/* Force default allocator, since _PyRuntimeState_Fini() must

use the same allocator than this function. */

PyMemAllocatorEx old_alloc;

_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

PyThread_type_lock lock1 = PyThread_allocate_lock();

if (lock1 == NULL) {

return -1;

}

PyThread_type_lock lock2 = PyThread_allocate_lock();

if (lock2 == NULL) {

PyThread_free_lock(lock1);

return -1;

}

PyThread_type_lock lock3 = PyThread_allocate_lock();

if (lock3 == NULL) {

PyThread_free_lock(lock1);

PyThread_free_lock(lock2);

return -1;

}

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

*plock1 = lock1;

*plock2 = lock2;

*plock3 = lock3;

return 0;

}

static void

init_runtime(_PyRuntimeState *runtime,

void *open_code_hook, void *open_code_userdata,

_Py_AuditHookEntry *audit_hook_head,

Py_ssize_t unicode_next_index,

PyThread_type_lock unicode_ids_mutex,

PyThread_type_lock interpreters_mutex,

PyThread_type_lock xidregistry_mutex)

{

if (runtime->_initialized) {

_PyRuntimeState_reset(runtime);

assert(!runtime->initialized);

}

runtime->open_code_hook = open_code_hook;

runtime->open_code_userdata = open_code_userdata;

runtime->audit_hook_head = audit_hook_head;

_PyEval_InitRuntimeState(&runtime->ceval);

PyPreConfig_InitPythonConfig(&runtime->preconfig);

runtime->gilstate.check_enabled = 1;

/* A TSS key must be initialized with Py_tss_NEEDS_INIT

in accordance with the specification. */

Py_tss_t initial = Py_tss_NEEDS_INIT;

runtime->gilstate.autoTSSkey = initial;

runtime->interpreters.mutex = interpreters_mutex;

// This prevents interpreters from getting created

// until _PyInterpreterState_Enable() is called.

runtime->interpreters.next_id = -1;

runtime->xidregistry.mutex = xidregistry_mutex;

// Set it to the ID of the main thread of the main interpreter.

runtime->main_thread = PyThread_get_thread_ident();

runtime->unicode_ids.next_index = unicode_next_index;

runtime->unicode_ids.lock = unicode_ids_mutex;

runtime->_initialized = 1;

}

PyStatus

_PyRuntimeState_Init(_PyRuntimeState *runtime)

{

/* We preserve the hook across init, because there is

currently no public API to set it between runtime

initialization and interpreter initialization. */

void *open_code_hook = runtime->open_code_hook;

void *open_code_userdata = runtime->open_code_userdata;

_Py_AuditHookEntry *audit_hook_head = runtime->audit_hook_head;

// bpo-42882: Preserve next_index value if Py_Initialize()/Py_Finalize()

// is called multiple times.

Py_ssize_t unicode_next_index = runtime->unicode_ids.next_index;

PyThread_type_lock lock1, lock2, lock3;

if (alloc_for_runtime(&lock1, &lock2, &lock3) != 0) {

return _PyStatus_NO_MEMORY();

}

init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head,

unicode_next_index, lock1, lock2, lock3);

return _PyStatus_OK();

}

void

_PyRuntimeState_Fini(_PyRuntimeState *runtime)

{

/* Force the allocator used by _PyRuntimeState_Init(). */

PyMemAllocatorEx old_alloc;

_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

#define FREE_LOCK(LOCK) \

if (LOCK != NULL) { \

PyThread_free_lock(LOCK); \

LOCK = NULL; \

}

FREE_LOCK(runtime->interpreters.mutex);

FREE_LOCK(runtime->xidregistry.mutex);

FREE_LOCK(runtime->unicode_ids.lock);

#undef FREE_LOCK

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

}

#ifdef HAVE_FORK

/* This function is called from PyOS_AfterFork_Child to ensure that

newly created child processes do not share locks with the parent. */

PyStatus

_PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)

{

// This was initially set in _PyRuntimeState_Init().

runtime->main_thread = PyThread_get_thread_ident();

/* Force default allocator, since _PyRuntimeState_Fini() must

use the same allocator than this function. */

PyMemAllocatorEx old_alloc;

_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

int reinit_interp = _PyThread_at_fork_reinit(&runtime->interpreters.mutex);

int reinit_xidregistry = _PyThread_at_fork_reinit(&runtime->xidregistry.mutex);

int reinit_unicode_ids = _PyThread_at_fork_reinit(&runtime->unicode_ids.lock);

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

/* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does

* not force the default allocator. */

int reinit_main_id = _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex);

if (reinit_interp < 0

|| reinit_main_id < 0

|| reinit_xidregistry < 0

|| reinit_unicode_ids < 0)

{

return _PyStatus_ERR("Failed to reinitialize runtime locks");

}

return _PyStatus_OK();

}

#endif

#define HEAD_LOCK(runtime) \

PyThread_acquire_lock((runtime)->interpreters.mutex, WAIT_LOCK)

#define HEAD_UNLOCK(runtime) \

PyThread_release_lock((runtime)->interpreters.mutex)

/* Forward declaration */

static void _PyGILState_NoteThreadState(

struct _gilstate_runtime_state *gilstate, PyThreadState* tstate);

PyStatus

_PyInterpreterState_Enable(_PyRuntimeState *runtime)

{

struct pyinterpreters *interpreters = &runtime->interpreters;

interpreters->next_id = 0;

/* Py_Finalize() calls _PyRuntimeState_Fini() which clears the mutex.

Create a new mutex if needed. */

if (interpreters->mutex == NULL) {

/* Force default allocator, since _PyRuntimeState_Fini() must

use the same allocator than this function. */

PyMemAllocatorEx old_alloc;

_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

interpreters->mutex = PyThread_allocate_lock();

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

if (interpreters->mutex == NULL) {

return _PyStatus_ERR("Can't initialize threads for interpreter");

}

}

return _PyStatus_OK();

}

static PyInterpreterState *

alloc_interpreter(void)

{

return PyMem_RawCalloc(1, sizeof(PyInterpreterState));

}

static void

free_interpreter(PyInterpreterState *interp)

{

PyMem_RawFree(interp);

}

/* Get the interpreter state to a minimal consistent state.

Further init happens in pylifecycle.c before it can be used.

All fields not initialized here are expected to be zeroed out,

e.g. by PyMem_RawCalloc() or memset().

The runtime state is not manipulated. Instead it is assumed that

the interpreter is getting added to the runtime.

*/

static void

init_interpreter(PyInterpreterState *interp,

_PyRuntimeState *runtime, int64_t id,

PyInterpreterState *next,

PyThread_type_lock pending_lock)

{

if (interp->_initialized) {

Py_FatalError("interpreter already initialized");

}

assert(runtime != NULL);

interp->runtime = runtime;

assert(id > 0 || (id == 0 && interp == runtime->interpreters.main));

interp->id = id;

interp->id_refcount = -1;

assert(runtime->interpreters.head == interp);

assert(next != NULL || (interp == runtime->interpreters.main));

interp->next = next;

_PyEval_InitState(&interp->ceval, pending_lock);

_PyGC_InitState(&interp->gc);

PyConfig_InitPythonConfig(&interp->config);

_PyType_InitCache(interp);

interp->eval_frame = NULL;

#ifdef HAVE_DLOPEN

#if HAVE_DECL_RTLD_NOW

interp->dlopenflags = RTLD_NOW;

#else

interp->dlopenflags = RTLD_LAZY;

#endif

#endif

interp->_initialized = 1;

}

PyInterpreterState *

PyInterpreterState_New(void)

{

PyInterpreterState *interp;

PyThreadState *tstate = _PyThreadState_GET();

/* tstate is NULL when Py_InitializeFromConfig() calls

PyInterpreterState_New() to create the main interpreter. */

if (_PySys_Audit(tstate, "cpython.PyInterpreterState_New", NULL) < 0) {

return NULL;

}

PyThread_type_lock pending_lock = PyThread_allocate_lock();

if (pending_lock == NULL) {

if (tstate != NULL) {

_PyErr_NoMemory(tstate);

}

return NULL;

}

/* Don't get runtime from tstate since tstate can be NULL. */

_PyRuntimeState *runtime = &_PyRuntime;

struct pyinterpreters *interpreters = &runtime->interpreters;

/* We completely serialize creation of multiple interpreters, since

it simplifies things here and blocking concurrent calls isn't a problem.

Regardless, we must fully block subinterpreter creation until

after the main interpreter is created. */

HEAD_LOCK(runtime);

int64_t id = interpreters->next_id;

interpreters->next_id += 1;

// Allocate the interpreter and add it to the runtime state.

PyInterpreterState *old_head = interpreters->head;

if (old_head == NULL) {

// We are creating the main interpreter.

assert(interpreters->main == NULL);

assert(id == 0);

interp = alloc_interpreter();

if (interp == NULL) {

goto error;

}

assert(interp->id == 0);

assert(interp->next == NULL);

interpreters->main = interp;

}

else {

assert(id != 0);

assert(interpreters->main != NULL);

interp = alloc_interpreter();

if (interp == NULL) {

goto error;

}

if (id < 0) {

/* overflow or Py_Initialize() not called yet! */

if (tstate != NULL) {

_PyErr_SetString(tstate, PyExc_RuntimeError,

"failed to get an interpreter ID");

}

goto error;

}

}

interpreters->head = interp;

init_interpreter(interp, runtime, id, old_head, pending_lock);

HEAD_UNLOCK(runtime);

return interp;

error:

HEAD_UNLOCK(runtime);

PyThread_free_lock(pending_lock);

if (interp != NULL) {

free_interpreter(interp);

}

return NULL;

}

static void

interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)

{

_PyRuntimeState *runtime = interp->runtime;

if (_PySys_Audit(tstate, "cpython.PyInterpreterState_Clear", NULL) < 0) {

_PyErr_Clear(tstate);

}

HEAD_LOCK(runtime);

for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) {

PyThreadState_Clear(p);

}

HEAD_UNLOCK(runtime);

Py_CLEAR(interp->audit_hooks);

PyConfig_Clear(&interp->config);

Py_CLEAR(interp->codec_search_path);

Py_CLEAR(interp->codec_search_cache);

Py_CLEAR(interp->codec_error_registry);

Py_CLEAR(interp->modules);

Py_CLEAR(interp->modules_by_index);

Py_CLEAR(interp->builtins_copy);

Py_CLEAR(interp->importlib);

Py_CLEAR(interp->import_func);

Py_CLEAR(interp->dict);

#ifdef HAVE_FORK

Py_CLEAR(interp->before_forkers);

Py_CLEAR(interp->after_forkers_parent);

Py_CLEAR(interp->after_forkers_child);

#endif

_PyAST_Fini(interp);

_PyWarnings_Fini(interp);

_PyAtExit_Fini(interp);

// All Python types must be destroyed before the last GC collection. Python

// types create a reference cycle to themselves in their in their

// PyTypeObject.tp_mro member (the tuple contains the type).

/* Last garbage collection on this interpreter */

_PyGC_CollectNoFail(tstate);

_PyGC_Fini(interp);

/* We don't clear sysdict and builtins until the end of this function.

Because clearing other attributes can execute arbitrary Python code

which requires sysdict and builtins. */

PyDict_Clear(interp->sysdict);

PyDict_Clear(interp->builtins);

Py_CLEAR(interp->sysdict);

Py_CLEAR(interp->builtins);

// XXX Once we have one allocator per interpreter (i.e.

// per-interpreter GC) we must ensure that all of the interpreter's

// objects have been cleaned up at the point.

}

void

PyInterpreterState_Clear(PyInterpreterState *interp)

{

// Use the current Python thread state to call audit hooks and to collect

// garbage. It can be different than the current Python thread state

// of 'interp'.

PyThreadState *current_tstate = _PyThreadState_GET();

interpreter_clear(interp, current_tstate);

}

void

_PyInterpreterState_Clear(PyThreadState *tstate)

{

interpreter_clear(tstate->interp, tstate);

}

static void

zapthreads(PyInterpreterState *interp, int check_current)

{

PyThreadState *tstate;

/* No need to lock the mutex here because this should only happen

when the threads are all really dead (XXX famous last words). */

while ((tstate = interp->threads.head) != NULL) {

_PyThreadState_Delete(tstate, check_current);

}

}

void

PyInterpreterState_Delete(PyInterpreterState *interp)

{

_PyRuntimeState *runtime = interp->runtime;

struct pyinterpreters *interpreters = &runtime->interpreters;

zapthreads(interp, 0);

_PyEval_FiniState(&interp->ceval);

/* Delete current thread. After this, many C API calls become crashy. */

_PyThreadState_Swap(&runtime->gilstate, NULL);

HEAD_LOCK(runtime);

PyInterpreterState **p;

for (p = &interpreters->head; ; p = &(*p)->next) {

if (*p == NULL) {

Py_FatalError("NULL interpreter");

}

if (*p == interp) {

break;

}

}

if (interp->threads.head != NULL) {

Py_FatalError("remaining threads");

}

*p = interp->next;

if (interpreters->main == interp) {

interpreters->main = NULL;

if (interpreters->head != NULL) {

Py_FatalError("remaining subinterpreters");

}

}

HEAD_UNLOCK(runtime);

if (interp->id_mutex != NULL) {

PyThread_free_lock(interp->id_mutex);

}

free_interpreter(interp);

}

#ifdef HAVE_FORK

/*

* Delete all interpreter states except the main interpreter. If there

* is a current interpreter state, it *must* be the main interpreter.

*/

PyStatus

_PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime)

{

struct _gilstate_runtime_state *gilstate = &runtime->gilstate;

struct pyinterpreters *interpreters = &runtime->interpreters;

PyThreadState *tstate = _PyThreadState_Swap(gilstate, NULL);

if (tstate != NULL && tstate->interp != interpreters->main) {

return _PyStatus_ERR("not main interpreter");

}

HEAD_LOCK(runtime);

PyInterpreterState *interp = interpreters->head;

interpreters->head = NULL;

while (interp != NULL) {

if (interp == interpreters->main) {

interpreters->main->next = NULL;

interpreters->head = interp;

interp = interp->next;

continue;

}

PyInterpreterState_Clear(interp); // XXX must activate?

zapthreads(interp, 1);

if (interp->id_mutex != NULL) {

PyThread_free_lock(interp->id_mutex);

}

PyInterpreterState *prev_interp = interp;

interp = interp->next;

free_interpreter(prev_interp);

}

HEAD_UNLOCK(runtime);

if (interpreters->head == NULL) {

return _PyStatus_ERR("missing main interpreter");

}

_PyThreadState_Swap(gilstate, tstate);

return _PyStatus_OK();

}

#endif

PyInterpreterState *

PyInterpreterState_Get(void)

{

PyThreadState *tstate = _PyThreadState_GET();

_Py_EnsureTstateNotNULL(tstate);

PyInterpreterState *interp = tstate->interp;

if (interp == NULL) {

Py_FatalError("no current interpreter");

}

return interp;

}

int64_t

PyInterpreterState_GetID(PyInterpreterState *interp)

{

if (interp == NULL) {

PyErr_SetString(PyExc_RuntimeError, "no interpreter provided");

return -1;

}

return interp->id;

}

static PyInterpreterState *

interp_look_up_id(_PyRuntimeState *runtime, int64_t requested_id)

{

PyInterpreterState *interp = runtime->interpreters.head;

while (interp != NULL) {

int64_t id = PyInterpreterState_GetID(interp);

if (id < 0) {

return NULL;

}

if (requested_id == id) {

return interp;

}

interp = PyInterpreterState_Next(interp);

}

return NULL;

}

PyInterpreterState *

_PyInterpreterState_LookUpID(int64_t requested_id)

{

PyInterpreterState *interp = NULL;

if (requested_id >= 0) {

_PyRuntimeState *runtime = &_PyRuntime;

HEAD_LOCK(runtime);

interp = interp_look_up_id(runtime, requested_id);

HEAD_UNLOCK(runtime);

}

if (interp == NULL && !PyErr_Occurred()) {

PyErr_Format(PyExc_RuntimeError,

"unrecognized interpreter ID %lld", requested_id);

}

return interp;

}

int

_PyInterpreterState_IDInitref(PyInterpreterState *interp)

{

if (interp->id_mutex != NULL) {

return 0;

}

interp->id_mutex = PyThread_allocate_lock();

if (interp->id_mutex == NULL) {

PyErr_SetString(PyExc_RuntimeError,

"failed to create init interpreter ID mutex");

return -1;

}

interp->id_refcount = 0;

return 0;

}

int

_PyInterpreterState_IDIncref(PyInterpreterState *interp)

{

if (_PyInterpreterState_IDInitref(interp) < 0) {

return -1;

}

PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK);

interp->id_refcount += 1;

PyThread_release_lock(interp->id_mutex);

return 0;

}

void

_PyInterpreterState_IDDecref(PyInterpreterState *interp)

{

assert(interp->id_mutex != NULL);

struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate;

PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK);

assert(interp->id_refcount != 0);

interp->id_refcount -= 1;

int64_t refcount = interp->id_refcount;

PyThread_release_lock(interp->id_mutex);

if (refcount == 0 && interp->requires_idref) {

// XXX Using the "head" thread isn't strictly correct.

PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);

// XXX Possible GILState issues?

PyThreadState *save_tstate = _PyThreadState_Swap(gilstate, tstate);

Py_EndInterpreter(tstate);

_PyThreadState_Swap(gilstate, save_tstate);

}

}

int

_PyInterpreterState_RequiresIDRef(PyInterpreterState *interp)

{

return interp->requires_idref;

}

void

_PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required)

{

interp->requires_idref = required ? 1 : 0;

}

PyObject *

_PyInterpreterState_GetMainModule(PyInterpreterState *interp)

{

if (interp->modules == NULL) {

PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized");

return NULL;

}

return PyMapping_GetItemString(interp->modules, "__main__");

}

PyObject *

PyInterpreterState_GetDict(PyInterpreterState *interp)

{

if (interp->dict == NULL) {

interp->dict = PyDict_New();

if (interp->dict == NULL) {

PyErr_Clear();

}

}

/* Returning NULL means no per-interpreter dict is available. */

return interp->dict;

}

/* Minimum size of data stack chunk */

#define DATA_STACK_CHUNK_SIZE (16*1024)

static _PyStackChunk*

allocate_chunk(int size_in_bytes, _PyStackChunk* previous)

{

assert(size_in_bytes % sizeof(PyObject **) == 0);

_PyStackChunk *res = _PyObject_VirtualAlloc(size_in_bytes);

if (res == NULL) {

return NULL;

}

res->previous = previous;

res->size = size_in_bytes;

res->top = 0;

return res;

}

static PyThreadState *

alloc_threadstate(void)

{

return PyMem_RawCalloc(1, sizeof(PyThreadState));

}

static void

free_threadstate(PyThreadState *tstate)

{

PyMem_RawFree(tstate);

}

/* Get the thread state to a minimal consistent state.

Further init happens in pylifecycle.c before it can be used.

All fields not initialized here are expected to be zeroed out,

e.g. by PyMem_RawCalloc() or memset().

The interpreter state is not manipulated. Instead it is assumed that

the thread is getting added to the interpreter.

*/

static void

init_threadstate(PyThreadState *tstate,

PyInterpreterState *interp, uint64_t id,

PyThreadState *next,

_PyStackChunk *datastack_chunk)

{

if (tstate->_initialized) {

Py_FatalError("thread state already initialized");

}

assert(interp != NULL);

tstate->interp = interp;

assert(id > 0);

tstate->id = id;

assert(interp->threads.head == tstate);

assert((next != NULL && id != 1) || (next == NULL && id == 1));

if (next != NULL) {

assert(next->prev == NULL || next->prev == tstate);

next->prev = tstate;

}

tstate->next = next;

tstate->prev = NULL;

tstate->thread_id = PyThread_get_thread_ident();

#ifdef PY_HAVE_THREAD_NATIVE_ID

tstate->native_thread_id = PyThread_get_thread_native_id();

#endif

tstate->context_ver = 1;

tstate->recursion_limit = interp->ceval.recursion_limit,

tstate->recursion_remaining = interp->ceval.recursion_limit,

tstate->exc_info = &tstate->exc_state;

tstate->cframe = &tstate->root_cframe;

assert(datastack_chunk != NULL);

tstate->datastack_chunk = datastack_chunk;

/* If top points to entry 0, then _PyThreadState_PopFrame will try to pop this chunk */

tstate->datastack_top = &tstate->datastack_chunk->data[1];

tstate->datastack_limit = (PyObject **)(((char *)tstate->datastack_chunk) + DATA_STACK_CHUNK_SIZE);

tstate->_initialized = 1;

}

static PyThreadState *

new_threadstate(PyInterpreterState *interp)

{

PyThreadState *tstate;

_PyRuntimeState *runtime = interp->runtime;

_PyStackChunk *datastack_chunk = allocate_chunk(DATA_STACK_CHUNK_SIZE, NULL);

if (datastack_chunk == NULL) {

return NULL;

}

/* We serialize concurrent creation to protect global state. */

HEAD_LOCK(runtime);

interp->threads.next_unique_id += 1;

uint64_t id = interp->threads.next_unique_id;

// Allocate the thread state and add it to the interpreter.

PyThreadState *old_head = interp->threads.head;

if (old_head == NULL) {

// It's the interpreter's initial thread state.

assert(id == 1);

tstate = alloc_threadstate();

if (tstate == NULL) {

goto error;

}

}

else {

// Every valid interpreter must have at least one thread.

assert(id > 1);

assert(old_head->prev == NULL);

tstate = alloc_threadstate();

if (tstate == NULL) {

goto error;

}

}

interp->threads.head = tstate;

init_threadstate(tstate, interp, id, old_head, datastack_chunk);

HEAD_UNLOCK(runtime);

return tstate;

error:

HEAD_UNLOCK(runtime);

_PyObject_VirtualFree(datastack_chunk, datastack_chunk->size);

return NULL;

}

PyThreadState *

PyThreadState_New(PyInterpreterState *interp)

{

PyThreadState *tstate = new_threadstate(interp);

_PyThreadState_SetCurrent(tstate);

return tstate;

}

PyThreadState *

_PyThreadState_Prealloc(PyInterpreterState *interp)

{

return new_threadstate(interp);

}

// We keep this around for (accidental) stable ABI compatibility.

// Realisically, no extensions are using it.

void

_PyThreadState_Init(PyThreadState *tstate)

{

Py_FatalError("_PyThreadState_Init() is for internal use only");

}

void

_PyThreadState_SetCurrent(PyThreadState *tstate)

{

_PyGILState_NoteThreadState(&tstate->interp->runtime->gilstate, tstate);

}

PyObject*

PyState_FindModule(struct PyModuleDef* module)

{

Py_ssize_t index = module->m_base.m_index;

PyInterpreterState *state = _PyInterpreterState_GET();

PyObject *res;

if (module->m_slots) {

return NULL;

}

if (index == 0)

return NULL;

if (state->modules_by_index == NULL)

return NULL;

if (index >= PyList_GET_SIZE(state->modules_by_index))

return NULL;

res = PyList_GET_ITEM(state->modules_by_index, index);

return res==Py_None ? NULL : res;

}

int

_PyState_AddModule(PyThreadState *tstate, PyObject* module, struct PyModuleDef* def)

{

if (!def) {

assert(_PyErr_Occurred(tstate));

return -1;

}

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_AddModule called on module with slots");

return -1;

}

PyInterpreterState *interp = tstate->interp;

if (!interp->modules_by_index) {

interp->modules_by_index = PyList_New(0);

if (!interp->modules_by_index) {

return -1;

}

}

while (PyList_GET_SIZE(interp->modules_by_index) <= def->m_base.m_index) {

if (PyList_Append(interp->modules_by_index, Py_None) < 0) {

return -1;

}

}

Py_INCREF(module);

return PyList_SetItem(interp->modules_by_index,

def->m_base.m_index, module);

}

int

PyState_AddModule(PyObject* module, struct PyModuleDef* def)

{

if (!def) {

Py_FatalError("module definition is NULL");

return -1;

}

PyThreadState *tstate = _PyThreadState_GET();

PyInterpreterState *interp = tstate->interp;

Py_ssize_t index = def->m_base.m_index;

if (interp->modules_by_index &&

index < PyList_GET_SIZE(interp->modules_by_index) &&

module == PyList_GET_ITEM(interp->modules_by_index, index))

{

_Py_FatalErrorFormat(__func__, "module %p already added", module);

return -1;

}

return _PyState_AddModule(tstate, module, def);

}

int

PyState_RemoveModule(struct PyModuleDef* def)

{

PyThreadState *tstate = _PyThreadState_GET();

PyInterpreterState *interp = tstate->interp;

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_RemoveModule called on module with slots");

return -1;

}

Py_ssize_t index = def->m_base.m_index;

if (index == 0) {

Py_FatalError("invalid module index");

}

if (interp->modules_by_index == NULL) {

Py_FatalError("Interpreters module-list not accessible.");

}

if (index > PyList_GET_SIZE(interp->modules_by_index)) {

Py_FatalError("Module index out of bounds.");

}

Py_INCREF(Py_None);

return PyList_SetItem(interp->modules_by_index, index, Py_None);

}

// Used by finalize_modules()

void

_PyInterpreterState_ClearModules(PyInterpreterState *interp)

{

if (!interp->modules_by_index) {

return;

}

Py_ssize_t i;

for (i = 0; i < PyList_GET_SIZE(interp->modules_by_index); i++) {

PyObject *m = PyList_GET_ITEM(interp->modules_by_index, i);

if (PyModule_Check(m)) {

/* cleanup the saved copy of module dicts */

PyModuleDef *md = PyModule_GetDef(m);

if (md) {

Py_CLEAR(md->m_base.m_copy);

}

}

}

/* Setting modules_by_index to NULL could be dangerous, so we

clear the list instead. */

if (PyList_SetSlice(interp->modules_by_index,

0, PyList_GET_SIZE(interp->modules_by_index),

NULL)) {