/* Execute compiled code */

/* XXX TO DO:

XXX speed up searching for keywords by using a dictionary

XXX document it!

*/

/* enable more aggressive intra-module optimizations, where available */

#define PY_LOCAL_AGGRESSIVE

#include "Python.h"

#include "pycore_abstract.h" // _PyIndex_Check()

#include "pycore_call.h"

#include "pycore_ceval.h"

#include "pycore_code.h"

#include "pycore_initconfig.h"

#include "pycore_object.h"

#include "pycore_pyerrors.h"

#include "pycore_pylifecycle.h"

#include "pycore_pymem.h" // _PyMem_IsPtrFreed()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_sysmodule.h"

#include "pycore_tupleobject.h"

#include "code.h"

#include "dictobject.h"

#include "frameobject.h"

#include "opcode.h"

#include "pydtrace.h"

#include "setobject.h"

#include <ctype.h>

#ifdef Py_DEBUG

/* For debugging the interpreter: */

#define LLTRACE 1 /* Low-level trace feature */

#define CHECKEXC 1 /* Double-check exception checking */

#endif

#if !defined(Py_BUILD_CORE)

# error "ceval.c must be build with Py_BUILD_CORE define for best performance"

#endif

_Py_IDENTIFIER(__name__);

/* Forward declarations */

Py_LOCAL_INLINE(PyObject *) call_function(

PyThreadState *tstate, PyObject ***pp_stack,

Py_ssize_t oparg, PyObject *kwnames);

static PyObject * do_call_core(

PyThreadState *tstate, PyObject *func,

PyObject *callargs, PyObject *kwdict);

#ifdef LLTRACE

static int lltrace;

static int prtrace(PyThreadState *, PyObject *, const char *);

#endif

static int call_trace(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *,

int, PyObject *);

static int call_trace_protected(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *,

int, PyObject *);

static void call_exc_trace(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *);

static int maybe_call_line_trace(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *,

int *, int *, int *);

static void maybe_dtrace_line(PyFrameObject *, int *, int *, int *);

static void dtrace_function_entry(PyFrameObject *);

static void dtrace_function_return(PyFrameObject *);

static PyObject * import_name(PyThreadState *, PyFrameObject *,

PyObject *, PyObject *, PyObject *);

static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);

static int import_all_from(PyThreadState *, PyObject *, PyObject *);

static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);

static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);

static PyObject * unicode_concatenate(PyThreadState *, PyObject *, PyObject *,

PyFrameObject *, const _Py_CODEUNIT *);

static PyObject * special_lookup(PyThreadState *, PyObject *, _Py_Identifier *);

static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);

static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);

static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int);

#define NAME_ERROR_MSG \

"name '%.200s' is not defined"

#define UNBOUNDLOCAL_ERROR_MSG \

"local variable '%.200s' referenced before assignment"

#define UNBOUNDFREE_ERROR_MSG \

"free variable '%.200s' referenced before assignment" \

" in enclosing scope"

/* Dynamic execution profile */

#ifdef DYNAMIC_EXECUTION_PROFILE

#ifdef DXPAIRS

static long dxpairs[257][256];

#define dxp dxpairs[256]

#else

static long dxp[256];

#endif

#endif

/* per opcode cache */

#ifdef Py_DEBUG

// --with-pydebug is used to find memory leak. opcache makes it harder.

// So we disable opcache when Py_DEBUG is defined.

// See bpo-37146

#define OPCACHE_MIN_RUNS 0 /* disable opcache */

#else

#define OPCACHE_MIN_RUNS 1024 /* create opcache when code executed this time */

#endif

#define OPCACHE_STATS 0 /* Enable stats */

#if OPCACHE_STATS

static size_t opcache_code_objects = 0;

static size_t opcache_code_objects_extra_mem = 0;

static size_t opcache_global_opts = 0;

static size_t opcache_global_hits = 0;

static size_t opcache_global_misses = 0;

#endif

#ifndef NDEBUG

/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and

PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen

when a thread continues to run after Python finalization, especially

daemon threads. */

static int

is_tstate_valid(PyThreadState *tstate)

{

assert(!_PyMem_IsPtrFreed(tstate));

assert(!_PyMem_IsPtrFreed(tstate->interp));

return 1;

}

#endif

/* This can set eval_breaker to 0 even though gil_drop_request became

1. We believe this is all right because the eval loop will release

the GIL eventually anyway. */

static inline void

COMPUTE_EVAL_BREAKER(PyInterpreterState *interp,

struct _ceval_runtime_state *ceval,

struct _ceval_state *ceval2)

{

_Py_atomic_store_relaxed(&ceval2->eval_breaker,

_Py_atomic_load_relaxed(&ceval->gil_drop_request)

| (_Py_atomic_load_relaxed(&ceval->signals_pending)

&& _Py_ThreadCanHandleSignals(interp))

| (_Py_atomic_load_relaxed(&ceval2->pending.calls_to_do)

&& _Py_ThreadCanHandlePendingCalls())

| ceval2->pending.async_exc);

}

static inline void

SET_GIL_DROP_REQUEST(PyInterpreterState *interp)

{

struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

struct _ceval_state *ceval2 = &interp->ceval;

_Py_atomic_store_relaxed(&ceval->gil_drop_request, 1);

_Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);

}

static inline void

RESET_GIL_DROP_REQUEST(PyInterpreterState *interp)

{

struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

struct _ceval_state *ceval2 = &interp->ceval;

_Py_atomic_store_relaxed(&ceval->gil_drop_request, 0);

COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);

}

static inline void

SIGNAL_PENDING_CALLS(PyInterpreterState *interp)

{

struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

struct _ceval_state *ceval2 = &interp->ceval;

_Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 1);

COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);

}

static inline void

UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp)

{

struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

struct _ceval_state *ceval2 = &interp->ceval;

_Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0);

COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);

}

static inline void

SIGNAL_PENDING_SIGNALS(PyInterpreterState *interp)

{

struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

struct _ceval_state *ceval2 = &interp->ceval;

_Py_atomic_store_relaxed(&ceval->signals_pending, 1);

/* eval_breaker is not set to 1 if thread_can_handle_signals() is false */

COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);

}

static inline void

UNSIGNAL_PENDING_SIGNALS(PyInterpreterState *interp)

{

struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

struct _ceval_state *ceval2 = &interp->ceval;

_Py_atomic_store_relaxed(&ceval->signals_pending, 0);

COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);

}

static inline void

SIGNAL_ASYNC_EXC(PyInterpreterState *interp)

{

struct _ceval_state *ceval2 = &interp->ceval;

ceval2->pending.async_exc = 1;

_Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);

}

static inline void

UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp)

{

struct _ceval_runtime_state *ceval = &interp->runtime->ceval;

struct _ceval_state *ceval2 = &interp->ceval;

ceval2->pending.async_exc = 0;

COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);

}

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#include "ceval_gil.h"

static void

ensure_tstate_not_null(const char *func, PyThreadState *tstate)

{

if (tstate == NULL) {

_Py_FatalErrorFunc(func,

"current thread state is NULL (released GIL?)");

}

}

int

_PyEval_ThreadsInitialized(_PyRuntimeState *runtime)

{

return gil_created(&runtime->ceval.gil);

}

int

PyEval_ThreadsInitialized(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

return _PyEval_ThreadsInitialized(runtime);

}

PyStatus

_PyEval_InitGIL(PyThreadState *tstate)

{

if (!_Py_IsMainInterpreter(tstate)) {

/* Currently, the GIL is shared by all interpreters,

and only the main interpreter is responsible to create

and destroy it. */

return _PyStatus_OK();

}

struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;

assert(!gil_created(gil));

PyThread_init_thread();

create_gil(gil);

take_gil(tstate);

assert(gil_created(gil));

return _PyStatus_OK();

}

void

_PyEval_FiniGIL(PyThreadState *tstate)

{

if (!_Py_IsMainInterpreter(tstate)) {

/* Currently, the GIL is shared by all interpreters,

and only the main interpreter is responsible to create

and destroy it. */

return;

}

struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;

if (!gil_created(gil)) {

/* First Py_InitializeFromConfig() call: the GIL doesn't exist

yet: do nothing. */

return;

}

destroy_gil(gil);

assert(!gil_created(gil));

}

void

PyEval_InitThreads(void)

{

/* Do nothing: kept for backward compatibility */

}

void

_PyEval_Fini(void)

{

#if OPCACHE_STATS

fprintf(stderr, "-- Opcode cache number of objects = %zd\n",

opcache_code_objects);

fprintf(stderr, "-- Opcode cache total extra mem = %zd\n",

opcache_code_objects_extra_mem);

fprintf(stderr, "\n");

fprintf(stderr, "-- Opcode cache LOAD_GLOBAL hits = %zd (%d%%)\n",

opcache_global_hits,

(int) (100.0 * opcache_global_hits /

(opcache_global_hits + opcache_global_misses)));

fprintf(stderr, "-- Opcode cache LOAD_GLOBAL misses = %zd (%d%%)\n",

opcache_global_misses,

(int) (100.0 * opcache_global_misses /

(opcache_global_hits + opcache_global_misses)));

fprintf(stderr, "-- Opcode cache LOAD_GLOBAL opts = %zd\n",

opcache_global_opts);

fprintf(stderr, "\n");

#endif

}

void

PyEval_AcquireLock(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

ensure_tstate_not_null(__func__, tstate);

take_gil(tstate);

}

void

PyEval_ReleaseLock(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

/* This function must succeed when the current thread state is NULL.

We therefore avoid PyThreadState_Get() which dumps a fatal error

in debug mode. */

drop_gil(&runtime->ceval, tstate);

}

void

_PyEval_ReleaseLock(PyThreadState *tstate)

{

struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval;

drop_gil(ceval, tstate);

}

void

PyEval_AcquireThread(PyThreadState *tstate)

{

ensure_tstate_not_null(__func__, tstate);

take_gil(tstate);

struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;

if (_PyThreadState_Swap(gilstate, tstate) != NULL) {

Py_FatalError("non-NULL old thread state");

}

}

void

PyEval_ReleaseThread(PyThreadState *tstate)

{

assert(is_tstate_valid(tstate));

_PyRuntimeState *runtime = tstate->interp->runtime;

PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);

if (new_tstate != tstate) {

Py_FatalError("wrong thread state");

}

drop_gil(&runtime->ceval, tstate);

}

#ifdef HAVE_FORK

/* This function is called from PyOS_AfterFork_Child to destroy all threads

* which are not running in the child process, and clear internal locks

* which might be held by those threads.

*/

void

_PyEval_ReInitThreads(_PyRuntimeState *runtime)

{

struct _gil_runtime_state *gil = &runtime->ceval.gil;

if (!gil_created(gil)) {

return;

}

recreate_gil(gil);

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

ensure_tstate_not_null(__func__, tstate);

take_gil(tstate);

struct _pending_calls *pending = &tstate->interp->ceval.pending;

if (_PyThread_at_fork_reinit(&pending->lock) < 0) {

Py_FatalError("Can't initialize threads for pending calls");

}

/* Destroy all threads except the current one */

_PyThreadState_DeleteExcept(runtime, tstate);

}

#endif

/* This function is used to signal that async exceptions are waiting to be

raised. */

void

_PyEval_SignalAsyncExc(PyThreadState *tstate)

{

assert(is_tstate_valid(tstate));

SIGNAL_ASYNC_EXC(tstate->interp);

}

PyThreadState *

PyEval_SaveThread(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

struct _ceval_runtime_state *ceval = &runtime->ceval;

PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);

ensure_tstate_not_null(__func__, tstate);

assert(gil_created(&ceval->gil));

drop_gil(ceval, tstate);

return tstate;

}

void

PyEval_RestoreThread(PyThreadState *tstate)

{

ensure_tstate_not_null(__func__, tstate);

take_gil(tstate);

struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;

_PyThreadState_Swap(gilstate, tstate);

}

/* Mechanism whereby asynchronously executing callbacks (e.g. UNIX

signal handlers or Mac I/O completion routines) can schedule calls

to a function to be called synchronously.

The synchronous function is called with one void* argument.

It should return 0 for success or -1 for failure -- failure should

be accompanied by an exception.

If registry succeeds, the registry function returns 0; if it fails

(e.g. due to too many pending calls) it returns -1 (without setting

an exception condition).

Note that because registry may occur from within signal handlers,

or other asynchronous events, calling malloc() is unsafe!

Any thread can schedule pending calls, but only the main thread

will execute them.

There is no facility to schedule calls to a particular thread, but

that should be easy to change, should that ever be required. In

that case, the static variables here should go into the python

threadstate.

*/

void

_PyEval_SignalReceived(PyInterpreterState *interp)

{

/* bpo-30703: Function called when the C signal handler of Python gets a

signal. We cannot queue a callback using _PyEval_AddPendingCall() since

that function is not async-signal-safe. */

SIGNAL_PENDING_SIGNALS(interp);

}

/* Push one item onto the queue while holding the lock. */

static int

_push_pending_call(struct _pending_calls *pending,

int (*func)(void *), void *arg)

{

int i = pending->last;

int j = (i + 1) % NPENDINGCALLS;

if (j == pending->first) {

return -1; /* Queue full */

}

pending->calls[i].func = func;

pending->calls[i].arg = arg;

pending->last = j;

return 0;

}

/* Pop one item off the queue while holding the lock. */

static void

_pop_pending_call(struct _pending_calls *pending,

int (**func)(void *), void **arg)

{

int i = pending->first;

if (i == pending->last) {

return; /* Queue empty */

}

*func = pending->calls[i].func;

*arg = pending->calls[i].arg;

pending->first = (i + 1) % NPENDINGCALLS;

}

/* This implementation is thread-safe. It allows

scheduling to be made from any thread, and even from an executing

callback.

*/

int

_PyEval_AddPendingCall(PyInterpreterState *interp,

int (*func)(void *), void *arg)

{

struct _pending_calls *pending = &interp->ceval.pending;

/* Ensure that _PyEval_InitPendingCalls() was called

and that _PyEval_FiniPendingCalls() is not called yet. */

assert(pending->lock != NULL);

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

int result = _push_pending_call(pending, func, arg);

PyThread_release_lock(pending->lock);

/* signal main loop */

SIGNAL_PENDING_CALLS(interp);

return result;

}

int

Py_AddPendingCall(int (*func)(void *), void *arg)

{

/* Best-effort to support subinterpreters and calls with the GIL released.

First attempt _PyThreadState_GET() since it supports subinterpreters.

If the GIL is released, _PyThreadState_GET() returns NULL . In this

case, use PyGILState_GetThisThreadState() which works even if the GIL

is released.

Sadly, PyGILState_GetThisThreadState() doesn't support subinterpreters:

see bpo-10915 and bpo-15751.

Py_AddPendingCall() doesn't require the caller to hold the GIL. */

PyThreadState *tstate = _PyThreadState_GET();

if (tstate == NULL) {

tstate = PyGILState_GetThisThreadState();

}

PyInterpreterState *interp;

if (tstate != NULL) {

interp = tstate->interp;

}

else {

/* Last resort: use the main interpreter */

interp = _PyRuntime.interpreters.main;

}

return _PyEval_AddPendingCall(interp, func, arg);

}

static int

handle_signals(PyThreadState *tstate)

{

assert(is_tstate_valid(tstate));

if (!_Py_ThreadCanHandleSignals(tstate->interp)) {

return 0;

}

UNSIGNAL_PENDING_SIGNALS(tstate->interp);

if (_PyErr_CheckSignalsTstate(tstate) < 0) {

/* On failure, re-schedule a call to handle_signals(). */

SIGNAL_PENDING_SIGNALS(tstate->interp);

return -1;

}

return 0;

}

static int

make_pending_calls(PyThreadState *tstate)

{

assert(is_tstate_valid(tstate));

/* only execute pending calls on main thread */

if (!_Py_ThreadCanHandlePendingCalls()) {

return 0;

}

/* don't perform recursive pending calls */

static int busy = 0;

if (busy) {

return 0;

}

busy = 1;

/* unsignal before starting to call callbacks, so that any callback

added in-between re-signals */

UNSIGNAL_PENDING_CALLS(tstate->interp);

int res = 0;

/* perform a bounded number of calls, in case of recursion */

struct _pending_calls *pending = &tstate->interp->ceval.pending;

for (int i=0; i<NPENDINGCALLS; i++) {

int (*func)(void *) = NULL;

void *arg = NULL;

/* pop one item off the queue while holding the lock */

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

_pop_pending_call(pending, &func, &arg);

PyThread_release_lock(pending->lock);

/* having released the lock, perform the callback */

if (func == NULL) {

break;

}

res = func(arg);

if (res) {

goto error;

}

}

busy = 0;

return res;

error:

busy = 0;

SIGNAL_PENDING_CALLS(tstate->interp);

return res;

}

void

_Py_FinishPendingCalls(PyThreadState *tstate)

{

assert(PyGILState_Check());

struct _pending_calls *pending = &tstate->interp->ceval.pending;

if (!_Py_atomic_load_relaxed(&(pending->calls_to_do))) {

return;

}

if (make_pending_calls(tstate) < 0) {

PyObject *exc, *val, *tb;

_PyErr_Fetch(tstate, &exc, &val, &tb);

PyErr_BadInternalCall();

_PyErr_ChainExceptions(exc, val, tb);

_PyErr_Print(tstate);

}

}

/* Py_MakePendingCalls() is a simple wrapper for the sake

of backward-compatibility. */

int

Py_MakePendingCalls(void)

{

assert(PyGILState_Check());

PyThreadState *tstate = _PyThreadState_GET();

/* Python signal handler doesn't really queue a callback: it only signals

that a signal was received, see _PyEval_SignalReceived(). */

int res = handle_signals(tstate);

if (res != 0) {

return res;

}

res = make_pending_calls(tstate);

if (res != 0) {

return res;

}

return 0;

}

/* The interpreter's recursion limit */

#ifndef Py_DEFAULT_RECURSION_LIMIT

#define Py_DEFAULT_RECURSION_LIMIT 1000

#endif

int _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;

void

_PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval)

{

ceval->recursion_limit = Py_DEFAULT_RECURSION_LIMIT;

_Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;

_gil_initialize(&ceval->gil);

}

int

_PyEval_InitState(struct _ceval_state *ceval)

{

struct _pending_calls *pending = &ceval->pending;

assert(pending->lock == NULL);

pending->lock = PyThread_allocate_lock();

if (pending->lock == NULL) {

return -1;

}

return 0;

}

void

_PyEval_FiniState(struct _ceval_state *ceval)

{

struct _pending_calls *pending = &ceval->pending;

if (pending->lock != NULL) {

PyThread_free_lock(pending->lock);

pending->lock = NULL;

}

}

int

Py_GetRecursionLimit(void)

{

struct _ceval_runtime_state *ceval = &_PyRuntime.ceval;

return ceval->recursion_limit;

}

void

Py_SetRecursionLimit(int new_limit)

{

struct _ceval_runtime_state *ceval = &_PyRuntime.ceval;

ceval->recursion_limit = new_limit;

_Py_CheckRecursionLimit = new_limit;

}

/* The function _Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall()

if the recursion_depth reaches _Py_CheckRecursionLimit.

If USE_STACKCHECK, the macro decrements _Py_CheckRecursionLimit

to guarantee that _Py_CheckRecursiveCall() is regularly called.

Without USE_STACKCHECK, there is no need for this. */

int

_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)

{

_PyRuntimeState *runtime = tstate->interp->runtime;

int recursion_limit = runtime->ceval.recursion_limit;

#ifdef USE_STACKCHECK

tstate->stackcheck_counter = 0;

if (PyOS_CheckStack()) {

--tstate->recursion_depth;

_PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow");

return -1;

}

/* Needed for ABI backwards-compatibility (see bpo-31857) */

_Py_CheckRecursionLimit = recursion_limit;

#endif

if (tstate->recursion_critical)

/* Somebody asked that we don't check for recursion. */

return 0;

if (tstate->overflowed) {

if (tstate->recursion_depth > recursion_limit + 50) {

/* Overflowing while handling an overflow. Give up. */

Py_FatalError("Cannot recover from stack overflow.");

}

return 0;

}

if (tstate->recursion_depth > recursion_limit) {

--tstate->recursion_depth;

tstate->overflowed = 1;

_PyErr_Format(tstate, PyExc_RecursionError,

"maximum recursion depth exceeded%s",

where);

return -1;

}

return 0;

}

static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);

static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);

#define _Py_TracingPossible(ceval) ((ceval)->tracing_possible)

PyObject *

PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)

{

return PyEval_EvalCodeEx(co,

globals, locals,

(PyObject **)NULL, 0,

(PyObject **)NULL, 0,

(PyObject **)NULL, 0,

NULL, NULL);

}

/* Interpreter main loop */

PyObject *

PyEval_EvalFrame(PyFrameObject *f)

{

/* Function kept for backward compatibility */

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f, 0);

}

PyObject *

PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)

{

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f, throwflag);

}

/* Handle signals, pending calls, GIL drop request

and asynchronous exception */

static int

eval_frame_handle_pending(PyThreadState *tstate)

{

_PyRuntimeState * const runtime = &_PyRuntime;

struct _ceval_runtime_state *ceval = &runtime->ceval;

/* Pending signals */

if (_Py_atomic_load_relaxed(&ceval->signals_pending)) {

if (handle_signals(tstate) != 0) {

return -1;

}

}

/* Pending calls */

struct _ceval_state *ceval2 = &tstate->interp->ceval;

if (_Py_atomic_load_relaxed(&ceval2->pending.calls_to_do)) {

if (make_pending_calls(tstate) != 0) {

return -1;

}

}

/* GIL drop request */

if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) {

/* Give another thread a chance */

if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) {

Py_FatalError("tstate mix-up");

}

drop_gil(ceval, tstate);

/* Other threads may run now */

take_gil(tstate);

if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {

Py_FatalError("orphan tstate");

}

}

/* Check for asynchronous exception. */

if (tstate->async_exc != NULL) {

PyObject *exc = tstate->async_exc;

tstate->async_exc = NULL;

UNSIGNAL_ASYNC_EXC(tstate->interp);

_PyErr_SetNone(tstate, exc);

Py_DECREF(exc);

return -1;

}

return 0;

}

PyObject* _Py_HOT_FUNCTION

_PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)

{

ensure_tstate_not_null(__func__, tstate);

#ifdef DXPAIRS

int lastopcode = 0;

#endif

PyObject **stack_pointer; /* Next free slot in value stack */

const _Py_CODEUNIT *next_instr;

int opcode; /* Current opcode */

int oparg; /* Current opcode argument, if any */

PyObject **fastlocals, **freevars;

PyObject *retval = NULL; /* Return value */

struct _ceval_state * const ceval2 = &tstate->interp->ceval;

_Py_atomic_int * const eval_breaker = &ceval2->eval_breaker;

PyCodeObject *co;

/* when tracing we set things up so that

not (instr_lb <= current_bytecode_offset < instr_ub)

is true when the line being executed has changed. The

initial values are such as to make this false the first

time it is tested. */

int instr_ub = -1, instr_lb = 0, instr_prev = -1;

const _Py_CODEUNIT *first_instr;

PyObject *names;

PyObject *consts;

_PyOpcache *co_opcache;

#ifdef LLTRACE

_Py_IDENTIFIER(__ltrace__);

#endif

/* Computed GOTOs, or

the-optimization-commonly-but-improperly-known-as-"threaded code"

using gcc's labels-as-values extension

(http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).

The traditional bytecode evaluation loop uses a "switch" statement, which

decent compilers will optimize as a single indirect branch instruction

combined with a lookup table of jump addresses. However, since the

indirect jump instruction is shared by all opcodes, the CPU will have a

hard time making the right prediction for where to jump next (actually,

it will be always wrong except in the uncommon case of a sequence of

several identical opcodes).

"Threaded code" in contrast, uses an explicit jump table and an explicit

indirect jump instruction at the end of each opcode. Since the jump

instruction is at a different address for each opcode, the CPU will make a

separate prediction for each of these instructions, which is equivalent to

predicting the second opcode of each opcode pair. These predictions have

a much better chance to turn out valid, especially in small bytecode loops.

A mispredicted branch on a modern CPU flushes the whole pipeline and

can cost several CPU cycles (depending on the pipeline depth),

and potentially many more instructions (depending on the pipeline width).

A correctly predicted branch, however, is nearly free.

At the time of this writing, the "threaded code" version is up to 15-20%

faster than the normal "switch" version, depending on the compiler and the

CPU architecture.

We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,

because it would render the measurements invalid.

NOTE: care must be taken that the compiler doesn't try to "optimize" the

indirect jumps by sharing them between all opcodes. Such optimizations

can be disabled on gcc by using the -fno-gcse flag (or possibly

-fno-crossjumping).

*/

#ifdef DYNAMIC_EXECUTION_PROFILE

#undef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 0

#endif

#ifdef HAVE_COMPUTED_GOTOS

#ifndef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 1

#endif

#else

#if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS

#error "Computed gotos are not supported on this compiler."

#endif

#undef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 0

#endif

#if USE_COMPUTED_GOTOS

/* Import the static jump table */

#include "opcode_targets.h"

#define TARGET(op) \

op: \

TARGET_##op

#ifdef LLTRACE

#define FAST_DISPATCH() \

{ \

if (!lltrace && !_Py_TracingPossible(ceval2) && !PyDTrace_LINE_ENABLED()) { \

f->f_lasti = INSTR_OFFSET(); \

NEXTOPARG(); \

goto *opcode_targets[opcode]; \

} \

goto fast_next_opcode; \

}

#else

#define FAST_DISPATCH() \

{ \

if (!_Py_TracingPossible(ceval2) && !PyDTrace_LINE_ENABLED()) { \

f->f_lasti = INSTR_OFFSET(); \

NEXTOPARG(); \

goto *opcode_targets[opcode]; \

} \

goto fast_next_opcode; \

}

#endif