// This file contains instruction definitions.

// It is read by Tools/cases_generator/generate_cases.py

// to generate Python/generated_cases.c.h.

// Note that there is some dummy C code at the top and bottom of the file

// to fool text editors like VS Code into believing this is valid C code.

// The actual instruction definitions start at // BEGIN BYTECODES //.

// See Tools/cases_generator/README.md for more information.

#include "Python.h"

#include "pycore_abstract.h" // _PyIndex_Check()

#include "pycore_call.h" // _PyObject_FastCallDictTstate()

#include "pycore_ceval.h" // _PyEval_SignalAsyncExc()

#include "pycore_code.h"

#include "pycore_function.h"

#include "pycore_intrinsics.h"

#include "pycore_long.h" // _PyLong_GetZero()

#include "pycore_object.h" // _PyObject_GC_TRACK()

#include "pycore_moduleobject.h" // PyModuleObject

#include "pycore_opcode.h" // EXTRA_CASES

#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()

#include "pycore_pymem.h" // _PyMem_IsPtrFreed()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_range.h" // _PyRangeIterObject

#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs

#include "pycore_sysmodule.h" // _PySys_Audit()

#include "pycore_tuple.h" // _PyTuple_ITEMS()

#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS

#include "pycore_dict.h"

#include "dictobject.h"

#include "pycore_frame.h"

#include "opcode.h"

#include "pydtrace.h"

#include "setobject.h"

#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX

#define USE_COMPUTED_GOTOS 0

#include "ceval_macros.h"

/* Flow control macros */

#define DEOPT_IF(cond, instname) ((void)0)

#define ERROR_IF(cond, labelname) ((void)0)

#define GO_TO_INSTRUCTION(instname) ((void)0)

#define PREDICT(opname) ((void)0)

#define inst(name, ...) case name:

#define op(name, ...) /* NAME is ignored */

#define macro(name) static int MACRO_##name

#define super(name) static int SUPER_##name

#define family(name, ...) static int family_##name

// Dummy variables for stack effects.

static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub;

static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2;

static PyObject *list, *tuple, *dict, *owner, *set, *str, *tup, *map, *keys;

static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter;

static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc;

static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from;

static PyObject **pieces, **values;

static size_t jump;

// Dummy variables for cache effects

static uint16_t invert, counter, index, hint;

static uint32_t type_version;

static PyObject *

dummy_func(

PyThreadState *tstate,

_PyInterpreterFrame *frame,

unsigned char opcode,

unsigned int oparg,

_Py_atomic_int * const eval_breaker,

_PyCFrame cframe,

PyObject *names,

PyObject *consts,

_Py_CODEUNIT *next_instr,

PyObject **stack_pointer,

PyObject *kwnames,

int throwflag,

binaryfunc binary_ops[]

)

{

_PyInterpreterFrame entry_frame;

switch (opcode) {

// BEGIN BYTECODES //

inst(NOP, (--)) {

}

inst(RESUME, (--)) {

assert(tstate->cframe == &cframe);

assert(frame == cframe.current_frame);

if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {

goto handle_eval_breaker;

}

}

inst(LOAD_CLOSURE, (-- value)) {

/* We keep LOAD_CLOSURE so that the bytecode stays more readable. */

value = GETLOCAL(oparg);

ERROR_IF(value == NULL, unbound_local_error);

Py_INCREF(value);

}

inst(LOAD_FAST_CHECK, (-- value)) {

value = GETLOCAL(oparg);

ERROR_IF(value == NULL, unbound_local_error);

Py_INCREF(value);

}

inst(LOAD_FAST, (-- value)) {

value = GETLOCAL(oparg);

assert(value != NULL);

Py_INCREF(value);

}

inst(LOAD_CONST, (-- value)) {

value = GETITEM(consts, oparg);

Py_INCREF(value);

}

inst(STORE_FAST, (value --)) {

SETLOCAL(oparg, value);

}

super(LOAD_FAST__LOAD_FAST) = LOAD_FAST + LOAD_FAST;

super(LOAD_FAST__LOAD_CONST) = LOAD_FAST + LOAD_CONST;

super(STORE_FAST__LOAD_FAST) = STORE_FAST + LOAD_FAST;

super(STORE_FAST__STORE_FAST) = STORE_FAST + STORE_FAST;

super(LOAD_CONST__LOAD_FAST) = LOAD_CONST + LOAD_FAST;

inst(POP_TOP, (value --)) {

DECREF_INPUTS();

}

inst(PUSH_NULL, (-- res)) {

res = NULL;

}

macro(END_FOR) = POP_TOP + POP_TOP;

inst(UNARY_NEGATIVE, (value -- res)) {

res = PyNumber_Negative(value);

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

inst(UNARY_NOT, (value -- res)) {

int err = PyObject_IsTrue(value);

DECREF_INPUTS();

ERROR_IF(err < 0, error);

if (err == 0) {

res = Py_True;

}

else {

res = Py_False;

}

Py_INCREF(res);

}

inst(UNARY_INVERT, (value -- res)) {

res = PyNumber_Invert(value);

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

family(binary_op, INLINE_CACHE_ENTRIES_BINARY_OP) = {

BINARY_OP,

BINARY_OP_ADD_FLOAT,

BINARY_OP_ADD_INT,

BINARY_OP_ADD_UNICODE,

// BINARY_OP_INPLACE_ADD_UNICODE, // This is an odd duck.

BINARY_OP_MULTIPLY_FLOAT,

BINARY_OP_MULTIPLY_INT,

BINARY_OP_SUBTRACT_FLOAT,

BINARY_OP_SUBTRACT_INT,

};

inst(BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- prod)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);

DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);

STAT_INC(BINARY_OP, hit);

prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);

_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);

_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);

ERROR_IF(prod == NULL, error);

}

inst(BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- prod)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);

DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);

STAT_INC(BINARY_OP, hit);

double dprod = ((PyFloatObject *)left)->ob_fval *

((PyFloatObject *)right)->ob_fval;

DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod);

}

inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);

DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);

STAT_INC(BINARY_OP, hit);

sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);

_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);

_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);

ERROR_IF(sub == NULL, error);

}

inst(BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- sub)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);

DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);

STAT_INC(BINARY_OP, hit);

double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval;

DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub);

}

inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);

DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);

STAT_INC(BINARY_OP, hit);

res = PyUnicode_Concat(left, right);

_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);

_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);

ERROR_IF(res == NULL, error);

}

// This is a subtle one. It's a super-instruction for

// BINARY_OP_ADD_UNICODE followed by STORE_FAST

// where the store goes into the left argument.

// So the inputs are the same as for all BINARY_OP

// specializations, but there is no output.

// At the end we just skip over the STORE_FAST.

inst(BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);

DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);

_Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];

assert(true_next.op.code == STORE_FAST ||

true_next.op.code == STORE_FAST__LOAD_FAST);

PyObject **target_local = &GETLOCAL(true_next.op.arg);

DEOPT_IF(*target_local != left, BINARY_OP);

STAT_INC(BINARY_OP, hit);

/* Handle `left = left + right` or `left += right` for str.

*

* When possible, extend `left` in place rather than

* allocating a new PyUnicodeObject. This attempts to avoid

* quadratic behavior when one neglects to use str.join().

*

* If `left` has only two references remaining (one from

* the stack, one in the locals), DECREFing `left` leaves

* only the locals reference, so PyUnicode_Append knows

* that the string is safe to mutate.

*/

assert(Py_REFCNT(left) >= 2);

_Py_DECREF_NO_DEALLOC(left);

PyUnicode_Append(target_local, right);

_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);

ERROR_IF(*target_local == NULL, error);

// The STORE_FAST is already done.

JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1);

}

inst(BINARY_OP_ADD_FLOAT, (unused/1, left, right -- sum)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);

DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);

STAT_INC(BINARY_OP, hit);

double dsum = ((PyFloatObject *)left)->ob_fval +

((PyFloatObject *)right)->ob_fval;

DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum);

}

inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);

DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);

STAT_INC(BINARY_OP, hit);

sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);

_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);

_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);

ERROR_IF(sum == NULL, error);

}

family(binary_subscr, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = {

BINARY_SUBSCR,

BINARY_SUBSCR_DICT,

BINARY_SUBSCR_GETITEM,

BINARY_SUBSCR_LIST_INT,

BINARY_SUBSCR_TUPLE_INT,

};

inst(BINARY_SUBSCR, (unused/4, container, sub -- res)) {

#if ENABLE_SPECIALIZATION

_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;

if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {

assert(cframe.use_tracing == 0);

next_instr--;

_Py_Specialize_BinarySubscr(container, sub, next_instr);

DISPATCH_SAME_OPARG();

}

STAT_INC(BINARY_SUBSCR, deferred);

DECREMENT_ADAPTIVE_COUNTER(cache->counter);

#endif /* ENABLE_SPECIALIZATION */

res = PyObject_GetItem(container, sub);

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

inst(BINARY_SLICE, (container, start, stop -- res)) {

PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);

// Can't use ERROR_IF() here, because we haven't

// DECREF'ed container yet, and we still own slice.

if (slice == NULL) {

res = NULL;

}

else {

res = PyObject_GetItem(container, slice);

Py_DECREF(slice);

}

Py_DECREF(container);

ERROR_IF(res == NULL, error);

}

inst(STORE_SLICE, (v, container, start, stop -- )) {

PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);

int err;

if (slice == NULL) {

err = 1;

}

else {

err = PyObject_SetItem(container, slice, v);

Py_DECREF(slice);

}

Py_DECREF(v);

Py_DECREF(container);

ERROR_IF(err, error);

}

inst(BINARY_SUBSCR_LIST_INT, (unused/4, list, sub -- res)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);

DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);

// Deopt unless 0 <= sub < PyList_Size(list)

DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR);

assert(((PyLongObject *)_PyLong_GetZero())->long_value.ob_digit[0] == 0);

Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];

DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);

STAT_INC(BINARY_SUBSCR, hit);

res = PyList_GET_ITEM(list, index);

assert(res != NULL);

Py_INCREF(res);

_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);

Py_DECREF(list);

}

inst(BINARY_SUBSCR_TUPLE_INT, (unused/4, tuple, sub -- res)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);

DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);

// Deopt unless 0 <= sub < PyTuple_Size(list)

DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), BINARY_SUBSCR);

assert(((PyLongObject *)_PyLong_GetZero())->long_value.ob_digit[0] == 0);

Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];

DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);

STAT_INC(BINARY_SUBSCR, hit);

res = PyTuple_GET_ITEM(tuple, index);

assert(res != NULL);

Py_INCREF(res);

_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);

Py_DECREF(tuple);

}

inst(BINARY_SUBSCR_DICT, (unused/4, dict, sub -- res)) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);

STAT_INC(BINARY_SUBSCR, hit);

res = PyDict_GetItemWithError(dict, sub);

if (res == NULL) {

if (!_PyErr_Occurred(tstate)) {

_PyErr_SetKeyError(sub);

}

Py_DECREF(dict);

Py_DECREF(sub);

ERROR_IF(true, error);

}

Py_INCREF(res); // Do this before DECREF'ing dict, sub

DECREF_INPUTS();

}

inst(BINARY_SUBSCR_GETITEM, (unused/1, type_version/2, func_version/1, container, sub -- unused)) {

PyTypeObject *tp = Py_TYPE(container);

DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR);

assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);

PyObject *cached = ((PyHeapTypeObject *)tp)->_spec_cache.getitem;

assert(PyFunction_Check(cached));

PyFunctionObject *getitem = (PyFunctionObject *)cached;

DEOPT_IF(getitem->func_version != func_version, BINARY_SUBSCR);

PyCodeObject *code = (PyCodeObject *)getitem->func_code;

assert(code->co_argcount == 2);

DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);

STAT_INC(BINARY_SUBSCR, hit);

Py_INCREF(getitem);

_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);

STACK_SHRINK(2);

new_frame->localsplus[0] = container;

new_frame->localsplus[1] = sub;

JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);

DISPATCH_INLINED(new_frame);

}

inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) {

ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error);

PREDICT(JUMP_BACKWARD);

}

inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) {

int err = PySet_Add(set, v);

Py_DECREF(v);

ERROR_IF(err, error);

PREDICT(JUMP_BACKWARD);

}

family(store_subscr, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = {

STORE_SUBSCR,

STORE_SUBSCR_DICT,

STORE_SUBSCR_LIST_INT,

};

inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) {

#if ENABLE_SPECIALIZATION

if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {

assert(cframe.use_tracing == 0);

next_instr--;

_Py_Specialize_StoreSubscr(container, sub, next_instr);

DISPATCH_SAME_OPARG();

}

STAT_INC(STORE_SUBSCR, deferred);

_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;

DECREMENT_ADAPTIVE_COUNTER(cache->counter);

#else

(void)counter; // Unused.

#endif /* ENABLE_SPECIALIZATION */

/* container[sub] = v */

int err = PyObject_SetItem(container, sub, v);

DECREF_INPUTS();

ERROR_IF(err, error);

}

inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);

DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);

// Ensure nonnegative, zero-or-one-digit ints.

DEOPT_IF(!_PyLong_IsPositiveSingleDigit(sub), STORE_SUBSCR);

Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];

// Ensure index < len(list)

DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);

STAT_INC(STORE_SUBSCR, hit);

PyObject *old_value = PyList_GET_ITEM(list, index);

PyList_SET_ITEM(list, index, value);

assert(old_value != NULL);

Py_DECREF(old_value);

_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);

Py_DECREF(list);

}

inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) {

assert(cframe.use_tracing == 0);

DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);

STAT_INC(STORE_SUBSCR, hit);

int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);

Py_DECREF(dict);

ERROR_IF(err, error);

}

inst(DELETE_SUBSCR, (container, sub --)) {

/* del container[sub] */

int err = PyObject_DelItem(container, sub);

DECREF_INPUTS();

ERROR_IF(err, error);

}

inst(CALL_INTRINSIC_1, (value -- res)) {

assert(oparg <= MAX_INTRINSIC_1);

res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value);

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

inst(CALL_INTRINSIC_2, (value2, value1 -- res)) {

assert(oparg <= MAX_INTRINSIC_2);

res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1);

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

inst(RAISE_VARARGS, (args[oparg] -- )) {

PyObject *cause = NULL, *exc = NULL;

switch (oparg) {

case 2:

cause = args[1];

/* fall through */

case 1:

exc = args[0];

/* fall through */

case 0:

ERROR_IF(do_raise(tstate, exc, cause), exception_unwind);

break;

default:

_PyErr_SetString(tstate, PyExc_SystemError,

"bad RAISE_VARARGS oparg");

break;

}

ERROR_IF(true, error);

}

inst(INTERPRETER_EXIT, (retval --)) {

assert(frame == &entry_frame);

assert(_PyFrame_IsIncomplete(frame));

STACK_SHRINK(1); // Since we're not going to DISPATCH()

assert(EMPTY());

/* Restore previous cframe and return. */

tstate->cframe = cframe.previous;

tstate->cframe->use_tracing = cframe.use_tracing;

assert(tstate->cframe->current_frame == frame->previous);

assert(!_PyErr_Occurred(tstate));

_Py_LeaveRecursiveCallTstate(tstate);

return retval;

}

inst(RETURN_VALUE, (retval --)) {

STACK_SHRINK(1);

assert(EMPTY());

_PyFrame_SetStackPointer(frame, stack_pointer);

TRACE_FUNCTION_EXIT();

DTRACE_FUNCTION_EXIT();

_Py_LeaveRecursiveCallPy(tstate);

assert(frame != &entry_frame);

// GH-99729: We need to unlink the frame *before* clearing it:

_PyInterpreterFrame *dying = frame;

frame = cframe.current_frame = dying->previous;

_PyEvalFrameClearAndPop(tstate, dying);

_PyFrame_StackPush(frame, retval);

goto resume_frame;

}

inst(RETURN_CONST, (--)) {

PyObject *retval = GETITEM(consts, oparg);

Py_INCREF(retval);

assert(EMPTY());

_PyFrame_SetStackPointer(frame, stack_pointer);

TRACE_FUNCTION_EXIT();

DTRACE_FUNCTION_EXIT();

_Py_LeaveRecursiveCallPy(tstate);

assert(frame != &entry_frame);

// GH-99729: We need to unlink the frame *before* clearing it:

_PyInterpreterFrame *dying = frame;

frame = cframe.current_frame = dying->previous;

_PyEvalFrameClearAndPop(tstate, dying);

_PyFrame_StackPush(frame, retval);

goto resume_frame;

}

inst(GET_AITER, (obj -- iter)) {

unaryfunc getter = NULL;

PyTypeObject *type = Py_TYPE(obj);

if (type->tp_as_async != NULL) {

getter = type->tp_as_async->am_aiter;

}

if (getter == NULL) {

_PyErr_Format(tstate, PyExc_TypeError,

"'async for' requires an object with "

"__aiter__ method, got %.100s",

type->tp_name);

DECREF_INPUTS();

ERROR_IF(true, error);

}

iter = (*getter)(obj);

DECREF_INPUTS();

ERROR_IF(iter == NULL, error);

if (Py_TYPE(iter)->tp_as_async == NULL ||

Py_TYPE(iter)->tp_as_async->am_anext == NULL) {

_PyErr_Format(tstate, PyExc_TypeError,

"'async for' received an object from __aiter__ "

"that does not implement __anext__: %.100s",

Py_TYPE(iter)->tp_name);

Py_DECREF(iter);

ERROR_IF(true, error);

}

}

inst(GET_ANEXT, (aiter -- aiter, awaitable)) {

unaryfunc getter = NULL;

PyObject *next_iter = NULL;

PyTypeObject *type = Py_TYPE(aiter);

if (PyAsyncGen_CheckExact(aiter)) {

awaitable = type->tp_as_async->am_anext(aiter);

if (awaitable == NULL) {

goto error;

}

} else {

if (type->tp_as_async != NULL){

getter = type->tp_as_async->am_anext;

}

if (getter != NULL) {

next_iter = (*getter)(aiter);

if (next_iter == NULL) {

goto error;

}

}

else {

_PyErr_Format(tstate, PyExc_TypeError,

"'async for' requires an iterator with "

"__anext__ method, got %.100s",

type->tp_name);

goto error;

}

awaitable = _PyCoro_GetAwaitableIter(next_iter);

if (awaitable == NULL) {

_PyErr_FormatFromCause(

PyExc_TypeError,

"'async for' received an invalid object "

"from __anext__: %.100s",

Py_TYPE(next_iter)->tp_name);

Py_DECREF(next_iter);

goto error;

} else {

Py_DECREF(next_iter);

}

}

PREDICT(LOAD_CONST);

}

inst(GET_AWAITABLE, (iterable -- iter)) {

iter = _PyCoro_GetAwaitableIter(iterable);

if (iter == NULL) {

format_awaitable_error(tstate, Py_TYPE(iterable), oparg);

}

DECREF_INPUTS();

if (iter != NULL && PyCoro_CheckExact(iter)) {

PyObject *yf = _PyGen_yf((PyGenObject*)iter);

if (yf != NULL) {

/* `iter` is a coroutine object that is being

awaited, `yf` is a pointer to the current awaitable

being awaited on. */

Py_DECREF(yf);

Py_CLEAR(iter);

_PyErr_SetString(tstate, PyExc_RuntimeError,

"coroutine is being awaited already");

/* The code below jumps to `error` if `iter` is NULL. */

}

}

ERROR_IF(iter == NULL, error);

PREDICT(LOAD_CONST);

}

family(for_iter, INLINE_CACHE_ENTRIES_FOR_ITER) = {

SEND,

SEND_GEN,

};

inst(SEND, (unused/1, receiver, v -- receiver, retval)) {

#if ENABLE_SPECIALIZATION

_PySendCache *cache = (_PySendCache *)next_instr;

if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {

assert(cframe.use_tracing == 0);

next_instr--;

_Py_Specialize_Send(receiver, next_instr);

DISPATCH_SAME_OPARG();

}

STAT_INC(SEND, deferred);

DECREMENT_ADAPTIVE_COUNTER(cache->counter);

#endif /* ENABLE_SPECIALIZATION */

assert(frame != &entry_frame);

if (Py_IsNone(v) && PyIter_Check(receiver)) {

retval = Py_TYPE(receiver)->tp_iternext(receiver);

}

else {

retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);

}

if (retval == NULL) {

if (tstate->c_tracefunc != NULL

&& _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))

call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);

if (_PyGen_FetchStopIterationValue(&retval) == 0) {

assert(retval != NULL);

JUMPBY(oparg);

}

else {

assert(retval == NULL);

goto error;

}

}

else {

assert(retval != NULL);

}

Py_DECREF(v);

}

inst(SEND_GEN, (unused/1, receiver, v -- receiver)) {

assert(cframe.use_tracing == 0);

PyGenObject *gen = (PyGenObject *)receiver;

DEOPT_IF(Py_TYPE(gen) != &PyGen_Type &&

Py_TYPE(gen) != &PyCoro_Type, SEND);

DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);

STAT_INC(SEND, hit);

_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;

frame->yield_offset = oparg;

STACK_SHRINK(1);

_PyFrame_StackPush(gen_frame, v);

gen->gi_frame_state = FRAME_EXECUTING;

gen->gi_exc_state.previous_item = tstate->exc_info;

tstate->exc_info = &gen->gi_exc_state;

JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg);

DISPATCH_INLINED(gen_frame);

}

inst(YIELD_VALUE, (retval -- unused)) {

// NOTE: It's important that YIELD_VALUE never raises an exception!

// The compiler treats any exception raised here as a failed close()

// or throw() call.

assert(frame != &entry_frame);

PyGenObject *gen = _PyFrame_GetGenerator(frame);

gen->gi_frame_state = FRAME_SUSPENDED;

_PyFrame_SetStackPointer(frame, stack_pointer - 1);

TRACE_FUNCTION_EXIT();

DTRACE_FUNCTION_EXIT();

tstate->exc_info = gen->gi_exc_state.previous_item;

gen->gi_exc_state.previous_item = NULL;

_Py_LeaveRecursiveCallPy(tstate);

_PyInterpreterFrame *gen_frame = frame;

frame = cframe.current_frame = frame->previous;

gen_frame->previous = NULL;

frame->prev_instr -= frame->yield_offset;

_PyFrame_StackPush(frame, retval);

goto resume_frame;

}

inst(POP_EXCEPT, (exc_value -- )) {

_PyErr_StackItem *exc_info = tstate->exc_info;

Py_XSETREF(exc_info->exc_value, exc_value);

}

inst(RERAISE, (values[oparg], exc -- values[oparg])) {

assert(oparg >= 0 && oparg <= 2);

if (oparg) {

PyObject *lasti = values[0];

if (PyLong_Check(lasti)) {

frame->prev_instr = _PyCode_CODE(frame->f_code) + PyLong_AsLong(lasti);

assert(!_PyErr_Occurred(tstate));

}

else {

assert(PyLong_Check(lasti));

_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");

goto error;

}

}

assert(exc && PyExceptionInstance_Check(exc));

Py_INCREF(exc);

PyObject *typ = Py_NewRef(PyExceptionInstance_Class(exc));

PyObject *tb = PyException_GetTraceback(exc);

_PyErr_Restore(tstate, typ, exc, tb);

goto exception_unwind;

}

inst(END_ASYNC_FOR, (awaitable, exc -- )) {

assert(exc && PyExceptionInstance_Check(exc));

if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) {

DECREF_INPUTS();

}

else {

Py_INCREF(exc);

PyObject *typ = Py_NewRef(PyExceptionInstance_Class(exc));

PyObject *tb = PyException_GetTraceback(exc);

_PyErr_Restore(tstate, typ, exc, tb);

goto exception_unwind;

}

}

inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) {

assert(throwflag);

assert(exc_value && PyExceptionInstance_Check(exc_value));

if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) {

value = Py_NewRef(((PyStopIterationObject *)exc_value)->value);

DECREF_INPUTS();

none = Py_NewRef(Py_None);

}

else {

_PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));

goto exception_unwind;

}

}

inst(LOAD_ASSERTION_ERROR, ( -- value)) {

value = Py_NewRef(PyExc_AssertionError);

}

inst(LOAD_BUILD_CLASS, ( -- bc)) {

if (PyDict_CheckExact(BUILTINS())) {

bc = _PyDict_GetItemWithError(BUILTINS(),

&_Py_ID(__build_class__));

if (bc == NULL) {

if (!_PyErr_Occurred(tstate)) {

_PyErr_SetString(tstate, PyExc_NameError,

"__build_class__ not found");

}

ERROR_IF(true, error);

}

Py_INCREF(bc);

}

else {

bc = PyObject_GetItem(BUILTINS(), &_Py_ID(__build_class__));

if (bc == NULL) {

if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError))

_PyErr_SetString(tstate, PyExc_NameError,

"__build_class__ not found");

ERROR_IF(true, error);

}

}

}

inst(STORE_NAME, (v -- )) {

PyObject *name = GETITEM(names, oparg);

PyObject *ns = LOCALS();

int err;

if (ns == NULL) {

_PyErr_Format(tstate, PyExc_SystemError,

"no locals found when storing %R", name);

DECREF_INPUTS();

ERROR_IF(true, error);

}

if (PyDict_CheckExact(ns))

err = PyDict_SetItem(ns, name, v);

else

err = PyObject_SetItem(ns, name, v);

DECREF_INPUTS();

ERROR_IF(err, error);

}

inst(DELETE_NAME, (--)) {

PyObject *name = GETITEM(names, oparg);

PyObject *ns = LOCALS();

int err;

if (ns == NULL) {

_PyErr_Format(tstate, PyExc_SystemError,

"no locals when deleting %R", name);

goto error;

}

err = PyObject_DelItem(ns, name);

// Can't use ERROR_IF here.

if (err != 0) {

format_exc_check_arg(tstate, PyExc_NameError,

NAME_ERROR_MSG,

name);

goto error;

}

}

family(unpack_sequence, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE) = {

UNPACK_SEQUENCE,

UNPACK_SEQUENCE_TWO_TUPLE,

UNPACK_SEQUENCE_TUPLE,

UNPACK_SEQUENCE_LIST,

};

inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) {

#if ENABLE_SPECIALIZATION

_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;

if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {

assert(cframe.use_tracing == 0);

next_instr--;

_Py_Specialize_UnpackSequence(seq, next_instr, oparg);

DISPATCH_SAME_OPARG();

}

STAT_INC(UNPACK_SEQUENCE, deferred);

DECREMENT_ADAPTIVE_COUNTER(cache->counter);

#endif /* ENABLE_SPECIALIZATION */

PyObject **top = stack_pointer + oparg - 1;

int res = unpack_iterable(tstate, seq, oparg, -1, top);

Py_DECREF(seq);

ERROR_IF(res == 0, error);

}

inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) {

DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);

DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);

assert(oparg == 2);

STAT_INC(UNPACK_SEQUENCE, hit);

values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1));

values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0));

Py_DECREF(seq);

}

inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) {

DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);

DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);

STAT_INC(UNPACK_SEQUENCE, hit);

PyObject **items = _PyTuple_ITEMS(seq);

for (int i = oparg; --i >= 0; ) {

*values++ = Py_NewRef(items[i]);

}

Py_DECREF(seq);

}

inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) {

DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);

DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);

STAT_INC(UNPACK_SEQUENCE, hit);

PyObject **items = _PyList_ITEMS(seq);

for (int i = oparg; --i >= 0; ) {

*values++ = Py_NewRef(items[i]);

}

Py_DECREF(seq);

}

inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) {

int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);

PyObject **top = stack_pointer + totalargs - 1;

int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);

Py_DECREF(seq);

ERROR_IF(res == 0, error);

}

family(store_attr, INLINE_CACHE_ENTRIES_STORE_ATTR) = {

STORE_ATTR,

STORE_ATTR_INSTANCE_VALUE,

STORE_ATTR_SLOT,

STORE_ATTR_WITH_HINT,

};

inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) {

#if ENABLE_SPECIALIZATION

if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {

assert(cframe.use_tracing == 0);

PyObject *name = GETITEM(names, oparg);

next_instr--;

_Py_Specialize_StoreAttr(owner, next_instr, name);

DISPATCH_SAME_OPARG();

}

STAT_INC(STORE_ATTR, deferred);

_PyAttrCache *cache = (_PyAttrCache *)next_instr;

DECREMENT_ADAPTIVE_COUNTER(cache->counter);

#else

(void)counter; // Unused.

#endif /* ENABLE_SPECIALIZATION */

PyObject *name = GETITEM(names, oparg);

int err = PyObject_SetAttr(owner, name, v);

Py_DECREF(v);

Py_DECREF(owner);

ERROR_IF(err, error);

}

inst(DELETE_ATTR, (owner --)) {

PyObject *name = GETITEM(names, oparg);

int err = PyObject_SetAttr(owner, name, (PyObject *)NULL);

Py_DECREF(owner);

ERROR_IF(err, error);

}

inst(STORE_GLOBAL, (v --)) {

PyObject *name = GETITEM(names, oparg);

int err = PyDict_SetItem(GLOBALS(), name, v);

Py_DECREF(v);

ERROR_IF(err, error);

}

inst(DELETE_GLOBAL, (--)) {

PyObject *name = GETITEM(names, oparg);

int err;

err = PyDict_DelItem(GLOBALS(), name);

// Can't use ERROR_IF here.

if (err != 0) {

if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {

format_exc_check_arg(tstate, PyExc_NameError,

NAME_ERROR_MSG, name);

}