Skip to content

GH-118093: Make CALL_ALLOC_AND_ENTER_INIT suitable for tier 2. #123140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
markshannon merged 11 commits into from
Aug 20, 2024
Merged

GH-118093: Make CALL_ALLOC_AND_ENTER_INIT suitable for tier 2. #123140

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ec4b278
Project traces through CALL_ALLOC_AND_ENTER_INIT. Work in progress
markshannon Jul 17, 2024
2b94491
Convert CALL_ALLOC_AND_ENTER_INIT to micro-ops such that tier 2 suppo…
markshannon Aug 19, 2024
5cea41f
Allow inexact arguments for CALL_ALLOC_AND_ENTER_INIT.
markshannon Aug 19, 2024
8fc89ca
Merge branch 'main' into call-class-tier-2
markshannon Aug 19, 2024
2a83bbb
Fill in optimizer bytecodes for new uops
markshannon Aug 19, 2024
af5ed4b
Export symbol for JIT
markshannon Aug 19, 2024
ae8208f
Export symbol for JIT
markshannon Aug 20, 2024
882bff7
Tell globals checker to ignore a false positive
markshannon Aug 20, 2024
d4135c4
Update Misc/NEWS.d/next/Core_and_Builtins/2024-08-19-15-13-13.gh-issu…
markshannon Aug 20, 2024
8b7fe5c
Formatting fixups
markshannon Aug 20, 2024
2e15a65
Merge remote-tracking branch 'faster/call-class-tier-2' into call-cla…
markshannon Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Convert CALL_ALLOC_AND_ENTER_INIT to micro-ops such that tier 2 suppo… 
…rts it
  • Loading branch information
@markshannon
markshannon committed Aug 19, 2024
commit 2b944914fda2da00ebb0c14c499cb538b82dfe2b
13 changes: 8 additions & 5 deletions Include/internal/pycore_frame.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,9 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
static inline void
_PyFrame_Initialize(
_PyInterpreterFrame *frame, PyFunctionObject *func,
PyObject *locals, PyCodeObject *code, int null_locals_from)
PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous)
{
frame->previous = previous;
frame->f_funcobj = (PyObject *)func;
frame->f_executable = Py_NewRef(code);
frame->f_builtins = func->func_builtins;
Expand Down Expand Up @@ -277,26 +278,27 @@ PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFr
* Must be guarded by _PyThreadState_HasStackSpace()
* Consumes reference to func. */
static inline _PyInterpreterFrame *
_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from)
_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from, _PyInterpreterFrame * previous)
{
CALL_STAT_INC(frames_pushed);
PyCodeObject *code = (PyCodeObject *)func->func_code;
_PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top;
tstate->datastack_top += code->co_framesize;
assert(tstate->datastack_top < tstate->datastack_limit);
_PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from);
_PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from, previous);
return new_frame;
}

/* Pushes a trampoline frame without checking for space.
* Must be guarded by _PyThreadState_HasStackSpace() */
static inline _PyInterpreterFrame *
_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth)
_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrame * previous)
{
CALL_STAT_INC(frames_pushed);
_PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top;
tstate->datastack_top += code->co_framesize;
assert(tstate->datastack_top < tstate->datastack_limit);
frame->previous = previous;
frame->f_funcobj = Py_None;
frame->f_executable = Py_NewRef(code);
#ifdef Py_DEBUG
Expand All @@ -315,7 +317,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
PyAPI_FUNC(_PyInterpreterFrame *)
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, _PyStackRef const* args,
size_t argcount, PyObject *kwnames);
size_t argcount, PyObject *kwnames,
_PyInterpreterFrame *previous);

#ifdef __cplusplus
}
Expand Down
2 changes: 1 addition & 1 deletion Include/internal/pycore_opcode_metadata.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Include/internal/pycore_uop_ids.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions Objects/frameobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1771,8 +1771,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals)
{
PyCodeObject *code = (PyCodeObject *)func->func_code;
_PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func),
Py_XNewRef(locals), code, 0);
frame->previous = NULL;
Py_XNewRef(locals), code, 0, NULL);
}

PyFrameObject*
Expand Down
83 changes: 17 additions & 66 deletions Python/bytecodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -774,7 +774,7 @@ dummy_func(
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(BINARY_SUBSCR, hit);
Py_INCREF(getitem);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
STACK_SHRINK(2);
new_frame->localsplus[0] = container_st;
new_frame->localsplus[1] = sub_st;
Expand Down Expand Up @@ -1125,6 +1125,8 @@ dummy_func(
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
assert(gen_frame->previous == NULL);
gen_frame->previous = frame;
DISPATCH_INLINED(gen_frame);
}
if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {
Expand Down Expand Up @@ -1168,6 +1170,8 @@ dummy_func(
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
gen_frame->previous = frame;
DISPATCH_INLINED(gen_frame);
DISPATCH_INLINED(gen_frame);
}

Expand Down Expand Up @@ -2282,7 +2286,7 @@ dummy_func(
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
Expand All @@ -2309,7 +2313,7 @@ dummy_func(

PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
Py_INCREF(f);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
Expand Down Expand Up @@ -3091,6 +3095,7 @@ dummy_func(
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
gen_frame->previous = frame;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
}
Expand Down Expand Up @@ -3368,7 +3373,7 @@ dummy_func(
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
args, total_args, NULL, frame
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
Expand Down Expand Up @@ -3438,7 +3443,7 @@ dummy_func(
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
args, total_args, NULL, frame
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
Expand Down Expand Up @@ -3581,7 +3586,7 @@ dummy_func(
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
Expand All @@ -3595,7 +3600,7 @@ dummy_func(
assert(tstate->interp->eval_frame == NULL);
SYNC_SP();
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
Expand Down Expand Up @@ -3699,22 +3704,19 @@ dummy_func(

op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _PyInterpreterFrame *)) {
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame);
assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
/* Push self onto stack of shim */
shim->localsplus[0] = PyStackRef_DUP(self);
PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init);
init_frame = _PyFrame_PushUnchecked(tstate, init_func, oparg+1);
init_frame = _PyFrame_PushUnchecked(tstate, init_func, oparg+1, shim);
/* Copy self followed by args to __init__ frame */
init_frame->localsplus[0] = self;
for (int i = 0; i < oparg; i++) {
init_frame->localsplus[i+1] = args[i];
}
frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
SYNC_SP();
/* Link shim frame */
shim->previous = frame;
frame = shim;
/* Account for pushing the extra frame.
* We don't check recursion depth here,
* as it will be checked after start_frame */
Expand All @@ -3723,62 +3725,11 @@ dummy_func(

macro(CALL_ALLOC_AND_ENTER_INIT) =
unused/1 +
_CHECK_PEP_523 +
_CHECK_AND_ALLOCATE_OBJECT +
_CREATE_INIT_FRAME +
_PUSH_FRAME;

inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
/* This instruction does the following:
* 1. Creates the object (by calling ``object.__new__``)
* 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
* 3. Pushes the frame for ``__init__`` to the frame stack
* */
_PyCallCache *cache = (_PyCallCache *)&this_instr[1];
DEOPT_IF(!PyStackRef_IsNull(null));
DEOPT_IF(!PyType_Check(callable_o));
PyTypeObject *tp = (PyTypeObject *)callable_o;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version));
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
DEOPT_IF(code->co_argcount != oparg+1);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize));
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
ERROR_NO_POP();
}
PyStackRef_CLOSE(callable);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
/* Push self onto stack of shim */
Py_INCREF(self);
shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
Py_INCREF(init);
_PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
/* Copy self followed by args to __init__ frame */
init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
for (int i = 0; i < oparg; i++) {
init_frame->localsplus[i+1] = args[i];
}
frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
STACK_SHRINK(oparg+2);
_PyFrame_SetStackPointer(frame, stack_pointer);
/* Link frames */
init_frame->previous = shim;
shim->previous = frame;
frame = tstate->current_frame = init_frame;
CALL_STAT_INC(inlined_py_calls);
/* Account for pushing the extra frame.
* We don't check recursion depth here,
* as it will be checked after start_frame */
tstate->py_recursion_remaining--;
goto start_frame;
}

inst(EXIT_INIT_CHECK, (should_be_none -- )) {
assert(STACK_LEVEL() == 2);
if (!PyStackRef_Is(should_be_none, PyStackRef_None)) {
Expand Down Expand Up @@ -4229,7 +4180,7 @@ dummy_func(
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, positional_args, kwnames_o
args, positional_args, kwnames_o, frame
);
PyStackRef_CLOSE(kwnames);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
Expand Down Expand Up @@ -4344,7 +4295,7 @@ dummy_func(

_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
(PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
nargs, callargs, kwargs);
nargs, callargs, kwargs, frame);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
Expand Down
18 changes: 9 additions & 9 deletions Python/ceval.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs);
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous);

#ifdef HAVE_ERRNO_H
#include <errno.h>
Expand Down Expand Up @@ -1736,15 +1736,15 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame)
_PyInterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, _PyStackRef const* args,
size_t argcount, PyObject *kwnames)
size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous)
{
PyCodeObject * code = (PyCodeObject *)func->func_code;
CALL_STAT_INC(frames_pushed);
_PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize);
if (frame == NULL) {
goto fail;
}
_PyFrame_Initialize(frame, func, locals, code, 0);
_PyFrame_Initialize(frame, func, locals, code, 0, previous);
if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) {
assert(frame->owner == FRAME_OWNED_BY_THREAD);
clear_thread_frame(tstate, frame);
Expand All @@ -1771,7 +1771,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, PyObject *const* args,
size_t argcount, PyObject *kwnames)
size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous)
{
#if defined(Py_GIL_DISABLED)
size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames);
Expand All @@ -1787,11 +1787,11 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
for (size_t i = 0; i < kw_count; i++) {
tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]);
}
_PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames);
_PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames, previous);
PyMem_Free(tagged_args_buffer);
return res;
#else
return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames);
return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames, previous);
#endif
}

Expand All @@ -1800,7 +1800,7 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
*/
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs)
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous)
{
bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0);
PyObject *kwnames = NULL;
Expand All @@ -1821,7 +1821,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
}
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged(
tstate, (PyFunctionObject *)func, locals,
newargs, nargs, kwnames
newargs, nargs, kwnames, previous
);
if (has_dict) {
_PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames);
Expand Down Expand Up @@ -1858,7 +1858,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
}
}
_PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged(
tstate, func, locals, args, argcount, kwnames);
tstate, func, locals, args, argcount, kwnames, NULL);
if (frame == NULL) {
return NULL;
}
Expand Down
2 changes: 1 addition & 1 deletion Python/ceval_macros.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ do { \
do { \
assert(tstate->interp->eval_frame == NULL); \
_PyFrame_SetStackPointer(frame, stack_pointer); \
(NEW_FRAME)->previous = frame; \
assert((NEW_FRAME)->previous == frame); \
frame = tstate->current_frame = (NEW_FRAME); \
CALL_STAT_INC(inlined_py_calls); \
goto start_frame; \
Expand Down
Loading