Skip to content

Commit 7199584

Browse files
markshannonmarkshannon
authored
pythonGH-100987: Allow objects other than code objects as the "executable" of an internal frame. (pythonGH-105727)
* Add table describing possible executable classes for out-of-process debuggers. * Remove shim code object creation code as it is no longer needed. * Make lltrace a bit more robust w.r.t. non-standard frames.
1 parent ad56340 commit 7199584

28 files changed

Lines changed: 542 additions & 607 deletions

‎Include/internal/pycore_code.h‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -447,19 +447,6 @@ adaptive_counter_backoff(uint16_t counter) {
447447
return adaptive_counter_bits(value, backoff);
448448
}
449449

450-
451-
/* Line array cache for tracing */
452-
453-
typedef struct _PyShimCodeDef {
454-
const uint8_t *code;
455-
int codelen;
456-
int stacksize;
457-
const char *cname;
458-
} _PyShimCodeDef;
459-
460-
extern PyCodeObject *
461-
_Py_MakeShimCode(const _PyShimCodeDef *code);
462-
463450
extern uint32_t _Py_next_func_version;
464451

465452

‎Include/internal/pycore_frame.h‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ enum _frameowner {
4747
};
4848

4949
typedef struct _PyInterpreterFrame {
50-
PyCodeObject *f_code; /* Strong reference */
50+
PyObject *f_executable; /* Strong reference */
5151
struct _PyInterpreterFrame *previous;
5252
PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */
5353
PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
@@ -73,20 +73,25 @@ typedef struct _PyInterpreterFrame {
7373
} _PyInterpreterFrame;
7474

7575
#define _PyInterpreterFrame_LASTI(IF) \
76-
((int)((IF)->prev_instr - _PyCode_CODE((IF)->f_code)))
76+
((int)((IF)->prev_instr - _PyCode_CODE(_PyFrame_GetCode(IF))))
77+
78+
static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
79+
assert(PyCode_Check(f->f_executable));
80+
return (PyCodeObject *)f->f_executable;
81+
}
7782

7883
static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
79-
return f->localsplus + f->f_code->co_nlocalsplus;
84+
return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus;
8085
}
8186

8287
static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) {
83-
assert(f->stacktop > f->f_code->co_nlocalsplus);
88+
assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
8489
assert(f->localsplus[f->stacktop-1] != NULL);
8590
return f->localsplus[f->stacktop-1];
8691
}
8792

8893
static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) {
89-
assert(f->stacktop > f->f_code->co_nlocalsplus);
94+
assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
9095
f->stacktop--;
9196
return f->localsplus[f->stacktop];
9297
}
@@ -119,7 +124,7 @@ _PyFrame_Initialize(
119124
PyObject *locals, PyCodeObject *code, int null_locals_from)
120125
{
121126
frame->f_funcobj = (PyObject *)func;
122-
frame->f_code = (PyCodeObject *)Py_NewRef(code);
127+
frame->f_executable = Py_NewRef(code);
123128
frame->f_builtins = func->func_builtins;
124129
frame->f_globals = func->func_globals;
125130
frame->f_locals = locals;
@@ -172,8 +177,11 @@ _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
172177
static inline bool
173178
_PyFrame_IsIncomplete(_PyInterpreterFrame *frame)
174179
{
180+
if (frame->owner == FRAME_OWNED_BY_CSTACK) {
181+
return true;
182+
}
175183
return frame->owner != FRAME_OWNED_BY_GENERATOR &&
176-
frame->prev_instr < _PyCode_CODE(frame->f_code) + frame->f_code->_co_firsttraceable;
184+
frame->prev_instr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable;
177185
}
178186

179187
static inline _PyInterpreterFrame *
@@ -272,6 +280,14 @@ PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame)
272280
return (PyGenObject *)(((char *)frame) - offset_in_gen);
273281
}
274282

283+
#define PY_EXECUTABLE_KIND_SKIP 0
284+
#define PY_EXECUTABLE_KIND_PY_FUNCTION 1
285+
#define PY_EXECUTABLE_KIND_BUILTIN_FUNCTION 3
286+
#define PY_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4
287+
#define PY_EXECUTABLE_KINDS 5
288+
289+
PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PY_EXECUTABLE_KINDS+1];
290+
275291
#ifdef __cplusplus
276292
}
277293
#endif

‎Include/internal/pycore_global_objects_fini_generated.h‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ struct _Py_global_strings {
5151
STRUCT_FOR_STR(newline, "\n")
5252
STRUCT_FOR_STR(open_br, "{")
5353
STRUCT_FOR_STR(percent, "%")
54-
STRUCT_FOR_STR(shim_name, "<shim>")
5554
STRUCT_FOR_STR(type_params, ".type_params")
5655
STRUCT_FOR_STR(utf_8, "utf-8")
5756
} literals;

‎Include/internal/pycore_interp.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,6 @@ struct _is {
159159
struct ast_state ast;
160160
struct types_state types;
161161
struct callable_cache callable_cache;
162-
PyCodeObject *interpreter_trampoline;
163162
_PyOptimizerObject *optimizer;
164163
uint16_t optimizer_resume_threshold;
165164
uint16_t optimizer_backedge_threshold;

‎Include/internal/pycore_runtime_init_generated.h‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Misc/NEWS.d/next/Core and Builtins/2023-06-09-10-48-17.gh-issue-100987.mK-xny.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Allow objects other than code objects as the "executable" in internal
2+
frames. In the long term, this can help tools like Cython and PySpy interact
3+
more efficiently. In the shorter term, it allows us to perform some
4+
optimizations more simply.

‎Objects/codeobject.c‎

Lines changed: 0 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -2316,76 +2316,3 @@ _PyStaticCode_Init(PyCodeObject *co)
23162316
}
23172317

23182318
#define MAX_CODE_UNITS_PER_LOC_ENTRY 8
2319-
2320-
PyCodeObject *
2321-
_Py_MakeShimCode(const _PyShimCodeDef *codedef)
2322-
{
2323-
PyObject *name = NULL;
2324-
PyObject *co_code = NULL;
2325-
PyObject *lines = NULL;
2326-
PyCodeObject *codeobj = NULL;
2327-
uint8_t *loc_table = NULL;
2328-
2329-
name = _PyUnicode_FromASCII(codedef->cname, strlen(codedef->cname));
2330-
if (name == NULL) {
2331-
goto cleanup;
2332-
}
2333-
co_code = PyBytes_FromStringAndSize(
2334-
(const char *)codedef->code, codedef->codelen);
2335-
if (co_code == NULL) {
2336-
goto cleanup;
2337-
}
2338-
int code_units = codedef->codelen / sizeof(_Py_CODEUNIT);
2339-
int loc_entries = (code_units + MAX_CODE_UNITS_PER_LOC_ENTRY - 1) /
2340-
MAX_CODE_UNITS_PER_LOC_ENTRY;
2341-
loc_table = PyMem_Malloc(loc_entries);
2342-
if (loc_table == NULL) {
2343-
PyErr_NoMemory();
2344-
goto cleanup;
2345-
}
2346-
for (int i = 0; i < loc_entries-1; i++) {
2347-
loc_table[i] = 0x80 | (PY_CODE_LOCATION_INFO_NONE << 3) | 7;
2348-
code_units -= MAX_CODE_UNITS_PER_LOC_ENTRY;
2349-
}
2350-
assert(loc_entries > 0);
2351-
assert(code_units > 0 && code_units <= MAX_CODE_UNITS_PER_LOC_ENTRY);
2352-
loc_table[loc_entries-1] = 0x80 |
2353-
(PY_CODE_LOCATION_INFO_NONE << 3) | (code_units-1);
2354-
lines = PyBytes_FromStringAndSize((const char *)loc_table, loc_entries);
2355-
PyMem_Free(loc_table);
2356-
if (lines == NULL) {
2357-
goto cleanup;
2358-
}
2359-
_Py_DECLARE_STR(shim_name, "<shim>");
2360-
struct _PyCodeConstructor con = {
2361-
.filename = &_Py_STR(shim_name),
2362-
.name = name,
2363-
.qualname = name,
2364-
.flags = CO_NEWLOCALS | CO_OPTIMIZED,
2365-
2366-
.code = co_code,
2367-
.firstlineno = 1,
2368-
.linetable = lines,
2369-
2370-
.consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
2371-
.names = (PyObject *)&_Py_SINGLETON(tuple_empty),
2372-
2373-
.localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
2374-
.localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
2375-
2376-
.argcount = 0,
2377-
.posonlyargcount = 0,
2378-
.kwonlyargcount = 0,
2379-
2380-
.stacksize = codedef->stacksize,
2381-
2382-
.exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
2383-
};
2384-
2385-
codeobj = _PyCode_New(&con);
2386-
cleanup:
2387-
Py_XDECREF(name);
2388-
Py_XDECREF(co_code);
2389-
Py_XDECREF(lines);
2390-
return codeobj;
2391-
}

‎Objects/frameobject.c‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,7 @@ _PyFrame_GetState(PyFrameObject *frame)
642642
static int
643643
frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignored))
644644
{
645+
PyCodeObject *code = _PyFrame_GetCode(f->f_frame);
645646
if (p_new_lineno == NULL) {
646647
PyErr_SetString(PyExc_AttributeError, "cannot delete attribute");
647648
return -1;
@@ -719,7 +720,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
719720
}
720721
new_lineno = (int)l_new_lineno;
721722

722-
if (new_lineno < f->f_frame->f_code->co_firstlineno) {
723+
if (new_lineno < code->co_firstlineno) {
723724
PyErr_Format(PyExc_ValueError,
724725
"line %d comes before the current code block",
725726
new_lineno);
@@ -728,8 +729,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
728729

729730
/* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this
730731
* should never overflow. */
731-
int len = (int)Py_SIZE(f->f_frame->f_code);
732-
int *lines = marklines(f->f_frame->f_code, len);
732+
int len = (int)Py_SIZE(code);
733+
int *lines = marklines(code, len);
733734
if (lines == NULL) {
734735
return -1;
735736
}
@@ -743,7 +744,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
743744
return -1;
744745
}
745746

746-
int64_t *stacks = mark_stacks(f->f_frame->f_code, len);
747+
int64_t *stacks = mark_stacks(code, len);
747748
if (stacks == NULL) {
748749
PyMem_Free(lines);
749750
return -1;
@@ -788,7 +789,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
788789
// in the new location. Rather than crashing or changing co_code, just bind
789790
// None instead:
790791
int unbound = 0;
791-
for (int i = 0; i < f->f_frame->f_code->co_nlocalsplus; i++) {
792+
for (int i = 0; i < code->co_nlocalsplus; i++) {
792793
// Counting every unbound local is overly-cautious, but a full flow
793794
// analysis (like we do in the compiler) is probably too expensive:
794795
unbound += f->f_frame->localsplus[i] == NULL;
@@ -801,7 +802,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
801802
}
802803
// Do this in a second pass to avoid writing a bunch of Nones when
803804
// warnings are being treated as errors and the previous bit raises:
804-
for (int i = 0; i < f->f_frame->f_code->co_nlocalsplus; i++) {
805+
for (int i = 0; i < code->co_nlocalsplus; i++) {
805806
if (f->f_frame->localsplus[i] == NULL) {
806807
f->f_frame->localsplus[i] = Py_NewRef(Py_None);
807808
unbound--;
@@ -832,7 +833,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
832833
}
833834
/* Finally set the new lasti and return OK. */
834835
f->f_lineno = 0;
835-
f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr;
836+
f->f_frame->prev_instr = _PyCode_CODE(code) + best_addr;
836837
return 0;
837838
}
838839

@@ -886,15 +887,15 @@ frame_dealloc(PyFrameObject *f)
886887
}
887888

888889
Py_TRASHCAN_BEGIN(f, frame_dealloc);
889-
PyCodeObject *co = NULL;
890+
PyObject *co = NULL;
890891

891892
/* Kill all local variables including specials, if we own them */
892893
if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) {
893894
assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data);
894895
_PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data;
895896
/* Don't clear code object until the end */
896-
co = frame->f_code;
897-
frame->f_code = NULL;
897+
co = frame->f_executable;
898+
frame->f_executable = NULL;
898899
Py_CLEAR(frame->f_funcobj);
899900
Py_CLEAR(frame->f_locals);
900901
PyObject **locals = _PyFrame_GetLocalsArray(frame);
@@ -968,7 +969,7 @@ frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored))
968969
{
969970
Py_ssize_t res;
970971
res = offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus);
971-
PyCodeObject *code = f->f_frame->f_code;
972+
PyCodeObject *code = _PyFrame_GetCode(f->f_frame);
972973
res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *);
973974
return PyLong_FromSsize_t(res);
974975
}
@@ -980,7 +981,7 @@ static PyObject *
980981
frame_repr(PyFrameObject *f)
981982
{
982983
int lineno = PyFrame_GetLineNumber(f);
983-
PyCodeObject *code = f->f_frame->f_code;
984+
PyCodeObject *code = _PyFrame_GetCode(f->f_frame);
984985
return PyUnicode_FromFormat(
985986
"<frame at %p, file %R, line %d, code %S>",
986987
f, code->co_filename, lineno, code->co_name);
@@ -1102,7 +1103,7 @@ _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg)
11021103
// This only works when opcode is a non-quickened form:
11031104
assert(_PyOpcode_Deopt[opcode] == opcode);
11041105
int check_oparg = 0;
1105-
for (_Py_CODEUNIT *instruction = _PyCode_CODE(frame->f_code);
1106+
for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame));
11061107
instruction < frame->prev_instr; instruction++)
11071108
{
11081109
int check_opcode = _PyOpcode_Deopt[instruction->op.code];
@@ -1128,7 +1129,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame)
11281129
{
11291130
// COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt
11301131
// here:
1131-
PyCodeObject *co = frame->f_code;
1132+
PyCodeObject *co = _PyFrame_GetCode(frame);
11321133
int lasti = _PyInterpreterFrame_LASTI(frame);
11331134
if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS
11341135
&& PyFunction_Check(frame->f_funcobj)))
@@ -1145,7 +1146,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame)
11451146
frame->localsplus[offset + i] = Py_NewRef(o);
11461147
}
11471148
// COPY_FREE_VARS doesn't have inline CACHEs, either:
1148-
frame->prev_instr = _PyCode_CODE(frame->f_code);
1149+
frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame));
11491150
}
11501151

11511152

@@ -1213,7 +1214,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame)
12131214

12141215
frame_init_get_vars(frame);
12151216

1216-
PyCodeObject *co = frame->f_code;
1217+
PyCodeObject *co = _PyFrame_GetCode(frame);
12171218
for (int i = 0; i < co->co_nlocalsplus; i++) {
12181219
PyObject *value; // borrowed reference
12191220
if (!frame_get_var(frame, co, i, &value)) {
@@ -1257,7 +1258,7 @@ PyFrame_GetVar(PyFrameObject *frame_obj, PyObject *name)
12571258
_PyInterpreterFrame *frame = frame_obj->f_frame;
12581259
frame_init_get_vars(frame);
12591260

1260-
PyCodeObject *co = frame->f_code;
1261+
PyCodeObject *co = _PyFrame_GetCode(frame);
12611262
for (int i = 0; i < co->co_nlocalsplus; i++) {
12621263
PyObject *var_name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
12631264
if (!_PyUnicode_Equal(var_name, name)) {
@@ -1331,7 +1332,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear)
13311332
return;
13321333
}
13331334
fast = _PyFrame_GetLocalsArray(frame);
1334-
co = frame->f_code;
1335+
co = _PyFrame_GetCode(frame);
13351336

13361337
PyObject *exc = PyErr_GetRaisedException();
13371338
for (int i = 0; i < co->co_nlocalsplus; i++) {
@@ -1417,7 +1418,7 @@ PyFrame_GetCode(PyFrameObject *frame)
14171418
{
14181419
assert(frame != NULL);
14191420
assert(!_PyFrame_IsIncomplete(frame->f_frame));
1420-
PyCodeObject *code = frame->f_frame->f_code;
1421+
PyCodeObject *code = _PyFrame_GetCode(frame->f_frame);
14211422
assert(code != NULL);
14221423
return (PyCodeObject*)Py_NewRef(code);
14231424
}

0 commit comments

Comments
 (0)