#include "Python.h"

#include "opcode_ids.h"

#include "pycore_bitutils.h" // _Py_popcount32

#include "pycore_call.h"

#include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS

#include "pycore_code.h" // _PyCode_Clear_Executors()

#include "pycore_critical_section.h"

#include "pycore_frame.h"

#include "pycore_interp.h"

#include "pycore_long.h"

#include "pycore_modsupport.h" // _PyModule_CreateInitialized()

#include "pycore_namespace.h"

#include "pycore_object.h"

#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches

#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE

#include "pycore_pyerrors.h"

#include "pycore_pystate.h" // _PyInterpreterState_GET()

/* Uncomment this to dump debugging output when assertions fail */

// #define INSTRUMENT_DEBUG 1

#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)

#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \

if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \

_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \

}

#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped);

#else

#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj)

#define ASSERT_WORLD_STOPPED()

#endif

#ifdef Py_GIL_DISABLED

#define LOCK_CODE(code) \

assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \

Py_BEGIN_CRITICAL_SECTION(code)

#define UNLOCK_CODE() Py_END_CRITICAL_SECTION()

#else

#define LOCK_CODE(code)

#define UNLOCK_CODE()

#endif

PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type);

PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type);

static const int8_t EVENT_FOR_OPCODE[256] = {

[RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN,

[INSTRUMENTED_RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN,

[RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,

[INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,

[CALL] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL,

[CALL_KW] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL_KW] = PY_MONITORING_EVENT_CALL,

[CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,

[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,

[RESUME] = -1,

[YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,

[INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,

[JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,

[JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,

[POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH,

[POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH,

[POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH,

[POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,

[INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH,

[FOR_ITER] = PY_MONITORING_EVENT_BRANCH,

[INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH,

[END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,

[INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,

[END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,

[INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,

};

static const uint8_t DE_INSTRUMENT[256] = {

[INSTRUMENTED_RESUME] = RESUME,

[INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,

[INSTRUMENTED_RETURN_CONST] = RETURN_CONST,

[INSTRUMENTED_CALL] = CALL,

[INSTRUMENTED_CALL_KW] = CALL_KW,

[INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,

[INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,

[INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,

[INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,

[INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,

[INSTRUMENTED_FOR_ITER] = FOR_ITER,

[INSTRUMENTED_END_FOR] = END_FOR,

[INSTRUMENTED_END_SEND] = END_SEND,

[INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,

};

static const uint8_t INSTRUMENTED_OPCODES[256] = {

[RETURN_CONST] = INSTRUMENTED_RETURN_CONST,

[INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST,

[RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,

[INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,

[CALL] = INSTRUMENTED_CALL,

[INSTRUMENTED_CALL] = INSTRUMENTED_CALL,

[CALL_KW] = INSTRUMENTED_CALL_KW,

[INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW,

[CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,

[INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,

[YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,

[INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,

[RESUME] = INSTRUMENTED_RESUME,

[INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME,

[JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,

[INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,

[JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,

[INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,

[POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,

[POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,

[POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,

[INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,

[POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,

[END_FOR] = INSTRUMENTED_END_FOR,

[INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR,

[END_SEND] = INSTRUMENTED_END_SEND,

[INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND,

[FOR_ITER] = INSTRUMENTED_FOR_ITER,

[INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER,

[LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,

[INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,

[INSTRUMENTED_LINE] = INSTRUMENTED_LINE,

[INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION,

};

static inline bool

opcode_has_event(int opcode)

{

return (

opcode != INSTRUMENTED_LINE &&

INSTRUMENTED_OPCODES[opcode] > 0

);

}

static inline bool

is_instrumented(int opcode)

{

assert(opcode != 0);

assert(opcode != RESERVED);

return opcode >= MIN_INSTRUMENTED_OPCODE;

}

#ifndef NDEBUG

static inline bool

monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (a.tools[i] != b.tools[i]) {

return false;

}

}

return true;

}

#endif

static inline _Py_LocalMonitors

monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] & ~b.tools[i];

}

return res;

}

#ifndef NDEBUG

static inline _Py_LocalMonitors

monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] & b.tools[i];

}

return res;

}

#endif

/* The union of the *local* events in a and b.

* Global events like RAISE are ignored.

* Used for instrumentation, as only local

* events get instrumented.

*/

static inline _Py_LocalMonitors

local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] | b.tools[i];

}

return res;

}

static inline bool

monitors_are_empty(_Py_LocalMonitors m)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (m.tools[i]) {

return false;

}

}

return true;

}

static inline bool

multiple_tools(_Py_LocalMonitors *m)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (_Py_popcount32(m->tools[i]) > 1) {

return true;

}

}

return false;

}

static inline _PyMonitoringEventSet

get_local_events(_Py_LocalMonitors *m, int tool_id)

{

_PyMonitoringEventSet result = 0;

for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {

if ((m->tools[e] >> tool_id) & 1) {

result |= (1 << e);

}

}

return result;

}

static inline _PyMonitoringEventSet

get_events(_Py_GlobalMonitors *m, int tool_id)

{

_PyMonitoringEventSet result = 0;

for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {

if ((m->tools[e] >> tool_id) & 1) {

result |= (1 << e);

}

}

return result;

}

/* Line delta.

* 8 bit value.

* if line_delta == -128:

* line = None # represented as -1

* elif line_delta == -127 or line_delta == -126:

* line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));

* else:

* line = first_line + (offset >> OFFSET_SHIFT) + line_delta;

*/

#define NO_LINE -128

#define COMPUTED_LINE_LINENO_CHANGE -127

#define COMPUTED_LINE -126

#define OFFSET_SHIFT 4

static int8_t

compute_line_delta(PyCodeObject *code, int offset, int line)

{

if (line < 0) {

return NO_LINE;

}

int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT);

if (delta <= INT8_MAX && delta > COMPUTED_LINE) {

return delta;

}

return COMPUTED_LINE;

}

static int

compute_line(PyCodeObject *code, int offset, int8_t line_delta)

{

if (line_delta > COMPUTED_LINE) {

return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta;

}

if (line_delta == NO_LINE) {

return -1;

}

assert(line_delta == COMPUTED_LINE || line_delta == COMPUTED_LINE_LINENO_CHANGE);

/* Look it up */

return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));

}

int

_PyInstruction_GetLength(PyCodeObject *code, int offset)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

int opcode = _PyCode_CODE(code)[offset].op.code;

assert(opcode != 0);

assert(opcode != RESERVED);

if (opcode == INSTRUMENTED_LINE) {

opcode = code->_co_monitoring->lines[offset].original_opcode;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = code->_co_monitoring->per_instruction_opcodes[offset];

}

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented) {

opcode = deinstrumented;

}

else {

opcode = _PyOpcode_Deopt[opcode];

}

assert(opcode != 0);

assert(!is_instrumented(opcode));

if (opcode == ENTER_EXECUTOR) {

int exec_index = _PyCode_CODE(code)[offset].op.arg;

_PyExecutorObject *exec = code->co_executors->executors[exec_index];

opcode = _PyOpcode_Deopt[exec->vm_data.opcode];

}

assert(opcode != ENTER_EXECUTOR);

assert(opcode == _PyOpcode_Deopt[opcode]);

return 1 + _PyOpcode_Caches[opcode];

}

#ifdef INSTRUMENT_DEBUG

static void

dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out)

{

if (tools == NULL) {

fprintf(out, "tools = NULL");

}

else {

fprintf(out, "tools = %d", tools[i]);

}

}

static void

dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData *lines, int i, FILE*out)

{

if (lines == NULL) {

fprintf(out, ", lines = NULL");

}

else if (lines[i].original_opcode == 0) {

fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta);

}

else {

fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta);

}

}

static void

dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, int i, FILE*out)

{

if (line_tools == NULL) {

fprintf(out, ", line_tools = NULL");

}

else {

fprintf(out, ", line_tools = %d", line_tools[i]);

}

}

static void

dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out)

{

if (data->per_instruction_opcodes == NULL) {

fprintf(out, ", per-inst opcode = NULL");

}

else {

fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]);

}

if (data->per_instruction_tools == NULL) {

fprintf(out, ", per-inst tools = NULL");

}

else {

fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]);

}

}

static void

dump_global_monitors(const char *prefix, _Py_GlobalMonitors monitors, FILE*out)

{

fprintf(out, "%s monitors:\n", prefix);

for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) {

fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);

}

}

static void

dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out)

{

fprintf(out, "%s monitors:\n", prefix);

for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) {

fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);

}

}

/* No error checking -- Don't use this for anything but experimental debugging */

static void

dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)

{

_PyCoMonitoringData *data = code->_co_monitoring;

fprintf(out, "\n");

PyObject_Print(code->co_name, out, Py_PRINT_RAW);

fprintf(out, "\n");

if (data == NULL) {

fprintf(out, "NULL\n");

return;

}

dump_global_monitors("Global", _PyInterpreterState_GET()->monitors, out);

dump_local_monitors("Code", data->local_monitors, out);

dump_local_monitors("Active", data->active_monitors, out);

int code_len = (int)Py_SIZE(code);

bool starred = false;

for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) {

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

if (i == star) {

fprintf(out, "** ");

starred = true;

}

fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]);

dump_instrumentation_data_tools(code, data->tools, i, out);

dump_instrumentation_data_lines(code, data->lines, i, out);

dump_instrumentation_data_line_tools(code, data->line_tools, i, out);

dump_instrumentation_data_per_instruction(code, data, i, out);

fprintf(out, "\n");

;

}

if (!starred && star >= 0) {

fprintf(out, "Error offset not at valid instruction offset: %d\n", star);

fprintf(out, " ");

dump_instrumentation_data_tools(code, data->tools, star, out);

dump_instrumentation_data_lines(code, data->lines, star, out);

dump_instrumentation_data_line_tools(code, data->line_tools, star, out);

dump_instrumentation_data_per_instruction(code, data, star, out);

fprintf(out, "\n");

}

}

#define CHECK(test) do { \

if (!(test)) { \

dump_instrumentation_data(code, i, stderr); \

} \

assert(test); \

} while (0)

static bool

valid_opcode(int opcode)

{

if (opcode == INSTRUMENTED_LINE) {

return true;

}

if (IS_VALID_OPCODE(opcode) &&

opcode != CACHE &&

opcode != RESERVED &&

opcode < 255)

{

return true;

}

return false;

}

static void

sanity_check_instrumentation(PyCodeObject *code)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

_PyCoMonitoringData *data = code->_co_monitoring;

if (data == NULL) {

return;

}

_Py_GlobalMonitors global_monitors = _PyInterpreterState_GET()->monitors;

_Py_LocalMonitors active_monitors;

if (code->_co_monitoring) {

_Py_LocalMonitors local_monitors = code->_co_monitoring->local_monitors;

active_monitors = local_union(global_monitors, local_monitors);

}

else {

_Py_LocalMonitors empty = (_Py_LocalMonitors) { 0 };

active_monitors = local_union(global_monitors, empty);

}

assert(monitors_equals(

code->_co_monitoring->active_monitors,

active_monitors));

int code_len = (int)Py_SIZE(code);

for (int i = 0; i < code_len;) {

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

int base_opcode = _Py_GetBaseOpcode(code, i);

CHECK(valid_opcode(opcode));

CHECK(valid_opcode(base_opcode));

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = data->per_instruction_opcodes[i];

if (!is_instrumented(opcode)) {

CHECK(_PyOpcode_Deopt[opcode] == opcode);

}

}

if (opcode == INSTRUMENTED_LINE) {

CHECK(data->lines);

CHECK(valid_opcode(data->lines[i].original_opcode));

opcode = data->lines[i].original_opcode;

CHECK(opcode != END_FOR);

CHECK(opcode != RESUME);

CHECK(opcode != RESUME_CHECK);

CHECK(opcode != INSTRUMENTED_RESUME);

if (!is_instrumented(opcode)) {

CHECK(_PyOpcode_Deopt[opcode] == opcode);

}

CHECK(opcode != INSTRUMENTED_LINE);

}

else if (data->lines) {

/* If original_opcode is INSTRUMENTED_INSTRUCTION

* *and* we are executing a INSTRUMENTED_LINE instruction

* that has de-instrumented itself, then we will execute

* an invalid INSTRUMENTED_INSTRUCTION */

CHECK(data->lines[i].original_opcode != INSTRUMENTED_INSTRUCTION);

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

CHECK(data->per_instruction_opcodes[i] != 0);

opcode = data->per_instruction_opcodes[i];

}

if (is_instrumented(opcode)) {

CHECK(DE_INSTRUMENT[opcode] == base_opcode);

int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]];

if (event < 0) {

/* RESUME fixup */

event = instr->op.arg ? 1: 0;

}

CHECK(active_monitors.tools[event] != 0);

}

if (data->lines && base_opcode != END_FOR) {

int line1 = compute_line(code, i, data->lines[i].line_delta);

int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT));

CHECK(line1 == line2);

}

CHECK(valid_opcode(opcode));

if (data->tools) {

uint8_t local_tools = data->tools[i];

if (opcode_has_event(base_opcode)) {

int event = EVENT_FOR_OPCODE[base_opcode];

if (event == -1) {

/* RESUME fixup */

event = _PyCode_CODE(code)[i].op.arg;

}

CHECK((active_monitors.tools[event] & local_tools) == local_tools);

}

else {

CHECK(local_tools == 0xff);

}

}

i += _PyInstruction_GetLength(code, i);

assert(i <= code_len);

}

}

#else

#define CHECK(test) assert(test)

#endif

/* Get the underlying opcode, stripping instrumentation */

int _Py_GetBaseOpcode(PyCodeObject *code, int i)

{

int opcode = _PyCode_CODE(code)[i].op.code;

if (opcode == INSTRUMENTED_LINE) {

opcode = code->_co_monitoring->lines[i].original_opcode;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = code->_co_monitoring->per_instruction_opcodes[i];

}

CHECK(opcode != INSTRUMENTED_INSTRUCTION);

CHECK(opcode != INSTRUMENTED_LINE);

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented) {

return deinstrumented;

}

return _PyOpcode_Deopt[opcode];

}

static void

de_instrument(PyCodeObject *code, int i, int event)

{

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(event != PY_MONITORING_EVENT_LINE);

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

assert(opcode != ENTER_EXECUTOR);

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = &code->_co_monitoring->lines[i].original_opcode;

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i];

opcode = *opcode_ptr;

}

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented == 0) {

return;

}

CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);

FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented);

if (_PyOpcode_Caches[deinstrumented]) {

FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter,

adaptive_counter_warmup().as_counter);

}

}

static void

de_instrument_line(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

if (opcode != INSTRUMENTED_LINE) {

return;

}

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

int original_opcode = lines->original_opcode;

if (original_opcode == INSTRUMENTED_INSTRUCTION) {

lines->original_opcode = code->_co_monitoring->per_instruction_opcodes[i];

}

CHECK(original_opcode != 0);

CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);

instr->op.code = original_opcode;

if (_PyOpcode_Caches[original_opcode]) {

instr[1].counter = adaptive_counter_warmup();

}

assert(instr->op.code != INSTRUMENTED_LINE);

}

static void

de_instrument_per_instruction(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = &code->_co_monitoring->lines[i].original_opcode;

opcode = *opcode_ptr;

}

if (opcode != INSTRUMENTED_INSTRUCTION) {

return;

}

int original_opcode = code->_co_monitoring->per_instruction_opcodes[i];

CHECK(original_opcode != 0);

CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);

*opcode_ptr = original_opcode;

if (_PyOpcode_Caches[original_opcode]) {

instr[1].counter = adaptive_counter_warmup();

}

assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);

assert(instr->op.code != INSTRUMENTED_INSTRUCTION);

}

static void

instrument(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode =*opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

opcode_ptr = &lines->original_opcode;

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i];

opcode = *opcode_ptr;

CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE);

CHECK(opcode == _PyOpcode_Deopt[opcode]);

}

CHECK(opcode != 0);

if (!is_instrumented(opcode)) {

int deopt = _PyOpcode_Deopt[opcode];

int instrumented = INSTRUMENTED_OPCODES[deopt];

assert(instrumented);

FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented);

if (_PyOpcode_Caches[deopt]) {

FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter,

adaptive_counter_warmup().as_counter);

instr[1].counter = adaptive_counter_warmup();

}

}

}

static void

instrument_line(PyCodeObject *code, int i)

{

uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

return;

}

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

lines->original_opcode = _PyOpcode_Deopt[opcode];

CHECK(lines->original_opcode > 0);

*opcode_ptr = INSTRUMENTED_LINE;

}

static void

instrument_per_instruction(PyCodeObject *code, int i)

{

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

_PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i];

opcode_ptr = &lines->original_opcode;

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

assert(code->_co_monitoring->per_instruction_opcodes[i] > 0);

return;

}

CHECK(opcode != 0);

if (is_instrumented(opcode)) {

code->_co_monitoring->per_instruction_opcodes[i] = opcode;

}

else {

assert(opcode != 0);

assert(_PyOpcode_Deopt[opcode] != 0);

assert(_PyOpcode_Deopt[opcode] != RESUME);

code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];

}

assert(code->_co_monitoring->per_instruction_opcodes[i] > 0);

*opcode_ptr = INSTRUMENTED_INSTRUCTION;

}

static void

remove_tools(PyCodeObject * code, int offset, int event, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(event != PY_MONITORING_EVENT_LINE);

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));

assert(opcode_has_event(_Py_GetBaseOpcode(code, offset)));

_PyCoMonitoringData *monitoring = code->_co_monitoring;

if (monitoring && monitoring->tools) {

monitoring->tools[offset] &= ~tools;

if (monitoring->tools[offset] == 0) {

de_instrument(code, offset, event);

}

}

else {

/* Single tool */

uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event];

assert(_Py_popcount32(single_tool) <= 1);

if (((single_tool & tools) == single_tool)) {

de_instrument(code, offset, event);

}

}

}

#ifndef NDEBUG

static bool

tools_is_subset_for_event(PyCodeObject * code, int event, int tools)

{

int global_tools = _PyInterpreterState_GET()->monitors.tools[event];

int local_tools = code->_co_monitoring->local_monitors.tools[event];

return tools == ((global_tools | local_tools) & tools);

}

#endif

static void

remove_line_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(code->_co_monitoring);

if (code->_co_monitoring->line_tools)

{

uint8_t *toolsptr = &code->_co_monitoring->line_tools[offset];

*toolsptr &= ~tools;

if (*toolsptr == 0 ) {

de_instrument_line(code, offset);

}

}

else {

/* Single tool */

uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE];

assert(_Py_popcount32(single_tool) <= 1);

if (((single_tool & tools) == single_tool)) {

de_instrument_line(code, offset);

}

}

}

static void

add_tools(PyCodeObject * code, int offset, int event, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(event != PY_MONITORING_EVENT_LINE);

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));

assert(code->_co_monitoring);

if (code->_co_monitoring &&

code->_co_monitoring->tools

) {

code->_co_monitoring->tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

assert(tools_is_subset_for_event(code, event, tools));

}

instrument(code, offset);

}

static void

add_line_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools));

assert(code->_co_monitoring);

if (code->_co_monitoring->line_tools) {

code->_co_monitoring->line_tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

}

instrument_line(code, offset);

}

static void

add_per_instruction_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools));

assert(code->_co_monitoring);

if (code->_co_monitoring->per_instruction_tools) {

code->_co_monitoring->per_instruction_tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

}

instrument_per_instruction(code, offset);

}

static void

remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(code->_co_monitoring);

if (code->_co_monitoring->per_instruction_tools) {

uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset];

*toolsptr &= ~tools;

if (*toolsptr == 0) {

de_instrument_per_instruction(code, offset);

}

}

else {

/* Single tool */

uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];

assert(_Py_popcount32(single_tool) <= 1);

if (((single_tool & tools) == single_tool)) {

de_instrument_per_instruction(code, offset);

}

}

}

/* Return 1 if DISABLE returned, -1 if error, 0 otherwise */

static int

call_one_instrument(

PyInterpreterState *interp, PyThreadState *tstate, PyObject **args,

size_t nargsf, int8_t tool, int event)

{

assert(0 <= tool && tool < 8);

assert(tstate->tracing == 0);

PyObject *instrument = interp->monitoring_callables[tool][event];

if (instrument == NULL) {

return 0;

}

int old_what = tstate->what_event;

tstate->what_event = event;

tstate->tracing++;

PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL);

tstate->tracing--;

tstate->what_event = old_what;

if (res == NULL) {

return -1;

}

Py_DECREF(res);

return (res == &_PyInstrumentation_DISABLE);

}

static const int8_t MOST_SIGNIFICANT_BITS[16] = {

-1, 0, 1, 1,

2, 2, 2, 2,

3, 3, 3, 3,

3, 3, 3, 3,

};

/* We could use _Py_bit_length here, but that is designed for larger (32/64)

* bit ints, and can perform relatively poorly on platforms without the

* necessary intrinsics. */

static inline int most_significant_bit(uint8_t bits) {

assert(bits != 0);

if (bits > 15) {

return MOST_SIGNIFICANT_BITS[bits>>4]+4;

}

return MOST_SIGNIFICANT_BITS[bits];

}

static uint32_t

global_version(PyInterpreterState *interp)

{

uint32_t version = (uint32_t)_Py_atomic_load_uintptr_relaxed(

&interp->ceval.instrumentation_version);

#ifdef Py_DEBUG

PyThreadState *tstate = _PyThreadState_GET();

uint32_t thread_version =

(uint32_t)(_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &

~_PY_EVAL_EVENTS_MASK);

assert(thread_version == version);

#endif

return version;

}

/* Atomically set the given version in the given location, without touching

anything in _PY_EVAL_EVENTS_MASK. */

static void

set_version_raw(uintptr_t *ptr, uint32_t version)

{

uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr);

uintptr_t new;

do {

new = (old & _PY_EVAL_EVENTS_MASK) | version;

} while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new));

}

static void

set_global_version(PyThreadState *tstate, uint32_t version)

{

assert((version & _PY_EVAL_EVENTS_MASK) == 0);

PyInterpreterState *interp = tstate->interp;

set_version_raw(&interp->ceval.instrumentation_version, version);

#ifdef Py_GIL_DISABLED

// Set the version on all threads in free-threaded builds.

_PyRuntimeState *runtime = &_PyRuntime;

HEAD_LOCK(runtime);

for (tstate = interp->threads.head; tstate;

tstate = PyThreadState_Next(tstate)) {

set_version_raw(&tstate->eval_breaker, version);

};

HEAD_UNLOCK(runtime);

#else

// Normal builds take the current version from instrumentation_version when

// attaching a thread, so we only have to set the current thread's version.

set_version_raw(&tstate->eval_breaker, version);

#endif

}

static bool

is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

return global_version(interp) == code->_co_instrumentation_version;

}

#ifndef NDEBUG

static bool

instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

_Py_LocalMonitors expected = local_union(

interp->monitors,

code->_co_monitoring->local_monitors);

return monitors_equals(code->_co_monitoring->active_monitors, expected);