From 82516ed4047d539952f7670660f8138fbb4efec7 Mon Sep 17 00:00:00 2001 From: Aniket Singh Yadav Date: Sun, 5 Jul 2026 11:29:00 +0000 Subject: [PATCH] Cache the offset of the inline values 'valid' byte in LOAD_ATTR --- Include/internal/pycore_code.h | 8 ++ Include/internal/pycore_opcode_metadata.h | 60 +++++------ Modules/_testinternalcapi/test_cases.c.h | 118 ++++++++++++---------- Python/bytecodes.c | 23 +++-- Python/executor_cases.c.h | 16 ++- Python/generated_cases.c.h | 118 ++++++++++++---------- Python/specialize.c | 30 +++++- 7 files changed, 218 insertions(+), 155 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 5b1fddbe15b98b8..b4170340b24e89e 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -108,6 +108,7 @@ typedef struct { typedef struct { _Py_BackoffCounter counter; uint16_t type_version[2]; + uint16_t value_offset; union { uint16_t keys_version[2]; uint16_t dict_offset; @@ -121,6 +122,13 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) +typedef struct { + _Py_BackoffCounter counter; + uint16_t version[2]; + uint16_t value_offset; + uint16_t index; +} _PyLoadAttrInstanceValueCache; + typedef struct { _Py_BackoffCounter counter; uint16_t func_version[2]; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 6086dfd21f0839a..51020f5cbaa00c3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1032,7 +1032,7 @@ enum InstructionFormat { INSTR_FMT_IBC00 = 3, INSTR_FMT_IBC000 = 4, INSTR_FMT_IBC0000 = 5, - INSTR_FMT_IBC00000000 = 6, + INSTR_FMT_IBC000000000 = 6, INSTR_FMT_IX = 7, INSTR_FMT_IXC = 8, INSTR_FMT_IXC00 = 9, @@ -1233,20 +1233,20 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, @@ -1457,20 +1457,20 @@ _PyOpcode_macro_expansion[256] = { [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 2, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, - [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 4, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, 2, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 4, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 6 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 10 } } }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 4 }, { _LOAD_ATTR_CLASS, 4, 6 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 10 } } }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, 2, 4 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, OPERAND1_4, 6 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 10 }, { _PUSH_FRAME, OPARG_SIMPLE, 10 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 1, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 4 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 10 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 4 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 6 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 6 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 1, 3 }, { _GUARD_KEYS_VERSION, 2, 4 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 6 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 10 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 6 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 1, 3 }, { _GUARD_KEYS_VERSION, 2, 4 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 6 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 4 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 6 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 10 }, { _PUSH_FRAME, OPARG_SIMPLE, 10 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 10 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 10 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, OPARG_SIMPLE, 0 } } }, [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, OPARG_SIMPLE, 0 } } }, [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, OPARG_SIMPLE, 0 } } }, @@ -1815,7 +1815,7 @@ const uint8_t _PyOpcode_Caches[256] = { [STORE_ATTR] = 4, [LOAD_GLOBAL] = 4, [LOAD_SUPER_ATTR] = 1, - [LOAD_ATTR] = 9, + [LOAD_ATTR] = 10, [COMPARE_OP] = 1, [CONTAINS_OP] = 1, [JUMP_BACKWARD] = 1, diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 708aa4d966e5357..00982ac31af0ef6 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -8677,10 +8677,10 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; + _Py_CODEUNIT* const this_instr = next_instr - 11; (void)this_instr; _PyStackRef owner; _PyStackRef attr; @@ -8704,7 +8704,7 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } - /* Skip 8 cache entries */ + /* Skip 9 cache entries */ // _LOAD_ATTR { self_or_null = &stack_pointer[0]; @@ -8750,9 +8750,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef *null; @@ -8774,10 +8774,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - /* Skip 2 cache entries */ + /* Skip 3 cache entries */ // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -8809,9 +8809,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef *null; @@ -8828,9 +8828,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { - uint32_t type_version = read_u32(&this_instr[4].cache); + uint32_t type_version = read_u32(&this_instr[5].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); @@ -8846,7 +8847,7 @@ } // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -8878,9 +8879,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; /* Skip 1 cache entry */ @@ -8904,10 +8905,11 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME { - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); + uint32_t func_version = read_u32(&this_instr[5].cache); + PyObject *getattribute = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; @@ -8973,9 +8975,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -8996,10 +8998,12 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { + uint16_t value_offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); @@ -9007,7 +9011,7 @@ } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t offset = read_u16(&this_instr[4].cache); + uint16_t offset = read_u16(&this_instr[5].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); @@ -9061,9 +9065,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -9080,9 +9084,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _CHECK_ATTR_METHOD_LAZY_DICT { - uint16_t dictoffset = read_u16(&this_instr[4].cache); + uint16_t dictoffset = read_u16(&this_instr[5].cache); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -9094,7 +9099,7 @@ /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -9117,9 +9122,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -9136,10 +9141,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - /* Skip 2 cache entries */ + /* Skip 3 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -9163,9 +9168,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -9184,10 +9189,12 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { + uint16_t value_offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); @@ -9195,7 +9202,7 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&this_instr[4].cache); + uint32_t keys_version = read_u32(&this_instr[5].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -9207,7 +9214,7 @@ } // _LOAD_ATTR_METHOD_WITH_VALUES { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -9230,9 +9237,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -9291,7 +9298,7 @@ PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } - /* Skip 5 cache entries */ + /* Skip 6 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; @@ -9312,9 +9319,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; /* Skip 1 cache entry */ @@ -9330,10 +9337,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - /* Skip 2 cache entries */ + /* Skip 3 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -9360,9 +9367,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; /* Skip 1 cache entry */ @@ -9380,10 +9387,12 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { + uint16_t value_offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); @@ -9391,7 +9400,7 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&this_instr[4].cache); + uint32_t keys_version = read_u32(&this_instr[5].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -9403,7 +9412,7 @@ } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -9429,9 +9438,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; /* Skip 1 cache entry */ @@ -9455,10 +9464,11 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _LOAD_ATTR_PROPERTY_FRAME { - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); + uint32_t func_version = read_u32(&this_instr[5].cache); + PyObject *fget = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -9519,9 +9529,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -9573,7 +9583,7 @@ PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } - /* Skip 5 cache entries */ + /* Skip 6 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; @@ -9594,9 +9604,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -9691,7 +9701,7 @@ PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } - /* Skip 5 cache entries */ + /* Skip 6 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6d5ae1b074db5b7..e9d3d5f16188dfe 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2866,7 +2866,7 @@ dummy_func( macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + - unused/8 + + unused/9 + _LOAD_ATTR; op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { @@ -2890,11 +2890,12 @@ dummy_func( EXIT_IF(tp != (PyTypeObject *)type); } - op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { + op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (value_offset/1, owner -- owner)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - EXIT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)); + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + EXIT_IF(!FT_ATOMIC_LOAD_UINT8(*valid_ptr)); } op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, o)) { @@ -2954,7 +2955,7 @@ dummy_func( unused/1 + _LOAD_ATTR_MODULE + POP_TOP + - unused/5 + + unused/6 + _PUSH_NULL_CONDITIONAL; op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, o)) { @@ -3003,7 +3004,7 @@ dummy_func( _GUARD_TYPE_VERSION + _LOAD_ATTR_WITH_HINT + POP_TOP + - unused/5 + + unused/6 + _PUSH_NULL_CONDITIONAL; op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, o)) { @@ -3029,7 +3030,7 @@ dummy_func( _GUARD_TYPE_VERSION + _LOAD_ATTR_SLOT + // NOTE: This action may also deopt POP_TOP + - unused/5 + + unused/6 + _PUSH_NULL_CONDITIONAL; op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { @@ -3051,7 +3052,7 @@ dummy_func( unused/1 + _RECORD_TOS + _CHECK_ATTR_CLASS + - unused/2 + + unused/3 + _LOAD_ATTR_CLASS + _PUSH_NULL_CONDITIONAL; @@ -3059,6 +3060,7 @@ dummy_func( unused/1 + _RECORD_TOS + _GUARD_TYPE_VERSION + + unused/1 + _CHECK_ATTR_CLASS + _LOAD_ATTR_CLASS + _PUSH_NULL_CONDITIONAL; @@ -3082,6 +3084,7 @@ dummy_func( _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + _CHECK_PEP_523 + + unused/1 + _LOAD_ATTR_PROPERTY_FRAME + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -3110,6 +3113,7 @@ dummy_func( _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + _CHECK_PEP_523 + + unused/1 + _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4276,7 +4280,7 @@ dummy_func( unused/1 + _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + - unused/2 + + unused/3 + _LOAD_ATTR_METHOD_NO_DICT; op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { @@ -4308,7 +4312,7 @@ dummy_func( unused/1 + _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + - unused/2 + + unused/3 + _LOAD_ATTR_NONDESCRIPTOR_NO_DICT; op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { @@ -4332,6 +4336,7 @@ dummy_func( unused/1 + _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + + unused/1 + _CHECK_ATTR_METHOD_LAZY_DICT + unused/1 + _LOAD_ATTR_METHOD_LAZY_DICT; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6be53213966678e..96466a657ce985b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -12054,10 +12054,12 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; + uint16_t value_offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -12076,10 +12078,12 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; + uint16_t value_offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); @@ -12098,10 +12102,12 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; + uint16_t value_offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = owner; _tos_cache0 = _stack_item_0; @@ -12123,10 +12129,12 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; + uint16_t value_offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UOP_STAT_INC(uopcode, miss); _tos_cache2 = owner; _tos_cache1 = _stack_item_1; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d336b9fbcd57124..3e5a5a8d5ed3d64 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8676,10 +8676,10 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; + _Py_CODEUNIT* const this_instr = next_instr - 11; (void)this_instr; _PyStackRef owner; _PyStackRef attr; @@ -8703,7 +8703,7 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } - /* Skip 8 cache entries */ + /* Skip 9 cache entries */ // _LOAD_ATTR { self_or_null = &stack_pointer[0]; @@ -8749,9 +8749,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef *null; @@ -8773,10 +8773,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - /* Skip 2 cache entries */ + /* Skip 3 cache entries */ // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -8808,9 +8808,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef *null; @@ -8827,9 +8827,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { - uint32_t type_version = read_u32(&this_instr[4].cache); + uint32_t type_version = read_u32(&this_instr[5].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); @@ -8845,7 +8846,7 @@ } // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -8877,9 +8878,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; /* Skip 1 cache entry */ @@ -8903,10 +8904,11 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME { - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); + uint32_t func_version = read_u32(&this_instr[5].cache); + PyObject *getattribute = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; @@ -8972,9 +8974,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -8995,10 +8997,12 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { + uint16_t value_offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); @@ -9006,7 +9010,7 @@ } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t offset = read_u16(&this_instr[4].cache); + uint16_t offset = read_u16(&this_instr[5].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); @@ -9060,9 +9064,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -9079,9 +9083,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _CHECK_ATTR_METHOD_LAZY_DICT { - uint16_t dictoffset = read_u16(&this_instr[4].cache); + uint16_t dictoffset = read_u16(&this_instr[5].cache); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -9093,7 +9098,7 @@ /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -9116,9 +9121,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -9135,10 +9140,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - /* Skip 2 cache entries */ + /* Skip 3 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -9162,9 +9167,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -9183,10 +9188,12 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { + uint16_t value_offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); @@ -9194,7 +9201,7 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&this_instr[4].cache); + uint32_t keys_version = read_u32(&this_instr[5].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -9206,7 +9213,7 @@ } // _LOAD_ATTR_METHOD_WITH_VALUES { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -9229,9 +9236,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -9290,7 +9297,7 @@ PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } - /* Skip 5 cache entries */ + /* Skip 6 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; @@ -9311,9 +9318,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; /* Skip 1 cache entry */ @@ -9329,10 +9336,10 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - /* Skip 2 cache entries */ + /* Skip 3 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -9359,9 +9366,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; /* Skip 1 cache entry */ @@ -9379,10 +9386,12 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { + uint16_t value_offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + uint8_t *valid_ptr = (uint8_t *)owner_o + value_offset; + if (!FT_ATOMIC_LOAD_UINT8(*valid_ptr)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); @@ -9390,7 +9399,7 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&this_instr[4].cache); + uint32_t keys_version = read_u32(&this_instr[5].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -9402,7 +9411,7 @@ } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { - PyObject *descr = read_obj(&this_instr[6].cache); + PyObject *descr = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -9428,9 +9437,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; /* Skip 1 cache entry */ @@ -9454,10 +9463,11 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } + /* Skip 1 cache entry */ // _LOAD_ATTR_PROPERTY_FRAME { - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); + uint32_t func_version = read_u32(&this_instr[5].cache); + PyObject *fget = read_obj(&this_instr[7].cache); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -9518,9 +9528,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -9572,7 +9582,7 @@ PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } - /* Skip 5 cache entries */ + /* Skip 6 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; @@ -9593,9 +9603,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 10; + next_instr += 11; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 10, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -9690,7 +9700,7 @@ PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } - /* Skip 5 cache entries */ + /* Skip 6 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; diff --git a/Python/specialize.c b/Python/specialize.c index 6aaae6f383d22f5..7593de78af523b3 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -646,15 +646,30 @@ specialize_dict_access_inline( return 0; } assert(index >= 0); - assert(_PyObject_InlineValues(owner)->valid); - char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + PyDictValues *values = _PyObject_InlineValues(owner); + assert(values->valid); + char *value_addr = (char *)&values->values[index]; Py_ssize_t offset = value_addr - (char *)owner; if (offset != (uint16_t)offset) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } - cache->index = (uint16_t)offset; - write_u32(cache->version, tp_version); + if (base_op == LOAD_ATTR) { + Py_ssize_t valid_offset = (char *)&values->valid - (char *)owner; + if (valid_offset != (uint16_t)valid_offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + _PyLoadAttrInstanceValueCache *iv_cache = + (_PyLoadAttrInstanceValueCache *)cache; + iv_cache->value_offset = (uint16_t)valid_offset; + iv_cache->index = (uint16_t)offset; + write_u32(iv_cache->version, tp_version); + } + else { + cache->index = (uint16_t)offset; + write_u32(cache->version, tp_version); + } specialize(instr, values_op); return 1; } @@ -1286,6 +1301,13 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } + Py_ssize_t valid_offset = + (char *)&_PyObject_InlineValues(owner)->valid - (char *)owner; + if (valid_offset != (uint16_t)valid_offset) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->value_offset = (uint16_t)valid_offset; write_u32(cache->keys_version, shared_keys_version); specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); }