Skip to content

Align specialization guards and caching with CPython #7341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 31 commits into from
Mar 4, 2026
Merged

Align specialization guards and caching with CPython #7341

Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
c6b6842
fix jit
youknowone Mar 4, 2026
b379239
vm: complete specialized opcode dispatch paths
youknowone Mar 4, 2026
bdeca2b
vm: cache LOAD_GLOBAL with dict entry hints
youknowone Mar 4, 2026
ca3814a
vm: align adaptive specialization counters with CPython backoff
youknowone Mar 4, 2026
f04bf22
vm: apply cooldown counter on specialization success paths
youknowone Mar 4, 2026
02a1495
vm: retain LOAD_GLOBAL specializations on misses
youknowone Mar 4, 2026
13125d8
vm: keep attr and call specializations on guard misses
youknowone Mar 4, 2026
c7007b5
vm: retain store-attr and store-subscr specializations on misses
youknowone Mar 4, 2026
453294d
vm: retain specialization opcodes on generic fallback paths
youknowone Mar 4, 2026
b8f6cc2
vm: align jump-backward specialization defaults with CPython
youknowone Mar 4, 2026
7bfa961
vm: retain exact-args call specializations on misses
youknowone Mar 4, 2026
c852994
vm: retain SEND_GEN specialization on non-coroutine sends
youknowone Mar 4, 2026
1a1bbf5
vm: specialize list.append calls like CPython CALL_LIST_APPEND
youknowone Mar 4, 2026
bdd87f4
vm: set cooldown on LOAD_ATTR_CLASS specialization
youknowone Mar 4, 2026
696e57a
vm: specialize bound method object CALL paths
youknowone Mar 4, 2026
3dc64cc
vm: specialize CALL_KW for bound method objects
youknowone Mar 4, 2026
f220a24
vm: use current-state function version for CALL_KW specialization
youknowone Mar 4, 2026
a66c706
vm: align CALL/CALL_KW pyfunction specialization with CPython
youknowone Mar 4, 2026
59545f5
vm: drop call-site identity caches in generic CALL specializations
youknowone Mar 4, 2026
f8eebec
vm: align builtin type call specializations with CPython guards
youknowone Mar 4, 2026
58bb1ea
vm: align builtin CALL guards with CPython self_or_null semantics
youknowone Mar 4, 2026
157a2c3
vm: require exact list in CALL_LIST_APPEND fast path
youknowone Mar 4, 2026
cae26d4
vm: align CALL builtin/class specialization flow with CPython
youknowone Mar 4, 2026
0d01f7e
vm: tighten len/isinstance CALL specializations to builtin guards
youknowone Mar 4, 2026
22cccae
vm: gate CALL_BUILTIN_CLASS on type vectorcall like CPython
youknowone Mar 4, 2026
30e8a75
vm: run non-py CALL specializations via direct vectorcall
youknowone Mar 4, 2026
da36859
vm: align class-call specialization branching with CPython
youknowone Mar 4, 2026
42d81ff
Fix CI: disable ForIterGen, tighten CALL guards
youknowone Mar 4, 2026
d464119
vm: restore FOR_ITER_GEN specialization and tuple index parity
youknowone Mar 4, 2026
0f044d6
Add datastack-backed FastLocals for non-generator frames
youknowone Mar 4, 2026
cc3fd09
Drop read lock before key_eq in dict get_hint
youknowone Mar 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
vm: drop call-site identity caches in generic CALL specializations
  • Loading branch information
@youknowone
youknowone committed Mar 4, 2026
commit 59545f5a109c6be2d16a87c4a40ea3eff9edf5f4
123 changes: 33 additions & 90 deletions crates/vm/src/frame.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3620,17 +3620,13 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallBuiltinO => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
if nargs == 1 {
let obj = self.pop_value();
let _null = self.pop_value_opt();
let callable = self.pop_value();
let callable_tag = &*callable as *const PyObject as u32;
if cached_tag == callable_tag
&& let Some(native) = callable.downcast_ref::<PyNativeFunction>()
if let Some(native) = callable.downcast_ref::<PyNativeFunction>()
&& native.zelf.is_none()
{
let args = FuncArgs {
args: vec![obj],
Expand All @@ -3648,18 +3644,13 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallBuiltinFast => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 1);
let callable_tag = callable as *const PyObject as u32;
let func = if cached_tag == callable_tag {
let func = {
callable
.downcast_ref::<PyNativeFunction>()
.filter(|n| n.zelf.is_none())
.map(|n| n.value.func)
} else {
None
};
if let Some(func) = func {
let positional_args: Vec<PyObjectRef> =
Expand Down Expand Up @@ -3751,21 +3742,24 @@ impl ExecutingFrame<'_> {
}
}
Instruction::CallListAppend => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
if nargs == 1 {
// Stack: [callable, self_or_null, item]
let stack = &self.state.stack;
let stack_len = stack.len();
let self_or_null_is_some = stack[stack_len - 2].is_some();
let callable = self.nth_value(2);
let callable_tag = callable as *const PyObject as u32;
let self_is_list = stack[stack_len - 2]
.as_ref()
.is_some_and(|obj| obj.downcast_ref::<PyList>().is_some());
if cached_tag == callable_tag && self_or_null_is_some && self_is_list {
let is_list_append =
callable
.downcast_ref::<PyMethodDescriptor>()
.is_some_and(|descr| {
descr.method.name == "append"
&& descr.objclass.is(vm.ctx.types.list_type)
});
if is_list_append && self_or_null_is_some && self_is_list {
let item = self.pop_value();
let self_or_null = self.pop_value_opt();
let callable = self.pop_value();
Expand All @@ -3789,18 +3783,14 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallMethodDescriptorNoargs => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
if nargs == 0 {
// Stack: [callable, self_or_null] — peek to get func ptr
let stack = &self.state.stack;
let stack_len = stack.len();
let self_or_null_is_some = stack[stack_len - 1].is_some();
let callable = self.nth_value(1);
let callable_tag = callable as *const PyObject as u32;
let func = if cached_tag == callable_tag && self_or_null_is_some {
let func = if self_or_null_is_some {
callable
.downcast_ref::<PyMethodDescriptor>()
.map(|d| d.method.func)
Expand All @@ -3823,18 +3813,14 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallMethodDescriptorO => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
if nargs == 1 {
// Stack: [callable, self_or_null, arg1]
let stack = &self.state.stack;
let stack_len = stack.len();
let self_or_null_is_some = stack[stack_len - 2].is_some();
let callable = self.nth_value(2);
let callable_tag = callable as *const PyObject as u32;
let func = if cached_tag == callable_tag && self_or_null_is_some {
let func = if self_or_null_is_some {
callable
.downcast_ref::<PyMethodDescriptor>()
.map(|d| d.method.func)
Expand All @@ -3858,16 +3844,12 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallMethodDescriptorFast => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 1);
let callable_tag = callable as *const PyObject as u32;
let stack = &self.state.stack;
let stack_len = stack.len();
let self_or_null_is_some = stack[stack_len - nargs as usize - 1].is_some();
let func = if cached_tag == callable_tag && self_or_null_is_some {
let func = if self_or_null_is_some {
callable
.downcast_ref::<PyMethodDescriptor>()
.map(|d| d.method.func)
Expand All @@ -3894,13 +3876,9 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallBuiltinClass => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 1);
let callable_tag = callable as *const PyObject as u32;
if cached_tag == callable_tag && callable.downcast_ref::<PyType>().is_some() {
if callable.downcast_ref::<PyType>().is_some() {
let args = self.collect_positional_args(nargs);
let self_or_null = self.pop_value_opt();
let callable = self.pop_value();
Expand Down Expand Up @@ -3978,16 +3956,12 @@ impl ExecutingFrame<'_> {
}
Instruction::CallMethodDescriptorFastWithKeywords => {
// Native function interface is uniform regardless of keyword support
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 1);
let callable_tag = callable as *const PyObject as u32;
let stack = &self.state.stack;
let stack_len = stack.len();
let self_or_null_is_some = stack[stack_len - nargs as usize - 1].is_some();
let func = if cached_tag == callable_tag && self_or_null_is_some {
let func = if self_or_null_is_some {
callable
.downcast_ref::<PyMethodDescriptor>()
.map(|d| d.method.func)
Expand Down Expand Up @@ -4015,18 +3989,13 @@ impl ExecutingFrame<'_> {
}
Instruction::CallBuiltinFastWithKeywords => {
// Native function interface is uniform regardless of keyword support
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 1);
let callable_tag = callable as *const PyObject as u32;
let func = if cached_tag == callable_tag {
let func = {
callable
.downcast_ref::<PyNativeFunction>()
.filter(|n| n.zelf.is_none())
.map(|n| n.value.func)
} else {
None
};
if let Some(func) = func {
let positional_args: Vec<PyObjectRef> =
Expand All @@ -4045,13 +4014,11 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallNonPyGeneral => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 1);
let callable_tag = callable as *const PyObject as u32;
if cached_tag == callable_tag {
if callable.downcast_ref::<PyFunction>().is_some()
|| callable.downcast_ref::<PyBoundMethod>().is_some()
{
let args = self.collect_positional_args(nargs);
return self.execute_call(args, vm);
}
Expand Down Expand Up @@ -4149,13 +4116,11 @@ impl ExecutingFrame<'_> {
self.execute_call(args, vm)
}
Instruction::CallKwNonPy => {
let instr_idx = self.lasti() as usize - 1;
let cache_base = instr_idx + 1;
let cached_tag = self.code.instructions.read_cache_u32(cache_base + 1);
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 2);
let callable_tag = callable as *const PyObject as u32;
if cached_tag == callable_tag {
if callable.downcast_ref::<PyFunction>().is_some()
|| callable.downcast_ref::<PyBoundMethod>().is_some()
{
let args = self.collect_keyword_args(nargs);
return self.execute_call(args, vm);
}
Expand Down Expand Up @@ -6920,7 +6885,6 @@ impl ExecutingFrame<'_> {

// Try to specialize method descriptor calls
if self_or_null_is_some && let Some(descr) = callable.downcast_ref::<PyMethodDescriptor>() {
let callable_tag = callable as *const PyObject as u32;
let call_cache_entries = Instruction::CallListAppend.cache_entries();
let next_idx = cache_base + call_cache_entries;
let next_is_pop_top = if next_idx < self.code.instructions.len() {
Expand All @@ -6943,11 +6907,6 @@ impl ExecutingFrame<'_> {
_ => Instruction::CallMethodDescriptorFast,
}
};
unsafe {
self.code
.instructions
.write_cache_u32(cache_base + 1, callable_tag);
}
self.specialize_at(instr_idx, cache_base, new_op);
return;
}
Expand All @@ -6966,10 +6925,12 @@ impl ExecutingFrame<'_> {
};
let new_op = Some(new_op);
if let Some(new_op) = new_op {
unsafe {
self.code
.instructions
.write_cache_u32(cache_base + 1, callable_tag);
if matches!(new_op, Instruction::CallLen | Instruction::CallIsinstance) {
unsafe {
self.code
.instructions
.write_cache_u32(cache_base + 1, callable_tag);
}
}
self.specialize_at(instr_idx, cache_base, new_op);
return;
Expand Down Expand Up @@ -7027,24 +6988,12 @@ impl ExecutingFrame<'_> {
}
}
// General builtin class call (any type with Callable)
let callable_tag = callable as *const PyObject as u32;
unsafe {
self.code
.instructions
.write_cache_u32(cache_base + 1, callable_tag);
}
self.specialize_at(instr_idx, cache_base, Instruction::CallBuiltinClass);
return;
}
}

// General fallback: cache callable identity to skip re-specialization
let callable_tag = callable as *const PyObject as u32;
unsafe {
self.code
.instructions
.write_cache_u32(cache_base + 1, callable_tag);
}
// General fallback: specialized non-Python callable path
self.specialize_at(instr_idx, cache_base, Instruction::CallNonPyGeneral);
}

Expand Down Expand Up @@ -7116,13 +7065,7 @@ impl ExecutingFrame<'_> {
return;
}

// General fallback
let callable_tag = callable as *const PyObject as u32;
unsafe {
self.code
.instructions
.write_cache_u32(cache_base + 1, callable_tag);
}
// General fallback: specialized non-Python callable path
self.specialize_at(instr_idx, cache_base, Instruction::CallKwNonPy);
}

Expand Down