From c09c47596a84110bd8c3d5ecedefdd20bae6ee86 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 29 Apr 2026 08:29:20 +0900 Subject: [PATCH 01/76] Align CFG cleanup bytecode with CPython --- .cspell.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.cspell.json b/.cspell.json index f05f2adcd6..c78ff79ff8 100644 --- a/.cspell.json +++ b/.cspell.json @@ -69,6 +69,7 @@ "jitted", "jitting", "kwonly", + "lolcatz", "lossily", "mcache", "oparg", From fe642009a57152bc4119bfd1868ed7ea81a5d234 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 30 Apr 2026 22:46:36 +0900 Subject: [PATCH 02/76] remove test --- .cspell.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index c78ff79ff8..f05f2adcd6 100644 --- a/.cspell.json +++ b/.cspell.json @@ -69,7 +69,6 @@ "jitted", "jitting", "kwonly", - "lolcatz", "lossily", "mcache", "oparg", From 94e2ae3abdafd596ecb07527dbae66a3db2fa09c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 5 May 2026 14:21:45 +0900 Subject: [PATCH 03/76] Align bytecode CFG cleanup with CPython --- crates/codegen/src/compile.rs | 42 +-- crates/codegen/src/ir.rs | 467 ++++++++++++++++++++++++---------- 2 files changed, 329 insertions(+), 180 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 6147368736..7711258898 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3384,8 +3384,6 @@ impl Compiler { )?; } - let else_block = self.new_block(); - // if handlers is empty, compile body directly // without wrapping in TryExcept (only FinallyTry is needed) if handlers.is_empty() { @@ -3487,19 +3485,11 @@ impl Compiler { self.compile_statements(body)?; emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.remove_last_no_location_nop(); self.pop_fblock(FBlockType::TryExcept); - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: else_block } - ); - self.set_no_location(); - self.remove_last_no_location_nop(); let cleanup_block = self.new_block(); // We successfully ran the try block: // else: - self.switch_to_block(else_block); self.compile_statements(orelse)?; emit!( @@ -5839,17 +5829,10 @@ impl Compiler { if is_async { self.set_source_range(with_range); } - let preserve_pop_block_nop = !is_async && self.current_block_follows_end_async_for(); - if preserve_pop_block_nop { - emit!(self, Instruction::Nop); - self.preserve_last_redundant_nop(); - } emit!(self, PseudoInstruction::PopBlock); if !is_async { self.set_no_location(); - if !preserve_pop_block_nop { - self.remove_last_no_location_nop(); - } + self.remove_last_no_location_nop(); self.set_source_range(with_range); } self.pop_fblock(if is_async { @@ -10877,29 +10860,6 @@ impl Compiler { &mut info.blocks[info.current_block] } - fn current_block_follows_end_async_for(&mut self) -> bool { - let info = self.current_code_info(); - let current = info.current_block; - if info.blocks[current] - .instructions - .iter() - .rev() - .find_map(|info| info.instr.real()) - .is_some_and(|instr| matches!(instr, Instruction::EndAsyncFor)) - { - return true; - } - info.blocks.iter().any(|block| { - block.next == current - && block - .instructions - .iter() - .rev() - .find_map(|info| info.instr.real()) - .is_some_and(|instr| matches!(instr, Instruction::EndAsyncFor)) - }) - } - fn new_block(&mut self) -> BlockIdx { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8d77b4688b..81b66962f1 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -331,7 +331,6 @@ impl CodeInfo { self.eliminate_unreachable_blocks(); resolve_line_numbers(&mut self.blocks); materialize_empty_conditional_exit_targets(&mut self.blocks); - materialize_exception_cleanup_jump_targets(&mut self.blocks); redirect_empty_block_targets(&mut self.blocks); duplicate_end_returns(&mut self.blocks, &self.metadata); duplicate_shared_jump_back_targets(&mut self.blocks); @@ -474,6 +473,13 @@ impl CodeInfo { if matches!(instr.instr.real(), Some(Instruction::Nop)) { if lineno < 0 || prev_lineno == lineno { remove = true; + } else if src > 0 + && matches!( + src_instructions[src - 1].instr.real(), + Some(Instruction::PopIter) + ) + { + remove = true; } else if src < src_instructions.len() - 1 { if src_instructions[src + 1].instr.is_block_push() { remove = false; @@ -555,6 +561,21 @@ impl CodeInfo { // The offset (in code units) of END_SEND from SEND in the yield-from sequence. const END_SEND_OFFSET: u32 = 5; loop { + for block in blocks.iter_mut() { + let mut i = 1; + while i < block.instructions.len() { + if matches!( + block.instructions[i - 1].instr.real(), + Some(Instruction::PopIter) + ) && matches!(block.instructions[i].instr.real(), Some(Instruction::Nop)) + { + block.instructions.remove(i); + } else { + i += 1; + } + } + } + let mut num_instructions = 0; for (idx, block) in iter_blocks(&blocks) { block_to_offset[idx.idx()] = Label::from_u32(num_instructions as u32); @@ -591,6 +612,7 @@ impl CodeInfo { op = Opcode::Nop.into(); info.instr = op.into(); info.target = BlockIdx::NULL; + recompile = true; let updated_cache = op.cache_entries() as u32; recompile |= updated_cache != old_cache_entries; info.cache_entries = updated_cache; @@ -2595,7 +2617,7 @@ impl CodeInfo { Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), _ => None, }; - for block in &mut self.blocks { + for (block_idx, block) in self.blocks.iter_mut().enumerate() { let mut i = 0; while i + 1 < block.instructions.len() { let curr = &block.instructions[i]; @@ -2645,6 +2667,22 @@ impl CodeInfo { _ => unreachable!(), }; set_to_nop(&mut block.instructions[i]); + let preserves_pure_self_loop_anchor = i == 0 + && block.instructions[i + 1..].iter().all(|info| { + if info.target == BlockIdx(block_idx as u32) + && info.instr.is_unconditional_jump() + { + return true; + } + matches!(info.instr.real(), Some(Instruction::Nop)) + }) + && block.instructions[i + 1..].iter().any(|info| { + info.target == BlockIdx(block_idx as u32) + && info.instr.is_unconditional_jump() + }); + if preserves_pure_self_loop_anchor { + block.instructions[i].preserve_block_start_no_location_nop = true; + } if is_true == jump_if_true { block.instructions[i + 1].instr = PseudoInstruction::Jump { delta: Arg::marker(), @@ -2966,13 +3004,24 @@ impl CodeInfo { /// Remove NOP instructions from all blocks, but keep NOPs that introduce /// a new source line (they serve as line markers for monitoring LINE events). fn remove_nops(&mut self) { - for block in &mut self.blocks { + let keep_target_start_nops: Vec<_> = (0..self.blocks.len()) + .map(|idx| keep_target_start_no_location_nop(&self.blocks, BlockIdx(idx as u32))) + .collect(); + for (block_idx, block) in self.blocks.iter_mut().enumerate() { let mut prev_line = None; let mut src = 0usize; block.instructions.retain(|ins| { let keep = 'keep: { if matches!(ins.instr.real(), Some(Instruction::Nop)) { - if ins.remove_no_location_nop && instruction_lineno(ins) < 0 { + let keep_target_start = src == 0 + && keep_target_start_nops + .get(block_idx) + .copied() + .unwrap_or(false); + if ins.remove_no_location_nop + && instruction_lineno(ins) < 0 + && !keep_target_start + { break 'keep false; } let line = ins.location.line.get() as i32; @@ -9137,18 +9186,205 @@ impl CodeInfo { } } + fn fast_scan_many_locals( + &mut self, + nlocals: usize, + merged_cell_local: &impl Fn(usize) -> Option, + ) { + debug_assert!(nlocals > 64); + let mut states = vec![0usize; nlocals - 64]; + let mut blocknum = 0usize; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + blocknum += 1; + let old_instructions = self.blocks[current.idx()].instructions.clone(); + let mut new_instructions = Vec::with_capacity(old_instructions.len()); + let mut changed = false; + + for mut info in old_instructions { + match info.instr.real() { + Some( + Instruction::DeleteFast { var_num } + | Instruction::LoadFastAndClear { var_num }, + ) => { + let idx = usize::from(var_num.get(info.arg)); + if idx >= 64 && idx < nlocals { + states[idx - 64] = blocknum - 1; + } + new_instructions.push(info); + } + None if matches!( + info.instr.pseudo(), + Some(PseudoInstruction::StoreFastMaybeNull { .. }) + ) => + { + let Some(PseudoInstruction::StoreFastMaybeNull { var_num }) = + info.instr.pseudo() + else { + unreachable!(); + }; + let idx = var_num.get(info.arg) as usize; + if idx >= 64 && idx < nlocals { + states[idx - 64] = blocknum - 1; + } + new_instructions.push(info); + } + Some(Instruction::DeleteDeref { i }) => { + let cell_relative = usize::from(i.get(info.arg)); + if let Some(idx) = merged_cell_local(cell_relative) + && idx >= 64 + && idx < nlocals + { + states[idx - 64] = blocknum - 1; + } + new_instructions.push(info); + } + Some(Instruction::StoreFast { var_num }) => { + let idx = usize::from(var_num.get(info.arg)); + if idx >= 64 && idx < nlocals { + states[idx - 64] = blocknum; + } + new_instructions.push(info); + } + Some(Instruction::StoreDeref { i }) => { + let cell_relative = usize::from(i.get(info.arg)); + if let Some(idx) = merged_cell_local(cell_relative) + && idx >= 64 + && idx < nlocals + { + states[idx - 64] = blocknum; + } + new_instructions.push(info); + } + Some(Instruction::StoreFastStoreFast { var_nums }) => { + let packed = var_nums.get(info.arg); + let (idx1, idx2) = packed.indexes(); + let idx1 = usize::from(idx1); + let idx2 = usize::from(idx2); + if idx1 >= 64 && idx1 < nlocals { + states[idx1 - 64] = blocknum; + } + if idx2 >= 64 && idx2 < nlocals { + states[idx2 - 64] = blocknum; + } + new_instructions.push(info); + } + Some(Instruction::StoreFastLoadFast { var_nums }) => { + let packed = var_nums.get(info.arg); + let (store_idx, load_idx) = packed.indexes(); + let store_idx = usize::from(store_idx); + let load_idx = usize::from(load_idx); + if store_idx >= 64 && store_idx < nlocals { + states[store_idx - 64] = blocknum; + } + if load_idx >= 64 && load_idx < nlocals { + if states[load_idx - 64] != blocknum { + let mut first = info; + first.instr = Instruction::StoreFast { + var_num: Arg::marker(), + } + .into(); + first.arg = OpArg::new(store_idx as u32); + + let mut second = info; + second.instr = Opcode::LoadFastCheck.into(); + second.arg = OpArg::new(load_idx as u32); + + new_instructions.push(first); + new_instructions.push(second); + changed = true; + } else { + new_instructions.push(info); + } + states[load_idx - 64] = blocknum; + } else { + new_instructions.push(info); + } + } + Some(Instruction::LoadFast { var_num }) => { + let idx = usize::from(var_num.get(info.arg)); + if idx >= 64 && idx < nlocals { + if states[idx - 64] != blocknum { + info.instr = Opcode::LoadFastCheck.into(); + changed = true; + } + states[idx - 64] = blocknum; + } + new_instructions.push(info); + } + Some(Instruction::LoadFastLoadFast { var_nums }) => { + let packed = var_nums.get(info.arg); + let (idx1, idx2) = packed.indexes(); + let idx1 = usize::from(idx1); + let idx2 = usize::from(idx2); + let needs_check_1 = + idx1 >= 64 && idx1 < nlocals && states[idx1 - 64] != blocknum; + if idx1 >= 64 && idx1 < nlocals { + states[idx1 - 64] = blocknum; + } + let needs_check_2 = + idx2 >= 64 && idx2 < nlocals && states[idx2 - 64] != blocknum; + if idx2 >= 64 && idx2 < nlocals { + states[idx2 - 64] = blocknum; + } + + if needs_check_1 || needs_check_2 { + let mut first = info; + first.instr = if needs_check_1 { + Opcode::LoadFastCheck + } else { + Opcode::LoadFast + } + .into(); + first.arg = OpArg::new(idx1 as u32); + + let mut second = info; + second.instr = if needs_check_2 { + Opcode::LoadFastCheck.into() + } else { + Opcode::LoadFast.into() + }; + second.arg = OpArg::new(idx2 as u32); + + new_instructions.push(first); + new_instructions.push(second); + changed = true; + } else { + new_instructions.push(info); + } + } + Some(Instruction::LoadFastCheck { var_num }) => { + let idx = usize::from(var_num.get(info.arg)); + if idx >= 64 && idx < nlocals { + states[idx - 64] = blocknum; + } + new_instructions.push(info); + } + _ => new_instructions.push(info), + } + } + + if changed { + self.blocks[current.idx()].instructions = new_instructions; + } + current = self.blocks[current.idx()].next; + } + } + fn add_checks_for_loads_of_uninitialized_variables(&mut self) { - let nlocals = self.metadata.varnames.len(); + let mut nlocals = self.metadata.varnames.len(); if nlocals == 0 { return; } - let merged_cell_local = |cell_relative: usize| { - self.metadata - .cellvars - .get_index(cell_relative) - .and_then(|name| self.metadata.varnames.get_index_of(name.as_str())) - }; + let cell_to_local: Vec<_> = self + .metadata + .cellvars + .iter() + .map(|name| self.metadata.varnames.get_index_of(name.as_str())) + .collect(); + let merged_cell_local = + |cell_relative: usize| cell_to_local.get(cell_relative).copied().flatten(); let mut nparams = self.metadata.argcount as usize + self.metadata.kwonlyargcount as usize; if self.flags.contains(CodeFlags::VARARGS) { @@ -9159,6 +9395,11 @@ impl CodeInfo { } nparams = nparams.min(nlocals); + if nlocals > 64 { + self.fast_scan_many_locals(nlocals, &merged_cell_local); + nlocals = 64; + } + let mut in_masks: Vec>> = vec![None; self.blocks.len()]; let mut start_mask = vec![false; nlocals]; for slot in start_mask.iter_mut().skip(nparams) { @@ -9597,7 +9838,6 @@ impl CodeInfo { )); materialize_empty_conditional_exit_targets(&mut self.blocks); - materialize_exception_cleanup_jump_targets(&mut self.blocks); trace.push(( "after_materialize_empty_conditional_exit_targets".to_owned(), self.debug_block_dump(), @@ -10909,47 +11149,16 @@ fn compute_target_predecessor_flags(blocks: &[Block]) -> TargetPredecessorFlags fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { let mut changes = 0; - let TargetPredecessorFlags { - targeted: targeted_blocks, - plain_jump: plain_jump_targets, - .. - } = compute_target_predecessor_flags(blocks); + let plain_jump_targets = compute_target_predecessor_flags(blocks).plain_jump; let mut block_order = Vec::new(); let mut current = BlockIdx(0); while current != BlockIdx::NULL { block_order.push(current); current = blocks[current.idx()].next; } - let mut fallthrough_prev_lineno = vec![None; blocks.len()]; - let mut fallthrough_prev_pop_top_lineno = vec![None; blocks.len()]; - let mut prev_nonempty = BlockIdx::NULL; - for &block_idx in &block_order { - if blocks[block_idx.idx()].instructions.is_empty() { - continue; - } - if prev_nonempty != BlockIdx::NULL - && !targeted_blocks[block_idx.idx()] - && block_has_fallthrough(&blocks[prev_nonempty.idx()]) - && next_nonempty_block(blocks, blocks[prev_nonempty.idx()].next) == block_idx - { - fallthrough_prev_lineno[block_idx.idx()] = blocks[prev_nonempty.idx()] - .instructions - .last() - .map(instruction_lineno); - } - if prev_nonempty != BlockIdx::NULL - && block_has_fallthrough(&blocks[prev_nonempty.idx()]) - && next_nonempty_block(blocks, blocks[prev_nonempty.idx()].next) == block_idx - && let Some(last) = blocks[prev_nonempty.idx()].instructions.last() - && matches!(last.instr.real(), Some(Instruction::PopTop)) - { - fallthrough_prev_pop_top_lineno[block_idx.idx()] = Some(instruction_lineno(last)); - } - prev_nonempty = block_idx; - } - for block_idx in block_order { let bi = block_idx.idx(); + let keep_target_start_nop = keep_target_start_no_location_nop(blocks, block_idx); let mut src_instructions = core::mem::take(&mut blocks[bi].instructions); let mut kept = Vec::with_capacity(src_instructions.len()); let mut prev_lineno = -1i32; @@ -10962,19 +11171,25 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { if matches!(instr.instr.real(), Some(Instruction::Nop)) { if instr.preserve_block_start_no_location_nop { remove = false; - } else if lineno < 0 - || (src == 0 - && fallthrough_prev_lineno[block_idx.idx()] - .is_some_and(|prev_lineno| prev_lineno == lineno)) - || (src == 0 - && src_instructions.len() == 1 - && fallthrough_prev_pop_top_lineno[block_idx.idx()] - .is_some_and(|prev_lineno| prev_lineno == lineno)) + } else if lineno < 0 { + remove = true; + } else if src > 0 + && matches!( + src_instructions[src - 1].instr.real(), + Some(Instruction::PopIter) + ) + { + remove = true; + } else if src == 0 + && !keep_target_start_nop + && layout_predecessor_ends_with_pop_iter(blocks, block_idx) { remove = true; } else if instr.remove_no_location_nop && src == 0 && plain_jump_targets[block_idx.idx()] + && instr.lineno_override.is_some() + && !keep_target_start_nop { let next_lineno = src_instructions[src + 1..].iter().find_map(|next_instr| { let line = instruction_lineno(next_instr); @@ -11243,92 +11458,6 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { } } -fn materialize_exception_cleanup_jump_targets(blocks: &mut [Block]) { - let mut inserts = Vec::new(); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let Some(last) = blocks[idx].instructions.last() else { - current = blocks[idx].next; - continue; - }; - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { - current = blocks[idx].next; - continue; - } - - let mut cursor = blocks[idx].next; - let mut saw_cleanup = false; - while cursor != BlockIdx::NULL && cursor != last.target { - let block = &blocks[cursor.idx()]; - if !block_is_exceptional(block) && !is_exception_cleanup_block(block) { - break; - } - saw_cleanup = true; - cursor = block.next; - } - if saw_cleanup - && cursor == last.target - && !blocks[last.target.idx()].instructions.is_empty() - && !block_starts_with_with_exit_none_call(&blocks[last.target.idx()]) - && !matches!( - blocks[last.target.idx()] - .instructions - .first() - .and_then(|info| info.instr.real()), - Some(Instruction::Nop) - ) - { - inserts.push(last.target); - } - - current = blocks[idx].next; - } - - inserts.sort_by_key(|idx| idx.idx()); - inserts.dedup(); - for target in inserts { - let Some(first) = blocks[target.idx()].instructions.first().copied() else { - continue; - }; - blocks[target.idx()].instructions.insert( - 0, - InstructionInfo { - instr: Instruction::Nop.into(), - arg: OpArg::NULL, - target: BlockIdx::NULL, - location: first.location, - end_location: first.end_location, - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: Some(-1), - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - preserve_block_start_no_location_nop: true, - }, - ); - } -} - -fn block_starts_with_with_exit_none_call(block: &Block) -> bool { - let real_instrs: Vec<_> = block - .instructions - .iter() - .filter_map(|info| info.instr.real()) - .take(4) - .collect(); - matches!( - real_instrs.as_slice(), - [ - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::Call { .. }, - ] - ) -} - fn merge_unsafe_mask(slot: &mut Option>, incoming: &[bool]) -> bool { match slot { Some(existing) => { @@ -11571,6 +11700,13 @@ fn is_loop_cleanup_block(block: &Block) -> bool { }) } +fn is_async_loop_cleanup_block(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::EndAsyncFor))) +} + fn is_exception_cleanup_block(block: &Block) -> bool { block .instructions @@ -11582,6 +11718,59 @@ fn is_exception_cleanup_block(block: &Block) -> bool { .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::Reraise { .. }))) } +fn block_starts_with_with_exit_none_call(block: &Block) -> bool { + let real_instrs: Vec<_> = block + .instructions + .iter() + .filter_map(|info| { + let instr = info.instr.real()?; + (!matches!(instr, Instruction::Nop)).then_some(instr) + }) + .take(4) + .collect(); + matches!( + real_instrs.as_slice(), + [ + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) +} + +fn keep_target_start_no_location_nop(blocks: &[Block], target: BlockIdx) -> bool { + if target == BlockIdx::NULL { + return false; + } + let Some(first) = blocks[target.idx()].instructions.first() else { + return false; + }; + if !matches!(first.instr.real(), Some(Instruction::Nop)) { + return false; + } + let layout_pred = find_layout_predecessor(blocks, target); + if layout_pred == BlockIdx::NULL { + return false; + } + if is_async_loop_cleanup_block(&blocks[layout_pred.idx()]) { + return true; + } + is_exception_cleanup_block(&blocks[layout_pred.idx()]) + && !block_starts_with_with_exit_none_call(&blocks[target.idx()]) +} + +fn layout_predecessor_ends_with_pop_iter(blocks: &[Block], target: BlockIdx) -> bool { + let layout_pred = find_layout_predecessor(blocks, target); + layout_pred != BlockIdx::NULL + && block_has_fallthrough(&blocks[layout_pred.idx()]) + && next_nonempty_block(blocks, blocks[layout_pred.idx()].next) == target + && blocks[layout_pred.idx()] + .instructions + .last() + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::PopIter))) +} + fn is_with_suppress_exit_block(block: &Block) -> bool { let real_instrs: Vec<_> = block .instructions From a743ffc2c0aeb2dbc473ff782d0c7935cbd17466 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 5 May 2026 19:43:27 +0900 Subject: [PATCH 04/76] fix --- Lib/test/test_peepholer.py | 1 - crates/codegen/src/compile.rs | 250 ++++++++++++++++++++++++++++-- crates/codegen/src/ir.rs | 228 ++++++++++++++++++++++----- crates/codegen/src/symboltable.rs | 48 +++--- 4 files changed, 445 insertions(+), 82 deletions(-) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 6654a8830b..6c2e4c0811 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -933,7 +933,6 @@ def f(): self.assertInBytecode(f, 'LOAD_FAST_CHECK') self.assertNotInBytecode(f, 'LOAD_FAST') - @unittest.expectedFailure # TODO: RUSTPYTHON; RETURN_VALUE def test_load_fast_too_many_locals(self): # When there get to be too many locals to analyze completely, # later locals are all converted to LOAD_FAST_CHECK, except diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 7711258898..6c50f4eb08 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -2842,7 +2842,11 @@ impl Compiler { statement.range(), )); } - let folded_constant = self.try_fold_constant_expr(v)?; + let folded_constant = if v.is_constant() { + self.try_fold_constant_expr(v)? + } else { + None + }; let preserve_tos = folded_constant.is_none(); if preserve_tos { self.compile_expression(v)?; @@ -7796,22 +7800,7 @@ impl Compiler { for value in prefix_values { let continue_block = self.new_block(); - match value { - ast::Expr::BoolOp(ast::ExprBoolOp { - op: inner_op, - values, - .. - }) if inner_op != op => { - let (last_inner_value, inner_prefix_values) = values.split_last().unwrap(); - for inner_value in inner_prefix_values { - self.compile_expression(inner_value)?; - self.emit_short_circuit_test(inner_op, continue_block); - emit!(self, Instruction::PopTop); - } - self.compile_expression(last_inner_value)?; - } - _ => self.compile_expression(value)?, - } + self.compile_expression(value)?; self.emit_short_circuit_test(op, after_block); self.switch_to_block(continue_block); emit!(self, Instruction::PopTop); @@ -14620,6 +14609,132 @@ def f(c, encodeO, encodeWS): ); } + #[test] + fn test_nested_opposite_boolop_threads_to_fallthrough_like_cpython() { + for source in [ + "\ +def f(a, b, c): + return ((a and b) + or c) +", + "\ +def f(a, b, c): + return ((a or b) + and c) +", + ] { + let code = compile_exec(source); + let f = find_code(&code, "f").expect("missing f code"); + let jumps: Vec<_> = f + .instructions + .iter() + .filter(|unit| { + matches!( + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + }) + .collect(); + + assert_eq!( + jumps.len(), + 2, + "expected two conditional jumps, got {jumps:?}" + ); + assert!( + u8::from(jumps[0].arg) > u8::from(jumps[1].arg), + "expected CPython-style first jump to bypass the opposite short-circuit test, got jumps={jumps:?}" + ); + } + } + + #[test] + fn test_loop_or_continue_keeps_boolop_true_edge_to_continue() { + let code = compile_exec( + "\ +def f(numpy_array, lshape, rshape, litems, fmt, tl): + for _ in range(3): + if numpy_array: + if 0 in lshape or 0 in rshape: + continue + zl = numpy_array_from_structure(litems, fmt, tl) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::LoadSmallInt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + ] + ) + }), + "expected CPython-style `or` continue test to keep first true edge to continue, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadSmallInt { .. }, + ] + ) + }), + "unexpected inverted first `or` continue condition before second operand, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_and_or_expression_threads_same_false_short_circuit() { + let code = compile_exec( + "\ +def f(fmt, MEMORYVIEW): + x = len(fmt) + return ((x == 1 or (x == 2 and fmt[0] == '@')) and + fmt[x - 1] in MEMORYVIEW) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let false_jumps: Vec<_> = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::PopJumpIfFalse { .. })) + .collect(); + + assert!( + false_jumps.len() >= 2, + "expected nested boolop false jumps, got ops={:?}", + f.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert!( + u8::from(false_jumps[0].arg) > u8::from(false_jumps[1].arg), + "expected CPython-style same-false short-circuit threading to outer end, got false_jumps={false_jumps:?}" + ); + } + #[test] fn test_broad_exception_import_keeps_borrow_in_common_tail() { let code = compile_exec( @@ -16801,6 +16916,38 @@ def f(p, s): ); } + #[test] + fn test_for_return_unary_constant_preserves_value_over_iterator_cleanup() { + let code = compile_exec( + "\ +def f(xs): + for x in xs: + return -1 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let units: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + + assert!( + units.windows(4).any(|window| { + matches!( + window[0].op, + Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } + ) && matches!( + window[1].op, + Instruction::Swap { i } + if i.get(OpArg::new(u32::from(u8::from(window[1].arg)))) == 2 + ) && matches!(window[2].op, Instruction::PopTop) + && matches!(window[3].op, Instruction::ReturnValue) + }), + "expected CPython-style LOAD_CONST/SWAP/POP_TOP/RETURN_VALUE cleanup, got units={units:?}" + ); + } + #[test] fn test_try_else_if_return_keeps_conditional_target_nop() { let code = compile_exec( @@ -19234,6 +19381,43 @@ def f(xs): ); } + #[test] + fn test_loop_not_in_conditional_body_threads_true_path_to_jump_back() { + let code = compile_exec( + "\ +def f(native, array): + for k in native: + if not k in 'bBhHiIlLfd': + del array[k] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + ] + ) + }), + "expected CPython-style true path to jump back before not-in body, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { let code = compile_exec( @@ -19275,6 +19459,38 @@ def f(x, y): ); } + #[test] + fn test_constant_true_while_pass_keeps_loop_header_nop() { + let code = compile_exec( + "\ +def f(): + while 1: + pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(2).any(|window| { + matches!( + window, + [ + Instruction::Nop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style loop-header NOP before self backedge, got ops={ops:?}" + ); + } + #[test] fn test_nested_if_shared_jump_back_target_is_duplicated() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 81b66962f1..cef3c2b7a2 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -471,13 +471,13 @@ impl CodeInfo { let mut remove = false; if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if lineno < 0 || prev_lineno == lineno { - remove = true; - } else if src > 0 - && matches!( - src_instructions[src - 1].instr.real(), - Some(Instruction::PopIter) - ) + if lineno < 0 + || prev_lineno == lineno + || (src > 0 + && matches!( + src_instructions[src - 1].instr.real(), + Some(Instruction::PopIter) + )) { remove = true; } else if src < src_instructions.len() - 1 { @@ -561,21 +561,6 @@ impl CodeInfo { // The offset (in code units) of END_SEND from SEND in the yield-from sequence. const END_SEND_OFFSET: u32 = 5; loop { - for block in blocks.iter_mut() { - let mut i = 1; - while i < block.instructions.len() { - if matches!( - block.instructions[i - 1].instr.real(), - Some(Instruction::PopIter) - ) && matches!(block.instructions[i].instr.real(), Some(Instruction::Nop)) - { - block.instructions.remove(i); - } else { - i += 1; - } - } - } - let mut num_instructions = 0; for (idx, block) in iter_blocks(&blocks) { block_to_offset[idx.idx()] = Label::from_u32(num_instructions as u32); @@ -10579,6 +10564,53 @@ fn jump_threading_unconditional(blocks: &mut [Block]) { jump_threading_impl(blocks, false); } +fn opposite_short_circuit_target(block: &Block, source: AnyInstruction) -> bool { + let Some(cond_idx) = trailing_conditional_jump_index(block) else { + return false; + }; + let [first, second, ..] = block.instructions.as_slice() else { + return false; + }; + if !matches!(first.instr.real(), Some(Instruction::Copy { i }) if i.get(first.arg) == 1) + || !matches!(second.instr.real(), Some(Instruction::ToBool)) + { + return false; + } + matches!( + (source.real(), block.instructions[cond_idx].instr.real()), + ( + Some(Instruction::PopJumpIfFalse { .. }), + Some(Instruction::PopJumpIfTrue { .. }) + ) | ( + Some(Instruction::PopJumpIfTrue { .. }), + Some(Instruction::PopJumpIfFalse { .. }) + ) + ) +} + +fn same_short_circuit_target(block: &Block, source: AnyInstruction) -> Option { + let cond_idx = trailing_conditional_jump_index(block)?; + let [first, second, ..] = block.instructions.as_slice() else { + return None; + }; + if !matches!(first.instr.real(), Some(Instruction::Copy { i }) if i.get(first.arg) == 1) + || !matches!(second.instr.real(), Some(Instruction::ToBool)) + { + return None; + } + matches!( + (source.real(), block.instructions[cond_idx].instr.real()), + ( + Some(Instruction::PopJumpIfFalse { .. }), + Some(Instruction::PopJumpIfFalse { .. }) + ) | ( + Some(Instruction::PopJumpIfTrue { .. }), + Some(Instruction::PopJumpIfTrue { .. }) + ) + ) + .then_some(block.instructions[cond_idx].target) +} + #[derive(Clone, Copy, PartialEq, Eq)] enum JumpThreadKind { Plain, @@ -10674,6 +10706,28 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { if target == BlockIdx::NULL { continue; } + if include_conditional + && is_conditional_jump(&ins.instr) + && opposite_short_circuit_target(&blocks[target.idx()], ins.instr) + { + let final_target = next_nonempty_block(blocks, blocks[target.idx()].next); + if final_target != BlockIdx::NULL && ins.target != final_target { + blocks[bi].instructions[last_idx].target = final_target; + changed = true; + continue; + } + } + if include_conditional + && is_conditional_jump(&ins.instr) + && let Some(final_target) = + same_short_circuit_target(&blocks[target.idx()], ins.instr) + && final_target != BlockIdx::NULL + && ins.target != final_target + { + blocks[bi].instructions[last_idx].target = final_target; + changed = true; + continue; + } if include_conditional && is_conditional_jump(&ins.instr) { let source_pos = block_order[bi]; let target_pos = block_order.get(target.idx()).copied().unwrap_or(u32::MAX); @@ -11159,7 +11213,23 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { for block_idx in block_order { let bi = block_idx.idx(); let keep_target_start_nop = keep_target_start_no_location_nop(blocks, block_idx); + let follows_pop_iter_cleanup = layout_predecessor_ends_with_pop_iter(blocks, block_idx); let mut src_instructions = core::mem::take(&mut blocks[bi].instructions); + if !keep_target_start_nop + && matches!( + src_instructions.as_slice(), + [InstructionInfo { + instr: AnyInstruction::Real(Instruction::Nop), + .. + }] + ) + && follows_pop_iter_cleanup + { + changes += 1; + src_instructions.clear(); + blocks[bi].instructions = src_instructions; + continue; + } let mut kept = Vec::with_capacity(src_instructions.len()); let mut prev_lineno = -1i32; @@ -11169,21 +11239,17 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { let mut remove = false; if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if instr.preserve_block_start_no_location_nop { - remove = false; - } else if lineno < 0 { - remove = true; - } else if src > 0 + if (src > 0 && matches!( src_instructions[src - 1].instr.real(), Some(Instruction::PopIter) - ) + )) + || (src == 0 && !keep_target_start_nop && follows_pop_iter_cleanup) { remove = true; - } else if src == 0 - && !keep_target_start_nop - && layout_predecessor_ends_with_pop_iter(blocks, block_idx) - { + } else if instr.preserve_block_start_no_location_nop { + remove = false; + } else if lineno < 0 { remove = true; } else if instr.remove_no_location_nop && src == 0 @@ -11207,7 +11273,9 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { } else if src < src_instructions.len() - 1 { if src_instructions[src + 1].instr.is_block_push() { remove = false; - } else if src_instructions[src + 1].instr.is_unconditional_jump() { + } else if src_instructions[src + 1].instr.is_unconditional_jump() + && src_instructions[src + 1].target != block_idx + { src_instructions[src + 1].lineno_override = Some(lineno); remove = true; } else if src_instructions[src + 1].folded_from_nonliteral_expr { @@ -12818,7 +12886,7 @@ fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { } } -fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut [Block]) { +fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec) { fn jump_back_target(blocks: &[Block], block_idx: BlockIdx) -> Option { if block_idx == BlockIdx::NULL { return None; @@ -12866,6 +12934,30 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut [Block]) { None } + fn find_body_tail_before_jump( + blocks: &[Block], + body_start: BlockIdx, + jump_start: BlockIdx, + ) -> Option { + let mut cursor = body_start; + let mut tail = BlockIdx::NULL; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && cursor != jump_start { + if visited[cursor.idx()] { + return None; + } + visited[cursor.idx()] = true; + if block_is_exceptional(&blocks[cursor.idx()]) + || block_is_protected(&blocks[cursor.idx()]) + { + return None; + } + tail = cursor; + cursor = blocks[cursor.idx()].next; + } + (tail != BlockIdx::NULL && cursor == jump_start).then_some(tail) + } + fn body_segment_contains_for_iter( blocks: &[Block], body_start: BlockIdx, @@ -12893,6 +12985,17 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut [Block]) { false } + fn block_starts_loop_cleanup(blocks: &[Block], block_idx: BlockIdx) -> bool { + block_idx != BlockIdx::NULL + && matches!( + blocks[block_idx.idx()] + .instructions + .first() + .and_then(|info| info.instr.real()), + Some(Instruction::EndFor | Instruction::PopIter) + ) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -12902,6 +13005,55 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut [Block]) { continue; }; let cond = blocks[idx].instructions[cond_idx]; + let Some(reversed_cond) = reversed_conditional(&cond.instr) else { + current = next; + continue; + }; + + let true_jump_start = cond.target; + let true_jump = next_nonempty_block(blocks, true_jump_start); + let body_start = next; + let body = next_nonempty_block(blocks, body_start); + let true_jump_loop_target = jump_back_target(blocks, true_jump); + let true_jump_targets_loop_header = true_jump_loop_target.is_some_and(|loop_target| { + blocks[loop_target.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) + }); + if true_jump_targets_loop_header + && true_jump_start == true_jump + && body_start != BlockIdx::NULL + && body != BlockIdx::NULL + && true_jump != BlockIdx::NULL + && !block_is_exceptional(&blocks[idx]) + && !block_is_exceptional(&blocks[true_jump.idx()]) + && !block_is_protected(&blocks[idx]) + && !block_is_protected(&blocks[true_jump.idx()]) + && next_nonempty_block(blocks, blocks[true_jump.idx()].next) != body + && let Some(body_tail) = find_body_tail_before_jump(blocks, body, true_jump_start) + { + let after_jump = blocks[true_jump.idx()].next; + let body_tail_is_conditional = + trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); + let can_reorder = !body_tail_is_conditional + && (matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) + || body_segment_contains_for_iter(blocks, body, body_tail)); + if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { + let cloned_jump_idx = BlockIdx(blocks.len() as u32); + let mut cloned_jump = blocks[true_jump.idx()].clone(); + cloned_jump.next = body_start; + blocks.push(cloned_jump); + + blocks[idx].instructions[cond_idx].instr = reversed_cond; + blocks[idx].instructions[cond_idx].target = body_start; + blocks[idx].next = cloned_jump_idx; + blocks[body_tail.idx()].next = true_jump_start; + current = blocks[idx].next; + continue; + } + } + if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { current = next; continue; @@ -12936,12 +13088,14 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut [Block]) { current = next; continue; }; - if !body_segment_contains_for_iter(blocks, body, body_tail) { + let after_body = blocks[body_tail.idx()].next; + if after_body == BlockIdx::NULL || after_body == false_jump_start { current = next; continue; } - let after_body = blocks[body_tail.idx()].next; - if after_body == BlockIdx::NULL || after_body == false_jump_start { + if !body_segment_contains_for_iter(blocks, body, body_tail) + || block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_body)) + { current = next; continue; } diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 31da7c164e..23313b1a59 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -591,14 +591,7 @@ impl SymbolTableAnalyzer { } // Analyze symbols in current scope - let remove_owned_cells_from_free = matches!( - symbol_table.typ, - CompilerScope::Function - | CompilerScope::AsyncFunction - | CompilerScope::Lambda - | CompilerScope::Comprehension - | CompilerScope::Annotation - ); + let function_like_scope = SymbolTableBuilder::is_function_like_scope(symbol_table.typ); for symbol in symbol_table.symbols.values_mut() { self.analyze_symbol( symbol, @@ -611,7 +604,7 @@ impl SymbolTableAnalyzer { // CPython analyze_cells(): once a function-like scope owns a // child-requested name as a cell, that name is no longer free in // the enclosing scope. - if remove_owned_cells_from_free && symbol.scope == SymbolScope::Cell { + if function_like_scope && symbol.scope == SymbolScope::Cell { newfree.shift_remove(symbol.name.as_str()); } @@ -625,17 +618,9 @@ impl SymbolTableAnalyzer { // - only promote LOCAL -> CELL in function-like scopes, where // analyze_cells() runs. Module and class scopes keep their normal // scope and rely on DEF_COMP_CELL for comprehension-only cells. - let promote_inlined_cells_to_cell = matches!( - symbol_table.typ, - CompilerScope::Function - | CompilerScope::AsyncFunction - | CompilerScope::Lambda - | CompilerScope::Comprehension - | CompilerScope::Annotation - ); for symbol in symbol_table.symbols.values_mut() { if inlined_cells.contains(&symbol.name) - && promote_inlined_cells_to_cell + && function_like_scope && symbol.scope == SymbolScope::Local { symbol.scope = SymbolScope::Cell; @@ -1073,6 +1058,17 @@ impl SymbolTableBuilder { this } + fn is_function_like_scope(typ: CompilerScope) -> bool { + matches!( + typ, + CompilerScope::Function + | CompilerScope::AsyncFunction + | CompilerScope::Lambda + | CompilerScope::Comprehension + | CompilerScope::Annotation + ) + } + fn finish(mut self) -> Result { assert_eq!(self.tables.len(), 1); let mut symbol_table = self.tables.pop().unwrap(); @@ -1467,7 +1463,11 @@ impl SymbolTableBuilder { parameters, self.line_index_start(*range), has_return_annotation, - *is_async, + if *is_async { + CompilerScope::AsyncFunction + } else { + CompilerScope::Function + }, has_type_params, // skip_defaults: already scanned above )?; self.scan_statements(body)?; @@ -2136,10 +2136,9 @@ impl SymbolTableBuilder { parameters, self.line_index_start(expression.range()), false, // lambdas have no return annotation - false, // lambdas are never async + CompilerScope::Lambda, false, // don't skip defaults )?; - self.tables.last_mut().unwrap().typ = CompilerScope::Lambda; } else { self.enter_scope( "lambda", @@ -2549,7 +2548,7 @@ impl SymbolTableBuilder { parameters: &ast::Parameters, line_number: u32, has_return_annotation: bool, - is_async: bool, + scope_type: CompilerScope, skip_defaults: bool, ) -> SymbolTableResult { // Evaluate eventual default parameters (unless already scanned before type_param_block): @@ -2609,11 +2608,6 @@ impl SymbolTableBuilder { None }; - let scope_type = if is_async { - CompilerScope::AsyncFunction - } else { - CompilerScope::Function - }; self.enter_scope(name, scope_type, line_number); // Move annotation_block to function scope only if we have one From c998628e74d94b3d7806286860eda4111f4e21b7 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 5 May 2026 20:07:30 +0900 Subject: [PATCH 05/76] Align named-except borrow deopts with CPython --- crates/codegen/src/compile.rs | 158 +++++++++++++++++++++++++++++++++- crates/codegen/src/ir.rs | 58 +++++++++++-- 2 files changed, 209 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 6c50f4eb08..eae51572ed 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -11892,7 +11892,10 @@ mod ruff_tests { #[cfg(test)] mod tests { use super::*; - use rustpython_compiler_core::{SourceFileBuilder, bytecode::OpArg}; + use rustpython_compiler_core::{ + SourceFileBuilder, + bytecode::{CodeUnit, OpArg}, + }; fn assert_scope_exit_locations(code: &CodeObject) { for (instr, (location, _)) in code.instructions.iter().zip(code.locations.iter()) { @@ -22305,6 +22308,159 @@ def f(self): ); } + #[test] + fn test_named_except_cleanup_or_guard_tail_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, nd, slices): + listerr = None + try: + sliced = multislice(lst, slices) + except Exception as e: + listerr = e.__class__ + nderr = None + try: + ndsliced = nd[slices] + except Exception as e: + nderr = e.__class__ + if nderr or listerr: + self.assertIs(nderr, listerr) + else: + self.assertEqual(ndsliced.tolist(), sliced) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let assert_is_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "assertIs" + } + _ => false, + }) + .expect("missing assertIs LOAD_ATTR"); + let tail = &instructions[assert_is_idx.saturating_sub(1)..]; + + assert!( + matches!( + tail.first().map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "named-except `a or b` normal tail should keep borrowed receiver, got tail={tail:?}" + ); + assert!( + tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "named-except `a or b` normal tail should keep borrowed arguments, got tail={tail:?}" + ); + } + + #[test] + fn test_named_except_cleanup_slice_assign_tail_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, items, lslice, rslice, ndarray, memoryview, ValueError, + is_memoryview_format, fmt): + nd = ndarray(items, shape=[5], format=fmt) + ex = ndarray(items, shape=[5], format=fmt) + mv = memoryview(ex) + lsterr = None + diff_structure = None + lst = items[:] + try: + lval = lst[lslice] + rval = lst[rslice] + lst[lslice] = lst[rslice] + diff_structure = len(lval) != len(rval) + except Exception as e: + lsterr = e.__class__ + nderr = None + try: + nd[lslice] = nd[rslice] + except Exception as e: + nderr = e.__class__ + if diff_structure: + self.assertIs(nderr, ValueError) + else: + self.assertEqual(nd.tolist(), lst) + self.assertIs(nderr, lsterr) + if not is_memoryview_format(fmt): + return + mverr = None + try: + mv[lslice] = mv[rslice] + except Exception as e: + mverr = e.__class__ + if diff_structure: + self.assertIs(mverr, ValueError) + else: + self.assertEqual(mv.tolist(), lst) + self.assertEqual(mv, nd) + self.assertIs(mverr, lsterr) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .unwrap_or(instructions.len()); + let normal_path = &instructions[..handler_start]; + let load_fast_name = |unit: &CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let idx = usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg))))); + Some(f.varnames[idx].as_str()) + } + _ => None, + }; + + assert!( + normal_path.windows(3).any(|window| { + matches!(window[0].op, Instruction::LoadFastBorrow { .. }) + && load_fast_name(window[0]) == Some("diff_structure") + && matches!(window[1].op, Instruction::ToBool) + && matches!(window[2].op, Instruction::PopJumpIfFalse { .. }) + }), + "expected named-except resume guard to keep borrowed diff_structure load, got normal_path={normal_path:?}" + ); + assert!( + normal_path.windows(4).any(|window| { + matches!( + window[0].op, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) && matches!(window[1].op, Instruction::BinaryOp { .. }) + && matches!( + window[2].op, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + && matches!(window[3].op, Instruction::StoreSubscr) + }), + "expected slice assignment normal path to keep borrowed local loads, got normal_path={normal_path:?}" + ); + assert!( + normal_path.windows(2).any(|window| { + matches!(window[0].op, Instruction::LoadFastBorrow { .. }) + && matches!(window[1].op, Instruction::LoadAttr { .. }) + }), + "expected named-except slice-assign tail method calls to keep borrowed receivers, got normal_path={normal_path:?}" + ); + } + #[test] fn test_with_suppress_named_except_resume_tail_uses_strong_loads() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index cef3c2b7a2..a897c7b247 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4637,6 +4637,42 @@ impl CodeInfo { .collect() } + fn block_has_simple_scope_exit(block: &Block) -> bool { + for info in &block.instructions { + match info.instr.real() { + Some(instr) if instr.is_scope_exit() => return true, + Some( + Instruction::Nop + | Instruction::NotTaken + | Instruction::LoadConst { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::StoreFast { .. } + | Instruction::StoreName { .. } + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::BuildTuple { .. }, + ) => {} + Some(_) => return false, + None => {} + } + } + false + } + + fn leading_bool_guard_has_scope_exit_successor(blocks: &[Block], block: &Block) -> bool { + let Some(cond_idx) = trailing_conditional_jump_index(block) else { + return false; + }; + [block.instructions[cond_idx].target, block.next] + .into_iter() + .any(|successor| { + successor != BlockIdx::NULL + && block_has_simple_scope_exit(&blocks[successor.idx()]) + }) + } + fn path_reaches_named_cleanup( blocks: &[Block], start: BlockIdx, @@ -4827,6 +4863,7 @@ impl CodeInfo { let seed = BlockIdx::new(idx as u32); let mut segment = Vec::new(); let mut cursor = seed; + let seed_guard_local = leading_bool_guard_local(&self.blocks[seed.idx()]); let mut fallback_guard_local = None; while cursor != BlockIdx::NULL { let block = &self.blocks[cursor.idx()]; @@ -4836,6 +4873,12 @@ impl CodeInfo { if cursor != seed && let Some(local) = leading_bool_guard_local(block) { + if !leading_bool_guard_has_scope_exit_successor(&self.blocks, block) { + break; + } + if seed_guard_local.is_some_and(|seed_local| seed_local != local) { + break; + } match fallback_guard_local { None => fallback_guard_local = Some(local), Some(expected) if expected != local => break, @@ -4845,7 +4888,8 @@ impl CodeInfo { segment.push(cursor); cursor = block.next; } - if fallback_guard_local.is_none() && !named_cleanup_requires_deopt[idx] { + let requires_deopt = named_cleanup_requires_deopt[idx]; + if fallback_guard_local.is_none() && !requires_deopt { continue; } @@ -4861,6 +4905,9 @@ impl CodeInfo { let is_same_guard_fallback = fallback_guard_local.is_some_and(|local| { leading_bool_guard_local(&self.blocks[block_idx.idx()]) == Some(local) }); + if !requires_deopt && !is_same_guard_fallback { + continue; + } if block_idx != seed && !is_same_guard_fallback && predecessors[block_idx.idx()].iter().any(|pred| { @@ -7408,11 +7455,10 @@ impl CodeInfo { .any(|info| info.except_handler.is_some()) }); let has_handler_resume_predecessor = predecessors[idx].iter().any(|pred| { - is_handler_resume_block[pred.idx()] - || is_handler_resume_predecessor( - &self.blocks[pred.idx()], - BlockIdx::new(idx as u32), - ) + let pred_block = &self.blocks[pred.idx()]; + !is_named_except_cleanup_normal_exit_block(pred_block) + && (is_handler_resume_block[pred.idx()] + || is_handler_resume_predecessor(pred_block, BlockIdx::new(idx as u32))) }); let has_suppressing_with_resume_predecessor = predecessors[idx].iter().any(|pred| { From 68610b9d61d2da1eba2c09d9f3715e141f763db0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 5 May 2026 20:31:50 +0900 Subject: [PATCH 06/76] Remove duplicated Lib test codegen cases --- crates/codegen/src/compile.rs | 158 +--------------------------------- 1 file changed, 1 insertion(+), 157 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index eae51572ed..6c50f4eb08 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -11892,10 +11892,7 @@ mod ruff_tests { #[cfg(test)] mod tests { use super::*; - use rustpython_compiler_core::{ - SourceFileBuilder, - bytecode::{CodeUnit, OpArg}, - }; + use rustpython_compiler_core::{SourceFileBuilder, bytecode::OpArg}; fn assert_scope_exit_locations(code: &CodeObject) { for (instr, (location, _)) in code.instructions.iter().zip(code.locations.iter()) { @@ -22308,159 +22305,6 @@ def f(self): ); } - #[test] - fn test_named_except_cleanup_or_guard_tail_keeps_borrow() { - let code = compile_exec( - "\ -def f(self, nd, slices): - listerr = None - try: - sliced = multislice(lst, slices) - except Exception as e: - listerr = e.__class__ - nderr = None - try: - ndsliced = nd[slices] - except Exception as e: - nderr = e.__class__ - if nderr or listerr: - self.assertIs(nderr, listerr) - else: - self.assertEqual(ndsliced.tolist(), sliced) -", - ); - let f = find_code(&code, "f").expect("missing f code"); - let instructions: Vec<_> = f - .instructions - .iter() - .filter(|unit| !matches!(unit.op, Instruction::Cache)) - .collect(); - let assert_is_idx = instructions - .iter() - .position(|unit| match unit.op { - Instruction::LoadAttr { namei } => { - let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); - f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "assertIs" - } - _ => false, - }) - .expect("missing assertIs LOAD_ATTR"); - let tail = &instructions[assert_is_idx.saturating_sub(1)..]; - - assert!( - matches!( - tail.first().map(|unit| unit.op), - Some(Instruction::LoadFastBorrow { .. }) - ), - "named-except `a or b` normal tail should keep borrowed receiver, got tail={tail:?}" - ); - assert!( - tail.iter().any(|unit| { - matches!( - unit.op, - Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - }), - "named-except `a or b` normal tail should keep borrowed arguments, got tail={tail:?}" - ); - } - - #[test] - fn test_named_except_cleanup_slice_assign_tail_keeps_borrow() { - let code = compile_exec( - "\ -def f(self, items, lslice, rslice, ndarray, memoryview, ValueError, - is_memoryview_format, fmt): - nd = ndarray(items, shape=[5], format=fmt) - ex = ndarray(items, shape=[5], format=fmt) - mv = memoryview(ex) - lsterr = None - diff_structure = None - lst = items[:] - try: - lval = lst[lslice] - rval = lst[rslice] - lst[lslice] = lst[rslice] - diff_structure = len(lval) != len(rval) - except Exception as e: - lsterr = e.__class__ - nderr = None - try: - nd[lslice] = nd[rslice] - except Exception as e: - nderr = e.__class__ - if diff_structure: - self.assertIs(nderr, ValueError) - else: - self.assertEqual(nd.tolist(), lst) - self.assertIs(nderr, lsterr) - if not is_memoryview_format(fmt): - return - mverr = None - try: - mv[lslice] = mv[rslice] - except Exception as e: - mverr = e.__class__ - if diff_structure: - self.assertIs(mverr, ValueError) - else: - self.assertEqual(mv.tolist(), lst) - self.assertEqual(mv, nd) - self.assertIs(mverr, lsterr) -", - ); - let f = find_code(&code, "f").expect("missing f code"); - let instructions: Vec<_> = f - .instructions - .iter() - .filter(|unit| !matches!(unit.op, Instruction::Cache)) - .collect(); - let handler_start = instructions - .iter() - .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) - .unwrap_or(instructions.len()); - let normal_path = &instructions[..handler_start]; - let load_fast_name = |unit: &CodeUnit| match unit.op { - Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { - let idx = usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg))))); - Some(f.varnames[idx].as_str()) - } - _ => None, - }; - - assert!( - normal_path.windows(3).any(|window| { - matches!(window[0].op, Instruction::LoadFastBorrow { .. }) - && load_fast_name(window[0]) == Some("diff_structure") - && matches!(window[1].op, Instruction::ToBool) - && matches!(window[2].op, Instruction::PopJumpIfFalse { .. }) - }), - "expected named-except resume guard to keep borrowed diff_structure load, got normal_path={normal_path:?}" - ); - assert!( - normal_path.windows(4).any(|window| { - matches!( - window[0].op, - Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) && matches!(window[1].op, Instruction::BinaryOp { .. }) - && matches!( - window[2].op, - Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - && matches!(window[3].op, Instruction::StoreSubscr) - }), - "expected slice assignment normal path to keep borrowed local loads, got normal_path={normal_path:?}" - ); - assert!( - normal_path.windows(2).any(|window| { - matches!(window[0].op, Instruction::LoadFastBorrow { .. }) - && matches!(window[1].op, Instruction::LoadAttr { .. }) - }), - "expected named-except slice-assign tail method calls to keep borrowed receivers, got normal_path={normal_path:?}" - ); - } - #[test] fn test_with_suppress_named_except_resume_tail_uses_strong_loads() { let code = compile_exec( From c82c43f870e2393e0735aa9f5ca7f42bf1ecd429 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 6 May 2026 20:39:06 +0900 Subject: [PATCH 07/76] Cache CFG layout predecessors during NOP cleanup --- crates/codegen/src/ir.rs | 45 +++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a897c7b247..6f772ca6d4 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -2989,8 +2989,15 @@ impl CodeInfo { /// Remove NOP instructions from all blocks, but keep NOPs that introduce /// a new source line (they serve as line markers for monitoring LINE events). fn remove_nops(&mut self) { + let layout_predecessors = compute_layout_predecessors(&self.blocks); let keep_target_start_nops: Vec<_> = (0..self.blocks.len()) - .map(|idx| keep_target_start_no_location_nop(&self.blocks, BlockIdx(idx as u32))) + .map(|idx| { + keep_target_start_no_location_nop( + &self.blocks, + BlockIdx(idx as u32), + &layout_predecessors, + ) + }) .collect(); for (block_idx, block) in self.blocks.iter_mut().enumerate() { let mut prev_line = None; @@ -11250,6 +11257,7 @@ fn compute_target_predecessor_flags(blocks: &[Block]) -> TargetPredecessorFlags fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { let mut changes = 0; let plain_jump_targets = compute_target_predecessor_flags(blocks).plain_jump; + let layout_predecessors = compute_layout_predecessors(blocks); let mut block_order = Vec::new(); let mut current = BlockIdx(0); while current != BlockIdx::NULL { @@ -11258,8 +11266,10 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { } for block_idx in block_order { let bi = block_idx.idx(); - let keep_target_start_nop = keep_target_start_no_location_nop(blocks, block_idx); - let follows_pop_iter_cleanup = layout_predecessor_ends_with_pop_iter(blocks, block_idx); + let keep_target_start_nop = + keep_target_start_no_location_nop(blocks, block_idx, &layout_predecessors); + let follows_pop_iter_cleanup = + layout_predecessor_ends_with_pop_iter(blocks, block_idx, &layout_predecessors); let mut src_instructions = core::mem::take(&mut blocks[bi].instructions); if !keep_target_start_nop && matches!( @@ -11853,7 +11863,11 @@ fn block_starts_with_with_exit_none_call(block: &Block) -> bool { ) } -fn keep_target_start_no_location_nop(blocks: &[Block], target: BlockIdx) -> bool { +fn keep_target_start_no_location_nop( + blocks: &[Block], + target: BlockIdx, + layout_predecessors: &[BlockIdx], +) -> bool { if target == BlockIdx::NULL { return false; } @@ -11863,7 +11877,7 @@ fn keep_target_start_no_location_nop(blocks: &[Block], target: BlockIdx) -> bool if !matches!(first.instr.real(), Some(Instruction::Nop)) { return false; } - let layout_pred = find_layout_predecessor(blocks, target); + let layout_pred = layout_predecessors[target.idx()]; if layout_pred == BlockIdx::NULL { return false; } @@ -11874,8 +11888,12 @@ fn keep_target_start_no_location_nop(blocks: &[Block], target: BlockIdx) -> bool && !block_starts_with_with_exit_none_call(&blocks[target.idx()]) } -fn layout_predecessor_ends_with_pop_iter(blocks: &[Block], target: BlockIdx) -> bool { - let layout_pred = find_layout_predecessor(blocks, target); +fn layout_predecessor_ends_with_pop_iter( + blocks: &[Block], + target: BlockIdx, + layout_predecessors: &[BlockIdx], +) -> bool { + let layout_pred = layout_predecessors[target.idx()]; layout_pred != BlockIdx::NULL && block_has_fallthrough(&blocks[layout_pred.idx()]) && next_nonempty_block(blocks, blocks[layout_pred.idx()].next) == target @@ -13559,6 +13577,19 @@ fn find_layout_predecessor(blocks: &[Block], target: BlockIdx) -> BlockIdx { BlockIdx::NULL } +fn compute_layout_predecessors(blocks: &[Block]) -> Vec { + let mut predecessors = vec![BlockIdx::NULL; blocks.len()]; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let next = blocks[current.idx()].next; + if next != BlockIdx::NULL { + predecessors[next.idx()] = current; + } + current = next; + } + predecessors +} + fn has_unique_fallthrough_origin( blocks: &[Block], reachable: &[bool], From cd9a71146d85f771f35edd2c50553bed457ea2ae Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 19:14:53 +0900 Subject: [PATCH 08/76] Address bytecode parity review feedback --- crates/codegen/src/compile.rs | 71 +++++++++++++ crates/codegen/src/ir.rs | 182 +++++++++++++++++++++++++++------- 2 files changed, 219 insertions(+), 34 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 6c50f4eb08..adf546f1bc 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -13686,6 +13686,77 @@ def g2(x): ); } + #[test] + fn test_high_index_parameter_stays_initialized_in_fast_scan() { + let params = (0..65) + .map(|idx| format!("p{idx}")) + .collect::>() + .join(", "); + let code = compile_exec(&format!( + "\ +def f({params}): + return p64 +" + )); + let f = find_code(&code, "f").expect("missing f code"); + + assert!( + f.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } + if f.varnames + [usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))))] + == "p64" + )), + "expected high-index parameter p64 to use LOAD_FAST, got ops={:?}", + f.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert!( + !f.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::LoadFastCheck { var_num } + if f.varnames + [usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))))] + == "p64" + )), + "high-index parameter p64 should not use LOAD_FAST_CHECK before deletion" + ); + } + + #[test] + fn test_deleted_high_index_parameter_uses_load_fast_check() { + let params = (0..65) + .map(|idx| format!("p{idx}")) + .collect::>() + .join(", "); + let code = compile_exec(&format!( + "\ +def f({params}): + del p64 + return p64 +" + )); + let f = find_code(&code, "f").expect("missing f code"); + + assert!( + f.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::LoadFastCheck { var_num } + if f.varnames + [usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))))] + == "p64" + )), + "expected deleted high-index parameter p64 to use LOAD_FAST_CHECK, got ops={:?}", + f.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } + #[test] fn test_assert_without_message_raises_class_directly() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 6f772ca6d4..526b748d85 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -9227,10 +9227,22 @@ impl CodeInfo { fn fast_scan_many_locals( &mut self, nlocals: usize, + nparams: usize, merged_cell_local: &impl Fn(usize) -> Option, ) { + const PARAM_INITIALIZED: usize = usize::MAX; + debug_assert!(nlocals > 64); let mut states = vec![0usize; nlocals - 64]; + let high_params = nparams.saturating_sub(64).min(states.len()); + for state in states.iter_mut().take(high_params) { + *state = PARAM_INITIALIZED; + } + + let is_known = |idx: usize, state: usize, blocknum: usize| { + state == blocknum || (idx < nparams && state == PARAM_INITIALIZED) + }; + let mut blocknum = 0usize; let mut current = BlockIdx(0); while current != BlockIdx::NULL { @@ -9316,7 +9328,7 @@ impl CodeInfo { states[store_idx - 64] = blocknum; } if load_idx >= 64 && load_idx < nlocals { - if states[load_idx - 64] != blocknum { + if !is_known(load_idx, states[load_idx - 64], blocknum) { let mut first = info; first.instr = Instruction::StoreFast { var_num: Arg::marker(), @@ -9334,19 +9346,17 @@ impl CodeInfo { } else { new_instructions.push(info); } - states[load_idx - 64] = blocknum; } else { new_instructions.push(info); } } Some(Instruction::LoadFast { var_num }) => { let idx = usize::from(var_num.get(info.arg)); - if idx >= 64 && idx < nlocals { - if states[idx - 64] != blocknum { - info.instr = Opcode::LoadFastCheck.into(); - changed = true; - } + if idx >= 64 && idx < nlocals && !is_known(idx, states[idx - 64], blocknum) + { + info.instr = Opcode::LoadFastCheck.into(); states[idx - 64] = blocknum; + changed = true; } new_instructions.push(info); } @@ -9355,16 +9365,15 @@ impl CodeInfo { let (idx1, idx2) = packed.indexes(); let idx1 = usize::from(idx1); let idx2 = usize::from(idx2); - let needs_check_1 = - idx1 >= 64 && idx1 < nlocals && states[idx1 - 64] != blocknum; - if idx1 >= 64 && idx1 < nlocals { + let needs_check_1 = idx1 >= 64 + && idx1 < nlocals + && !is_known(idx1, states[idx1 - 64], blocknum); + if needs_check_1 { states[idx1 - 64] = blocknum; } - let needs_check_2 = - idx2 >= 64 && idx2 < nlocals && states[idx2 - 64] != blocknum; - if idx2 >= 64 && idx2 < nlocals { - states[idx2 - 64] = blocknum; - } + let needs_check_2 = idx2 >= 64 + && idx2 < nlocals + && !is_known(idx2, states[idx2 - 64], blocknum); if needs_check_1 || needs_check_2 { let mut first = info; @@ -9387,6 +9396,9 @@ impl CodeInfo { new_instructions.push(first); new_instructions.push(second); changed = true; + if needs_check_2 { + states[idx2 - 64] = blocknum; + } } else { new_instructions.push(info); } @@ -9434,7 +9446,7 @@ impl CodeInfo { nparams = nparams.min(nlocals); if nlocals > 64 { - self.fast_scan_many_locals(nlocals, &merged_cell_local); + self.fast_scan_many_locals(nlocals, nparams, &merged_cell_local); nlocals = 64; } @@ -10617,20 +10629,39 @@ fn jump_threading_unconditional(blocks: &mut [Block]) { jump_threading_impl(blocks, false); } -fn opposite_short_circuit_target(block: &Block, source: AnyInstruction) -> bool { - let Some(cond_idx) = trailing_conditional_jump_index(block) else { - return false; - }; +fn short_circuit_stub_conditional(block: &Block) -> Option { + let cond_idx = trailing_conditional_jump_index(block)?; + if cond_idx < 2 { + return None; + } let [first, second, ..] = block.instructions.as_slice() else { - return false; + return None; }; if !matches!(first.instr.real(), Some(Instruction::Copy { i }) if i.get(first.arg) == 1) || !matches!(second.instr.real(), Some(Instruction::ToBool)) { - return false; + return None; } + + let only_markers_between = block.instructions[2..cond_idx].iter().all(|info| { + matches!( + info.instr.real(), + None | Some(Instruction::Nop | Instruction::NotTaken) + ) + }); + if !only_markers_between { + return None; + } + + block.instructions[cond_idx].instr.real() +} + +fn opposite_short_circuit_target(block: &Block, source: AnyInstruction) -> bool { + let Some(conditional) = short_circuit_stub_conditional(block) else { + return false; + }; matches!( - (source.real(), block.instructions[cond_idx].instr.real()), + (source.real(), Some(conditional)), ( Some(Instruction::PopJumpIfFalse { .. }), Some(Instruction::PopJumpIfTrue { .. }) @@ -10642,17 +10673,9 @@ fn opposite_short_circuit_target(block: &Block, source: AnyInstruction) -> bool } fn same_short_circuit_target(block: &Block, source: AnyInstruction) -> Option { - let cond_idx = trailing_conditional_jump_index(block)?; - let [first, second, ..] = block.instructions.as_slice() else { - return None; - }; - if !matches!(first.instr.real(), Some(Instruction::Copy { i }) if i.get(first.arg) == 1) - || !matches!(second.instr.real(), Some(Instruction::ToBool)) - { - return None; - } + let conditional = short_circuit_stub_conditional(block)?; matches!( - (source.real(), block.instructions[cond_idx].instr.real()), + (source.real(), Some(conditional)), ( Some(Instruction::PopJumpIfFalse { .. }), Some(Instruction::PopJumpIfFalse { .. }) @@ -10661,7 +10684,7 @@ fn same_short_circuit_target(block: &Block, source: AnyInstruction) -> Option InstructionInfo { + InstructionInfo { + instr: instr.into(), + arg: OpArg::new(arg), + target, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + folded_from_nonliteral_expr: false, + lineno_override: None, + cache_entries: 0, + preserve_redundant_jump_as_nop: false, + remove_no_location_nop: false, + preserve_block_start_no_location_nop: false, + } + } + + #[test] + fn short_circuit_stub_allows_only_marker_instructions_before_jump() { + let final_target = BlockIdx(7); + let block = Block { + instructions: vec![ + instruction_info(Instruction::Copy { i: Arg::marker() }, 1, BlockIdx::NULL), + instruction_info(Instruction::ToBool, 0, BlockIdx::NULL), + instruction_info(Instruction::Nop, 0, BlockIdx::NULL), + instruction_info(Instruction::NotTaken, 0, BlockIdx::NULL), + instruction_info( + Instruction::PopJumpIfFalse { + delta: Arg::marker(), + }, + 0, + final_target, + ), + ], + ..Block::default() + }; + + assert_eq!( + same_short_circuit_target( + &block, + Instruction::PopJumpIfFalse { + delta: Arg::marker(), + } + .into(), + ), + Some(final_target) + ); + } + + #[test] + fn short_circuit_stub_rejects_real_instruction_before_jump() { + let block = Block { + instructions: vec![ + instruction_info(Instruction::Copy { i: Arg::marker() }, 1, BlockIdx::NULL), + instruction_info(Instruction::ToBool, 0, BlockIdx::NULL), + instruction_info(Instruction::PopTop, 0, BlockIdx::NULL), + instruction_info( + Instruction::PopJumpIfFalse { + delta: Arg::marker(), + }, + 0, + BlockIdx(7), + ), + ], + ..Block::default() + }; + + assert_eq!( + same_short_circuit_target( + &block, + Instruction::PopJumpIfFalse { + delta: Arg::marker(), + } + .into(), + ), + None + ); + assert!(!opposite_short_circuit_target( + &block, + Instruction::PopJumpIfTrue { + delta: Arg::marker(), + } + .into() + )); + } +} From 15a88dfb09a058d8a6d123e538164ce4d92cae4f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 19:47:10 +0900 Subject: [PATCH 09/76] Align nested with cleanup bytecode --- crates/codegen/src/compile.rs | 136 +++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 4 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index adf546f1bc..46689da908 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -28,9 +28,9 @@ use rustpython_compiler_core::{ Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, bytecode::{ self, AnyInstruction, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, CodeObject, - ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, IntrinsicFunction1, - Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, SpecialMethod, - UnpackExArgs, oparg, + ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, InstructionMetadata, + IntrinsicFunction1, Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, + SpecialMethod, UnpackExArgs, oparg, }, }; use rustpython_wtf8::Wtf8Buf; @@ -616,6 +616,16 @@ impl Compiler { } } + fn statements_end_with_with_cleanup_scope_exit(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| match stmt { + ast::Stmt::With(ast::StmtWith { body, .. }) => { + Self::statements_end_with_scope_exit(body) + || Self::statements_end_with_with_cleanup_scope_exit(body) + } + _ => false, + }) + } + fn preserves_finally_entry_nop(body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| match stmt { ast::Stmt::Try(ast::StmtTry { @@ -5827,6 +5837,10 @@ impl Compiler { self.compile_with(items, body, is_async)?; } + let preserve_outer_cleanup_target_nop = !is_async + && (Self::statements_end_with_with_cleanup_scope_exit(body) + || self.current_block_has_terminal_with_suppress_exit_predecessor()); + // Pop fblock before normal exit. CPython emits this POP_BLOCK with // no location for sync with, but with the with-item location for // async with. @@ -5836,7 +5850,11 @@ impl Compiler { emit!(self, PseudoInstruction::PopBlock); if !is_async { self.set_no_location(); - self.remove_last_no_location_nop(); + if preserve_outer_cleanup_target_nop { + self.preserve_last_redundant_nop(); + } else { + self.remove_last_no_location_nop(); + } self.set_source_range(with_range); } self.pop_fblock(if is_async { @@ -10006,6 +10024,44 @@ impl Compiler { } } + fn current_block_has_terminal_with_suppress_exit_predecessor(&self) -> bool { + let code = self.code_stack.last().expect("no code on stack"); + let target = code.current_block; + let mut has_suppress_exit = false; + let mut has_normal_exit = false; + + for block in &code.blocks { + let Some((last, prefix)) = block.instructions.split_last() else { + continue; + }; + if last.target != target { + continue; + } + match last.instr.pseudo() { + Some(PseudoInstruction::JumpNoInterrupt { .. }) => { + let real_instrs: Vec<_> = + prefix.iter().filter_map(|info| info.instr.real()).collect(); + has_suppress_exit |= matches!( + real_instrs.as_slice(), + [ + Instruction::PopTop, + Instruction::PopExcept, + Instruction::PopTop, + Instruction::PopTop, + Instruction::PopTop, + ] + ); + } + Some(PseudoInstruction::Jump { .. }) => { + has_normal_exit |= !prefix.iter().any(|info| info.instr.is_scope_exit()); + } + _ => {} + } + } + + has_suppress_exit && !has_normal_exit + } + fn remove_last_no_location_nop(&mut self) { if let Some(info) = self.current_block().instructions.last_mut() { info.remove_no_location_nop = true; @@ -20084,6 +20140,78 @@ async def foo(): ); } + #[test] + fn test_nested_terminal_with_keeps_outer_cleanup_target_nop() { + let code = compile_exec( + "\ +def f(): + with a(): + with b(): + raise E() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::Copy { .. }, + Instruction::PopExcept, + Instruction::Reraise { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "expected CPython-style outer with-exit target NOP after terminal nested with cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_nonterminal_with_drops_outer_cleanup_target_nop() { + let code = compile_exec( + "\ +def f(): + with a(): + with b(): + x() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::Copy { .. }, + Instruction::PopExcept, + Instruction::Reraise { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "unexpected outer with-exit target NOP for nested with with normal fallthrough, got ops={ops:?}" + ); + } + #[test] fn test_try_loop_elif_places_return_before_orelse_tail() { let code = compile_exec( From 4bd27bea96b20c590bd291e07b79a8bfd0c08b8a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 20:05:22 +0900 Subject: [PATCH 10/76] Align conditional raise loop backedge ordering --- crates/codegen/src/compile.rs | 49 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 16 +++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 46689da908..caec050f32 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -21775,6 +21775,55 @@ def f(items, limit): ); } + #[test] + fn test_simple_for_conditional_raise_orders_backedge_before_raise() { + let code = compile_exec( + "\ +def f(kw): + for k in ('stdout', 'check'): + if k in kw: + raise ValueError(f'{k} argument not allowed, it will be overridden.') +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected CPython-style true jump to raise path after loop backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "unexpected conditional raise body before loop backedge, got ops={ops:?}" + ); + } + #[test] fn test_exception_handler_loop_conditional_raise_orders_backedge_before_raise() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 526b748d85..f2c72035d3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -12543,6 +12543,20 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( && jump_targets_for_iter(blocks, jump_block) } + fn is_explicit_continue_after_conditional( + blocks: &[Block], + jump_block: BlockIdx, + cond: InstructionInfo, + ) -> bool { + if !is_explicit_continue_to_for_iter(blocks, jump_block) { + return false; + } + let Some(info) = blocks[jump_block.idx()].instructions.first() else { + return false; + }; + instruction_lineno(info) > instruction_lineno(&cond) + } + fn is_explicit_non_for_jump_back(blocks: &[Block], jump_block: BlockIdx) -> bool { if jump_block == BlockIdx::NULL || jump_targets_for_iter(blocks, jump_block) { return false; @@ -12629,7 +12643,7 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( || !is_scope_exit_block(&blocks[exit_block.idx()]) || !is_jump_only_block(&blocks[jump_block.idx()]) || (jump_targets_for_iter(blocks, jump_block) - && !is_explicit_continue_to_for_iter(blocks, jump_block)) + && !is_explicit_continue_after_conditional(blocks, jump_block, cond)) || next_nonempty_block(blocks, blocks[jump_block.idx()].next) != exit_block || !comes_before( blocks, From 9b257b16bd89d21134b56755da16fdc5f74b3f83 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 20:28:16 +0900 Subject: [PATCH 11/76] Align percent format optimization with CPython preprocess --- crates/codegen/src/compile.rs | 266 +++++++++++++------------------ crates/codegen/src/lib.rs | 1 + crates/codegen/src/preprocess.rs | 225 ++++++++++++++++++++++++++ 3 files changed, 341 insertions(+), 151 deletions(-) create mode 100644 crates/codegen/src/preprocess.rs diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index caec050f32..5e37f27214 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -13,6 +13,7 @@ use crate::{ IndexMap, IndexSet, ToPythonName, error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason}, ir::{self, BlockIdx}, + preprocess, symboltable::{self, CompilerScope, Symbol, SymbolFlags, SymbolScope, SymbolTable}, unparse::UnparseExpr, }; @@ -222,12 +223,6 @@ impl CompileContext { } } -/// Segment of a parsed %-format string for optimize_format_str. -struct FormatSegment { - literal: String, - conversion: Option, -} - #[derive(Debug, Clone, Copy, PartialEq)] enum ComprehensionType { Generator, @@ -266,11 +261,12 @@ fn validate_duplicate_params(params: &ast::Parameters) -> Result<(), CodegenErro /// Compile an Mod produced from ruff parser pub fn compile_top( - ast: ruff_python_ast::Mod, + mut ast: ruff_python_ast::Mod, source_file: SourceFile, mode: Mode, opts: CompileOpts, ) -> CompileResult { + preprocess::preprocess_mod(&mut ast); match ast { ruff_python_ast::Mod::Module(module) => match mode { Mode::Exec | Mode::Eval => compile_program(&module, source_file, opts), @@ -8192,15 +8188,6 @@ impl Compiler { ast::Expr::BinOp(ast::ExprBinOp { left, op, right, .. }) => { - // optimize_format_str: 'format' % (args,) → f-string bytecode - if matches!(op, ast::Operator::Mod) - && let ast::Expr::StringLiteral(s) = left.as_ref() - && let ast::Expr::Tuple(ast::ExprTuple { elts, .. }) = right.as_ref() - && !elts.iter().any(|e| matches!(e, ast::Expr::Starred(_))) - && self.try_optimize_format_str(s.value.to_str(), elts, range)? - { - return Ok(()); - } self.compile_expression(left)?; self.compile_expression(right)?; @@ -11012,6 +10999,7 @@ impl Compiler { } fn compile_expr_fstring(&mut self, fstring: &ast::ExprFString) -> CompileResult<()> { + let fstring_range = fstring.range; let fstring = fstring.value.as_slice(); if self.count_fstring_parts(fstring) > STACK_USE_GUIDELINE { return self.compile_fstring_parts_joined(fstring); @@ -11019,10 +11007,18 @@ impl Compiler { let mut element_count = 0; let mut pending_literal = None; + let mut pending_literal_no_location = false; for part in fstring { - self.compile_fstring_part_into(part, &mut pending_literal, &mut element_count, false)?; + self.compile_fstring_part_into( + part, + &mut pending_literal, + &mut pending_literal_no_location, + &mut element_count, + false, + )?; } - self.finish_fstring(pending_literal, element_count) + self.set_source_range(fstring_range); + self.finish_fstring(pending_literal, pending_literal_no_location, element_count) } fn compile_fstring_parts_joined(&mut self, fstring: &[ast::FStringPart]) -> CompileResult<()> { @@ -11035,10 +11031,17 @@ impl Compiler { let mut element_count = 0; let mut pending_literal = None; + let mut pending_literal_no_location = false; for part in fstring { - self.compile_fstring_part_into(part, &mut pending_literal, &mut element_count, true)?; + self.compile_fstring_part_into( + part, + &mut pending_literal, + &mut pending_literal_no_location, + &mut element_count, + true, + )?; } - self.finish_fstring_join(pending_literal, element_count); + self.finish_fstring_join(pending_literal, pending_literal_no_location, element_count); Ok(()) } @@ -11046,16 +11049,20 @@ impl Compiler { &mut self, part: &ast::FStringPart, pending_literal: &mut Option, + pending_literal_no_location: &mut bool, element_count: &mut u32, append_to_join_list: bool, ) -> CompileResult<()> { match part { ast::FStringPart::Literal(string) => { let value = self.compile_fstring_part_literal_value(string); - if let Some(pending) = pending_literal.as_mut() { - pending.push_wtf8(value.as_ref()); - } else { + if pending_literal.is_none() { + self.set_source_range(string.range); + *pending_literal_no_location = string.range == TextRange::default(); *pending_literal = Some(value); + } else if let Some(pending) = pending_literal.as_mut() { + *pending_literal_no_location &= string.range == TextRange::default(); + pending.push_wtf8(value.as_ref()); } Ok(()) } @@ -11063,6 +11070,7 @@ impl Compiler { fstring.flags, &fstring.elements, pending_literal, + pending_literal_no_location, element_count, append_to_join_list, ), @@ -11072,11 +11080,13 @@ impl Compiler { fn finish_fstring( &mut self, mut pending_literal: Option, + mut pending_literal_no_location: bool, mut element_count: u32, ) -> CompileResult<()> { let keep_empty = element_count == 0; self.emit_pending_fstring_literal( &mut pending_literal, + &mut pending_literal_no_location, &mut element_count, keep_empty, false, @@ -11101,11 +11111,13 @@ impl Compiler { fn finish_fstring_join( &mut self, mut pending_literal: Option, + mut pending_literal_no_location: bool, mut element_count: u32, ) { let keep_empty = element_count == 0; self.emit_pending_fstring_literal( &mut pending_literal, + &mut pending_literal_no_location, &mut element_count, keep_empty, true, @@ -11116,6 +11128,7 @@ impl Compiler { fn emit_pending_fstring_literal( &mut self, pending_literal: &mut Option, + pending_literal_no_location: &mut bool, element_count: &mut u32, keep_empty: bool, append_to_join_list: bool, @@ -11123,6 +11136,8 @@ impl Compiler { let Some(value) = pending_literal.take() else { return; }; + let no_location = *pending_literal_no_location; + *pending_literal_no_location = false; // CPython drops empty literal fragments when they are adjacent to // formatted values, but still emits an empty string for a fully-empty @@ -11132,6 +11147,9 @@ impl Compiler { } self.emit_load_const(ConstantData::Str { value }); + if no_location { + self.set_no_location(); + } *element_count += 1; if append_to_join_list { emit!(self, Instruction::ListAppend { i: 1 }); @@ -11189,125 +11207,6 @@ impl Compiler { *element_count += 1; } - /// Optimize `'format_str' % (args,)` into f-string bytecode. - /// Returns true if optimization was applied, false to fall back to normal BINARY_OP %. - /// Matches CPython's codegen.c `compiler_formatted_value` optimization. - fn try_optimize_format_str( - &mut self, - format_str: &str, - args: &[ast::Expr], - range: ruff_text_size::TextRange, - ) -> CompileResult { - // Parse format string into segments - let Some(segments) = Self::parse_percent_format(format_str) else { - return Ok(false); - }; - - // Verify arg count matches specifier count - let spec_count = segments.iter().filter(|s| s.conversion.is_some()).count(); - if spec_count != args.len() { - return Ok(false); - } - - self.set_source_range(range); - - // Special case: no specifiers, just %% escaping → constant fold - if spec_count == 0 { - let folded: String = segments.iter().map(|s| s.literal.as_str()).collect(); - self.emit_load_const(ConstantData::Str { - value: folded.into(), - }); - return Ok(true); - } - - // Emit f-string style bytecode - let mut part_count: u32 = 0; - let mut arg_idx = 0; - - for seg in &segments { - if !seg.literal.is_empty() { - self.emit_load_const(ConstantData::Str { - value: seg.literal.clone().into(), - }); - part_count += 1; - } - if let Some(conv) = seg.conversion { - self.compile_expression(&args[arg_idx])?; - self.set_source_range(range); - emit!(self, Instruction::ConvertValue { oparg: conv }); - emit!(self, Instruction::FormatSimple); - part_count += 1; - arg_idx += 1; - } - } - - if part_count == 0 { - self.emit_load_const(ConstantData::Str { - value: String::new().into(), - }); - } else if part_count > 1 { - emit!(self, Instruction::BuildString { count: part_count }); - } - - Ok(true) - } - - /// Parse a %-format string into segments of (literal_prefix, optional conversion). - /// Returns None if the format string contains unsupported specifiers. - fn parse_percent_format(format_str: &str) -> Option> { - let mut segments = Vec::new(); - let mut chars = format_str.chars().peekable(); - let mut current_literal = String::new(); - - while let Some(ch) = chars.next() { - if ch == '%' { - match chars.peek() { - Some('%') => { - chars.next(); - current_literal.push('%'); - } - Some('s') => { - chars.next(); - segments.push(FormatSegment { - literal: core::mem::take(&mut current_literal), - conversion: Some(oparg::ConvertValueOparg::Str), - }); - } - Some('r') => { - chars.next(); - segments.push(FormatSegment { - literal: core::mem::take(&mut current_literal), - conversion: Some(oparg::ConvertValueOparg::Repr), - }); - } - Some('a') => { - chars.next(); - segments.push(FormatSegment { - literal: core::mem::take(&mut current_literal), - conversion: Some(oparg::ConvertValueOparg::Ascii), - }); - } - _ => { - // Unsupported: %d, %f, %(name)s, %10s, etc. - return None; - } - } - } else { - current_literal.push(ch); - } - } - - // Trailing literal - if !current_literal.is_empty() { - segments.push(FormatSegment { - literal: current_literal, - conversion: None, - }); - } - - Some(segments) - } - fn compile_fstring_elements( &mut self, flags: ast::FStringFlags, @@ -11319,14 +11218,16 @@ impl Compiler { let mut element_count = 0; let mut pending_literal: Option = None; + let mut pending_literal_no_location = false; self.compile_fstring_elements_into( flags, fstring_elements, &mut pending_literal, + &mut pending_literal_no_location, &mut element_count, false, )?; - self.finish_fstring(pending_literal, element_count) + self.finish_fstring(pending_literal, pending_literal_no_location, element_count) } fn compile_fstring_elements_joined( @@ -11343,14 +11244,16 @@ impl Compiler { let mut element_count = 0; let mut pending_literal: Option = None; + let mut pending_literal_no_location = false; self.compile_fstring_elements_into( flags, fstring_elements, &mut pending_literal, + &mut pending_literal_no_location, &mut element_count, true, )?; - self.finish_fstring_join(pending_literal, element_count); + self.finish_fstring_join(pending_literal, pending_literal_no_location, element_count); Ok(()) } @@ -11359,6 +11262,7 @@ impl Compiler { flags: ast::FStringFlags, fstring_elements: &ast::InterpolatedStringElements, pending_literal: &mut Option, + pending_literal_no_location: &mut bool, element_count: &mut u32, append_to_join_list: bool, ) -> CompileResult<()> { @@ -11366,10 +11270,13 @@ impl Compiler { match element { ast::InterpolatedStringElement::Literal(string) => { let value = self.compile_fstring_literal_value(string, flags); - if let Some(pending) = pending_literal.as_mut() { - pending.push_wtf8(value.as_ref()); - } else { + if pending_literal.is_none() { + self.set_source_range(string.range); + *pending_literal_no_location = string.range == TextRange::default(); *pending_literal = Some(value); + } else if let Some(pending) = pending_literal.as_mut() { + *pending_literal_no_location &= string.range == TextRange::default(); + pending.push_wtf8(value.as_ref()); } } ast::InterpolatedStringElement::Interpolation(fstring_expr) => { @@ -11391,9 +11298,11 @@ impl Compiler { .concat(); let text: Wtf8Buf = text.into(); - pending_literal - .get_or_insert_with(Wtf8Buf::new) - .push_wtf8(text.as_ref()); + if pending_literal.is_none() { + *pending_literal_no_location = false; + *pending_literal = Some(Wtf8Buf::new()); + } + pending_literal.as_mut().unwrap().push_wtf8(text.as_ref()); // If debug text is present, apply repr conversion when no `format_spec` specified. // See action_helpers.c: fstring_find_expr_replacement @@ -11407,6 +11316,7 @@ impl Compiler { self.emit_pending_fstring_literal( pending_literal, + pending_literal_no_location, element_count, false, append_to_join_list, @@ -12003,7 +11913,8 @@ mod tests { ruff_python_parser::Mode::Module.into(), ) .unwrap(); - let ast = parsed.into_syntax(); + let mut ast = parsed.into_syntax(); + preprocess::preprocess_mod(&mut ast); let ast = match ast { ruff_python_ast::Mod::Module(stmts) => stmts, _ => unreachable!(), @@ -13310,6 +13221,59 @@ def f(msg): ); } + #[test] + fn test_try_percent_format_preprocess_removes_redundant_try_nop() { + let code = compile_exec( + "\ +def f(self, signal): + if self.returncode and self.returncode < 0: + try: + return \"Command '%s' died with %r.\" % ( + self.cmd, signal.Signals(-self.returncode)) + except ValueError: + return \"Command '%s' died with unknown signal %d.\" % ( + self.cmd, -self.returncode) + return \"Command '%s' returned non-zero exit status %d.\" % ( + self.cmd, self.returncode) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::NotTaken, + Instruction::LoadConst { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "expected preprocessed percent-format body immediately after condition, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::NotTaken, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "percent-format preprocessing should let CFG remove the try-line NOP, got ops={ops:?}" + ); + } + #[test] fn test_try_else_return_keeps_nop_before_final_call_return() { let code = compile_exec( diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs index c1a318c19c..8d6ad98435 100644 --- a/crates/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -14,6 +14,7 @@ type IndexSet = indexmap::IndexSet; pub mod compile; pub mod error; pub mod ir; +mod preprocess; mod string_parser; pub mod symboltable; mod unparse; diff --git a/crates/codegen/src/preprocess.rs b/crates/codegen/src/preprocess.rs new file mode 100644 index 0000000000..63246ff840 --- /dev/null +++ b/crates/codegen/src/preprocess.rs @@ -0,0 +1,225 @@ +use alloc::{boxed::Box, string::String, vec::Vec}; + +use ruff_python_ast::{ + self as ast, AtomicNodeIndex, ConversionFlag, Expr, ExprFString, FString, FStringFlags, + FStringValue, HasNodeIndex, InterpolatedElement, InterpolatedStringElement, + InterpolatedStringElements, InterpolatedStringFormatSpec, InterpolatedStringLiteralElement, + Operator, + visitor::transformer::{self, Transformer}, +}; +use ruff_text_size::{Ranged, TextRange}; + +const MAXDIGITS: usize = 3; +const F_LJUST: u8 = 1; + +pub(crate) fn preprocess_mod(module: &mut ast::Mod) { + let preprocessor = AstPreprocessor; + match module { + ast::Mod::Module(module) => preprocessor.visit_body(&mut module.body), + ast::Mod::Expression(expr) => preprocessor.visit_expr(&mut expr.body), + } +} + +struct AstPreprocessor; + +impl Transformer for AstPreprocessor { + fn visit_expr(&self, expr: &mut Expr) { + transformer::walk_expr(self, expr); + if let Some(optimized) = optimize_format(expr) { + *expr = optimized; + } + } +} + +fn optimize_format(expr: &Expr) -> Option { + let Expr::BinOp(binop) = expr else { + return None; + }; + if !matches!(binop.op, Operator::Mod) { + return None; + } + let Expr::StringLiteral(format) = binop.left.as_ref() else { + return None; + }; + let Expr::Tuple(tuple) = binop.right.as_ref() else { + return None; + }; + if tuple + .elts + .iter() + .any(|expr| matches!(expr, Expr::Starred(_))) + { + return None; + } + + let elements = parse_format(format.value.to_str(), &tuple.elts)?; + Some(Expr::FString(ExprFString { + node_index: binop.node_index.clone(), + range: binop.range, + value: FStringValue::single(FString { + range: binop.range, + node_index: binop.node_index.clone(), + elements: InterpolatedStringElements::from(elements), + flags: FStringFlags::empty(), + }), + })) +} + +fn parse_format(format: &str, args: &[Expr]) -> Option> { + let chars: Vec = format.chars().collect(); + let mut elements = Vec::with_capacity(args.len().saturating_mul(2).saturating_add(1)); + let mut pos = 0; + let mut arg_idx = 0; + + loop { + if let Some(literal) = parse_literal(&chars, &mut pos) { + elements.push(literal.into()); + } + if pos >= chars.len() { + break; + } + if arg_idx >= args.len() { + return None; + } + debug_assert_eq!(chars[pos], '%'); + pos += 1; + let formatted = parse_format_arg(&chars, &mut pos, args[arg_idx].clone())?; + elements.push(formatted.into()); + arg_idx += 1; + } + + (arg_idx == args.len()).then_some(elements) +} + +fn parse_literal(chars: &[char], pos: &mut usize) -> Option { + let start = *pos; + let mut has_percents = false; + while *pos < chars.len() { + if chars[*pos] != '%' { + *pos += 1; + } else if *pos + 1 < chars.len() && chars[*pos + 1] == '%' { + has_percents = true; + *pos += 2; + } else { + break; + } + } + if *pos == start { + return None; + } + + let mut value = String::new(); + let mut i = start; + while i < *pos { + if has_percents && chars[i] == '%' && i + 1 < *pos && chars[i + 1] == '%' { + value.push('%'); + i += 2; + } else { + value.push(chars[i]); + i += 1; + } + } + + Some(generated_literal(value)) +} + +fn parse_format_arg(chars: &[char], pos: &mut usize, arg: Expr) -> Option { + let (spec, flags, width, precision) = simple_format_arg_parse(chars, pos)?; + let conversion = match spec { + 's' => ConversionFlag::Str, + 'r' => ConversionFlag::Repr, + 'a' => ConversionFlag::Ascii, + _ => return None, + }; + + let mut format_spec = String::new(); + if flags & F_LJUST == 0 + && let Some(width) = width + && width > 0 + { + format_spec.push('>'); + } + if let Some(width) = width { + format_spec.push_str(&width.to_string()); + } + if let Some(precision) = precision { + format_spec.push('.'); + format_spec.push_str(&precision.to_string()); + } + + let range = arg.range(); + let format_spec = (!format_spec.is_empty()).then(|| { + Box::new(InterpolatedStringFormatSpec { + range: TextRange::default(), + node_index: AtomicNodeIndex::NONE, + elements: InterpolatedStringElements::from(vec![generated_literal(format_spec).into()]), + }) + }); + + Some(InterpolatedElement { + range, + node_index: arg.node_index().clone(), + expression: Box::new(arg), + debug_text: None, + conversion, + format_spec, + }) +} + +fn simple_format_arg_parse( + chars: &[char], + pos: &mut usize, +) -> Option<(char, u8, Option, Option)> { + let mut flags = 0; + let mut ch = next_char(chars, pos)?; + loop { + match ch { + '-' => flags |= F_LJUST, + '+' | ' ' | '#' | '0' => {} + _ => break, + } + ch = next_char(chars, pos)?; + } + + let width = parse_digits(chars, pos, &mut ch)?; + let precision = if ch == '.' { + ch = next_char(chars, pos)?; + parse_digits(chars, pos, &mut ch)? + } else { + None + }; + + Some((ch, flags, width, precision)) +} + +fn parse_digits(chars: &[char], pos: &mut usize, ch: &mut char) -> Option> { + if !ch.is_ascii_digit() { + return Some(None); + } + + let mut value = 0u16; + let mut digits = 0usize; + while ch.is_ascii_digit() { + value = value * 10 + (*ch as u16 - b'0' as u16); + *ch = next_char(chars, pos)?; + digits += 1; + if digits >= MAXDIGITS { + return None; + } + } + Some(Some(value)) +} + +fn next_char(chars: &[char], pos: &mut usize) -> Option { + let ch = chars.get(*pos).copied()?; + *pos += 1; + Some(ch) +} + +fn generated_literal(value: String) -> InterpolatedStringLiteralElement { + InterpolatedStringLiteralElement { + range: TextRange::default(), + node_index: AtomicNodeIndex::NONE, + value: value.into_boxed_str(), + } +} From 8fbe367eb5ed99431e7312483b6bbbde831a1ce7 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 20:48:03 +0900 Subject: [PATCH 12/76] Preserve shared finally reraises in CFG cleanup --- crates/codegen/src/compile.rs | 58 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 45 +++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 5e37f27214..935b6e55e5 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -13274,6 +13274,64 @@ def f(self, signal): ); } + #[test] + fn test_nested_try_except_in_finally_exception_path_shares_continuation() { + let code = compile_exec( + "\ +def f(self, exc_type, KeyboardInterrupt, TimeoutExpired): + try: + if self.stdin: + self.stdin.close() + finally: + if exc_type == KeyboardInterrupt: + if self._sigint_wait_secs > 0: + try: + self._wait(timeout=self._sigint_wait_secs) + except TimeoutExpired: + pass + self._sigint_wait_secs = 0 + else: + self.wait() +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let store_reraise_tails = ops + .windows(2) + .filter(|window| { + matches!( + window, + [Instruction::StoreAttr { .. }, Instruction::Reraise { .. },] + ) + }) + .count(); + + assert_eq!( + store_reraise_tails, 1, + "nested try/except inside an exceptional finally body should share the remaining finalbody tail before RERAISE, got ops={ops:?}" + ); + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadSmallInt { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::StoreAttr { .. }, + Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }), + "normal finally body should keep CPython-style borrowed load before STORE_ATTR, got ops={ops:?}" + ); + } + #[test] fn test_try_else_return_keeps_nop_before_final_call_return() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f2c72035d3..a0d2cf747e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5822,6 +5822,9 @@ impl CodeInfo { while cursor != BlockIdx::NULL && !visited[cursor.idx()] { visited[cursor.idx()] = true; let block = &blocks[cursor.idx()]; + if cursor != handler && block.except_handler { + return false; + } for info in &block.instructions { match info.instr.real() { Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) => { @@ -11500,6 +11503,39 @@ fn redirect_empty_block_targets(blocks: &mut [Block]) { } fn redirect_empty_unconditional_jump_targets(blocks: &mut [Block]) { + let block_exits_to_reraise = |block_idx: BlockIdx| { + let block = &blocks[block_idx.idx()]; + let Some(last) = block.instructions.last() else { + return false; + }; + if matches!(last.instr.real(), Some(Instruction::Reraise { .. })) { + return true; + } + if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { + return false; + } + let target = next_nonempty_block(blocks, last.target); + target != BlockIdx::NULL + && blocks[target.idx()] + .instructions + .last() + .is_some_and(|instr| { + matches!(instr.instr.real(), Some(Instruction::Reraise { .. })) + }) + }; + + let mut raw_predecessors = vec![0u32; blocks.len()]; + for block in blocks.iter() { + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + raw_predecessors[block.next.idx()] += 1; + } + for instr in &block.instructions { + if instr.target != BlockIdx::NULL { + raw_predecessors[instr.target.idx()] += 1; + } + } + } + let redirected_targets: Vec> = blocks .iter() .map(|block| { @@ -11510,6 +11546,15 @@ fn redirect_empty_unconditional_jump_targets(blocks: &mut [Block]) { if instr.target == BlockIdx::NULL || !instr.instr.is_unconditional_jump() { instr.target } else { + if blocks[instr.target.idx()].instructions.is_empty() + && raw_predecessors[instr.target.idx()] > 1 + && { + let target = next_nonempty_block(blocks, instr.target); + target != BlockIdx::NULL && block_exits_to_reraise(target) + } + { + return instr.target; + } let target = next_nonempty_block(blocks, instr.target); if matches!( jump_thread_kind(instr.instr), From 3ccda96213cf296744def175227865f96ece0ea4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 21:20:14 +0900 Subject: [PATCH 13/76] Align CFG cleanup with CPython finally layout --- crates/codegen/src/compile.rs | 159 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 48 ++++------ 2 files changed, 175 insertions(+), 32 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 935b6e55e5..07d3bfc7e0 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -591,6 +591,24 @@ impl Compiler { .is_some_and(Self::statement_ends_with_scope_exit) } + fn statements_end_with_finally_entry_scope_exit(body: &[ast::Stmt]) -> bool { + body.last() + .is_some_and(Self::statement_ends_with_finally_entry_scope_exit) + } + + fn statement_ends_with_finally_entry_scope_exit(stmt: &ast::Stmt) -> bool { + match stmt { + ast::Stmt::Return(_) + | ast::Stmt::Raise(_) + | ast::Stmt::Break(_) + | ast::Stmt::Continue(_) => true, + ast::Stmt::If(ast::StmtIf { body, .. }) => { + Self::statements_end_with_finally_entry_scope_exit(body) + } + _ => false, + } + } + fn statement_ends_with_scope_exit(stmt: &ast::Stmt) -> bool { match stmt { ast::Stmt::Return(_) | ast::Stmt::Raise(_) => true, @@ -633,6 +651,14 @@ impl Compiler { !finalbody.is_empty() || (!handlers.is_empty() && Self::statements_end_with_scope_exit(body)) } + ast::Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) => { + elif_else_clauses.is_empty() + && Self::statements_end_with_finally_entry_scope_exit(body) + } _ => false, }) } @@ -10805,6 +10831,7 @@ impl Compiler { } // Emit cleanup for each fblock + let mut jump_no_location = false; for action in unwind_actions { match action { UnwindAction::With { is_async, range } => { @@ -10825,6 +10852,7 @@ impl Compiler { emit!(self, Instruction::PopTop); self.set_source_range(saved_range); + jump_no_location = true; } UnwindAction::HandlerCleanup { ref name } => { // codegen_unwind_fblock(HANDLER_CLEANUP) @@ -10861,6 +10889,7 @@ impl Compiler { // this keeps the fblock stack consistent for error checking) let code = self.current_code_info(); code.fblock.insert(fblock_idx, saved_fblock); + jump_no_location = true; } UnwindAction::FinallyEnd => { // codegen_unwind_fblock(FINALLY_END) @@ -10883,6 +10912,9 @@ impl Compiler { // Jump to target let target = if is_break { exit_block } else { loop_block }; emit!(self, PseudoInstruction::Jump { delta: target }); + if jump_no_location { + self.set_no_location(); + } Ok(()) } @@ -13221,6 +13253,93 @@ def f(msg): ); } + #[test] + fn test_nested_try_line_nops_after_for_cleanup_are_preserved() { + let code = compile_exec( + "\ +def f(xs, env): + for x in xs: + pass + try: + try: + if env is not None: + env_list = [] + else: + env_list = None + finally: + pass + finally: + pass +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::Nop, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::PopJumpIfNone { .. }, + ] + ) + }), + "expected CPython-style outer and inner try-line NOPs after for cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_try_finally_if_break_false_edge_keeps_finalbody_entry_nop() { + let code = compile_exec( + "\ +def f(self, pid): + while True: + try: + if pid == self.pid: + self.h() + break + finally: + self.r() + self.g() + return self.x +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::ReturnValue, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "expected CPython-style if-line NOP before fallthrough finally body, got ops={ops:?}" + ); + } + #[test] fn test_try_percent_format_preprocess_removes_redundant_try_nop() { let code = compile_exec( @@ -14729,6 +14848,46 @@ def f(tar1, x): ); } + #[test] + fn test_with_break_cleanup_makes_following_jump_artificial() { + let code = compile_exec( + "\ +def f(self): + while self.returncode is None: + with self._waitpid_lock: + if self.returncode is not None: + break + self.work() + return self.returncode +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops_lines: Vec<_> = f + .instructions + .iter() + .zip(&f.locations) + .filter_map(|(unit, (location, _))| { + (!matches!(unit.op, Instruction::Cache)).then_some((unit.op, location.line.get())) + }) + .collect(); + + assert!( + !ops_lines.windows(2).any(|window| { + matches!( + window, + [ + (Instruction::Nop, 5), + ( + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + 7 + ), + ] + ) + }), + "expected CPython-style artificial jump after with-break cleanup, got ops_lines={ops_lines:?}", + ); + } + #[test] fn test_nested_boolop_same_or_prefixes_compile_without_extra_boolop_block() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a0d2cf747e..7091767366 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -11294,24 +11294,9 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { let bi = block_idx.idx(); let keep_target_start_nop = keep_target_start_no_location_nop(blocks, block_idx, &layout_predecessors); - let follows_pop_iter_cleanup = - layout_predecessor_ends_with_pop_iter(blocks, block_idx, &layout_predecessors); + let follows_same_line_pop_iter = + layout_predecessor_ends_with_pop_iter_on_line(blocks, block_idx, &layout_predecessors); let mut src_instructions = core::mem::take(&mut blocks[bi].instructions); - if !keep_target_start_nop - && matches!( - src_instructions.as_slice(), - [InstructionInfo { - instr: AnyInstruction::Real(Instruction::Nop), - .. - }] - ) - && follows_pop_iter_cleanup - { - changes += 1; - src_instructions.clear(); - blocks[bi].instructions = src_instructions; - continue; - } let mut kept = Vec::with_capacity(src_instructions.len()); let mut prev_lineno = -1i32; @@ -11321,12 +11306,10 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { let mut remove = false; if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if (src > 0 - && matches!( - src_instructions[src - 1].instr.real(), - Some(Instruction::PopIter) - )) - || (src == 0 && !keep_target_start_nop && follows_pop_iter_cleanup) + if src == 0 + && !keep_target_start_nop + && lineno > 0 + && follows_same_line_pop_iter == Some(lineno) { remove = true; } else if instr.preserve_block_start_no_location_nop { @@ -11956,19 +11939,20 @@ fn keep_target_start_no_location_nop( && !block_starts_with_with_exit_none_call(&blocks[target.idx()]) } -fn layout_predecessor_ends_with_pop_iter( +fn layout_predecessor_ends_with_pop_iter_on_line( blocks: &[Block], target: BlockIdx, layout_predecessors: &[BlockIdx], -) -> bool { +) -> Option { let layout_pred = layout_predecessors[target.idx()]; - layout_pred != BlockIdx::NULL - && block_has_fallthrough(&blocks[layout_pred.idx()]) - && next_nonempty_block(blocks, blocks[layout_pred.idx()].next) == target - && blocks[layout_pred.idx()] - .instructions - .last() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::PopIter))) + if layout_pred == BlockIdx::NULL + || !block_has_fallthrough(&blocks[layout_pred.idx()]) + || next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target + { + return None; + } + let last = blocks[layout_pred.idx()].instructions.last()?; + matches!(last.instr.real(), Some(Instruction::PopIter)).then_some(instruction_lineno(last)) } fn is_with_suppress_exit_block(block: &Block) -> bool { From 2aa954fb90b52e05c74f0839386f498fabfd19c4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 21:58:20 +0900 Subject: [PATCH 14/76] Align loop CFG anchors with CPython --- crates/codegen/src/compile.rs | 105 ++++++++++++++++++++++++ crates/codegen/src/ir.rs | 149 ++++++++++++++++++++++++++++++++-- 2 files changed, 245 insertions(+), 9 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 07d3bfc7e0..7d41e95c51 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -14888,6 +14888,48 @@ def f(self): ); } + #[test] + fn test_while_exit_before_with_cleanup_materializes_anchor_nop() { + let code = compile_exec( + "\ +def f(selector, self): + with selector: + while selector.get_map(): + pass + try: + self.wait() + except Exception: + pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops_lines: Vec<_> = f + .instructions + .iter() + .zip(&f.locations) + .filter_map(|(unit, (location, _))| { + (!matches!(unit.op, Instruction::Cache)).then_some((unit.op, location.line.get())) + }) + .collect(); + + assert!( + ops_lines.windows(6).any(|window| { + matches!( + window, + [ + (Instruction::JumpBackward { .. }, 4), + (Instruction::Nop, 3), + (Instruction::LoadConst { .. }, 2), + (Instruction::LoadConst { .. }, 2), + (Instruction::LoadConst { .. }, 2), + (Instruction::Call { .. }, 2), + ] + ) + }), + "expected CPython-style while-exit anchor NOP before with cleanup, got ops_lines={ops_lines:?}", + ); + } + #[test] fn test_nested_boolop_same_or_prefixes_compile_without_extra_boolop_block() { let code = compile_exec( @@ -22120,6 +22162,69 @@ def f(b, curr, curr_append, decoded_append, packI, curr_clear): ); } + #[test] + fn test_try_else_loop_if_body_keeps_cpython_fallthrough_before_backedge() { + let code = compile_exec( + "\ +def f(self, ready, selector, key, input_view, os, BrokenPipeError): + for key, events in ready: + if key.fileobj is self.stdin: + chunk = input_view[self._input_offset:self._input_offset + 1] + try: + self._input_offset += os.write(key.fd, chunk) + except BrokenPipeError: + selector.unregister(key.fileobj) + key.fileobj.close() + else: + if self._input_offset >= len(input_view): + selector.unregister(key.fileobj) + key.fileobj.close() + elif key.fileobj in (self.stdout, self.stderr): + self.read(key) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "expected CPython-style try-else if body fallthrough before loop backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "unexpected inverted try-else conditional with loop backedge before body, got ops={ops:?}" + ); + } + #[test] fn test_explicit_continue_after_return_orders_return_before_backedge() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7091767366..c9922ba1c7 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -10855,6 +10855,13 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { if chain_has_delete_subscr { continue; } + if target_ins.lineno_override.is_some_and(|lineno| lineno < 0) + && blocks[bi].instructions[..last_idx] + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Nop))) + { + continue; + } } if !include_conditional && source_pos < target_pos && final_target_pos < target_pos { @@ -11565,17 +11572,97 @@ fn redirect_empty_unconditional_jump_targets(blocks: &mut [Block]) { } fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { + fn block_starts_with_with_normal_exit(block: &Block) -> bool { + matches!( + block.instructions.as_slice(), + [ + InstructionInfo { + instr: AnyInstruction::Real(Instruction::LoadConst { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::LoadConst { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::LoadConst { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::Call { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::PopTop), + .. + }, + .. + ] + ) + } + + fn with_normal_exit_is_followed_by_try(blocks: &[Block], block_idx: BlockIdx) -> bool { + if block_idx == BlockIdx::NULL + || !block_starts_with_with_normal_exit(&blocks[block_idx.idx()]) + { + return false; + } + let next = next_nonempty_block(blocks, blocks[block_idx.idx()].next); + next != BlockIdx::NULL + && blocks[next.idx()].instructions.first().is_some_and(|info| { + matches!( + info.instr, + AnyInstruction::Pseudo(PseudoInstruction::SetupFinally { .. }) + | AnyInstruction::Real(Instruction::Nop) + ) + }) + } + + fn has_loop_backedge_to(blocks: &[Block], target: BlockIdx) -> bool { + blocks.iter().enumerate().any(|(source_idx, block)| { + let source = BlockIdx(source_idx as u32); + comes_before(blocks, target, source) + && block.instructions.iter().any(|info| { + info.instr.is_unconditional_jump() + && info.target != BlockIdx::NULL + && next_nonempty_block(blocks, info.target) == target + }) + }) + } + let mut jump_back_inserts = Vec::new(); let mut inserts = Vec::new(); + let mut target_start_inserts = Vec::new(); for (block_idx, block) in blocks.iter().enumerate() { - let Some(last) = block.instructions.last() else { + let source = BlockIdx(block_idx as u32); + let (last, allow_scope_exit_target) = if let Some(last) = block + .instructions + .last() + .filter(|info| is_conditional_jump(&info.instr)) + { + (last, true) + } else if let Some(cond_idx) = trailing_conditional_jump_index(block) { + (&block.instructions[cond_idx], false) + } else { continue; }; - if !is_conditional_jump(&last.instr) || last.target == BlockIdx::NULL { + if last.target == BlockIdx::NULL { continue; } let target = last.target; if !blocks[target.idx()].instructions.is_empty() { + if with_normal_exit_is_followed_by_try(blocks, target) + && has_loop_backedge_to(blocks, source) + && !matches!( + blocks[target.idx()] + .instructions + .first() + .and_then(|info| info.instr.real()), + Some(Instruction::Nop) + ) + { + target_start_inserts.push((*last, target)); + } continue; } let next = next_nonempty_block(blocks, blocks[target.idx()].next); @@ -11591,10 +11678,14 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { jump_back_inserts.push((BlockIdx(block_idx as u32), target, next)); continue; } - if next == BlockIdx::NULL || !is_scope_exit_block(&blocks[next.idx()]) { + if next == BlockIdx::NULL + || !((allow_scope_exit_target && is_scope_exit_block(&blocks[next.idx()])) + || (with_normal_exit_is_followed_by_try(blocks, next) + && has_loop_backedge_to(blocks, source))) + { continue; } - inserts.push((BlockIdx(block_idx as u32), target)); + inserts.push((*last, target)); } for (source, target, next) in jump_back_inserts { @@ -11613,15 +11704,12 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { if !blocks[target.idx()].instructions.is_empty() { continue; } - let Some(last) = blocks[source.idx()].instructions.last().copied() else { - continue; - }; blocks[target.idx()].instructions.push(InstructionInfo { instr: Instruction::Nop.into(), arg: OpArg::NULL, target: BlockIdx::NULL, - location: last.location, - end_location: last.end_location, + location: source.location, + end_location: source.end_location, except_handler: None, folded_from_nonliteral_expr: false, lineno_override: None, @@ -11631,6 +11719,37 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { preserve_block_start_no_location_nop: false, }); } + + for (source, target) in target_start_inserts.into_iter().rev() { + if !with_normal_exit_is_followed_by_try(blocks, target) + || matches!( + blocks[target.idx()] + .instructions + .first() + .and_then(|info| info.instr.real()), + Some(Instruction::Nop) + ) + { + continue; + } + blocks[target.idx()].instructions.insert( + 0, + InstructionInfo { + instr: Instruction::Nop.into(), + arg: OpArg::NULL, + target: BlockIdx::NULL, + location: source.location, + end_location: source.end_location, + except_handler: None, + folded_from_nonliteral_expr: false, + lineno_override: None, + cache_entries: 0, + preserve_redundant_jump_as_nop: false, + remove_no_location_nop: false, + preserve_block_start_no_location_nop: false, + }, + ); + } } fn merge_unsafe_mask(slot: &mut Option>, incoming: &[bool]) -> bool { @@ -12510,6 +12629,18 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { } let after_jump = next_nonempty_block(blocks, blocks[jump_block.idx()].next); + let jump_is_artificial = blocks[jump_block.idx()] + .instructions + .first() + .is_some_and(|info| matches!(info.lineno_override, Some(line) if line < 0)); + if is_generic_false_path_reorder + && jump_is_artificial + && after_jump != BlockIdx::NULL + && is_loop_cleanup_block(&blocks[after_jump.idx()]) + { + current = next; + continue; + } if nonempty_blocks == 1 && !is_jump_only_block(&blocks[chain_start.idx()]) && after_jump != BlockIdx::NULL From 742ff9bff2545a66365db99da5a8a0bb1da1c026 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 22:19:12 +0900 Subject: [PATCH 15/76] Preserve loop false-path CFG bodies --- crates/codegen/src/compile.rs | 121 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 53 ++++++++++++++- 2 files changed, 173 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 7d41e95c51..8f3b11a4fe 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -19652,6 +19652,127 @@ def f(new, old): ); } + #[test] + fn test_line_bearing_loop_if_false_backedge_keeps_body_before_jump_back() { + let code = compile_exec( + "\ +def f(self, replacement_pairs): + for n, d in [(19, '%OC'), (2, '%Ow')]: + if self.LC_alt_digits is None: + s = str(n) + replacement_pairs.append((s, d)) + if n < 10: + replacement_pairs.append((s[1], d)) + elif len(self.LC_alt_digits) > n: + replacement_pairs.append((self.LC_alt_digits[n], d)) + else: + replacement_pairs.append((d, d)) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "expected CPython-style line-bearing false target to keep body before backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "unexpected no-lineno-style inverted loop-if body, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_elif_nested_if_false_backedge_keeps_body_before_jump_back() { + let code = compile_exec( + "\ +def f(keys, parse_int, d, ampm, AM, PM): + hour = minute = 0 + for group_key in keys: + if group_key == 'I': + hour = parse_int(d['I']) + if ampm in ('', AM): + if hour == 12: + hour = 0 + elif ampm == PM: + if hour != 12: + hour += 12 + elif group_key == 'M': + minute = parse_int(d['M']) + return hour, minute +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadSmallInt { .. }, + ] + ) + }), + "expected CPython-style nested elif body before false backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadSmallInt { .. }, + ] + ) + }), + "unexpected inverted nested elif false path before body, got ops={ops:?}" + ); + } + #[test] fn test_loop_multiblock_conditional_body_keeps_body_before_jump_back() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index c9922ba1c7..8bb3372661 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -10862,6 +10862,43 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { { continue; } + let after_target = next_nonempty_block(blocks, blocks[target.idx()].next); + let mut scan = blocks[bi].next; + let mut nonempty_between = 0usize; + let mut first_nonempty_between = BlockIdx::NULL; + let mut reaches_target = false; + let mut seen = vec![false; blocks.len()]; + while scan != BlockIdx::NULL && !seen[scan.idx()] { + if scan == target { + reaches_target = true; + break; + } + seen[scan.idx()] = true; + if !blocks[scan.idx()].instructions.is_empty() { + if first_nonempty_between == BlockIdx::NULL { + first_nonempty_between = scan; + } + nonempty_between += 1; + } + scan = blocks[scan.idx()].next; + } + if reaches_target + && nonempty_between <= 2 + && first_nonempty_between != BlockIdx::NULL + && !is_marker_jump_only_block(&blocks[first_nonempty_between.idx()]) + && !is_pop_top_jump_block(&blocks[first_nonempty_between.idx()]) + && !is_scope_exit_block(&blocks[first_nonempty_between.idx()]) + && !is_loop_cleanup_block(&blocks[first_nonempty_between.idx()]) + && after_target != BlockIdx::NULL + && !blocks[after_target.idx()].cold + && !block_is_exceptional(&blocks[after_target.idx()]) + && !is_marker_jump_only_block(&blocks[after_target.idx()]) + && !is_pop_top_jump_block(&blocks[after_target.idx()]) + && !is_scope_exit_block(&blocks[after_target.idx()]) + && !is_loop_cleanup_block(&blocks[after_target.idx()]) + { + continue; + } } if !include_conditional && source_pos < target_pos && final_target_pos < target_pos { @@ -11946,6 +11983,21 @@ fn is_jump_only_block(block: &Block) -> bool { instr.instr.is_unconditional_jump() && instr.target != BlockIdx::NULL } +fn is_marker_jump_only_block(block: &Block) -> bool { + let mut real_instrs = block.instructions.iter().filter(|info| { + !matches!( + info.instr.real(), + Some(Instruction::Nop | Instruction::NotTaken) + ) + }); + let Some(instr) = real_instrs.next() else { + return false; + }; + real_instrs.next().is_none() + && instr.instr.is_unconditional_jump() + && instr.target != BlockIdx::NULL +} + fn is_jump_back_only_block(blocks: &[Block], block_idx: BlockIdx) -> bool { if block_idx == BlockIdx::NULL || !is_jump_only_block(&blocks[block_idx.idx()]) { return false; @@ -12627,7 +12679,6 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; } - let after_jump = next_nonempty_block(blocks, blocks[jump_block.idx()].next); let jump_is_artificial = blocks[jump_block.idx()] .instructions From 6cfa6f2570509e0b2ca0cd596ba48f58d9aa94d6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 22:42:23 +0900 Subject: [PATCH 16/76] Align protected store-subscript CFG bytecode --- crates/codegen/src/compile.rs | 110 +++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 118 +++++++++++++++++++++++++++++++++- 2 files changed, 227 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 8f3b11a4fe..5d86502139 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -13012,6 +13012,88 @@ def f(kwonlyargs, kw_only_defaults, arg2value): ); } + #[test] + fn test_protected_store_subscr_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(cache, lock, format): + with lock: + format_regex = cache.get(format) + if not format_regex: + try: + format_regex = cache.compile(format) + except KeyError as err: + bad_directive = err.args[0] + del err + raise ValueError(bad_directive) from None + cache[format] = format_regex + return format_regex.match('x') +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFastLoadFast { .. }, + Instruction::LoadFast { .. }, + Instruction::StoreSubscr, + Instruction::LoadConst { .. }, + ] + ) + }), + "expected CPython-style strong loads before protected STORE_SUBSCR tail, got ops={ops:?}" + ); + + let code = compile_exec( + "\ +cache = {} +def g(lock, format): + with lock: + format_regex = cache.get(format) + if not format_regex: + try: + format_regex = compile(format) + except KeyError as err: + bad_directive = err.args[0] + del err + raise ValueError(bad_directive) from None + cache[format] = format_regex + return format_regex.match('x') +", + ); + let g = find_code(&code, "g").expect("missing function code"); + let ops: Vec<_> = g + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadGlobal { .. }, + Instruction::LoadFast { .. }, + Instruction::StoreSubscr, + ] + ) + }), + "expected CPython-style strong value/key loads around global STORE_SUBSCR tail, got ops={ops:?}" + ); + } + #[test] fn test_augassign_two_part_slice_uses_slice_opcodes() { let code = compile_exec( @@ -19771,6 +19853,34 @@ def f(keys, parse_int, d, ampm, AM, PM): }), "unexpected inverted nested elif false path before body, got ops={ops:?}" ); + assert!( + ops.windows(15).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadSmallInt { .. }, + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadSmallInt { .. }, + Instruction::BinaryOp { .. }, + Instruction::StoreFast { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style duplicated body/false loop exits for nested elif, got ops={ops:?}" + ); } #[test] diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8bb3372661..2da621abd4 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -333,6 +333,7 @@ impl CodeInfo { materialize_empty_conditional_exit_targets(&mut self.blocks); redirect_empty_block_targets(&mut self.blocks); duplicate_end_returns(&mut self.blocks, &self.metadata); + duplicate_fallthrough_jump_back_targets(&mut self.blocks); duplicate_shared_jump_back_targets(&mut self.blocks); self.dce(); // truncate after terminal in blocks that got return duplicated self.eliminate_unreachable_blocks(); // remove now-unreachable last block @@ -7960,6 +7961,36 @@ impl CodeInfo { }) } + fn protected_store_subscr_operand_start(block: &Block) -> Option { + let store_idx = block.instructions.iter().position(|info| { + matches!(info.instr.real(), Some(Instruction::StoreSubscr)) + && info.except_handler.is_some() + })?; + + let mut stack_items = 0; + for start in (0..store_idx).rev() { + let produced = match block.instructions[start].instr.real() { + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadGlobal { .. } + | Instruction::LoadName { .. } + | Instruction::LoadDeref { .. }, + ) => 1, + Some( + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. }, + ) => 2, + _ => return None, + }; + stack_items += produced; + if stack_items >= 3 { + return Some(start); + } + } + None + } + fn block_has_attr_named(block: &Block, names: &IndexSet, attr: &str) -> bool { block.instructions.iter().any(|info| { let raw = u32::from(info.arg) as usize; @@ -8167,7 +8198,24 @@ impl CodeInfo { } let mut to_deopt = Vec::new(); - for block in &self.blocks { + let has_exception_match_handler = self.blocks.iter().any(|block| { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + }); + if has_exception_match_handler { + for (block_idx, block) in self.blocks.iter().enumerate() { + if block_has_protected_instructions(block) + && let Some(start) = protected_store_subscr_operand_start(block) + { + to_deopt.push((BlockIdx::new(block_idx as u32), start)); + } + } + } + for (block_idx, block) in self.blocks.iter().enumerate() { if block_is_exceptional(block) || !block .instructions @@ -8187,6 +8235,10 @@ impl CodeInfo { { continue; } + if let Some(start) = protected_store_subscr_operand_start(block) { + to_deopt.push((BlockIdx::new(block_idx as u32), start)); + continue; + } let same_block_tail_start = first_unprotected_suffix(block); if same_block_tail_start.is_some() { continue; @@ -13949,6 +14001,70 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { } } +fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { + let predecessors = compute_predecessors(blocks); + let mut clones = Vec::new(); + + let mut layout_pred = BlockIdx(0); + while layout_pred != BlockIdx::NULL { + if !block_has_fallthrough(&blocks[layout_pred.idx()]) { + layout_pred = blocks[layout_pred.idx()].next; + continue; + } + + let target = next_nonempty_block(blocks, blocks[layout_pred.idx()].next); + if target == BlockIdx::NULL + || predecessors[target.idx()] < 2 + || !is_jump_back_only_block(blocks, target) + { + layout_pred = blocks[layout_pred.idx()].next; + continue; + } + + let jump_target = next_nonempty_block(blocks, blocks[target.idx()].instructions[0].target); + if jump_target == BlockIdx::NULL + || !has_non_exception_loop_backedge_to(blocks, target, jump_target) + { + layout_pred = blocks[layout_pred.idx()].next; + continue; + } + + let has_non_layout_jump_predecessor = blocks.iter().enumerate().any(|(idx, block)| { + let block_idx = BlockIdx(idx as u32); + block_idx != layout_pred + && block_idx != target + && block.instructions.iter().any(|info| { + is_jump_instruction(info) + && info.target != BlockIdx::NULL + && next_nonempty_block(blocks, info.target) == target + }) + }); + if has_non_layout_jump_predecessor { + clones.push((layout_pred, target)); + } + + layout_pred = blocks[layout_pred.idx()].next; + } + + for (layout_pred, target) in clones.into_iter().rev() { + if next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target { + continue; + } + let Some(last) = blocks[layout_pred.idx()].instructions.last().copied() else { + continue; + }; + + let new_idx = BlockIdx(blocks.len() as u32); + let mut cloned = blocks[target.idx()].clone(); + if let Some(first) = cloned.instructions.first_mut() { + overwrite_location(first, last.location, last.end_location); + } + cloned.next = blocks[layout_pred.idx()].next; + blocks.push(cloned); + blocks[layout_pred.idx()].next = new_idx; + } +} + /// Duplicate `LOAD_CONST None + RETURN_VALUE` for blocks that fall through /// to the final return block. fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { From cc59c4678dad8948b209a9e023d29aebb93a1934 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 23:10:18 +0900 Subject: [PATCH 17/76] Align borrow deopts with CPython CFG --- crates/codegen/src/compile.rs | 97 ++++++++++++++++- crates/codegen/src/ir.rs | 191 +++------------------------------- 2 files changed, 107 insertions(+), 181 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 5d86502139..8babbb82c0 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -15355,6 +15355,49 @@ def f(tarfile, tarinfo, self): ); } + #[test] + fn test_protected_store_finally_cleanup_keeps_borrow_tail() { + let code = compile_exec( + "\ +def f(re, f): + try: + try: + m = re.search('x', f.read()) + finally: + f.close() + if m is not None: + return m.group(1) + except OSError: + pass + return None +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let is_m_borrow = |unit: &bytecode::CodeUnit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))].as_str() == "m" + } + _ => false, + }; + + assert!( + ops.windows(3).any(|window| { + is_m_borrow(window[0]) + && matches!(window[1].op, Instruction::PopJumpIfNone { .. }) + && matches!(window[2].op, Instruction::NotTaken) + }) && ops.windows(2).any(|window| { + is_m_borrow(window[0]) && matches!(window[1].op, Instruction::LoadAttr { .. }) + }), + "finally cleanup RERAISE should not make the outer except deopt the normal m tail, got ops={ops:?}" + ); + } + #[test] fn test_generator_protected_store_subscr_tail_uses_strong_loads() { let code = compile_exec( @@ -16817,7 +16860,7 @@ def f(self, size): } #[test] - fn test_assertion_success_join_deopts_following_debug_tail() { + fn test_assertion_success_join_keeps_following_debug_tail_borrowed() { let code = compile_exec( "\ def f(self, typ, dat): @@ -16863,14 +16906,17 @@ def f(self, typ, dat): assert!( matches!( instructions[debug_attr - 1].op, - Instruction::LoadFast { .. } + Instruction::LoadFastBorrow { .. } ), - "CPython uses strong LOAD_FAST after assertion success join, got ops={:?}", + "CPython keeps LOAD_FAST_BORROW after assertion success join, got ops={:?}", instructions.iter().map(|unit| unit.op).collect::>() ); assert!( - matches!(instructions[mesg_attr - 1].op, Instruction::LoadFast { .. }), - "CPython uses strong LOAD_FAST in assertion-success debug body, got ops={:?}", + matches!( + instructions[mesg_attr - 1].op, + Instruction::LoadFastBorrow { .. } + ), + "CPython keeps LOAD_FAST_BORROW in assertion-success debug body, got ops={:?}", instructions.iter().map(|unit| unit.op).collect::>() ); } @@ -23805,6 +23851,47 @@ def f(items, decoded, b32rev): ); } + #[test] + fn test_terminal_except_loop_backedge_keeps_header_borrows() { + let code = compile_exec( + "\ +def f(self, value, start=0, stop=None): + i = start + while stop is None or i < stop: + try: + v = self[i] + except IndexError: + break + if v is value or v == value: + return i + i += 1 + raise ValueError +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + + assert!( + instructions.windows(7).any(|window| { + matches!(window[0].op, Instruction::StoreFast { .. }) + && matches!(window[1].op, Instruction::LoadFastBorrow { .. }) + && matches!(window[2].op, Instruction::PopJumpIfNone { .. }) + && matches!(window[3].op, Instruction::NotTaken) + && matches!( + window[4].op, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + && matches!(window[5].op, Instruction::CompareOp { .. }) + && matches!(window[6].op, Instruction::PopJumpIfFalse { .. }) + }), + "terminal-except loop backedge deopt should not cross into the loop header, got instructions={instructions:?}" + ); + } + #[test] fn test_except_handler_with_conditional_raise_and_resume_keeps_borrow() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 2da621abd4..f350b9e67c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -371,7 +371,6 @@ impl CodeInfo { // before optimize_load_fast. convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); remove_redundant_nops_and_jumps(&mut self.blocks); - self.mark_assertion_success_tail_borrow_disabled(); self.mark_unprotected_debug_four_tails_borrow_disabled(); self.compute_load_fast_start_depths(); // optimize_load_fast: after normalize_jumps @@ -3586,180 +3585,6 @@ impl CodeInfo { } } - fn mark_assertion_success_tail_borrow_disabled(&mut self) { - fn starts_with_fast_load(block: &Block) -> bool { - block - .instructions - .iter() - .find(|info| { - info.instr.real().is_some_and(|instr| { - !matches!(instr, Instruction::Nop | Instruction::NotTaken) - }) - }) - .is_some_and(|info| { - matches!( - info.instr.real(), - Some( - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - ) - }) - } - - fn starts_with_assertion_error(block: &Block) -> bool { - block - .instructions - .iter() - .find(|info| { - info.instr.real().is_some_and(|instr| { - !matches!(instr, Instruction::Nop | Instruction::NotTaken) - }) - }) - .is_some_and(|info| { - matches!( - info.instr.real(), - Some(Instruction::LoadCommonConstant { idx }) - if idx.get(info.arg) == oparg::CommonConstant::AssertionError - ) - }) - } - - fn block_contains_assertion_error(block: &Block) -> bool { - block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some(Instruction::LoadCommonConstant { idx }) - if idx.get(info.arg) == oparg::CommonConstant::AssertionError - ) - }) - } - - fn assert_check_success_target(blocks: &[Block], block: &Block) -> Option { - if block.next != BlockIdx::NULL - && starts_with_assertion_error(&blocks[block.next.idx()]) - { - return block - .instructions - .iter() - .find(|info| is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL) - .map(|info| info.target); - } - - let conditional = block.instructions.iter().enumerate().find(|(_, info)| { - is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL - }); - let (cond_idx, conditional) = conditional?; - let has_inline_assertion_failure = - block.instructions[cond_idx + 1..].iter().any(|info| { - matches!( - info.instr.real(), - Some(Instruction::LoadCommonConstant { idx }) - if idx.get(info.arg) == oparg::CommonConstant::AssertionError - ) - }) && block.instructions[cond_idx + 1..].iter().any(|info| { - matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. })) - }); - has_inline_assertion_failure.then_some(conditional.target) - } - - fn block_stores_fast(block: &Block) -> bool { - block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some( - Instruction::StoreFast { .. } - | Instruction::StoreFastLoadFast { .. } - | Instruction::StoreFastStoreFast { .. } - ) - ) - }) - } - - let mut predecessors = vec![Vec::new(); self.blocks.len()]; - for (pred_idx, block) in self.blocks.iter().enumerate() { - if block.next != BlockIdx::NULL { - predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); - } - for info in &block.instructions { - if info.target != BlockIdx::NULL { - predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); - } - } - } - - let mut assertion_success_tail = vec![false; self.blocks.len()]; - for (tail_idx, tail) in self.blocks.iter().enumerate() { - if tail.cold - || block_is_exceptional(tail) - || !starts_with_fast_load(tail) - || block_contains_assertion_error(tail) - || assert_check_success_target(&self.blocks, tail).is_some() - { - continue; - } - let tail_idx = BlockIdx::new(tail_idx as u32); - for pred in &predecessors[tail_idx.idx()] { - if assert_check_success_target(&self.blocks, &self.blocks[pred.idx()]) - == Some(tail_idx) - && trailing_conditional_jump_index(tail).is_some() - && !block_contains_assertion_error(tail) - { - assertion_success_tail[tail_idx.idx()] = true; - break; - } - for assert_check in &predecessors[pred.idx()] { - if assert_check_success_target(&self.blocks, &self.blocks[assert_check.idx()]) - == Some(*pred) - { - assertion_success_tail[tail_idx.idx()] = true; - break; - } - } - if assertion_success_tail[tail_idx.idx()] { - break; - } - } - } - - let mut visited = vec![false; self.blocks.len()]; - for (idx, is_assertion_success_tail) in assertion_success_tail.iter().enumerate() { - if !*is_assertion_success_tail { - continue; - } - let mut segment = Vec::new(); - let mut cursor = BlockIdx::new(idx as u32); - while cursor != BlockIdx::NULL && !visited[cursor.idx()] { - let block = &self.blocks[cursor.idx()]; - if block.cold || block_is_exceptional(block) { - break; - } - segment.push(cursor); - if block_stores_fast(block) { - segment.clear(); - break; - } - if block - .instructions - .last() - .is_some_and(|info| info.instr.is_scope_exit()) - { - break; - } - cursor = self.blocks[cursor.idx()].next; - } - for block_idx in segment { - if visited[block_idx.idx()] { - continue; - } - visited[block_idx.idx()] = true; - self.blocks[block_idx.idx()].disable_load_fast_borrow = true; - } - } - } - fn mark_unprotected_debug_four_tails_borrow_disabled(&mut self) { fn block_has_protected_instructions(block: &Block) -> bool { block @@ -5933,6 +5758,14 @@ impl CodeInfo { if last.instr.is_scope_exit() { return Vec::new(); } + if matches!( + last.instr.real(), + Some( + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) { + return Vec::new(); + } if last.instr.is_unconditional_jump() { return (last.target != BlockIdx::NULL) .then_some(last.target) @@ -10010,7 +9843,6 @@ impl CodeInfo { let _ = self.max_stackdepth()?; convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); remove_redundant_nops_and_jumps(&mut self.blocks); - self.mark_assertion_success_tail_borrow_disabled(); self.mark_unprotected_debug_four_tails_borrow_disabled(); trace.push(( "after_convert_pseudo_ops".to_owned(), @@ -12272,6 +12104,13 @@ fn block_has_exception_match_handler(blocks: &[Block], block: &Block) -> bool { }) { return true; } + if blocks[cursor.idx()] + .instructions + .iter() + .any(|info| info.instr.is_scope_exit()) + { + break; + } cursor = blocks[cursor.idx()].next; } } From 891e972860a340c1e3217077184f12d5cae2a7ad Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 23:31:11 +0900 Subject: [PATCH 18/76] Align delete-loop CFG with CPython --- crates/codegen/src/compile.rs | 137 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 131 +++++++++++++++++++++++++++----- 2 files changed, 249 insertions(+), 19 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 8babbb82c0..bb1f864f96 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -15398,6 +15398,54 @@ def f(re, f): ); } + #[test] + fn test_try_else_finally_cleanup_keeps_borrow_tail() { + let code = compile_exec( + "\ +def f(re, open): + global _SYSTEM_VERSION + if _SYSTEM_VERSION is None: + _SYSTEM_VERSION = '' + try: + f = open('/System/Library/CoreServices/SystemVersion.plist', encoding='utf-8') + except OSError: + pass + else: + try: + m = re.search('x', f.read()) + finally: + f.close() + if m is not None: + _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2]) + return _SYSTEM_VERSION +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let is_m_borrow = |unit: &bytecode::CodeUnit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))].as_str() == "m" + } + _ => false, + }; + + assert!( + ops.windows(3).any(|window| { + is_m_borrow(window[0]) + && matches!(window[1].op, Instruction::PopJumpIfNone { .. }) + && matches!(window[2].op, Instruction::NotTaken) + }) && ops.windows(2).any(|window| { + is_m_borrow(window[0]) && matches!(window[1].op, Instruction::LoadAttr { .. }) + }), + "try/else finally cleanup should keep CPython-style borrowed m tail, got ops={ops:?}" + ); + } + #[test] fn test_generator_protected_store_subscr_tail_uses_strong_loads() { let code = compile_exec( @@ -23892,6 +23940,95 @@ def f(self, value, start=0, stop=None): ); } + #[test] + fn test_loop_if_implicit_continue_places_body_after_jumpback() { + let code = compile_exec( + "\ +def f(_config_vars, _INITPRE): + for k in list(_config_vars): + if k.startswith(_INITPRE): + del _config_vars[k] +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::DeleteSubscr, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::EndFor, + ] + ) + }), + "loop if with implicit continue should use CPython body-after-jumpback layout, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_nested_if_delete_slice_places_body_after_jumpback() { + let code = compile_exec( + "\ +def f(compiler_so): + for idx in reversed(range(len(compiler_so))): + if compiler_so[idx] == '-arch' and compiler_so[idx + 1] == 'arm64': + del compiler_so[idx:idx + 2] +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(15).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::LoadSmallInt { .. }, + Instruction::BinaryOp { .. }, + Instruction::BinaryOp { .. }, + Instruction::LoadConst { .. }, + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "nested loop delete-slice condition should put false jump-back before body, got ops={ops:?}" + ); + } + #[test] fn test_except_handler_with_conditional_raise_and_resume_keeps_borrow() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f350b9e67c..b8baec08c5 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5726,23 +5726,28 @@ impl CodeInfo { false } - fn protected_block_has_terminal_exception_handler(blocks: &[Block], block: &Block) -> bool { - let mut seen_handlers = Vec::new(); - for handler in block - .instructions - .iter() - .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) - { - if seen_handlers.contains(&handler) { - continue; - } - seen_handlers.push(handler); - let has_exception_match = handler_reaches_match_before_terminal(blocks, handler); - if has_exception_match - && !handler_chain_has_multiple_handled_returns(blocks, handler) - && !handler_chain_resumes_normally(blocks, handler) - { - return true; + fn handler_is_terminal_exception_handler(blocks: &[Block], handler: BlockIdx) -> bool { + handler_reaches_match_before_terminal(blocks, handler) + && !handler_chain_has_multiple_handled_returns(blocks, handler) + && !handler_chain_resumes_normally(blocks, handler) + } + + fn trailing_protected_tail_has_terminal_exception_handler( + blocks: &[Block], + block: &Block, + ) -> bool { + for info in block.instructions.iter().rev() { + match info.instr.real() { + Some(Instruction::Nop | Instruction::NotTaken | Instruction::PopTop) => {} + Some(_) => { + let Some(handler) = + info.except_handler.map(|handler| handler.handler_block) + else { + return false; + }; + return handler_is_terminal_exception_handler(blocks, handler); + } + None => {} } } false @@ -5929,7 +5934,10 @@ impl CodeInfo { seen[pred.idx()] = true; let pred_block = &self.blocks[pred.idx()]; if block_has_protected_instructions(pred_block) { - if protected_block_has_terminal_exception_handler(&self.blocks, pred_block) { + if trailing_protected_tail_has_terminal_exception_handler( + &self.blocks, + pred_block, + ) { seeds.push(( BlockIdx::new(idx as u32), (has_protected_call_predecessor || has_call_store_tail) @@ -12411,6 +12419,57 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { false }; + fn is_single_delete_subscr_body(block: &Block) -> bool { + let real: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + .collect(); + real.iter() + .filter(|instr| matches!(instr, Instruction::DeleteSubscr)) + .count() + == 1 + && matches!(real.last(), Some(Instruction::DeleteSubscr)) + && real.iter().all(|instr| { + matches!( + instr, + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::BinaryOp { .. } + | Instruction::BuildSlice { .. } + | Instruction::DeleteSubscr + ) + }) + } + + fn chain_is_conditional_single_delete_body( + blocks: &[Block], + chain_start: BlockIdx, + jump_start: BlockIdx, + ) -> bool { + if chain_start == BlockIdx::NULL || jump_start == BlockIdx::NULL { + return false; + } + let Some(chain_cond_idx) = trailing_conditional_jump_index(&blocks[chain_start.idx()]) + else { + return false; + }; + let chain_cond = blocks[chain_start.idx()].instructions[chain_cond_idx]; + if !is_false_path_conditional_jump(&chain_cond.instr) || chain_cond.target != jump_start { + return false; + } + let body = next_nonempty_block(blocks, blocks[chain_start.idx()].next); + body != BlockIdx::NULL + && !block_is_exceptional(&blocks[body.idx()]) + && !block_is_protected(&blocks[body.idx()]) + && next_nonempty_block(blocks, blocks[body.idx()].next) == jump_start + && is_single_delete_subscr_body(&blocks[body.idx()]) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -12535,7 +12594,12 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; } - if is_generic_false_path_reorder && nonempty_blocks > 1 { + let chain_is_conditional_single_delete_body = + chain_is_conditional_single_delete_body(blocks, chain_start, jump_start); + if is_generic_false_path_reorder + && nonempty_blocks > 1 + && !chain_is_conditional_single_delete_body + { current = next; continue; } @@ -12579,6 +12643,7 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { && jump_is_artificial && after_jump != BlockIdx::NULL && is_loop_cleanup_block(&blocks[after_jump.idx()]) + && !chain_is_conditional_single_delete_body { current = next; continue; @@ -13199,6 +13264,33 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec ) } + fn is_single_delete_subscr_body(block: &Block) -> bool { + let real: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + .collect(); + real.iter() + .filter(|instr| matches!(instr, Instruction::DeleteSubscr)) + .count() + == 1 + && matches!(real.last(), Some(Instruction::DeleteSubscr)) + && real.iter().all(|instr| { + matches!( + instr, + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::BinaryOp { .. } + | Instruction::BuildSlice { .. } + | Instruction::DeleteSubscr + ) + }) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -13241,6 +13333,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); let can_reorder = !body_tail_is_conditional && (matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) + || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) || body_segment_contains_for_iter(blocks, body, body_tail)); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { let cloned_jump_idx = BlockIdx(blocks.len() as u32); From 19b74273372d7353aeb9ab6eee560c971d98abc9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 7 May 2026 23:52:14 +0900 Subject: [PATCH 19/76] Align CFG inlining with CPython jumps --- crates/codegen/src/compile.rs | 92 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 60 ++++++++++++++++++++++- 2 files changed, 151 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index bb1f864f96..953a2e1f2e 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -12927,6 +12927,48 @@ def set_f(xs): ); } + #[test] + fn test_builtin_tuple_genexpr_try_assignment_uses_shared_tail() { + let code = compile_exec( + "\ +def f(xs): + global y + try: + y = tuple(int(i) for i in xs.split('.')) + except ValueError: + y = () + return y +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let intrinsic = ops + .iter() + .position(|op| matches!(op, Instruction::CallIntrinsic1 { .. })) + .expect("tuple(genexpr) fast path should emit LIST_TO_TUPLE"); + let first_fallback = ops[intrinsic + 1..] + .iter() + .position(|op| matches!(op, Instruction::PushNull)) + .map(|offset| intrinsic + 1 + offset) + .expect("shadowed tuple fallback call should remain after fast path"); + let first_store = ops[intrinsic + 1..] + .iter() + .position(|op| matches!(op, Instruction::StoreGlobal { .. })) + .map(|offset| intrinsic + 1 + offset) + .expect("tuple(genexpr) result should be stored after fast or fallback call"); + + assert!( + matches!(ops[intrinsic + 1], Instruction::JumpForward { .. }) + && first_fallback < first_store, + "tuple(genexpr) fast path should jump over fallback to CPython-style shared store tail, got ops={ops:?}" + ); + } + #[test] fn test_module_store_uses_store_global_when_nested_scope_declares_global() { let code = compile_exec( @@ -17288,6 +17330,56 @@ def f(names, cls): ); } + #[test] + fn test_except_break_preserves_plain_jump_when_inlining_no_lineno_tail() { + let code = compile_exec( + "\ +def f(compiler_so, cc_args): + strip_sysroot = True + if '-arch' in cc_args: + while True: + try: + index = compiler_so.index('-arch') + del compiler_so[index:index + 2] + except ValueError: + break + if strip_sysroot: + while True: + indices = [i for i, x in enumerate(compiler_so) if x.startswith('-isysroot')] + if not indices: + break + index = indices[0] + del compiler_so[index:index + 1] + return compiler_so +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(2).any(|window| { + matches!( + window, + [Instruction::PopExcept, Instruction::JumpBackward { .. }] + ) + }) && !ops.windows(2).any(|window| { + matches!( + window, + [ + Instruction::PopExcept, + Instruction::JumpBackwardNoInterrupt { .. } + ] + ) + }), + "except-break cleanup should preserve CPython's plain JUMP when a no-lineno tail is inlined, got ops={ops:?}" + ); + } + #[test] fn test_nested_with_bare_except_keeps_handler_cleanup_before_following_code() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index b8baec08c5..e37ca41864 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -11048,6 +11048,41 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { .iter() .any(|ins| ins.instr.is_block_push()) }; + let block_ends_with_list_to_tuple_jump = |block: &Block| { + let ops: Vec<_> = block + .instructions + .iter() + .filter(|info| !matches!(info.instr.real(), Some(Instruction::Nop))) + .collect(); + let Some((last, prefix)) = ops.split_last() else { + return false; + }; + if !last.instr.is_unconditional_jump() { + return false; + } + let Some(prev) = prefix.last() else { + return false; + }; + match prev.instr.real() { + Some(Instruction::CallIntrinsic1 { func }) => { + func.get(prev.arg) == IntrinsicFunction1::ListToTuple + } + _ => false, + } + }; + let block_starts_with_store_and_exits = |block: &Block| { + block.instructions.first().is_some_and(|info| { + matches!( + info.instr.real(), + Some( + Instruction::StoreFast { .. } + | Instruction::StoreGlobal { .. } + | Instruction::StoreName { .. } + | Instruction::StoreDeref { .. } + ) + ) + }) && block_exits_scope(block) + }; loop { let mut changes = false; let mut predecessors = vec![0usize; blocks.len()]; @@ -11091,7 +11126,15 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { && !instruction_has_lineno(&blocks[target.idx()].instructions[0]) && !instruction_has_lineno(&blocks[target.idx()].instructions[1]) && !instruction_has_lineno(&blocks[target.idx()].instructions[2]); - if !shared_artificial_expr_exit && (small_exit_block || no_lineno_no_fallthrough) { + let shared_tuple_genexpr_assignment_tail = small_exit_block + && predecessors[target.idx()] > 1 + && block_ends_with_list_to_tuple_jump(&blocks[current.idx()]) + && block_starts_with_store_and_exits(&blocks[target.idx()]); + if !shared_artificial_expr_exit + && !shared_tuple_genexpr_assignment_tail + && (small_exit_block || no_lineno_no_fallthrough) + { + let removed_jump_kind = jump_thread_kind(last.instr); let removed_jump_had_lineno = blocks[current.idx()] .instructions .last() @@ -11106,6 +11149,21 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { blocks[current.idx()] .instructions .extend(blocks[target.idx()].instructions.clone()); + if no_lineno_no_fallthrough && removed_jump_kind == Some(JumpThreadKind::Plain) { + if let Some(last) = blocks[current.idx()].instructions.last_mut() + && jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) + { + last.instr = match last.instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { + PseudoOpcode::Jump.into() + } + AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => { + Opcode::JumpBackward.into() + } + _ => last.instr, + }; + } + } changes = true; } From d734c060117238a1d1692e1f150cf5db84a455b9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 00:09:43 +0900 Subject: [PATCH 20/76] Align protected CFG jump threading --- crates/codegen/src/compile.rs | 40 ++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 43 +++++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 953a2e1f2e..ec70c8bca7 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -22464,6 +22464,46 @@ def f(kw): ); } + #[test] + fn test_protected_for_is_none_raise_threads_backedge_before_raise() { + let code = compile_exec( + "\ +def f(stacklevel, frame, skip_file_prefixes): + try: + for x in range(stacklevel - 1): + frame = _next_external_frame(frame, skip_file_prefixes) + if frame is None: + raise ValueError + except ValueError: + frame = None + return frame +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::PopJumpIfNone { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected protected is-None raise path to match CPython's backedge-before-raise layout, got ops={ops:?}" + ); + } + #[test] fn test_exception_handler_loop_conditional_raise_orders_backedge_before_raise() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e37ca41864..bdb68264eb 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -7529,15 +7529,18 @@ impl CodeInfo { fn handler_chain_returns(blocks: &[Block], handler_block: BlockIdx) -> bool { let mut cursor = handler_block; let mut visited = vec![false; blocks.len()]; + let mut after_pop_except = false; while cursor != BlockIdx::NULL && !visited[cursor.idx()] { visited[cursor.idx()] = true; - let mut saw_pop_except = false; for info in &blocks[cursor.idx()].instructions { match info.instr.real() { Some(Instruction::ReturnValue) => return true, - Some(Instruction::PopExcept) => saw_pop_except = true, + Some(Instruction::PopExcept) => after_pop_except = true, + Some(_) if after_pop_except && is_conditional_jump(&info.instr) => { + return false; + } Some(instr) - if saw_pop_except + if after_pop_except && (instr.is_unconditional_jump() || instr.is_scope_exit()) => { return false; @@ -7545,9 +7548,6 @@ impl CodeInfo { _ => {} } } - if saw_pop_except { - return false; - } cursor = blocks[cursor.idx()].next; } false @@ -10662,21 +10662,36 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { { continue; } + target = next_nonempty_block(blocks, target); + if target == BlockIdx::NULL { + continue; + } if include_conditional && is_conditional_jump(&ins.instr) { let next = next_nonempty_block(blocks, blocks[bi].next); - if next != BlockIdx::NULL + let next_is_scope_exit = next != BlockIdx::NULL && blocks[next.idx()] .instructions .last() - .is_some_and(|instr| instr.instr.is_scope_exit()) - { - continue; + .is_some_and(|instr| instr.instr.is_scope_exit()); + if next_is_scope_exit { + let next_raises = blocks[next.idx()].instructions.iter().any(|instr| { + matches!(instr.instr.real(), Some(Instruction::RaiseVarargs { .. })) + }); + let target_is_loop_backedge = blocks[target.idx()] + .instructions + .first() + .filter(|target_ins| target_ins.instr.is_unconditional_jump()) + .map(|target_ins| next_nonempty_block(blocks, target_ins.target)) + .is_some_and(|final_target| { + final_target == BlockIdx(bi as u32) + || comes_before(blocks, final_target, BlockIdx(bi as u32)) + }); + if !(block_is_protected(&blocks[bi]) && next_raises && target_is_loop_backedge) + { + continue; + } } } - target = next_nonempty_block(blocks, target); - if target == BlockIdx::NULL { - continue; - } if include_conditional && is_conditional_jump(&ins.instr) && opposite_short_circuit_target(&blocks[target.idx()], ins.instr) From 0e2b2264c1ce2dd30387a4131ce0c9f1cb441dff Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 00:38:21 +0900 Subject: [PATCH 21/76] Narrow handler resume borrow deopt --- crates/codegen/src/compile.rs | 84 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 30 +++++++++++-- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index ec70c8bca7..3bbe08c889 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -21782,6 +21782,90 @@ def f(self): ); } + #[test] + fn test_handler_resume_before_later_loop_keeps_borrowed_tail_loads() { + let code = compile_exec( + "\ +def f(msg, category): + s = 'x' + if msg.line is None: + try: + import linecache + line = linecache.getline(msg.filename, msg.lineno) + except Exception: + line = None + linecache = None + else: + line = msg.line + if line: + line = line.strip() + s += ' %s\\n' % line + if msg.source is not None: + try: + import tracemalloc + except Exception: + suggest_tracemalloc = False + tb = None + else: + try: + suggest_tracemalloc = not tracemalloc.is_tracing() + tb = tracemalloc.get_object_traceback(msg.source) + except Exception: + suggest_tracemalloc = False + tb = None + if tb is not None: + for frame in tb: + s += frame.filename + elif suggest_tracemalloc: + s += category + return s +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let is_borrow_load = |unit: &&bytecode::CodeUnit, name: &str| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }; + + assert!( + ops.windows(5).any(|window| { + is_borrow_load(&window[0], "line") + && matches!(window[1].op, Instruction::ToBool) + && matches!(window[2].op, Instruction::PopJumpIfFalse { .. }) + && matches!(window[3].op, Instruction::NotTaken) + && is_borrow_load(&window[4], "line") + }), + "handler resume before a later independent loop should preserve CPython-style borrowed line loads, got instructions={:?}", + f.instructions + ); + assert!( + ops.windows(2) + .filter(|window| { + is_borrow_load(&window[0], "tracemalloc") + && matches!(window[1].op, Instruction::LoadAttr { .. }) + }) + .count() + >= 2, + "independent later loop should not deopt tracemalloc loads before the loop, got instructions={:?}", + f.instructions + ); + assert!( + ops.windows(2).any(|window| { + is_borrow_load(&window[0], "s") && matches!(window[1].op, Instruction::ReturnValue) + }), + "final return should keep CPython-style borrowed s load, got instructions={:?}", + f.instructions + ); + } + #[test] fn test_async_early_return_send_tail_uses_strong_load_fast_after_entry() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index bdb68264eb..74fc0893d2 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -7085,6 +7085,23 @@ impl CodeInfo { }) } + fn block_has_loop_back_to_or_before( + blocks: &[Block], + block: &Block, + target_block: BlockIdx, + ) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) && (info.target == target_block + || comes_before(blocks, info.target, target_block)) + }) + } + fn block_has_for_iter(block: &Block) -> bool { block .instructions @@ -7336,12 +7353,18 @@ impl CodeInfo { let should_seed = (has_protected_predecessor && has_finally_except_loop_tail) || (has_bool_guard_tail && has_handler_resume_predecessor) || has_handler_resume_loop_tail; - should_seed.then_some((BlockIdx::new(idx as u32), has_handler_resume_loop_tail)) + let allow_any_loop_back = + has_finally_except_loop_tail || has_handler_resume_loop_tail; + should_seed.then_some(( + BlockIdx::new(idx as u32), + has_handler_resume_loop_tail, + allow_any_loop_back, + )) }) .collect(); let mut visited = vec![false; self.blocks.len()]; - for (seed, include_join_tail) in seeds { + for (seed, include_join_tail, allow_any_loop_back) in seeds { let mut segment = Vec::new(); let mut found_loop_back = false; let mut seen = vec![false; self.blocks.len()]; @@ -7357,7 +7380,8 @@ impl CodeInfo { } segment.push(cursor); if block_has_loop_back(block) { - found_loop_back = true; + found_loop_back |= allow_any_loop_back + || block_has_loop_back_to_or_before(&self.blocks, block, seed); continue; } for successor in tail_successors(&self.blocks, &predecessors, cursor) { From 3bb04305e8254dfc1368eae626db386cd6ba5fc4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 01:09:51 +0900 Subject: [PATCH 22/76] Preserve branch-local implicit continue targets --- crates/codegen/src/compile.rs | 59 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 23 ++++++++++++-- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 3bbe08c889..c0f9c8698d 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -19979,6 +19979,65 @@ def f(self, replacement_pairs): ); } + #[test] + fn test_branch_local_implicit_continue_keeps_body_before_jump_back() { + let code = compile_exec( + "\ +def f(items, outer, cond, sub, out): + for x in items: + if outer: + if cond: + out.append(x) + if sub: + out.append(1) + out.append(2) + else: + out.append(3) + return out +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "expected branch-local implicit continue target to stay after the body, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "unexpected direct-loop-body implicit continue layout for branch-local target, got ops={ops:?}" + ); + } + #[test] fn test_loop_elif_nested_if_false_backedge_keeps_body_before_jump_back() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 74fc0893d2..9de3f603ba 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -10774,6 +10774,7 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { if conditional && final_target_pos <= source_pos { let mut scan = blocks[bi].next; let mut chain_has_delete_subscr = false; + let mut chain_has_for_iter = false; let mut seen = vec![false; blocks.len()]; while scan != BlockIdx::NULL && scan != target && !seen[scan.idx()] { seen[scan.idx()] = true; @@ -10781,6 +10782,9 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { blocks[scan.idx()].instructions.iter().any(|info| { matches!(info.instr.real(), Some(Instruction::DeleteSubscr)) }); + chain_has_for_iter |= blocks[scan.idx()].instructions.iter().any(|info| { + matches!(info.instr.real(), Some(Instruction::ForIter { .. })) + }); scan = blocks[scan.idx()].next; } if chain_has_delete_subscr { @@ -10794,8 +10798,21 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { continue; } let after_target = next_nonempty_block(blocks, blocks[target.idx()].next); + let final_target_has_for_iter = blocks[final_target.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))); + let target_exits_current_loop = trailing_conditional_jump_index( + &blocks[final_target.idx()], + ) + .is_some_and(|cond_idx| { + !final_target_has_for_iter + && next_nonempty_block( + blocks, + blocks[final_target.idx()].instructions[cond_idx].target, + ) == after_target + }); let mut scan = blocks[bi].next; - let mut nonempty_between = 0usize; let mut first_nonempty_between = BlockIdx::NULL; let mut reaches_target = false; let mut seen = vec![false; blocks.len()]; @@ -10809,12 +10826,11 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { if first_nonempty_between == BlockIdx::NULL { first_nonempty_between = scan; } - nonempty_between += 1; } scan = blocks[scan.idx()].next; } if reaches_target - && nonempty_between <= 2 + && !chain_has_for_iter && first_nonempty_between != BlockIdx::NULL && !is_marker_jump_only_block(&blocks[first_nonempty_between.idx()]) && !is_pop_top_jump_block(&blocks[first_nonempty_between.idx()]) @@ -10827,6 +10843,7 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { && !is_pop_top_jump_block(&blocks[after_target.idx()]) && !is_scope_exit_block(&blocks[after_target.idx()]) && !is_loop_cleanup_block(&blocks[after_target.idx()]) + && !target_exits_current_loop { continue; } From 55ec30d2df46749763377140c8662a61cf265e1e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 01:34:38 +0900 Subject: [PATCH 23/76] Avoid duplicating boolop continue backedges --- crates/codegen/src/compile.rs | 52 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 6 ++++ 2 files changed, 58 insertions(+) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c0f9c8698d..7438d07044 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20038,6 +20038,58 @@ def f(items, outer, cond, sub, out): ); } + #[test] + fn test_boolop_continue_deduplicates_marker_jump_back() { + let code = compile_exec( + "\ +def f(ws, seen, more_than): + while ws: + w = ws.pop() + if w in seen or w <= more_than: + continue + seen.add(w) + return seen +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(2).any(|window| { + matches!( + window, + [ + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected adjacent equivalent continue backedges to be deduplicated, got ops={ops:?}" + ); + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "expected CPython-style boolop continue fallthrough before body, got ops={ops:?}" + ); + } + #[test] fn test_loop_elif_nested_if_false_backedge_keeps_body_before_jump_back() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 9de3f603ba..1609896296 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14066,6 +14066,12 @@ fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { layout_pred = blocks[layout_pred.idx()].next; continue; } + if !block_has_no_lineno(&blocks[target.idx()]) + && trailing_conditional_jump_index(&blocks[layout_pred.idx()]).is_some() + { + layout_pred = blocks[layout_pred.idx()].next; + continue; + } let jump_target = next_nonempty_block(blocks, blocks[target.idx()].instructions[0].target); if jump_target == BlockIdx::NULL From 95c4492a8e03e4c999cec979ab17d55fde02b412 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 01:53:01 +0900 Subject: [PATCH 24/76] Preserve same-line assert message borrows --- crates/codegen/src/compile.rs | 40 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 41 ++++++++++++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 7438d07044..1a1762f866 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -14190,6 +14190,46 @@ def f(fname): ); } + #[test] + fn test_chained_compare_assert_message_keeps_borrowed_load_fast() { + let code = compile_exec( + "\ +def f(month): + assert 1 <= month <= 12, f'month must be in 1..12, not {month}' +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let assertion_error = ops + .iter() + .position(|op| matches!(op, Instruction::LoadCommonConstant { .. })) + .expect("missing LOAD_COMMON_CONSTANT AssertionError"); + let raise = ops[assertion_error..] + .iter() + .position(|op| matches!(op, Instruction::RaiseVarargs { .. })) + .map(|idx| assertion_error + idx) + .expect("missing assert raise"); + let message_path = &ops[assertion_error..raise]; + + assert!( + message_path + .iter() + .any(|op| matches!(op, Instruction::LoadFastBorrow { .. })), + "CPython keeps assert message loads borrowed for same-line chained compare failure blocks, got {message_path:?}" + ); + assert!( + !message_path + .iter() + .any(|op| matches!(op, Instruction::LoadFast { .. })), + "same-line chained compare assert message should not be deoptimized to LOAD_FAST, got {message_path:?}" + ); + } + #[test] fn test_assert_message_after_condition_in_same_block_keeps_borrowed_loads() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 1609896296..dc3de988db 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3546,13 +3546,32 @@ impl CodeInfo { } } + fn has_same_line_target_predecessor( + blocks: &[Block], + incoming_origins: &[Vec], + block_idx: BlockIdx, + line: i32, + ) -> bool { + incoming_origins[block_idx.idx()].iter().any(|&pred| { + blocks[pred.idx()].instructions.iter().any(|info| { + info.target != BlockIdx::NULL + && next_nonempty_block(blocks, info.target) == block_idx + && instruction_lineno(info) == line + }) + }) + } + let target_flags = compute_target_predecessor_flags(&self.blocks); - for (block_idx, block) in self.blocks.iter_mut().enumerate() { + let reachable = compute_reachable_blocks(&self.blocks); + let incoming_origins = compute_incoming_origins(&self.blocks, &reachable); + for block_idx in 0..self.blocks.len() { if block_idx == 0 || !target_flags.targeted[block_idx] { continue; } + let block = &self.blocks[block_idx]; let mut assert_start = None; + let mut ranges = Vec::new(); for i in 0..block.instructions.len() { if is_assertion_error_load(&block.instructions[i]) { assert_start = Some(i); @@ -3577,11 +3596,27 @@ impl CodeInfo { continue; } - for info in &mut block.instructions[start + 1..i] { - deoptimize_borrow(info); + let assert_line = instruction_lineno(&block.instructions[start]); + if has_same_line_target_predecessor( + &self.blocks, + &incoming_origins, + BlockIdx::new(block_idx as u32), + assert_line, + ) { + assert_start = None; + continue; } + + ranges.push((start + 1, i)); assert_start = None; } + + let block = &mut self.blocks[block_idx]; + for (start, end) in ranges { + for info in &mut block.instructions[start..end] { + deoptimize_borrow(info); + } + } } } From 8ce66775fb9a8c48e6ab85586110595a09c7c003 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 02:30:31 +0900 Subject: [PATCH 25/76] Handle nested handler update tail borrows --- crates/codegen/src/compile.rs | 60 +++++++++++++ crates/codegen/src/ir.rs | 162 +++++++++++++++++++++++++++------- 2 files changed, 189 insertions(+), 33 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 1a1762f866..b718ae5dc8 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -15437,6 +15437,66 @@ def f(tarfile, tarinfo, self): ); } + #[test] + fn test_nested_exception_handler_resume_update_tail_uses_strong_load() { + let code = compile_exec( + "\ +def f(inpos, size, g, replacement): + while inpos < size: + try: + g() + except KeyError: + try: + for y in replacement: + g(y) + except KeyError: + raise ValueError(inpos) + inpos += 1 + return inpos +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let arg = |unit: &&bytecode::CodeUnit| OpArg::new(u32::from(u8::from(unit.arg))); + let is_inpos_load = |unit: &&bytecode::CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + f.varnames[usize::from(var_num.get(arg(unit)))].as_str() == "inpos" + } + _ => false, + }; + let is_inpos_store = |unit: &&bytecode::CodeUnit| match unit.op { + Instruction::StoreFast { var_num } => { + f.varnames[usize::from(var_num.get(arg(unit)))].as_str() == "inpos" + } + _ => false, + }; + let update = ops + .windows(4) + .find(|window| { + is_inpos_load(&window[0]) + && matches!( + window[1].op, + Instruction::LoadSmallInt { i } if i.get(arg(&window[1])) == 1 + ) + && matches!( + window[2].op, + Instruction::BinaryOp { op } + if op.get(arg(&window[2])) == BinaryOperator::InplaceAdd + ) + && is_inpos_store(&window[3]) + }) + .expect("missing inpos += 1 update"); + + assert!( + matches!(update[0].op, Instruction::LoadFast { .. }), + "CPython keeps a strong LOAD_FAST for nested-handler resumed inplace update, got update={update:?}" + ); + } + #[test] fn test_protected_store_finally_cleanup_keeps_borrow_tail() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index dc3de988db..d283af3a85 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -7846,6 +7846,44 @@ impl CodeInfo { false } + fn handler_chain_has_nested_exception_match(blocks: &[Block], block: &Block) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .collect(); + let mut matches_seen = 0; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + let handler = &blocks[cursor.idx()]; + for info in &handler.instructions { + if matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) { + matches_seen += 1; + if matches_seen > 1 { + return true; + } + } + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + if info.instr.is_unconditional_jump() && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if handler.next != BlockIdx::NULL { + stack.push(handler.next); + } + } + false + } + fn block_has_tail_deopt_trigger_from(block: &Block, start: usize) -> bool { block.instructions.iter().skip(start).any(|info| { matches!( @@ -7975,6 +8013,47 @@ impl CodeInfo { }) } + fn borrowed_inplace_local_update_start(block: &Block) -> Option { + for i in 0..block.instructions.len().saturating_sub(3) { + let local = match block.instructions[i].instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { + usize::from(var_num.get(block.instructions[i].arg)) + } + _ => continue, + }; + let Some(Instruction::BinaryOp { op }) = block.instructions[i + 2].instr.real() + else { + continue; + }; + if !matches!( + op.get(block.instructions[i + 2].arg), + oparg::BinaryOperator::InplaceAdd + | oparg::BinaryOperator::InplaceSubtract + | oparg::BinaryOperator::InplaceMultiply + | oparg::BinaryOperator::InplaceMatrixMultiply + | oparg::BinaryOperator::InplaceTrueDivide + | oparg::BinaryOperator::InplaceFloorDivide + | oparg::BinaryOperator::InplaceRemainder + | oparg::BinaryOperator::InplacePower + | oparg::BinaryOperator::InplaceLshift + | oparg::BinaryOperator::InplaceRshift + | oparg::BinaryOperator::InplaceAnd + | oparg::BinaryOperator::InplaceXor + | oparg::BinaryOperator::InplaceOr + ) { + continue; + } + if matches!( + block.instructions[i + 3].instr.real(), + Some(Instruction::StoreFast { var_num }) + if usize::from(var_num.get(block.instructions[i + 3].arg)) == local + ) { + return Some(i); + } + } + None + } + fn starts_with_borrowed_local_bool_guard(block: &Block, locals: &[usize]) -> bool { let mut reals = block .instructions @@ -8085,6 +8164,43 @@ impl CodeInfo { }) } + fn collect_unprotected_tail_segment( + blocks: &[Block], + tail: BlockIdx, + ) -> (Vec<(BlockIdx, usize)>, Vec) { + let mut in_segment = vec![false; blocks.len()]; + let mut segment = Vec::new(); + let mut cursor = tail; + if cursor == BlockIdx::NULL + || block_is_exceptional(&blocks[cursor.idx()]) + || block_has_protected_instructions(&blocks[cursor.idx()]) + { + return (segment, in_segment); + } + while cursor != BlockIdx::NULL { + let segment_block = &blocks[cursor.idx()]; + if block_is_exceptional(segment_block) + || block_has_protected_instructions(segment_block) + { + break; + } + segment.push((cursor, 0)); + in_segment[cursor.idx()] = true; + let last_real = segment_block + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()); + if last_real.is_some_and(|instr| { + instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() + }) { + break; + } + cursor = next_nonempty_block(blocks, segment_block.next); + } + (segment, in_segment) + } + let mut predecessors = vec![Vec::new(); self.blocks.len()]; for (pred_idx, block) in self.blocks.iter().enumerate() { if block.next != BlockIdx::NULL { @@ -8143,12 +8259,24 @@ impl CodeInfo { if same_block_tail_start.is_some() { continue; } + let tail = next_nonempty_block(&self.blocks, block.next); + let (segment, in_segment) = collect_unprotected_tail_segment(&self.blocks, tail); + if handler_chain_has_nested_exception_match(&self.blocks, block) + && handler_chain_can_resume_to_segment(&self.blocks, block, &in_segment) + { + for (block_idx, start) in &segment { + if let Some(update_start) = + borrowed_inplace_local_update_start(&self.blocks[block_idx.idx()]) + { + to_deopt.push((*block_idx, (*start).max(update_start))); + } + } + } let stored_locals = collect_stored_fast_locals_until(block, block.instructions.len()); if stored_locals.is_empty() { continue; } let handler_has_explicit_raise = handler_chain_has_explicit_raise(&self.blocks, block); - let tail = next_nonempty_block(&self.blocks, block.next); if tail != BlockIdx::NULL && !block_is_exceptional(&self.blocks[tail.idx()]) && !block_has_protected_instructions(&self.blocks[tail.idx()]) @@ -8177,38 +8305,6 @@ impl CodeInfo { } } } - let mut in_segment = vec![false; self.blocks.len()]; - let mut segment = Vec::new(); - let mut cursor = { - if tail == BlockIdx::NULL - || block_is_exceptional(&self.blocks[tail.idx()]) - || block_has_protected_instructions(&self.blocks[tail.idx()]) - { - continue; - } - tail - }; - while cursor != BlockIdx::NULL { - let segment_block = &self.blocks[cursor.idx()]; - if block_is_exceptional(segment_block) - || block_has_protected_instructions(segment_block) - { - break; - } - segment.push((cursor, 0)); - in_segment[cursor.idx()] = true; - let last_real = segment_block - .instructions - .iter() - .rev() - .find_map(|info| info.instr.real()); - if last_real.is_some_and(|instr| { - instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() - }) { - break; - } - cursor = next_nonempty_block(&self.blocks, segment_block.next); - } if segment.is_empty() || segment.iter().any(|(block_idx, _)| { contains_debug_four_guard(&self.blocks[block_idx.idx()], &self.metadata.names) From d37b510f90a1eb238f5b97f2401a26f1ba7af2c2 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 02:55:44 +0900 Subject: [PATCH 26/76] Align terminal handler borrow deopts with CPython CFG --- crates/codegen/src/compile.rs | 113 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 84 ++++++++++++++++++++----- 2 files changed, 180 insertions(+), 17 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index b718ae5dc8..09da045aeb 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -15318,6 +15318,57 @@ def f(): ); } + #[test] + fn test_try_import_continue_handler_deopts_loop_tail_borrow() { + let code = compile_exec( + "\ +def f(size): + pos = 0 + while pos < size: + try: + import unicodedata + except ImportError: + continue + if pos < size: + pos += 1 + return pos +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let import_idx = ops + .iter() + .position(|op| matches!(op, Instruction::ImportName { .. })) + .expect("missing IMPORT_NAME"); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &ops[import_idx + 1..handler_start]; + let return_idx = normal_tail + .iter() + .position(|op| matches!(op, Instruction::ReturnValue)) + .unwrap_or(normal_tail.len()); + let loop_tail = &normal_tail[..return_idx.saturating_sub(1)]; + + assert!( + !loop_tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "CPython keeps strong LOAD_FAST ops in the loop tail after protected import with continue handler, got tail={loop_tail:?}", + ); + } + #[test] fn test_try_import_pass_else_keeps_borrow() { let code = compile_exec( @@ -24214,6 +24265,68 @@ def f(curr, decoded_append, packI, curr_clear, Error): ); } + #[test] + fn test_terminal_except_following_if_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(s): + try: + t = s[1:] + d = g(s) + except ValueError: + raise ValueError('bad') from None + if t: + try: + a, b, c = h(t) + except ValueError: + raise ValueError('bad') from None + else: + if b: + x, y, z = d + if y <= 12 and z <= q(x, y): + z += 1 + d = [x, y, z] + else: + a = [0] + return k(*(d + a)) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail_start = ops + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } + if f.varnames[usize::from( + var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))) + )] == "t" + ) + }) + .expect("missing post-try t load"); + let tail = &ops[tail_start..handler_start]; + + assert!( + !tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "terminal except following conditional tail should use CPython-style strong LOAD_FAST ops, got tail={tail:?}" + ); + } + #[test] fn test_protected_method_call_after_terminal_except_tail_uses_strong_loads() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d283af3a85..708a0cfebe 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5998,10 +5998,7 @@ impl CodeInfo { continue; } let block = &self.blocks[block_idx.idx()]; - if block_is_exceptional(block) - || block.cold - || block_has_protected_instructions(block) - { + if block_is_exceptional(block) || block.cold { continue; } visited[block_idx.idx()] = true; @@ -7612,6 +7609,46 @@ impl CodeInfo { false } + fn handler_chain_continues_to_or_before( + blocks: &[Block], + handler_block: BlockIdx, + seed: BlockIdx, + block_order: &[u32], + ) -> bool { + let mut cursor = handler_block; + let mut visited = vec![false; blocks.len()]; + let mut after_pop_except = false; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + for info in &blocks[cursor.idx()].instructions { + match info.instr.real() { + Some(Instruction::PopExcept) => after_pop_except = true, + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ) if after_pop_except + && info.target != BlockIdx::NULL + && block_order[info.target.idx()] <= block_order[seed.idx()] => + { + return true; + } + Some(_) if after_pop_except && is_conditional_jump(&info.instr) => { + return false; + } + Some(instr) + if after_pop_except + && (instr.is_unconditional_jump() || instr.is_scope_exit()) => + { + return false; + } + _ => {} + } + } + cursor = blocks[cursor.idx()].next; + } + false + } + fn block_has_protected_instructions(block: &Block) -> bool { block .instructions @@ -7631,6 +7668,15 @@ impl CodeInfo { } } + let mut block_order = vec![u32::MAX; self.blocks.len()]; + let mut cursor = BlockIdx(0); + let mut pos = 0u32; + while cursor != BlockIdx::NULL { + block_order[cursor.idx()] = pos; + pos += 1; + cursor = self.blocks[cursor.idx()].next; + } + let seeds: Vec<_> = self.blocks .iter() @@ -7648,24 +7694,25 @@ impl CodeInfo { .is_some_and(|handler| { handler_chain_returns(&self.blocks, handler.handler_block) }); - if !handler_returns { + let handler_continues = block.instructions[import_idx] + .except_handler + .is_some_and(|handler| { + handler_chain_continues_to_or_before( + &self.blocks, + handler.handler_block, + BlockIdx::new(idx as u32), + &block_order, + ) + }); + if !handler_returns && !handler_continues { return None; } - Some((BlockIdx::new(idx as u32), import_idx)) + Some((BlockIdx::new(idx as u32), import_idx, handler_continues)) }) .collect(); - let mut block_order = vec![u32::MAX; self.blocks.len()]; - let mut cursor = BlockIdx(0); - let mut pos = 0u32; - while cursor != BlockIdx::NULL { - block_order[cursor.idx()] = pos; - pos += 1; - cursor = self.blocks[cursor.idx()].next; - } - let mut visited = vec![false; self.blocks.len()]; - for (seed, import_idx) in seeds { + for (seed, import_idx, handler_continues) in seeds { let mut protected_store_locals = vec![false; self.metadata.varnames.len()]; for info in self.blocks[seed.idx()] .instructions @@ -7688,7 +7735,9 @@ impl CodeInfo { let mut segment = vec![(seed, import_idx + 1)]; let mut cursor = self.blocks[seed.idx()].next; while cursor != BlockIdx::NULL && !block_is_exceptional(&self.blocks[cursor.idx()]) { - if block_has_protected_instructions(&self.blocks[cursor.idx()]) { + if !handler_continues + && block_has_protected_instructions(&self.blocks[cursor.idx()]) + { break; } if predecessors[cursor.idx()].iter().any(|pred| { @@ -7704,6 +7753,7 @@ impl CodeInfo { .instructions .iter() .any(|info| info.instr.real().is_some_and(|instr| instr.is_scope_exit())) + && !handler_continues { break; } From ac086bf2f19e4a62ffa60dee4df534ca9df8a237 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 03:19:45 +0900 Subject: [PATCH 27/76] Refine borrow deopts for reraise handler continuations --- crates/codegen/src/compile.rs | 81 +++++++++ crates/codegen/src/ir.rs | 298 +++++++++++++++++++++++++++++----- 2 files changed, 341 insertions(+), 38 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 09da045aeb..0ffa1a5470 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -24327,6 +24327,87 @@ def f(s): ); } + #[test] + fn test_bare_except_internal_condition_keeps_try_body_borrows() { + let code = compile_exec( + "\ +def f(buffering, raw, binary, result, BufferedReader): + try: + line_buffering = False + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): + buffering = -1 + line_buffering = True + if buffering < 0: + buffering = max(min(raw._blksize, 8192 * 1024), 8192) + if buffering < 0: + raise ValueError('invalid buffering size') + if buffering == 0: + if binary: + return result + raise ValueError(\"can't have unbuffered text I/O\") + buffer = BufferedReader(raw, buffering) + result = buffer + if binary: + return result + return result + except: + result.close() + raise +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_path = &ops[..handler_start]; + + let borrows_name = |name: &str| { + warm_path.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == name || f.varnames[usize::from(right)] == name + } + _ => false, + }) + }; + let strong_loads_name = |name: &str| { + warm_path.iter().any(|unit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + Instruction::LoadFastLoadFast { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == name || f.varnames[usize::from(right)] == name + } + _ => false, + }) + }; + + for name in ["buffering", "raw", "binary", "result", "BufferedReader"] { + assert!( + borrows_name(name), + "CPython keeps {name} borrowed inside the same bare-except protected region, got warm_path={warm_path:?}" + ); + assert!( + !strong_loads_name(name), + "same protected-region conditional tail should not be terminal-except deoptimized for {name}, got warm_path={warm_path:?}" + ); + } + } + #[test] fn test_protected_method_call_after_terminal_except_tail_uses_strong_loads() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 708a0cfebe..61f7db76a8 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4950,22 +4950,89 @@ impl CodeInfo { false } - fn block_has_nonresuming_reraise_handler(blocks: &[Block], block: &Block) -> bool { - let mut seen_handlers = Vec::new(); + fn nonresuming_reraise_handlers(blocks: &[Block], block: &Block) -> Vec { + let mut handlers = Vec::new(); for handler in block .instructions .iter() .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) { - if seen_handlers.contains(&handler) { + if handlers.contains(&handler) { continue; } - seen_handlers.push(handler); if handler_chain_has_explicit_reraise(blocks, handler) && !handler_chain_resumes_normally(blocks, handler) { + handlers.push(handler); + } + } + handlers + } + + fn normal_successors(block: &Block) -> Vec { + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if matches!( + last.instr.real(), + Some( + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + fn normal_path_reaches_handler( + blocks: &[Block], + start: BlockIdx, + handler: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + if block.instructions.iter().any(|info| { + info.except_handler + .is_some_and(|except_handler| except_handler.handler_block == handler) + }) { return true; } + stack.extend(normal_successors(block)); } false } @@ -5010,8 +5077,15 @@ impl CodeInfo { if block_jumps_backward_to(pred_block, BlockIdx::new(idx as u32)) { continue; } - follows_protected_body[idx] = - block_has_nonresuming_reraise_handler(&self.blocks, pred_block); + let handlers = nonresuming_reraise_handlers(&self.blocks, pred_block); + follows_protected_body[idx] = !handlers.is_empty() + && !handlers.iter().any(|handler| { + normal_path_reaches_handler( + &self.blocks, + BlockIdx::new(idx as u32), + *handler, + ) + }); break; } if !block_is_exceptional(pred_block) @@ -5346,6 +5420,94 @@ impl CodeInfo { false } + fn nonresuming_handlers(blocks: &[Block], block: &Block) -> Vec { + let mut handlers = Vec::new(); + for handler in block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + { + if handlers.contains(&handler) { + continue; + } + if handler_chain_has_explicit_raise(blocks, handler) + && !handler_chain_has_multiple_handled_returns(blocks, handler) + && !handler_chain_resumes_normally(blocks, handler) + { + handlers.push(handler); + } + } + handlers + } + + fn normal_successors(block: &Block) -> Vec { + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if matches!( + last.instr.real(), + Some( + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + fn normal_path_reaches_handler( + blocks: &[Block], + start: BlockIdx, + handler: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + if block.instructions.iter().any(|info| { + info.except_handler + .is_some_and(|except_handler| except_handler.handler_block == handler) + }) { + return true; + } + stack.extend(normal_successors(block)); + } + false + } + let mut predecessors = vec![Vec::new(); self.blocks.len()]; let mut is_handler_resume_block = vec![false; self.blocks.len()]; for (pred_idx, block) in self.blocks.iter().enumerate() { @@ -5378,6 +5540,17 @@ impl CodeInfo { && block_has_exception_match_handler(&self.blocks, pred_block) && block_has_nonresuming_exception_match_handler(&self.blocks, pred_block) }); + let prev_nonresuming_handlers: Vec<_> = predecessors[idx] + .iter() + .flat_map(|pred| { + let pred_block = &self.blocks[pred.idx()]; + if block_has_protected_instructions(pred_block) { + nonresuming_handlers(&self.blocks, pred_block) + } else { + Vec::new() + } + }) + .collect(); let has_unprotected_normal_predecessor = predecessors[idx].iter().any(|pred| { let pred_block = &self.blocks[pred.idx()]; !block_is_exceptional(pred_block) @@ -5405,9 +5578,13 @@ impl CodeInfo { } else { prev_protected }; + let same_handler_continuation = prev_nonresuming_handlers + .iter() + .any(|handler| normal_path_reaches_handler(&self.blocks, seed, *handler)); (!block_is_exceptional(block) && seed != BlockIdx::NULL && seed_enabled + && !same_handler_continuation && !has_unprotected_normal_predecessor) .then_some((seed, force_deopt)) }) @@ -5767,25 +5944,37 @@ impl CodeInfo { && !handler_chain_resumes_normally(blocks, handler) } - fn trailing_protected_tail_has_terminal_exception_handler( + fn trailing_protected_tail_terminal_exception_handler( blocks: &[Block], block: &Block, - ) -> bool { + ) -> Option { for info in block.instructions.iter().rev() { match info.instr.real() { Some(Instruction::Nop | Instruction::NotTaken | Instruction::PopTop) => {} Some(_) => { - let Some(handler) = - info.except_handler.map(|handler| handler.handler_block) - else { - return false; - }; - return handler_is_terminal_exception_handler(blocks, handler); + let handler = info.except_handler.map(|handler| handler.handler_block)?; + return handler_is_terminal_exception_handler(blocks, handler) + .then_some(handler); } None => {} } } - false + None + } + + fn protected_tail_ends_with_conditional(block: &Block) -> bool { + block + .instructions + .iter() + .rev() + .filter_map(|info| info.instr.real()) + .find(|instr| { + !matches!( + instr, + Instruction::Nop | Instruction::NotTaken | Instruction::PopTop + ) + }) + .is_some_and(|instr| is_conditional_jump(&AnyInstruction::Real(instr))) } fn normal_successors(block: &Block) -> Vec { @@ -5829,6 +6018,33 @@ impl CodeInfo { .collect() } + fn normal_path_reaches_handler( + blocks: &[Block], + start: BlockIdx, + handler: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + if block.instructions.iter().any(|info| { + info.except_handler + .is_some_and(|except_handler| except_handler.handler_block == handler) + }) { + return true; + } + stack.extend(normal_successors(block)); + } + false + } + fn has_call_store_before_trailing_conditional(block: &Block) -> bool { let Some(cond_idx) = trailing_conditional_jump_index(block) else { return false; @@ -5969,12 +6185,18 @@ impl CodeInfo { seen[pred.idx()] = true; let pred_block = &self.blocks[pred.idx()]; if block_has_protected_instructions(pred_block) { - if trailing_protected_tail_has_terminal_exception_handler( - &self.blocks, - pred_block, - ) { + if protected_tail_ends_with_conditional(pred_block) { + break; + } + if let Some(handler) = + trailing_protected_tail_terminal_exception_handler(&self.blocks, pred_block) + { + let seed = BlockIdx::new(idx as u32); + if normal_path_reaches_handler(&self.blocks, seed, handler) { + break; + } seeds.push(( - BlockIdx::new(idx as u32), + seed, (has_protected_call_predecessor || has_call_store_tail) && !has_structured_terminal_tail_shape, )); @@ -11003,10 +11225,10 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { break; } seen[scan.idx()] = true; - if !blocks[scan.idx()].instructions.is_empty() { - if first_nonempty_between == BlockIdx::NULL { - first_nonempty_between = scan; - } + if !blocks[scan.idx()].instructions.is_empty() + && first_nonempty_between == BlockIdx::NULL + { + first_nonempty_between = scan; } scan = blocks[scan.idx()].next; } @@ -11386,20 +11608,20 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { blocks[current.idx()] .instructions .extend(blocks[target.idx()].instructions.clone()); - if no_lineno_no_fallthrough && removed_jump_kind == Some(JumpThreadKind::Plain) { - if let Some(last) = blocks[current.idx()].instructions.last_mut() - && jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) - { - last.instr = match last.instr.into() { - AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { - PseudoOpcode::Jump.into() - } - AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => { - Opcode::JumpBackward.into() - } - _ => last.instr, - }; - } + if no_lineno_no_fallthrough + && removed_jump_kind == Some(JumpThreadKind::Plain) + && let Some(last) = blocks[current.idx()].instructions.last_mut() + && jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) + { + last.instr = match last.instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { + PseudoOpcode::Jump.into() + } + AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => { + Opcode::JumpBackward.into() + } + _ => last.instr, + }; } changes = true; } From 644d3e35e5a2ef87e6a11567c3b6fcfd2416a273 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 03:45:58 +0900 Subject: [PATCH 28/76] Refine try-else terminal handler borrow deopts --- crates/codegen/src/compile.rs | 165 +++++++++++++++++++++++++++++++++- crates/codegen/src/ir.rs | 28 +++++- 2 files changed, 191 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 0ffa1a5470..f889659570 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -499,6 +499,12 @@ impl Compiler { } } + fn mark_try_else_orelse_entry_block(&mut self, block: BlockIdx) { + if block != BlockIdx::NULL { + self.current_code_info().blocks[block.idx()].try_else_orelse_entry = true; + } + } + fn new(opts: CompileOpts, source_file: SourceFile, code_name: String) -> Self { let module_code = ir::CodeInfo { // CPython convention: top-level module / interactive / @@ -3789,6 +3795,12 @@ impl Compiler { }) ) }); + let has_terminal_raise_handlers = handlers.iter().all(|handler| { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = + handler; + body.last() + .is_some_and(|stmt| matches!(stmt, ast::Stmt::Raise(_))) + }); if has_bare_except { self.disable_load_fast_borrow_for_block(end_block); } @@ -3806,6 +3818,11 @@ impl Compiler { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); self.remove_last_no_location_nop(); + if !orelse.is_empty() && has_terminal_raise_handlers { + let orelse_block = self.new_block(); + self.switch_to_block(orelse_block); + self.mark_try_else_orelse_entry_block(orelse_block); + } self.compile_statements(orelse)?; emit!( self, @@ -11890,7 +11907,10 @@ mod ruff_tests { #[cfg(test)] mod tests { use super::*; - use rustpython_compiler_core::{SourceFileBuilder, bytecode::OpArg}; + use rustpython_compiler_core::{ + SourceFileBuilder, + bytecode::{CodeUnit, OpArg}, + }; fn assert_scope_exit_locations(code: &CodeObject) { for (instr, (location, _)) in code.instructions.iter().zip(code.locations.iter()) { @@ -24408,6 +24428,149 @@ def f(buffering, raw, binary, result, BufferedReader): } } + #[test] + fn test_try_except_else_terminal_handler_conditional_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(self, pos, whence): + try: + pos_index = pos.__index__ + except AttributeError: + raise TypeError(f'{pos!r} is not an integer') + else: + pos = pos_index() + if whence == 0: + if pos < 0: + raise ValueError(f'negative {pos!r}') + self._pos = pos + elif whence == 1: + self._pos = max(0, self._pos + pos) + return self._pos +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_path = &ops[..handler_start]; + + let op_mentions_name = |unit: &CodeUnit, name: &str| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + Instruction::LoadFastLoadFast { var_nums } + | Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == name || f.varnames[usize::from(right)] == name + } + _ => false, + }; + let is_borrow_for_name = |unit: &CodeUnit, name: &str| match unit.op { + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } => op_mentions_name(unit, name), + _ => false, + }; + let is_strong_for_name = |unit: &CodeUnit, name: &str| match unit.op { + Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. } => { + op_mentions_name(unit, name) + } + _ => false, + }; + + assert!( + warm_path + .iter() + .any(|unit| is_borrow_for_name(unit, "pos_index")), + "try-else call should remain borrowed before the terminal-handler tail, got warm_path={warm_path:?}" + ); + + let tail_start = warm_path + .iter() + .position(|unit| { + is_strong_for_name(unit, "whence") || is_borrow_for_name(unit, "whence") + }) + .expect("missing post-try conditional tail"); + let post_try_tail = &warm_path[tail_start..]; + for name in ["whence", "pos", "self"] { + assert!( + post_try_tail + .iter() + .any(|unit| is_strong_for_name(unit, name)), + "terminal except conditional tail should use strong loads for {name}, got tail={post_try_tail:?}" + ); + assert!( + !post_try_tail + .iter() + .any(|unit| is_borrow_for_name(unit, name)), + "terminal except conditional tail should not borrow {name}, got tail={post_try_tail:?}" + ); + } + } + + #[test] + fn test_try_except_else_internal_conditional_keeps_borrowed_loads() { + let code = compile_exec( + "\ +def f(pos): + try: + pos_index = pos.__index__ + except AttributeError: + raise TypeError(f'{pos!r} is not an integer') + else: + pos = pos_index() + if pos < 0: + raise ValueError(f'negative {pos!r}') +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_path = &ops[..handler_start]; + + let mentions_pos = |unit: &CodeUnit| match unit.op { + Instruction::LoadFast { var_num } + | Instruction::LoadFastBorrow { var_num } + | Instruction::StoreFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == "pos" + } + _ => false, + }; + let else_store = warm_path + .iter() + .position(|unit| matches!(unit.op, Instruction::StoreFast { .. }) && mentions_pos(unit)) + .expect("missing try-else pos store"); + let else_tail = &warm_path[else_store + 1..]; + + assert!( + else_tail.iter().any(|unit| { + matches!(unit.op, Instruction::LoadFastBorrow { .. }) && mentions_pos(unit) + }), + "try-else internal conditional should keep borrowed pos loads, got tail={else_tail:?}" + ); + assert!( + !else_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. }) && mentions_pos(unit)), + "try-else internal conditional should not deopt pos loads, got tail={else_tail:?}" + ); + } + #[test] fn test_protected_method_call_after_terminal_except_tail_uses_strong_loads() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 61f7db76a8..0c70b37bdd 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -194,6 +194,8 @@ pub struct Block { pub cold: bool, /// Whether LOAD_FAST borrow optimization should be suppressed for this block. pub disable_load_fast_borrow: bool, + /// Entry block for a try-else orelse suite split after POP_BLOCK. + pub try_else_orelse_entry: bool, } impl Default for Block { @@ -206,6 +208,7 @@ impl Default for Block { start_depth: None, cold: false, disable_load_fast_borrow: false, + try_else_orelse_entry: false, } } } @@ -5534,6 +5537,9 @@ impl CodeInfo { .enumerate() .filter_map(|(idx, block)| { let cond_idx = trailing_conditional_jump_index(block)?; + if block.try_else_orelse_entry { + return None; + } let prev_protected = predecessors[idx].iter().any(|pred| { let pred_block = &self.blocks[pred.idx()]; block_has_protected_instructions(pred_block) @@ -6119,6 +6125,15 @@ impl CodeInfo { has_pop_except && jumps_to_target } + fn is_unprotected_call_store_bridge_to(block: &Block, successor: BlockIdx) -> bool { + !block_is_exceptional(block) + && !block.cold + && !block_has_protected_instructions(block) + && has_call_and_store(block) + && trailing_conditional_jump_index(block).is_none() + && normal_successors(block).contains(&successor) + } + let mut predecessors = vec![Vec::new(); self.blocks.len()]; for (pred_idx, block) in self.blocks.iter().enumerate() { if block.next != BlockIdx::NULL { @@ -6156,6 +6171,7 @@ impl CodeInfo { && block.start_depth.is_some_and(|depth| depth > 0) && !has_protected_call_predecessor && !has_call_store_tail) + || block.try_else_orelse_entry || predecessors[idx].iter().any(|pred| { is_handler_resume_predecessor( &self.blocks[pred.idx()], @@ -6168,6 +6184,10 @@ impl CodeInfo { && !pred_block.cold && !block_has_protected_instructions(pred_block) && block_has_non_nop_real_instructions(pred_block) + && !is_unprotected_call_store_bridge_to( + pred_block, + BlockIdx::new(idx as u32), + ) }) || !(has_structured_terminal_tail_shape || has_protected_call_predecessor @@ -6203,6 +6223,10 @@ impl CodeInfo { } break; } + if is_unprotected_call_store_bridge_to(pred_block, BlockIdx::new(idx as u32)) { + stack.extend(predecessors[pred.idx()].iter().copied()); + continue; + } if !block_is_exceptional(pred_block) && !pred_block.cold && !block_has_non_nop_real_instructions(pred_block) @@ -6225,7 +6249,9 @@ impl CodeInfo { } visited[block_idx.idx()] = true; let successors = normal_successors(&self.blocks[block_idx.idx()]); - deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + if !self.blocks[block_idx.idx()].try_else_orelse_entry { + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } if direct_only { continue; } From 7adc9f419750aaa7623bed27e8b8a0d93ec627ba Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 04:01:27 +0900 Subject: [PATCH 29/76] Refine protected tail borrow parity --- crates/codegen/src/compile.rs | 218 +++++++++++++++++++++++++++++++--- crates/codegen/src/ir.rs | 87 ++++++++++++++ 2 files changed, 287 insertions(+), 18 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index f889659570..feee56117b 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -646,6 +646,22 @@ impl Compiler { }) } + fn statements_end_with_conditional_scope_exit(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| match stmt { + ast::Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) => { + Self::statements_end_with_scope_exit(body) + || elif_else_clauses + .iter() + .any(|clause| Self::statements_end_with_scope_exit(&clause.body)) + } + _ => false, + }) + } + fn preserves_finally_entry_nop(body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| match stmt { ast::Stmt::Try(ast::StmtTry { @@ -5878,6 +5894,7 @@ impl Compiler { let preserve_outer_cleanup_target_nop = !is_async && (Self::statements_end_with_with_cleanup_scope_exit(body) + || Self::statements_end_with_conditional_scope_exit(body) || self.current_block_has_terminal_with_suppress_exit_predecessor()); // Pop fblock before normal exit. CPython emits this POP_BLOCK with @@ -23715,6 +23732,145 @@ def f(self, cm, E): ); } + #[test] + fn test_with_named_except_return_value_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, b, BlockingIOError): + with self._write_lock: + written = len(self._write_buf) + if len(self._write_buf) > self.buffer_size: + try: + self._flush_unlocked() + except BlockingIOError as e: + if len(self._write_buf) > self.buffer_size: + overage = len(self._write_buf) - self.buffer_size + written -= overage + raise BlockingIOError(e.errno, e.strerror, written) + return written +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let with_exit_start = instructions + .windows(3) + .position(|window| { + matches!( + window, + [ + CodeUnit { + op: Instruction::Swap { .. }, + .. + }, + CodeUnit { + op: Instruction::Swap { .. }, + .. + }, + CodeUnit { + op: Instruction::LoadConst { .. }, + .. + }, + ] + ) + }) + .expect("missing with-exit cleanup"); + let return_value_load = instructions + .get(with_exit_start.saturating_sub(1)) + .expect("missing return value load"); + let arg = OpArg::new(u32::from(u8::from(return_value_load.arg))); + + assert!( + matches!( + return_value_load.op, + Instruction::LoadFastBorrow { var_num } + if f.varnames[usize::from(var_num.get(arg))] == "written" + ), + "return value loaded through with-exit cleanup should keep borrowed written, got instructions={instructions:?}" + ); + } + + #[test] + fn test_with_final_conditional_return_preserves_fallthrough_cleanup_nop() { + let code = compile_exec( + "\ +def f(self): + with self.lock: + if self.raw is None or self.closed: + return + self.flush() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .map(|unit| unit.op) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "with fallthrough cleanup should preserve the CPython POP_BLOCK NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_named_except_conditional_reraise_final_store_attr_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, fd, OSError, errno): + try: + os.lseek(fd, 0, SEEK_END) + except OSError as e: + if e.errno != errno.ESPIPE: + raise + self._fd = fd +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let store_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::StoreAttr { namei } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.names[usize::try_from(namei.get(arg)).unwrap()].as_str() == "_fd" + } + _ => false, + }) + .expect("missing _fd store"); + + assert!( + matches!( + instructions + .get(store_attr.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) + ), + "conditional reraise named-except final store tail should keep borrowed pair loads, got instructions={instructions:?}" + ); + } + #[test] fn test_with_except_else_with_resume_loop_tail_uses_strong_loads() { let code = compile_exec( @@ -24516,18 +24672,25 @@ def f(self, pos, whence): } #[test] - fn test_try_except_else_internal_conditional_keeps_borrowed_loads() { + fn test_try_except_else_outer_join_keeps_borrowed_loads() { let code = compile_exec( "\ -def f(pos): - try: - pos_index = pos.__index__ - except AttributeError: - raise TypeError(f'{pos!r} is not an integer') +def f(self, pos=None): + if self.closed: + raise ValueError('closed') + if pos is None: + pos = self._pos else: - pos = pos_index() + try: + pos_index = pos.__index__ + except AttributeError: + raise TypeError(f'{pos!r} is not an integer') + else: + pos = pos_index() if pos < 0: raise ValueError(f'negative {pos!r}') + del self._buffer[pos:] + return pos ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -24542,32 +24705,51 @@ def f(pos): .expect("missing handler entry"); let warm_path = &ops[..handler_start]; - let mentions_pos = |unit: &CodeUnit| match unit.op { + let mentions_name = |unit: &CodeUnit, name: &str| match unit.op { Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } | Instruction::StoreFast { var_num } => { let arg = OpArg::new(u32::from(u8::from(unit.arg))); - f.varnames[usize::from(var_num.get(arg))] == "pos" + f.varnames[usize::from(var_num.get(arg))] == name } _ => false, }; let else_store = warm_path .iter() - .position(|unit| matches!(unit.op, Instruction::StoreFast { .. }) && mentions_pos(unit)) + .position(|unit| { + matches!(unit.op, Instruction::StoreFast { .. }) && mentions_name(unit, "pos") + }) .expect("missing try-else pos store"); - let else_tail = &warm_path[else_store + 1..]; + let delete_subscr = warm_path + .iter() + .position(|unit| matches!(unit.op, Instruction::DeleteSubscr)) + .expect("missing join delete"); + let else_tail = &warm_path[else_store + 1..delete_subscr]; assert!( - else_tail.iter().any(|unit| { - matches!(unit.op, Instruction::LoadFastBorrow { .. }) && mentions_pos(unit) - }), - "try-else internal conditional should keep borrowed pos loads, got tail={else_tail:?}" + else_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. }) + && mentions_name(unit, "pos")), + "terminal try-else conditional should use strong pos loads, got tail={else_tail:?}" ); + + let join_tail = &warm_path[delete_subscr.saturating_sub(6)..]; + for name in ["self", "pos"] { + assert!( + join_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. }) + && mentions_name(unit, name)), + "outer if/else join should keep borrowed {name} loads, got tail={join_tail:?}" + ); + } assert!( - !else_tail + !join_tail .iter() - .any(|unit| matches!(unit.op, Instruction::LoadFast { .. }) && mentions_pos(unit)), - "try-else internal conditional should not deopt pos loads, got tail={else_tail:?}" + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. }) + && (mentions_name(unit, "self") || mentions_name(unit, "pos"))), + "outer if/else join should not inherit terminal try-else deopts, got tail={join_tail:?}" ); } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 0c70b37bdd..370cbfadd3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4532,6 +4532,60 @@ impl CodeInfo { false } + fn is_return_value_through_with_exit(block: &Block) -> bool { + let reals: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .collect(); + if !matches!( + reals.first(), + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) { + return false; + } + if !matches!(reals.last(), Some(Instruction::ReturnValue)) { + return false; + } + reals.iter().skip(1).all(|instr| { + matches!( + instr, + Instruction::Swap { .. } + | Instruction::LoadConst { .. } + | Instruction::Call { .. } + | Instruction::PopTop + | Instruction::ReturnValue + ) + }) + } + + fn is_simple_store_attr_tail(block: &Block) -> bool { + let reals: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + .collect(); + matches!( + reals.as_slice(), + [ + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::StoreAttr { .. }, + .. + ] | [ + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. }, + Instruction::StoreAttr { .. }, + .. + ] + ) + } + fn leading_bool_guard_has_scope_exit_successor(blocks: &[Block], block: &Block) -> bool { let Some(cond_idx) = trailing_conditional_jump_index(block) else { return false; @@ -4779,6 +4833,14 @@ impl CodeInfo { if !requires_deopt && !is_same_guard_fallback { continue; } + if requires_deopt + && is_return_value_through_with_exit(&self.blocks[block_idx.idx()]) + { + continue; + } + if requires_deopt && is_simple_store_attr_tail(&self.blocks[block_idx.idx()]) { + continue; + } if block_idx != seed && !is_same_guard_fallback && predecessors[block_idx.idx()].iter().any(|pred| { @@ -6238,6 +6300,20 @@ impl CodeInfo { let mut visited = vec![false; self.blocks.len()]; for (seed, direct_only) in seeds { + let mut reachable_from_seed = vec![false; self.blocks.len()]; + let mut reachability_stack = vec![seed]; + while let Some(block_idx) = reachability_stack.pop() { + if block_idx == BlockIdx::NULL || reachable_from_seed[block_idx.idx()] { + continue; + } + let block = &self.blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + reachable_from_seed[block_idx.idx()] = true; + reachability_stack.extend(normal_successors(block)); + } + let mut stack = vec![seed]; while let Some(block_idx) = stack.pop() { if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { @@ -6247,6 +6323,17 @@ impl CodeInfo { if block_is_exceptional(block) || block.cold { continue; } + if block_idx != seed + && predecessors[block_idx.idx()].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + !block_is_exceptional(pred_block) + && !pred_block.cold + && !reachable_from_seed[pred.idx()] + && normal_successors(pred_block).contains(&block_idx) + }) + { + continue; + } visited[block_idx.idx()] = true; let successors = normal_successors(&self.blocks[block_idx.idx()]); if !self.blocks[block_idx.idx()].try_else_orelse_entry { From 33ec4a630d5d9e683e112da2b9f1b6ff81849317 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 04:25:05 +0900 Subject: [PATCH 30/76] Refine exception borrow deopt parity --- crates/codegen/src/compile.rs | 88 +++++++++++++++++++++++++---------- crates/codegen/src/ir.rs | 38 ++++++++++++++- 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index feee56117b..0c1d2295eb 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -662,6 +662,17 @@ impl Compiler { }) } + fn has_resuming_bare_except(handlers: &[ast::ExceptHandler]) -> bool { + handlers.iter().any(|handler| { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + body, + .. + }) = handler; + type_.is_none() && !Self::statements_end_with_scope_exit(body) + }) + } + fn preserves_finally_entry_nop(body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| match stmt { ast::Stmt::Try(ast::StmtTry { @@ -3404,16 +3415,7 @@ impl Compiler { // End block - continuation point after try-finally // Normal path jumps here to skip exception path blocks let end_block = self.new_block(); - let has_bare_except = handlers.iter().any(|handler| { - matches!( - handler, - ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { - type_: None, - .. - }) - ) - }); - if has_bare_except { + if Self::has_resuming_bare_except(handlers) { self.disable_load_fast_borrow_for_block(end_block); } @@ -3802,22 +3804,13 @@ impl Compiler { let handler_block = self.new_block(); let cleanup_block = self.new_block(); let end_block = self.new_block(); - let has_bare_except = handlers.iter().any(|handler| { - matches!( - handler, - ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { - type_: None, - .. - }) - ) - }); let has_terminal_raise_handlers = handlers.iter().all(|handler| { let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = handler; body.last() .is_some_and(|stmt| matches!(stmt, ast::Stmt::Raise(_))) }); - if has_bare_except { + if Self::has_resuming_bare_except(handlers) { self.disable_load_fast_borrow_for_block(end_block); } @@ -23834,12 +23827,29 @@ def f(self): fn test_named_except_conditional_reraise_final_store_attr_keeps_borrow() { let code = compile_exec( "\ -def f(self, fd, OSError, errno): +def f(self, fd, file, closefd, owned_fd, OSError, AttributeError, errno, os, stat, _setmode): try: - os.lseek(fd, 0, SEEK_END) - except OSError as e: - if e.errno != errno.ESPIPE: - raise + self._closefd = closefd + self._stat_atopen = os.fstat(fd) + try: + if stat.S_ISDIR(self._stat_atopen.st_mode): + raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) + except AttributeError: + pass + if _setmode: + _setmode(fd, os.O_BINARY) + self.name = file + if self._appending: + try: + os.lseek(fd, 0, SEEK_END) + except OSError as e: + if e.errno != errno.ESPIPE: + raise + except: + self._stat_atopen = None + if owned_fd is not None: + os.close(owned_fd) + raise self._fd = fd ", ); @@ -23849,6 +23859,18 @@ def f(self, fd, OSError, errno): .iter() .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); + let load_attr_name = |unit: &&bytecode::CodeUnit, expected: &str| match unit.op { + Instruction::LoadAttr { namei } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let load_attr = namei.get(arg); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == expected + } + _ => false, + }; + let lseek_attr = instructions + .iter() + .position(|unit| load_attr_name(unit, "lseek")) + .expect("missing lseek load"); let store_attr = instructions .iter() .position(|unit| match unit.op { @@ -23860,6 +23882,22 @@ def f(self, fd, OSError, errno): }) .expect("missing _fd store"); + assert!( + matches!( + instructions + .get(lseek_attr.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "nested conditional reraise try body should keep borrowed os receiver, got instructions={instructions:?}" + ); + assert!( + matches!( + instructions.get(lseek_attr + 1).map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "nested conditional reraise try body should keep borrowed fd argument, got instructions={instructions:?}" + ); assert!( matches!( instructions diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 370cbfadd3..ac4a3748b9 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4393,7 +4393,6 @@ impl CodeInfo { for block_idx in &segment { in_segment[block_idx.idx()] = true; } - for block_idx in segment { if visited[block_idx.idx()] { continue; @@ -4822,7 +4821,6 @@ impl CodeInfo { for block_idx in &segment { in_segment[block_idx.idx()] = true; } - for block_idx in segment { if visited[block_idx.idx()] { continue; @@ -5130,6 +5128,15 @@ impl CodeInfo { if block_is_exceptional(block) || block.cold || !starts_with_fast_load(block) { continue; } + if block_has_protected_instructions(block) { + continue; + } + if predecessors[idx] + .iter() + .any(|pred| is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()])) + { + continue; + } let mut seen = vec![false; self.blocks.len()]; let mut stack = predecessors[idx].clone(); while let Some(pred) = stack.pop() { @@ -5173,6 +5180,13 @@ impl CodeInfo { if block_is_exceptional(block) || block.cold { break; } + if block_has_protected_instructions(block) + || predecessors[cursor.idx()].iter().any(|pred| { + is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()]) + }) + { + break; + } visited[cursor.idx()] = true; deoptimize_block_borrows(&mut self.blocks[cursor.idx()]); if self.blocks[cursor.idx()] @@ -5666,9 +5680,25 @@ impl CodeInfo { if block_is_exceptional(&self.blocks[cursor.idx()]) { break; } + if cursor != seed + && predecessors[cursor.idx()].iter().any(|pred| { + is_handler_resume_block[pred.idx()] + || is_handler_resume_predecessor(&self.blocks[pred.idx()], cursor) + || is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()]) + }) + { + break; + } segment.push(cursor); cursor = self.blocks[cursor.idx()].next; } + if segment.iter().any(|block_idx| { + predecessors[block_idx.idx()] + .iter() + .any(|pred| is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()])) + }) { + continue; + } let segment_ops: Vec<_> = segment .iter() @@ -5796,9 +5826,13 @@ impl CodeInfo { if predecessors[block_idx.idx()].iter().any(|pred| { is_handler_resume_block[pred.idx()] || is_handler_resume_predecessor(&self.blocks[pred.idx()], block_idx) + || is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()]) }) { continue; } + if block_has_protected_instructions(&self.blocks[block_idx.idx()]) { + continue; + } if !force_deopt && block_idx != seed && predecessors[block_idx.idx()] From de00801a54a9acce7d6ea60ec05295d56a146528 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 04:57:35 +0900 Subject: [PATCH 31/76] Align while loop CFG layout with CPython --- crates/codegen/src/compile.rs | 101 ++++++++++++++++++++++++++++++++-- crates/codegen/src/ir.rs | 55 +++++++++++++++--- 2 files changed, 145 insertions(+), 11 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 0c1d2295eb..7a3fa85c3a 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -5724,8 +5724,12 @@ impl Compiler { self.enter_conditional_block(); let while_block = self.new_block(); - let else_block = self.new_block(); let after_block = self.new_block(); + let else_block = if orelse.is_empty() { + after_block + } else { + self.new_block() + }; self.switch_to_block(while_block); self.push_fblock(FBlockType::WhileLoop, while_block, after_block)?; @@ -5744,7 +5748,9 @@ impl Compiler { self.pop_fblock(FBlockType::WhileLoop); self.compile_statements(orelse)?; - self.switch_to_block(after_block); + if !orelse.is_empty() { + self.switch_to_block(after_block); + } self.leave_conditional_block(); Ok(()) @@ -16854,6 +16860,49 @@ def f(): ); } + #[test] + fn test_try_except_while_body_preserves_while_exit_line_nop() { + let code = compile_exec( + "\ +def f(x, E): + try: + while x: + x -= 1 + except E: + if not x: + return None + assert x + return x +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let assertion_error = ops + .iter() + .position(|op| matches!(op, Instruction::LoadCommonConstant { .. })) + .expect("missing assertion error load"); + + assert!( + ops[..assertion_error].windows(4).any(|window| { + matches!( + window, + [ + Instruction::JumpBackward { .. }, + Instruction::Nop, + Instruction::LoadFastBorrow { .. }, + Instruction::ToBool, + ] + ) + }), + "try/except while body should preserve CPython while-exit NOP before following assert, got ops={ops:?}" + ); + } + #[test] fn test_named_except_cleanup_keeps_jump_over_cleanup_and_next_try() { let code = compile_exec( @@ -20446,7 +20495,7 @@ def f(xs): "\ def f(native, array): for k in native: - if not k in 'bBhHiIlLfd': + if k not in 'bBhHiIlLfd': del array[k] ", ); @@ -20464,7 +20513,7 @@ def f(native, array): window, [ Instruction::ContainsOp { .. }, - Instruction::PopJumpIfFalse { .. }, + Instruction::PopJumpIfTrue { .. }, Instruction::NotTaken, Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. }, @@ -20477,6 +20526,50 @@ def f(native, array): ); } + #[test] + fn test_while_implicit_continue_body_after_jumpback_for_boolop_call_arg() { + let code = compile_exec( + "\ +def f(source, state, verbose, nested): + items = [] + itemsappend = items.append + sourcematch = source.match + while True: + itemsappend(parse(source, state, verbose, nested + 1, + not nested and not items)) + if not sourcematch('|'): + break + if not nested: + verbose = state.flags & 64 + return verbose +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style while implicit-continue jumpback before conditional body, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index ac4a3748b9..73a71d255e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3002,6 +3002,44 @@ impl CodeInfo { ) }) .collect(); + let mut conditional_targets = vec![false; self.blocks.len()]; + for block in &self.blocks { + for instr in &block.instructions { + if instr.target != BlockIdx::NULL && is_conditional_jump(&instr.instr) { + let target = next_nonempty_block(&self.blocks, instr.target); + if target != BlockIdx::NULL { + conditional_targets[target.idx()] = true; + } + } + } + } + let preserve_loop_exit_pop_block_nops: Vec<_> = (0..self.blocks.len()) + .map(|idx| { + let block_idx = BlockIdx(idx as u32); + let block = &self.blocks[idx]; + let layout_pred = layout_predecessors[idx]; + block.instructions.first().is_some_and(|instr| { + matches!(instr.instr.real(), Some(Instruction::Nop)) + && instr.remove_no_location_nop + && instruction_lineno(instr) < 0 + && conditional_targets[idx] + && layout_pred != BlockIdx::NULL + && self.blocks[layout_pred.idx()] + .instructions + .last() + .is_some_and(|last| { + matches!( + last.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) && next_nonempty_block(&self.blocks, last.target) != block_idx + }) + }) + }) + .collect(); + for (block_idx, block) in self.blocks.iter_mut().enumerate() { let mut prev_line = None; let mut src = 0usize; @@ -3013,9 +3051,15 @@ impl CodeInfo { .get(block_idx) .copied() .unwrap_or(false); + let keep_loop_exit_pop_block = src == 0 + && preserve_loop_exit_pop_block_nops + .get(block_idx) + .copied() + .unwrap_or(false); if ins.remove_no_location_nop && instruction_lineno(ins) < 0 && !keep_target_start + && !keep_loop_exit_pop_block { break 'keep false; } @@ -4517,9 +4561,12 @@ impl CodeInfo { | Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } | Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } | Instruction::StoreName { .. } | Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastCheck { .. } | Instruction::LoadFastLoadFast { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } | Instruction::BuildTuple { .. }, @@ -13974,13 +14021,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec let body_start = next; let body = next_nonempty_block(blocks, body_start); let true_jump_loop_target = jump_back_target(blocks, true_jump); - let true_jump_targets_loop_header = true_jump_loop_target.is_some_and(|loop_target| { - blocks[loop_target.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) - }); - if true_jump_targets_loop_header + if true_jump_loop_target.is_some() && true_jump_start == true_jump && body_start != BlockIdx::NULL && body != BlockIdx::NULL From 309b93618e3dcd1f4b2e45aebf71d43e46525c2b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 05:19:55 +0900 Subject: [PATCH 32/76] Align loop backedge CFG with CPython --- crates/codegen/src/compile.rs | 171 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 19 ++-- 2 files changed, 184 insertions(+), 6 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 7a3fa85c3a..45ab9283c3 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20570,6 +20570,177 @@ def f(source, state, verbose, nested): ); } + #[test] + fn test_multiblock_elif_continue_keeps_next_test_before_backedge() { + let code = compile_exec( + "\ +def f(source, state, verbose, nested, subpatternappend, start, MAXGROUPS): + sourceget = source.get + sourcematch = source.match + while True: + this = source.next + sourceget() + if this == '(': + if sourcematch('?'): + char = sourceget() + if char in '=!<': + dir = 1 + if char == '<': + char = sourceget() + if char not in '=!': + raise source.error('unknown extension ?<' + char, len(char) + 2) + dir = -1 + lookbehindgroups = state.lookbehindgroups + if lookbehindgroups is None: + state.lookbehindgroups = state.groups + p = parse_sub(source, state, verbose, nested + 1) + if dir < 0: + if lookbehindgroups is None: + state.lookbehindgroups = None + if not sourcematch(')'): + raise source.error('missing ), unterminated subpattern', source.tell() - start) + if char == '=': + subpatternappend(('ASSERT', (dir, p))) + elif p: + subpatternappend(('ASSERT_NOT', (dir, p))) + else: + subpatternappend(('FAILURE', ())) + continue + elif char == '(': + condname = source.getuntil(')', 'group name') + if not (condname.isdecimal() and condname.isascii()): + source.checkgroupname(condname, 1) + condgroup = state.groupdict.get(condname) + if condgroup is None: + msg = 'unknown group name %r' % condname + raise source.error(msg, len(condname) + 1) + else: + condgroup = int(condname) + if not condgroup: + raise source.error('bad group number', len(condname) + 1) + if condgroup >= MAXGROUPS: + msg = 'invalid group reference %d' % condgroup + raise source.error(msg, len(condname) + 1) + state.checklookbehindgroup(condgroup, source) + item_yes = parse(source, state, verbose, nested + 1) + if source.match('|'): + item_no = parse(source, state, verbose, nested + 1) + if source.next == '|': + raise source.error('conditional backref with more than two branches') + else: + item_no = None + if not source.match(')'): + raise source.error('missing ), unterminated subpattern', source.tell() - start) + subpatternappend(('GROUPREF_EXISTS', (condgroup, item_yes, item_no))) + continue + elif char == '>': + capture = False + return +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(2).any(|window| { + matches!( + window, + [ + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style elif test between separate continue backedges, got ops={ops:?}" + ); + } + + #[test] + fn test_while_scope_exit_body_keeps_line_backedge_before_raise_body() { + let code = compile_exec( + "\ +FLAGS = {} +TYPE_FLAGS = 0 +GLOBAL_FLAGS = 0 + +def f(source, state, char): + sourceget = source.get + add_flags = 0 + del_flags = 0 + if char != '-': + while True: + flag = FLAGS[char] + add_flags |= flag + if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag: + msg = 'bad inline flags' + raise source.error(msg) + char = sourceget() + if char is None: + raise source.error('missing -, : or )') + if char in ')-:': + break + if char not in FLAGS: + msg = 'unknown flag' if char.isalpha() else 'missing -, : or )' + raise source.error(msg, len(char)) + if char == ')': + state.flags |= add_flags + return None + if add_flags & GLOBAL_FLAGS: + raise source.error('bad inline flags: cannot turn on global flag', 1) + if char == '-': + char = sourceget() + if char is None: + raise source.error('missing flag') + if char not in FLAGS: + msg = 'unknown flag' if char.isalpha() else 'missing flag' + raise source.error(msg, len(char)) + while True: + flag = FLAGS[char] + del_flags |= flag + char = sourceget() + if char is None: + raise source.error('missing :') + if char == ':': + break + if char not in FLAGS: + msg = 'unknown flag' if char.isalpha() else 'missing :' + raise source.error(msg, len(char)) + return add_flags, del_flags +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "expected CPython-style false-path loop backedge before raising body, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 73a71d255e..4b5dbd110e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -13717,6 +13717,10 @@ fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) || next_nonempty_block(blocks, loop_exit_start) == after_jump }) }); + let jump_has_lineno = blocks[jump_block.idx()] + .instructions + .first() + .is_some_and(|info| instruction_lineno(info) >= 0); if exit_start == BlockIdx::NULL || exit_block == BlockIdx::NULL || jump_start == BlockIdx::NULL @@ -13730,14 +13734,11 @@ fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) || block_is_protected(&blocks[jump_block.idx()]) || !is_scope_exit_block(&blocks[exit_block.idx()]) || !is_jump_back_only_block(blocks, jump_block) - || blocks[jump_block.idx()] - .instructions - .first() - .is_some_and(|info| info.lineno_override.is_some_and(|lineno| lineno >= 0)) || jumps_to_for_iter || (after_jump != BlockIdx::NULL && !blocks[after_jump.idx()].cold - && !jump_exits_to_loop_exit) + && !jump_exits_to_loop_exit + && !jump_has_lineno) || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block { current = next; @@ -14036,8 +14037,10 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec let after_jump = blocks[true_jump.idx()].next; let body_tail_is_conditional = trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); + let body_is_single_block = body == body_tail; let can_reorder = !body_tail_is_conditional - && (matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) + && ((body_is_single_block + && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) || body_segment_contains_for_iter(blocks, body, body_tail)); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { @@ -14663,6 +14666,10 @@ fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { layout_pred = blocks[layout_pred.idx()].next; continue; } + if !block_has_no_lineno(&blocks[target.idx()]) && blocks[layout_pred.idx()].next != target { + layout_pred = blocks[layout_pred.idx()].next; + continue; + } let jump_target = next_nonempty_block(blocks, blocks[target.idx()].instructions[0].target); if jump_target == BlockIdx::NULL From f3b8ba1c4d30b585c036ffd023c7c24f5e9b7c33 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 05:55:34 +0900 Subject: [PATCH 33/76] Handle multi-block scope-exit CFG segments --- crates/codegen/src/compile.rs | 24 ++++++++++--- crates/codegen/src/ir.rs | 68 +++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 45ab9283c3..11ad736ac8 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20677,6 +20677,14 @@ def f(source, state, char): if char != '-': while True: flag = FLAGS[char] + if source.istext: + if char == 'L': + msg = 'bad inline flags' + raise source.error(msg) + else: + if char == 'u': + msg = 'bad inline flags' + raise source.error(msg) add_flags |= flag if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag: msg = 'bad inline flags' @@ -20703,6 +20711,9 @@ def f(source, state, char): raise source.error(msg, len(char)) while True: flag = FLAGS[char] + if flag & TYPE_FLAGS: + msg = 'bad inline flags' + raise source.error(msg) del_flags |= flag char = sourceget() if char is None: @@ -20723,8 +20734,9 @@ def f(source, state, char): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - assert!( - ops.windows(5).any(|window| { + let cpython_style_not_in_raise_body_count = ops + .windows(6) + .filter(|window| { matches!( window, [ @@ -20734,10 +20746,14 @@ def f(source, state, char): Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. }, Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, ] ) - }), - "expected CPython-style false-path loop backedge before raising body, got ops={ops:?}" + }) + .count(); + assert!( + cpython_style_not_in_raise_body_count >= 2, + "expected both while-loop not-in checks to put the false-path backedge before the conditional-expression raise body, got ops={ops:?}" ); } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4b5dbd110e..7c3f2e8649 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -13675,6 +13675,67 @@ fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) } } + fn scope_exit_segment_tail_before_jump( + blocks: &[Block], + start: BlockIdx, + jump_start: BlockIdx, + ) -> Option { + let jump_block = next_nonempty_block(blocks, jump_start); + if jump_block == BlockIdx::NULL { + return None; + } + + let mut segment = Vec::new(); + let mut cursor = next_nonempty_block(blocks, start); + while cursor != BlockIdx::NULL && cursor != jump_block { + if segment.len() >= blocks.len() { + return None; + } + let block = &blocks[cursor.idx()]; + if block_is_exceptional(block) || block_is_protected(block) { + return None; + } + segment.push(cursor); + cursor = next_nonempty_block(blocks, block.next); + } + if cursor != jump_block || segment.is_empty() { + return None; + } + + let mut in_segment = vec![false; blocks.len()]; + for block_idx in &segment { + in_segment[block_idx.idx()] = true; + } + + let mut has_scope_exit = false; + for block_idx in &segment { + let block = &blocks[block_idx.idx()]; + if is_scope_exit_block(block) { + has_scope_exit = true; + continue; + } + + if block_has_fallthrough(block) { + let next = next_nonempty_block(blocks, block.next); + if next == BlockIdx::NULL || !in_segment[next.idx()] { + return None; + } + } + + for info in &block.instructions { + if info.target == BlockIdx::NULL { + continue; + } + let target = next_nonempty_block(blocks, info.target); + if target == BlockIdx::NULL || !in_segment[target.idx()] { + return None; + } + } + } + + has_scope_exit.then_some(*segment.last().expect("non-empty segment")) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -13721,6 +13782,7 @@ fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) .instructions .first() .is_some_and(|info| instruction_lineno(info) >= 0); + let exit_segment_tail = scope_exit_segment_tail_before_jump(blocks, exit_start, jump_start); if exit_start == BlockIdx::NULL || exit_block == BlockIdx::NULL || jump_start == BlockIdx::NULL @@ -13732,26 +13794,26 @@ fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) || block_is_protected(&blocks[idx]) || block_is_protected(&blocks[exit_block.idx()]) || block_is_protected(&blocks[jump_block.idx()]) - || !is_scope_exit_block(&blocks[exit_block.idx()]) + || exit_segment_tail.is_none() || !is_jump_back_only_block(blocks, jump_block) || jumps_to_for_iter || (after_jump != BlockIdx::NULL && !blocks[after_jump.idx()].cold && !jump_exits_to_loop_exit && !jump_has_lineno) - || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block { current = next; continue; } + let exit_segment_tail = exit_segment_tail.expect("checked above"); let after_jump = blocks[jump_block.idx()].next; blocks[idx].instructions[cond_idx].instr = reversed_conditional(&cond.instr).expect("PopJumpIfFalse has a reversed conditional"); blocks[idx].instructions[cond_idx].target = exit_start; blocks[idx].next = jump_start; blocks[jump_block.idx()].next = exit_start; - blocks[exit_block.idx()].next = after_jump; + blocks[exit_segment_tail.idx()].next = after_jump; current = next; } } From aa2ca06d249e61fc16c71635c0671e2a45accf5d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 06:18:55 +0900 Subject: [PATCH 34/76] Preserve CPython-normalized call-body CFG --- crates/codegen/src/compile.rs | 59 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 30 +++++++++++++----- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 11ad736ac8..9d30c8c36e 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20757,6 +20757,65 @@ def f(source, state, char): ); } + #[test] + fn test_call_body_implicit_continue_keeps_cpython_normalized_forward_jump() { + let code = compile_exec( + "\ +DIGITS = '0123456789' + +def f(s, sget, lappend, addgroup, this, c): + while True: + if c in DIGITS: + isoctal = False + if s.next in DIGITS: + this += sget() + if not isoctal: + addgroup(int(this[1:]), len(this) - 1) + else: + lappend(this) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "expected CPython-style forward jump over call body, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "call body should not be moved after an implicit continue backedge, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7c3f2e8649..8a127da567 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -11788,11 +11788,12 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { && (small_exit_block || no_lineno_no_fallthrough) { let removed_jump_kind = jump_thread_kind(last.instr); - let removed_jump_had_lineno = blocks[current.idx()] - .instructions - .last() - .is_some_and(instruction_has_lineno); - if removed_jump_had_lineno { + let keep_removed_jump_nop = removed_jump_kind == Some(JumpThreadKind::NoInterrupt) + || blocks[current.idx()] + .instructions + .last() + .is_some_and(instruction_has_lineno); + if keep_removed_jump_nop { if let Some(last_instr) = blocks[current.idx()].instructions.last_mut() { set_to_nop(last_instr); } @@ -14065,6 +14066,13 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec }) } + fn block_has_call(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -14084,6 +14092,10 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec let body_start = next; let body = next_nonempty_block(blocks, body_start); let true_jump_loop_target = jump_back_target(blocks, true_jump); + let normalized_forward_conditional = blocks[idx] + .instructions + .get(cond_idx + 1) + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::NotTaken))); if true_jump_loop_target.is_some() && true_jump_start == true_jump && body_start != BlockIdx::NULL @@ -14100,11 +14112,15 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec let body_tail_is_conditional = trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); let body_is_single_block = body == body_tail; + let body_has_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); + let normalized_single_block_can_reorder = + !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); let can_reorder = !body_tail_is_conditional - && ((body_is_single_block + && ((normalized_single_block_can_reorder + && body_is_single_block && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) - || body_segment_contains_for_iter(blocks, body, body_tail)); + || body_has_for_iter); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { let cloned_jump_idx = BlockIdx(blocks.len() as u32); let mut cloned_jump = blocks[true_jump.idx()].clone(); From cea61f2ce0fb648bdd28f68ec5551818ebeb5baa Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 06:44:49 +0900 Subject: [PATCH 35/76] Preserve CPython empty if-end return anchor --- crates/codegen/src/compile.rs | 48 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 19 ++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 9d30c8c36e..3d43b8eeaa 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20816,6 +20816,54 @@ def f(s, sget, lappend, addgroup, this, c): ); } + #[test] + fn test_empty_if_end_label_preserves_cpython_return_anchor_nop() { + let code = compile_exec( + "\ +SRE_FLAG_LOCALE = 1 +SRE_FLAG_ASCII = 2 +SRE_FLAG_UNICODE = 4 + +def f(src, flags): + if isinstance(src, str): + if flags & SRE_FLAG_LOCALE: + raise ValueError('cannot use LOCALE flag with a str pattern') + if not flags & SRE_FLAG_ASCII: + flags |= SRE_FLAG_UNICODE + elif flags & SRE_FLAG_UNICODE: + raise ValueError('ASCII and UNICODE flags are incompatible') + else: + if flags & SRE_FLAG_UNICODE: + raise ValueError('cannot use UNICODE flag with a bytes pattern') + if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII: + raise ValueError('ASCII and LOCALE flags are incompatible') + return flags +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::RaiseVarargs { .. }, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ReturnValue, + ] + ) + }), + "expected CPython-style NOP return anchor after elif raise body, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8a127da567..025bb9e82b 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -11736,6 +11736,16 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { ) }) && block_exits_scope(block) }; + let block_is_simple_fast_return = |block: &Block| { + matches!( + block.instructions.as_slice(), + [load, ret] + if matches!( + load.instr.real(), + Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) + ) && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) + ) + }; loop { let mut changes = false; let mut predecessors = vec![0usize; blocks.len()]; @@ -11794,8 +11804,17 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { .last() .is_some_and(instruction_has_lineno); if keep_removed_jump_nop { + let preserve_empty_end_label_nop = removed_jump_kind + == Some(JumpThreadKind::NoInterrupt) + && small_exit_block + && blocks[current.idx()].instructions.len() == 1 + && block_is_simple_fast_return(&blocks[target.idx()]); if let Some(last_instr) = blocks[current.idx()].instructions.last_mut() { set_to_nop(last_instr); + if preserve_empty_end_label_nop { + last_instr.lineno_override = None; + last_instr.preserve_block_start_no_location_nop = true; + } } } else { let _ = blocks[current.idx()].instructions.pop(); From 8be2a54f16fef8ea175eaee062ca2a9f039e3757 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 07:08:07 +0900 Subject: [PATCH 36/76] Match CPython borrow CFG boundaries --- crates/codegen/src/compile.rs | 142 +++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 180 ++++++++++++++++++++++++++++++++++ 2 files changed, 322 insertions(+) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 3d43b8eeaa..032540534c 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20864,6 +20864,148 @@ def f(src, flags): ); } + #[test] + fn test_nested_except_normal_exit_return_uses_strong_loads() { + let code = compile_exec( + "\ +LITERAL = 1 + +def f(source, escape): + try: + c = escape[1:2] + if c == 'N' and source.istext: + try: + c = ord(source.lookup()) + except (KeyError, TypeError): + raise source.error() from None + return LITERAL, c + except ValueError: + pass + raise source.error(escape) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadGlobal { .. }, + Instruction::LoadFast { .. }, + Instruction::BuildTuple { .. }, + Instruction::ReturnValue, + ] + ) + }), + "expected CPython-style strong LOAD_FAST in nested except normal-exit return, got ops={ops:?}" + ); + } + + #[test] + fn test_targeted_nop_after_prefix_for_else_uses_strong_for_tail_loads() { + let code = compile_exec( + "\ +LITERAL = 1 +IN = 2 +NEGATE = 3 + +def f(items): + while True: + prefix = None + for item in items: + if not item: + break + if prefix is None: + prefix = item[0] + elif item[0] != prefix: + break + else: + for item in items: + del item[0] + continue + break + set = [] + for item in items: + if len(item) != 1: + break + op, av = item[0] + if op is LITERAL: + set.append((op, av)) + elif op is IN and av[0][0] is not NEGATE: + set.extend(av) + else: + break + return set +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let get_iter_idx = ops + .iter() + .rposition(|op| matches!(op, Instruction::GetIter)) + .expect("missing GET_ITER"); + assert!( + matches!(ops[get_iter_idx - 1], Instruction::LoadFast { .. }), + "targeted NOP after prefix for/else should use strong iterable LOAD_FAST, got ops={ops:?}" + ); + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFastLoadFast { .. }, + Instruction::BuildTuple { .. }, + ] + ) + }), + "targeted NOP after prefix for/else should keep set.append tuple loads strong, got ops={ops:?}" + ); + } + + #[test] + fn test_plain_pass_before_for_tail_keeps_borrows() { + let code = compile_exec( + "\ +def f(xs): + pass + for x in xs: + pass + return xs +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let get_iter_idx = ops + .iter() + .position(|op| matches!(op, Instruction::GetIter)) + .expect("missing GET_ITER"); + assert!( + matches!(ops[get_iter_idx - 1], Instruction::LoadFastBorrow { .. }), + "plain pass before for-tail should keep borrowed iterable load, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 025bb9e82b..061ccbc91c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -375,6 +375,8 @@ impl CodeInfo { convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); remove_redundant_nops_and_jumps(&mut self.blocks); self.mark_unprotected_debug_four_tails_borrow_disabled(); + self.mark_exception_handler_transition_targets_borrow_disabled(); + self.mark_targeted_nop_for_tails_borrow_disabled(); self.compute_load_fast_start_depths(); // optimize_load_fast: after normalize_jumps self.optimize_load_fast_borrow(); @@ -3856,6 +3858,182 @@ impl CodeInfo { } } + fn mark_exception_handler_transition_targets_borrow_disabled(&mut self) { + fn first_real_handler(block: &Block) -> Option { + block + .instructions + .iter() + .find(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .and_then(|info| info.except_handler) + } + + fn last_real_handler(block: &Block) -> Option { + block + .instructions + .iter() + .rev() + .find(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .and_then(|info| info.except_handler) + } + + fn has_direct_tuple_return_tail(block: &Block) -> bool { + let reals: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + .collect(); + reals + .iter() + .any(|instr| matches!(instr, Instruction::BuildTuple { .. })) + && reals + .last() + .is_some_and(|instr| matches!(instr, Instruction::ReturnValue)) + && !reals.iter().any(|instr| { + matches!( + instr, + Instruction::Swap { .. } + | Instruction::PopExcept + | Instruction::Reraise { .. } + | Instruction::WithExceptStart + ) + }) + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (idx, block) in self.blocks.iter().enumerate() { + let block_idx = BlockIdx::new(idx as u32); + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(block_idx); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(block_idx); + } + } + } + + let mut to_disable = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + if !has_direct_tuple_return_tail(block) { + continue; + } + let Some(handler) = first_real_handler(block) else { + continue; + }; + if predecessors[idx].iter().any(|pred| { + last_real_handler(&self.blocks[pred.idx()]) + .is_some_and(|pred_handler| pred_handler != handler) + }) { + to_disable.push(idx); + } + } + + for idx in to_disable { + self.blocks[idx].disable_load_fast_borrow = true; + } + } + + fn mark_targeted_nop_for_tails_borrow_disabled(&mut self) { + fn is_nop_only_block(block: &Block) -> bool { + !block.instructions.is_empty() + && block.instructions.iter().all(|info| { + matches!( + info.instr.real(), + Some(Instruction::Nop | Instruction::NotTaken) + ) + }) + } + + fn starts_for_iter_tail(block: &Block) -> bool { + let mut saw_iterable = false; + for info in block.instructions.iter().filter(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + }) { + match info.instr.real() { + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadName { .. } + | Instruction::LoadGlobal { .. }, + ) if !saw_iterable => saw_iterable = true, + Some(Instruction::GetIter) if saw_iterable => return true, + Some(Instruction::BuildList { .. } | Instruction::StoreFast { .. }) + if !saw_iterable => {} + _ => return false, + } + } + false + } + + let mut fallthrough_predecessors = vec![Vec::new(); self.blocks.len()]; + let mut jump_predecessors = vec![Vec::new(); self.blocks.len()]; + for (idx, block) in self.blocks.iter().enumerate() { + let block_idx = BlockIdx::new(idx as u32); + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + fallthrough_predecessors[block.next.idx()].push(block_idx); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + jump_predecessors[info.target.idx()].push(block_idx); + } + } + } + + let mut seeds = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + if !starts_for_iter_tail(block) { + continue; + } + let has_targeted_nop_predecessor = fallthrough_predecessors[idx].iter().any(|pred| { + is_nop_only_block(&self.blocks[pred.idx()]) + && !jump_predecessors[pred.idx()].is_empty() + }); + if has_targeted_nop_predecessor { + seeds.push(BlockIdx::new(idx as u32)); + } + } + + let mut seen = vec![false; self.blocks.len()]; + for seed in seeds { + let mut stack = vec![seed]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || seen[block_idx.idx()] { + continue; + } + seen[block_idx.idx()] = true; + self.blocks[block_idx.idx()].disable_load_fast_borrow = true; + + let block = &self.blocks[block_idx.idx()]; + if block + .instructions + .last() + .is_some_and(|info| info.instr.is_scope_exit()) + { + continue; + } + if block.next != BlockIdx::NULL && block.next.idx() >= seed.idx() { + stack.push(block.next); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL && info.target.idx() >= seed.idx() { + stack.push(info.target); + } + } + } + } + } + fn deoptimize_borrow_for_handler_return_paths(&mut self) { for block in &mut self.blocks { let len = block.instructions.len(); @@ -10473,6 +10651,8 @@ impl CodeInfo { convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); remove_redundant_nops_and_jumps(&mut self.blocks); self.mark_unprotected_debug_four_tails_borrow_disabled(); + self.mark_exception_handler_transition_targets_borrow_disabled(); + self.mark_targeted_nop_for_tails_borrow_disabled(); trace.push(( "after_convert_pseudo_ops".to_owned(), self.debug_block_dump(), From f155436a49eeca7e34f3f3b2ca916063b4f1f5e5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 07:25:24 +0900 Subject: [PATCH 37/76] Match CPython tuple unpack constant folding --- crates/codegen/src/compile.rs | 30 ++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 25 +++++++++++++------------ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 032540534c..316438bb64 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -21809,6 +21809,36 @@ def f(self): ); } + #[test] + fn test_four_item_unpack_assignment_folds_tuple_constant_like_cpython() { + let code = compile_exec("a, b, c, d = 1, 2, 3, 4\n"); + + assert!( + code.instructions.iter().any(|unit| { + matches!(unit.op, Instruction::LoadConst { .. }) + && matches!( + code.constants.get(usize::from(u8::from(unit.arg))), + Some(ConstantData::Tuple { elements }) if elements.len() == 4 + ) + }), + "four-item unpack assignment should fold BUILD_TUPLE before UNPACK_SEQUENCE like CPython, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::UnpackSequence { .. })), + "four-item unpack assignment should keep UNPACK_SEQUENCE after tuple folding, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } + #[test] fn test_chained_unpack_assignment_keeps_constant_collection_folding() { let code = compile_exec("(a, b) = c = d = (1, 2)\n"); diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 061ccbc91c..7c819a9417 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1688,18 +1688,19 @@ impl CodeInfo { }; let tuple_size = u32::from(block.instructions[i].arg) as usize; - if block - .instructions - .get(i + 1) - .and_then(|next| next.instr.real()) - .is_some_and(|next| { - matches!( - next, - Instruction::UnpackSequence { .. } - if usize::try_from(u32::from(block.instructions[i + 1].arg)).ok() - == Some(tuple_size) - ) - }) + if tuple_size <= 3 + && block + .instructions + .get(i + 1) + .and_then(|next| next.instr.real()) + .is_some_and(|next| { + matches!( + next, + Instruction::UnpackSequence { .. } + if usize::try_from(u32::from(block.instructions[i + 1].arg)).ok() + == Some(tuple_size) + ) + }) { return false; } From 85c336742c9052dfbb3a66116563dfd2c2d9eacd Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 07:56:37 +0900 Subject: [PATCH 38/76] Match CPython implicit continue CFG layout --- crates/codegen/src/compile.rs | 102 +++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 112 +++++++++++++++++++++++++++++++++- 2 files changed, 211 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 316438bb64..3e9a681be1 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -25592,6 +25592,108 @@ def f(compiler_so): ); } + #[test] + fn test_loop_if_subscr_store_delete_places_body_after_jumpback() { + let code = compile_exec( + "\ +def f(chunks): + for k in range(len(chunks)-1, 0, -1): + if chunks[k-1][-1] > chunks[k][0]: + chunks[k-1] = chunks[k-1][:-1] + chunks[k][1:] + del chunks[k] +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::LoadSmallInt { .. }, + Instruction::BinaryOp { .. }, + ] + ) + }), + "loop if with subscript store/delete body should put false jump-back before body, got ops={ops:?}" + ); + } + + #[test] + fn test_final_elif_implicit_continue_places_jumpback_before_body() { + let code = compile_exec( + "\ +def f(state, nextchar, whitespace, token, posix, quoted, debug): + while True: + if state is None: + break + elif state == ' ': + if not nextchar: + state = None + break + elif nextchar in whitespace: + if debug >= 2: + print('x') + if token or (posix and quoted): + break + else: + continue + elif state in ('a', 'c'): + if not nextchar: + state = None + break + elif nextchar in whitespace: + if debug >= 2: + print('y') + state = ' ' + if token or (posix and quoted): + break + else: + continue + return token +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::LoadConst { .. }, + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + ] + ) + }), + "final elif with implicit continue should put false jump-back before body, got ops={ops:?}" + ); + } + #[test] fn test_except_handler_with_conditional_raise_and_resume_keeps_borrow() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7c819a9417..5f61885404 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -13351,8 +13351,13 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { | Instruction::LoadFastLoadFast { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } | Instruction::LoadSmallInt { .. } + | Instruction::LoadConst { .. } + | Instruction::Copy { .. } + | Instruction::Swap { .. } | Instruction::BinaryOp { .. } + | Instruction::BinarySlice | Instruction::BuildSlice { .. } + | Instruction::StoreSubscr | Instruction::DeleteSubscr ) }) @@ -14228,6 +14233,50 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec false } + fn body_segment_contains_scope_exit( + blocks: &[Block], + body_start: BlockIdx, + body_tail: BlockIdx, + ) -> bool { + let mut cursor = body_start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL { + if visited[cursor.idx()] { + return false; + } + visited[cursor.idx()] = true; + if blocks[cursor.idx()] + .instructions + .iter() + .any(|info| info.instr.is_scope_exit()) + { + return true; + } + if cursor == body_tail { + return false; + } + cursor = blocks[cursor.idx()].next; + } + false + } + + fn empty_chain_reaches(blocks: &[Block], start: BlockIdx, target: BlockIdx) -> bool { + let mut cursor = start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && cursor != target { + if visited[cursor.idx()] + || block_is_exceptional(&blocks[cursor.idx()]) + || block_is_protected(&blocks[cursor.idx()]) + || !blocks[cursor.idx()].instructions.is_empty() + { + return false; + } + visited[cursor.idx()] = true; + cursor = blocks[cursor.idx()].next; + } + cursor == target + } + fn block_starts_loop_cleanup(blocks: &[Block], block_idx: BlockIdx) -> bool { block_idx != BlockIdx::NULL && matches!( @@ -14259,8 +14308,13 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec | Instruction::LoadFastLoadFast { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } | Instruction::LoadSmallInt { .. } + | Instruction::LoadConst { .. } + | Instruction::Copy { .. } + | Instruction::Swap { .. } | Instruction::BinaryOp { .. } + | Instruction::BinarySlice | Instruction::BuildSlice { .. } + | Instruction::StoreSubscr | Instruction::DeleteSubscr ) }) @@ -14297,7 +14351,8 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec .get(cond_idx + 1) .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::NotTaken))); if true_jump_loop_target.is_some() - && true_jump_start == true_jump + && (true_jump_start == true_jump + || empty_chain_reaches(blocks, true_jump_start, true_jump)) && body_start != BlockIdx::NULL && body != BlockIdx::NULL && true_jump != BlockIdx::NULL @@ -14313,14 +14368,20 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); let body_is_single_block = body == body_tail; let body_has_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); + let body_has_scope_exit = body_segment_contains_scope_exit(blocks, body, body_tail); let normalized_single_block_can_reorder = !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); - let can_reorder = !body_tail_is_conditional + let trailing_implicit_continue_can_reorder = after_jump != BlockIdx::NULL + && next_nonempty_block(blocks, after_jump) != body + && !block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_jump)) + && !is_scope_exit_block(&blocks[body.idx()]); + let can_reorder = (!body_tail_is_conditional && ((normalized_single_block_can_reorder && body_is_single_block && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) - || body_has_for_iter); + || body_has_for_iter)) + || (trailing_implicit_continue_can_reorder && body_has_scope_exit); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { let cloned_jump_idx = BlockIdx(blocks.len() as u32); let mut cloned_jump = blocks[true_jump.idx()].clone(); @@ -14861,6 +14922,14 @@ fn comes_before(blocks: &[Block], first: BlockIdx, second: BlockIdx) -> bool { fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { let predecessors = compute_predecessors(blocks); let mut clones = Vec::new(); + let mut block_order = vec![usize::MAX; blocks.len()]; + let mut current = BlockIdx(0); + let mut pos = 0usize; + while current != BlockIdx::NULL { + block_order[current.idx()] = pos; + pos += 1; + current = blocks[current.idx()].next; + } for target in 0..blocks.len() { let target = BlockIdx(target as u32); @@ -14877,6 +14946,43 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { } let layout_pred = find_layout_predecessor(blocks, target); + if layout_pred != BlockIdx::NULL + && (!block_has_fallthrough(&blocks[layout_pred.idx()]) + || next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target) + && predecessors[target.idx()] >= 2 + { + let mut jump_predecessors = Vec::new(); + for block_idx in 0..blocks.len() { + let block_idx = BlockIdx(block_idx as u32); + for (instr_idx, info) in blocks[block_idx.idx()].instructions.iter().enumerate() { + if !is_jump_instruction(info) || info.target == BlockIdx::NULL { + continue; + } + if next_nonempty_block(blocks, info.target) == target { + jump_predecessors.push((block_idx, instr_idx)); + } + } + } + if jump_predecessors.len() >= 2 + && jump_predecessors.iter().all(|(block_idx, instr_idx)| { + is_conditional_jump(&blocks[block_idx.idx()].instructions[*instr_idx].instr) + && block_order[block_idx.idx()] < block_order[target.idx()] + }) + && let Some((keep_block, keep_instr)) = jump_predecessors + .iter() + .max_by_key(|(block_idx, _)| block_order[block_idx.idx()]) + .copied() + { + for (block_idx, instr_idx) in jump_predecessors { + if block_idx == keep_block && instr_idx == keep_instr { + continue; + } + clones.push((target, block_idx, instr_idx)); + } + continue; + } + } + if layout_pred == BlockIdx::NULL || !block_has_fallthrough(&blocks[layout_pred.idx()]) || next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target From bb7be661034bb81029004f3a3fa5799d055d7d1c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 08:30:49 +0900 Subject: [PATCH 39/76] Keep implicit continue CFG targets in layout --- crates/codegen/src/compile.rs | 108 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 108 +++++++++++++++++++++++++++++++++- 2 files changed, 215 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 3e9a681be1..bfe0e5cb02 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -25694,6 +25694,114 @@ def f(state, nextchar, whitespace, token, posix, quoted, debug): ); } + #[test] + fn test_final_attribute_elif_implicit_continue_places_jumpback_before_body() { + let code = compile_exec( + "\ +def f(self, nextchar, quoted): + while True: + if self.state is None: + break + elif self.state in ('a', 'c'): + if not nextchar: + self.state = None + break + elif nextchar in self.whitespace: + self.state = ' ' + if self.token or (self.posix and quoted): + break + else: + continue + return self.token +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::LoadConst { .. }, + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + ] + ) + }), + "final attribute elif with implicit continue should put false jump-back before body, got ops={ops:?}" + ); + } + + #[test] + fn test_inner_if_implicit_continue_keeps_line_bearing_body_before_backedge() { + let code = compile_exec( + "\ +def f(self, nextchar, quoted): + while True: + if self.state is None: + break + elif self.state in ('a', 'c'): + if not nextchar: + self.state = None + break + elif nextchar in self.whitespace: + self.state = ' ' + if self.token or (self.posix and quoted): + break + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + if self.posix: + self.state = ' ' + if self.token or (self.posix and quoted): + break + else: + continue + elif self.state == 'c': + break + return self.token +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::LoadAttr { .. }, + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadConst { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::StoreAttr { .. }, + ] + ) + }), + "line-bearing inner if with implicit continue should keep body before backedge, got ops={ops:?}" + ); + } + #[test] fn test_except_handler_with_conditional_raise_and_resume_keeps_borrow() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5f61885404..d0043024e4 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -13596,6 +13596,42 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( allow_for_iter_jump_targets: bool, allow_true_scope_exit_reorder: bool, ) { + fn retarget_empty_chain_targets( + blocks: &mut [Block], + chain_start: BlockIdx, + chain_end: BlockIdx, + replacement: BlockIdx, + ) { + if chain_start == BlockIdx::NULL + || chain_end == BlockIdx::NULL + || replacement == BlockIdx::NULL + { + return; + } + + let mut in_chain = vec![false; blocks.len()]; + let mut cursor = chain_start; + while cursor != BlockIdx::NULL && cursor != chain_end { + let block = &blocks[cursor.idx()]; + if !block.instructions.is_empty() { + return; + } + in_chain[cursor.idx()] = true; + cursor = block.next; + } + if cursor != chain_end { + return; + } + + for block in blocks { + for instr in &mut block.instructions { + if instr.target != BlockIdx::NULL && in_chain[instr.target.idx()] { + instr.target = replacement; + } + } + } + } + fn jump_targets_for_iter(blocks: &[Block], jump_block: BlockIdx) -> bool { if jump_block == BlockIdx::NULL { return false; @@ -13740,6 +13776,7 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( } let after_exit = blocks[exit_block.idx()].next; + retarget_empty_chain_targets(blocks, next, jump_block, jump_block); blocks[idx].instructions[cond_idx].instr = reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); blocks[idx].instructions[cond_idx].target = jump_block; @@ -14099,6 +14136,14 @@ fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { } } + fn jump_back_lineno(block: &Block) -> Option { + let [info] = block.instructions.as_slice() else { + return None; + }; + matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) + .then(|| instruction_lineno(info)) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let Some(target) = jump_back_target(&blocks[current.idx()]) else { @@ -14118,6 +14163,8 @@ fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { || block_is_exceptional(&blocks[duplicate.idx()]) || block_is_protected(&blocks[duplicate.idx()]) || jump_back_target(&blocks[duplicate.idx()]) != Some(target) + || jump_back_lineno(&blocks[duplicate.idx()]) + != jump_back_lineno(&blocks[current.idx()]) { current = blocks[current.idx()].next; continue; @@ -14260,6 +14307,30 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec false } + fn body_segment_contains_jump_back_to( + blocks: &[Block], + body_start: BlockIdx, + body_tail: BlockIdx, + target: BlockIdx, + ) -> bool { + let mut cursor = body_start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL { + if visited[cursor.idx()] { + return false; + } + visited[cursor.idx()] = true; + if jump_back_target(blocks, cursor) == Some(target) { + return true; + } + if cursor == body_tail { + return false; + } + cursor = blocks[cursor.idx()].next; + } + false + } + fn empty_chain_reaches(blocks: &[Block], start: BlockIdx, target: BlockIdx) -> bool { let mut cursor = start; let mut visited = vec![false; blocks.len()]; @@ -14327,6 +14398,30 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) } + fn block_is_pure_conditional_test(block: &Block) -> bool { + let Some(cond_idx) = trailing_conditional_jump_index(block) else { + return false; + }; + block.instructions[..cond_idx].iter().all(|info| { + matches!( + info.instr.real(), + Some( + Instruction::Nop + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadConst { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::LoadAttr { .. } + | Instruction::ContainsOp { .. } + | Instruction::CompareOp { .. } + | Instruction::ToBool + ) + ) + }) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -14363,14 +14458,22 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && next_nonempty_block(blocks, blocks[true_jump.idx()].next) != body && let Some(body_tail) = find_body_tail_before_jump(blocks, body, true_jump_start) { + let loop_target = true_jump_loop_target.expect("checked above"); let after_jump = blocks[true_jump.idx()].next; let body_tail_is_conditional = trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); let body_is_single_block = body == body_tail; let body_has_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); let body_has_scope_exit = body_segment_contains_scope_exit(blocks, body, body_tail); + let body_has_loop_backedge = + body_segment_contains_jump_back_to(blocks, body, body_tail, loop_target); let normalized_single_block_can_reorder = !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); + let after_jump_target = next_nonempty_block(blocks, after_jump); + let after_jump_continues_conditional_chain = after_jump_target != BlockIdx::NULL + && trailing_conditional_jump_index(&blocks[after_jump_target.idx()]).is_some(); + let body_starts_with_conditional_test = + block_is_pure_conditional_test(&blocks[body.idx()]); let trailing_implicit_continue_can_reorder = after_jump != BlockIdx::NULL && next_nonempty_block(blocks, after_jump) != body && !block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_jump)) @@ -14381,7 +14484,10 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) || body_has_for_iter)) - || (trailing_implicit_continue_can_reorder && body_has_scope_exit); + || (trailing_implicit_continue_can_reorder + && (body_has_scope_exit || body_has_loop_backedge) + && (!after_jump_continues_conditional_chain + || body_starts_with_conditional_test)); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { let cloned_jump_idx = BlockIdx(blocks.len() as u32); let mut cloned_jump = blocks[true_jump.idx()].clone(); From 058bc6d0a80a3dbad607ec9722d686ee692187f1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 08:41:16 +0900 Subject: [PATCH 40/76] Align try-except end label location with CPython --- crates/codegen/src/compile.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index bfe0e5cb02..b07e6d3aaa 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3797,10 +3797,6 @@ impl Compiler { handlers: &[ast::ExceptHandler], orelse: &[ast::Stmt], ) -> CompileResult<()> { - let normal_exit_range = orelse - .last() - .map(ast::Stmt::range) - .or_else(|| body.last().map(ast::Stmt::range)); let handler_block = self.new_block(); let cleanup_block = self.new_block(); let end_block = self.new_block(); @@ -3955,9 +3951,6 @@ impl Compiler { self.set_no_location(); self.switch_to_block(end_block); - if let Some(range) = normal_exit_range { - self.set_source_range(range); - } Ok(()) } From e4eb97c249d6a0abf81ffe5f913a82f97e86faa6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 09:03:25 +0900 Subject: [PATCH 41/76] Remove folded operand NOPs before line propagation --- crates/codegen/src/compile.rs | 39 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 27 +++++++++++++++++++----- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index b07e6d3aaa..63dbcabef8 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -1552,6 +1552,7 @@ impl Compiler { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }); } @@ -10041,6 +10042,7 @@ impl Compiler { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }); } @@ -16896,6 +16898,43 @@ def f(x, E): ); } + #[test] + fn test_try_except_continuation_folded_tuple_drops_operand_nop() { + let code = compile_exec( + "\ +def f(): + try: + import sqlite3 + except ImportError: + return + + attributes = ('sqlite_version',) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::StoreFast { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::StoreFast { .. }, + ] + ) + }), + "expected CPython nop_out-style folded tuple operand NOP to be removed, got ops={ops:?}", + ); + } + #[test] fn test_named_except_cleanup_keeps_jump_over_cleanup_and_next_try() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d0043024e4..45fa74c853 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -125,6 +125,8 @@ pub struct InstructionInfo { pub preserve_redundant_jump_as_nop: bool, /// Drop this NOP before line propagation if it still has no location. pub remove_no_location_nop: bool, + /// This no-location NOP was created by CPython-style nop_out() folding. + pub folded_operand_nop: bool, /// Keep this no-location NOP until line propagation when it starts a block. pub preserve_block_start_no_location_nop: bool, } @@ -148,6 +150,7 @@ fn set_to_nop(info: &mut InstructionInfo) { info.cache_entries = 0; info.preserve_redundant_jump_as_nop = false; info.remove_no_location_nop = false; + info.folded_operand_nop = false; info.preserve_block_start_no_location_nop = false; } @@ -155,6 +158,7 @@ fn nop_out_no_location(info: &mut InstructionInfo) { set_to_nop(info); info.lineno_override = Some(-1); info.remove_no_location_nop = true; + info.folded_operand_nop = true; } fn is_named_except_cleanup_normal_exit_block(block: &Block) -> bool { @@ -3049,20 +3053,20 @@ impl CodeInfo { block.instructions.retain(|ins| { let keep = 'keep: { if matches!(ins.instr.real(), Some(Instruction::Nop)) { - let keep_target_start = src == 0 - && keep_target_start_nops + let keep_loop_exit_pop_block = src == 0 + && preserve_loop_exit_pop_block_nops .get(block_idx) .copied() .unwrap_or(false); - let keep_loop_exit_pop_block = src == 0 - && preserve_loop_exit_pop_block_nops + let keep_target_start = src == 0 + && keep_target_start_nops .get(block_idx) .copied() .unwrap_or(false); if ins.remove_no_location_nop && instruction_lineno(ins) < 0 - && !keep_target_start && !keep_loop_exit_pop_block + && (!keep_target_start || ins.folded_operand_nop) { break 'keep false; } @@ -11218,6 +11222,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }); jump_block.next = blocks[cold_idx.idx()].next; @@ -11748,6 +11753,7 @@ fn normalize_jumps(blocks: &mut Vec) { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }; blocks[idx].instructions.push(not_taken); @@ -11783,6 +11789,7 @@ fn normalize_jumps(blocks: &mut Vec) { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }); new_block.instructions.push(InstructionInfo { @@ -11797,6 +11804,7 @@ fn normalize_jumps(blocks: &mut Vec) { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }); new_block.next = old_next; @@ -12280,10 +12288,12 @@ fn remove_redundant_jumps_in_blocks(blocks: &mut [Block]) -> usize { } let last_instr = blocks[idx].instructions.last_mut().unwrap(); let remove_no_location_nop = last_instr.remove_no_location_nop; + let folded_operand_nop = last_instr.folded_operand_nop; let preserve_block_start_no_location_nop = last_instr.preserve_block_start_no_location_nop; set_to_nop(last_instr); last_instr.remove_no_location_nop = remove_no_location_nop; + last_instr.folded_operand_nop = folded_operand_nop; last_instr.preserve_block_start_no_location_nop = preserve_block_start_no_location_nop; changes += 1; @@ -12557,6 +12567,7 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }); } @@ -12587,6 +12598,7 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, }, ); @@ -15622,10 +15634,12 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { stack.pop(); // POP_BLOCK → NOP let remove_no_location_nop = blocks[bi].instructions[i].remove_no_location_nop; + let folded_operand_nop = blocks[bi].instructions[i].folded_operand_nop; let preserve_block_start_no_location_nop = blocks[bi].instructions[i].preserve_block_start_no_location_nop; set_to_nop(&mut blocks[bi].instructions[i]); blocks[bi].instructions[i].remove_no_location_nop = remove_no_location_nop; + blocks[bi].instructions[i].folded_operand_nop = folded_operand_nop; blocks[bi].instructions[i].preserve_block_start_no_location_nop = preserve_block_start_no_location_nop; } else { @@ -15711,10 +15725,12 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) // label_exception_targets. Dead blocks may still have them. PseudoInstruction::PopBlock => { let remove_no_location_nop = info.remove_no_location_nop; + let folded_operand_nop = info.folded_operand_nop; let preserve_block_start_no_location_nop = info.preserve_block_start_no_location_nop; set_to_nop(info); info.remove_no_location_nop = remove_no_location_nop; + info.folded_operand_nop = folded_operand_nop; info.preserve_block_start_no_location_nop = preserve_block_start_no_location_nop; } @@ -15812,6 +15828,7 @@ mod tests { cache_entries: 0, preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, + folded_operand_nop: false, preserve_block_start_no_location_nop: false, } } From b0221119672d3016cd6ae57f5fad2a499dc4bb9d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 09:22:49 +0900 Subject: [PATCH 42/76] Align no-location return exit handling --- crates/codegen/src/compile.rs | 99 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 30 ++++++++++- 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 63dbcabef8..8efee49c4f 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -1553,6 +1553,7 @@ impl Compiler { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }); } @@ -10043,6 +10044,7 @@ impl Compiler { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }); } @@ -10117,6 +10119,12 @@ impl Compiler { } } + fn mark_last_no_location_exit(&mut self) { + if let Some(last) = self.current_block().instructions.last_mut() { + last.no_location_exit = true; + } + } + fn emit_no_arg>(&mut self, ins: I) { self._emit(ins, OpArg::NULL, BlockIdx::NULL) } @@ -10630,8 +10638,10 @@ impl Compiler { fn emit_return_const_no_location(&mut self, constant: ConstantData) { self.emit_load_const(constant); self.set_no_location(); + self.mark_last_no_location_exit(); emit!(self, Instruction::ReturnValue); self.set_no_location(); + self.mark_last_no_location_exit(); } fn emit_end_async_for(&mut self, send_target: BlockIdx) { @@ -16935,6 +16945,95 @@ def f(): ); } + #[test] + fn test_if_else_normal_fallthrough_end_label_drops_return_anchor_nop() { + let code = compile_exec( + "\ +def f(s): + if s[0] in (0o200, 0o377): + n = 0 + for i in range(len(s) - 1): + n <<= 8 + n += s[i + 1] + if s[0] == 0o377: + n = -(256 ** (len(s) - 1) - n) + else: + try: + s = nts(s, 'ascii', 'strict') + n = int(s.strip() or '0', 8) + except ValueError: + raise InvalidHeaderError('invalid header') + return n +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::StoreFast { .. }, + Instruction::Nop, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::ReturnValue, + ] + ) + }), + "normal fallthrough into an if-end final return should not keep a CPython return-anchor NOP, got ops={ops:?}", + ); + } + + #[test] + fn test_explicit_final_return_none_is_not_duplicated() { + let code = compile_exec( + "\ +def f(src, dst, length, exception, bufsize): + if length == 0: + return + if length is None: + copyfileobj(src, dst, bufsize) + return + + blocks, remainder = divmod(length, bufsize) + for b in range(blocks): + buf = src.read(bufsize) + if len(buf) < bufsize: + raise exception('unexpected end of data') + dst.write(buf) + + if remainder != 0: + buf = src.read(remainder) + if len(buf) < remainder: + raise exception('unexpected end of data') + dst.write(buf) + return +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let return_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::ReturnValue)) + .count(); + + assert_eq!( + return_count, + 3, + "explicit final return None should not be duplicated as a synthetic no-location epilogue, got ops={:?}", + f.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } + #[test] fn test_named_except_cleanup_keeps_jump_over_cleanup_and_next_try() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 45fa74c853..caacb5e707 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -127,6 +127,8 @@ pub struct InstructionInfo { pub remove_no_location_nop: bool, /// This no-location NOP was created by CPython-style nop_out() folding. pub folded_operand_nop: bool, + /// This instruction was emitted as part of a synthetic no-location exit. + pub no_location_exit: bool, /// Keep this no-location NOP until line propagation when it starts a block. pub preserve_block_start_no_location_nop: bool, } @@ -151,6 +153,7 @@ fn set_to_nop(info: &mut InstructionInfo) { info.preserve_redundant_jump_as_nop = false; info.remove_no_location_nop = false; info.folded_operand_nop = false; + info.no_location_exit = false; info.preserve_block_start_no_location_nop = false; } @@ -11223,6 +11226,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }); jump_block.next = blocks[cold_idx.idx()].next; @@ -11754,6 +11758,7 @@ fn normalize_jumps(blocks: &mut Vec) { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }; blocks[idx].instructions.push(not_taken); @@ -11790,6 +11795,7 @@ fn normalize_jumps(blocks: &mut Vec) { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }); new_block.instructions.push(InstructionInfo { @@ -11805,6 +11811,7 @@ fn normalize_jumps(blocks: &mut Vec) { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }); new_block.next = old_next; @@ -11935,6 +11942,19 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { ) && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) ) }; + let normal_layout_fallthrough_into = |blocks: &[Block], target: BlockIdx| { + let mut current = BlockIdx(0); + let mut previous_nonempty = BlockIdx::NULL; + while current != BlockIdx::NULL && current != target { + if !blocks[current.idx()].instructions.is_empty() { + previous_nonempty = current; + } + current = blocks[current.idx()].next; + } + previous_nonempty != BlockIdx::NULL + && block_has_fallthrough(&blocks[previous_nonempty.idx()]) + && next_nonempty_block(blocks, blocks[previous_nonempty.idx()].next) == target + }; loop { let mut changes = false; let mut predecessors = vec![0usize; blocks.len()]; @@ -11997,9 +12017,12 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { == Some(JumpThreadKind::NoInterrupt) && small_exit_block && blocks[current.idx()].instructions.len() == 1 - && block_is_simple_fast_return(&blocks[target.idx()]); + && block_is_simple_fast_return(&blocks[target.idx()]) + && !normal_layout_fallthrough_into(blocks, current); if let Some(last_instr) = blocks[current.idx()].instructions.last_mut() { + let lineno_override = last_instr.lineno_override; set_to_nop(last_instr); + last_instr.lineno_override = lineno_override; if preserve_empty_end_label_nop { last_instr.lineno_override = None; last_instr.preserve_block_start_no_location_nop = true; @@ -12568,6 +12591,7 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }); } @@ -12599,6 +12623,7 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, }, ); @@ -15250,6 +15275,8 @@ fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { AnyInstruction::Real(Instruction::LoadConst { .. }) ) && is_load_const_none(&last_insts[0], metadata) + && last_insts[0].no_location_exit + && last_insts[1].no_location_exit && matches!( last_insts[1].instr, AnyInstruction::Real(Instruction::ReturnValue) @@ -15829,6 +15856,7 @@ mod tests { preserve_redundant_jump_as_nop: false, remove_no_location_nop: false, folded_operand_nop: false, + no_location_exit: false, preserve_block_start_no_location_nop: false, } } From 5e4c096a9fb33b0e1eb9a49c0ede40c79c323da4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 09:50:56 +0900 Subject: [PATCH 43/76] Align nested protected import bytecode --- crates/codegen/src/compile.rs | 67 +++++++++++++++- crates/codegen/src/ir.rs | 141 ++++++++++++++++++++++++++++------ 2 files changed, 182 insertions(+), 26 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 8efee49c4f..8bc9325598 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -7267,7 +7267,8 @@ impl Compiler { self.compile_addcompare(last_op); let end = self.new_block(); - emit!(self, PseudoInstruction::Jump { delta: end }); + emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); + self.set_no_location(); // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. self.switch_to_block(cleanup); @@ -7318,7 +7319,8 @@ impl Compiler { self.compile_addcompare(last_op); emit!(self, Instruction::ToBool); self.emit_pop_jump_by_condition(condition, target_block); - emit!(self, PseudoInstruction::Jump { delta: end }); + emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); + self.set_no_location(); self.switch_to_block(cleanup); emit!(self, Instruction::PopTop); @@ -10949,10 +10951,13 @@ impl Compiler { // Jump to target let target = if is_break { exit_block } else { loop_block }; + let saved_range = self.current_source_range; + self.set_source_range(range); emit!(self, PseudoInstruction::Jump { delta: target }); if jump_no_location { self.set_no_location(); } + self.set_source_range(saved_range); Ok(()) } @@ -22999,6 +23004,64 @@ def f(s, size, pos, errors): ); } + #[test] + fn test_nested_protected_import_tail_keeps_strong_load_fast() { + let code = compile_exec( + "\ +def f(self, mode, compresslevel): + try: + try: + import zlib + except ImportError: + raise RuntimeError from None + self.zlib = zlib + self.crc = zlib.crc32(b'') + if mode == 'r': + self.exception = zlib.error + self._init_read_gz() + else: + self._init_write_gz(compresslevel) + except: + self.closed = True + raise +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let import_idx = ops + .iter() + .position(|op| matches!(op, Instruction::ImportName { .. })) + .expect("missing IMPORT_NAME"); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let nested_protected_tail = &ops[import_idx + 1..handler_start]; + + assert!( + !nested_protected_tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "CPython keeps strong LOAD_FAST ops after a nested protected import tail, got tail={nested_protected_tail:?}" + ); + assert!( + nested_protected_tail + .iter() + .any(|op| matches!(op, Instruction::LoadFastLoadFast { .. })), + "expected nested protected import tail to keep LOAD_FAST_LOAD_FAST, got tail={nested_protected_tail:?}" + ); + } + #[test] fn test_unprotected_import_before_with_keeps_borrow() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index caacb5e707..784bee9c6c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -8132,34 +8132,36 @@ impl CodeInfo { fn deoptimize_protected_block_borrows_from( block: &mut Block, start: usize, - protected_store_locals: &[bool], + protected_store_locals: Option<&[bool]>, ) { for info in block.instructions.iter_mut().skip(start) { if info.except_handler.is_none() { break; } - match info.instr.real() { - Some(Instruction::LoadFastBorrow { var_num }) => { - let local = usize::from(var_num.get(info.arg)); - if protected_store_locals.get(local).copied().unwrap_or(false) { - continue; + if let Some(protected_store_locals) = protected_store_locals { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { + let local = usize::from(var_num.get(info.arg)); + if protected_store_locals.get(local).copied().unwrap_or(false) { + continue; + } } - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { var_nums }) => { - let (left, right) = var_nums.get(info.arg).indexes(); - let skip_left = protected_store_locals - .get(usize::from(left)) - .copied() - .unwrap_or(false); - let skip_right = protected_store_locals - .get(usize::from(right)) - .copied() - .unwrap_or(false); - if skip_left && skip_right { - continue; + Some(Instruction::LoadFastBorrowLoadFastBorrow { var_nums }) => { + let (left, right) = var_nums.get(info.arg).indexes(); + let skip_left = protected_store_locals + .get(usize::from(left)) + .copied() + .unwrap_or(false); + let skip_right = protected_store_locals + .get(usize::from(right)) + .copied() + .unwrap_or(false); + if skip_left && skip_right { + continue; + } } + _ => {} } - _ => {} } deoptimize_borrow(info); } @@ -8258,6 +8260,49 @@ impl CodeInfo { .any(|info| info.except_handler.is_some()) } + fn push_normal_successors(stack: &mut Vec, block: &Block) { + if block.next != BlockIdx::NULL { + stack.push(block.next); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + } + + fn has_nested_protected_import_tail( + blocks: &[Block], + seed: BlockIdx, + import_idx: usize, + ) -> bool { + let Some(import_handler) = blocks[seed.idx()].instructions[import_idx].except_handler + else { + return false; + }; + let mut cursor = seed; + let mut start = import_idx + 1; + while cursor != BlockIdx::NULL && !block_is_exceptional(&blocks[cursor.idx()]) { + let block = &blocks[cursor.idx()]; + let mut saw_protected = false; + for info in block.instructions.iter().skip(start) { + let Some(handler) = info.except_handler else { + return false; + }; + saw_protected = true; + if handler.handler_block != import_handler.handler_block { + return true; + } + } + if !saw_protected { + return false; + } + cursor = block.next; + start = 0; + } + false + } + let mut predecessors = vec![Vec::new(); self.blocks.len()]; for (pred_idx, block) in self.blocks.iter().enumerate() { if block.next != BlockIdx::NULL { @@ -8306,15 +8351,25 @@ impl CodeInfo { &block_order, ) }); - if !handler_returns && !handler_continues { + let nested_protected_tail = has_nested_protected_import_tail( + &self.blocks, + BlockIdx::new(idx as u32), + import_idx, + ); + if !handler_returns && !handler_continues && !nested_protected_tail { return None; } - Some((BlockIdx::new(idx as u32), import_idx, handler_continues)) + Some(( + BlockIdx::new(idx as u32), + import_idx, + handler_continues, + nested_protected_tail, + )) }) .collect(); let mut visited = vec![false; self.blocks.len()]; - for (seed, import_idx, handler_continues) in seeds { + for (seed, import_idx, handler_continues, nested_protected_tail) in seeds { let mut protected_store_locals = vec![false; self.metadata.varnames.len()]; for info in self.blocks[seed.idx()] .instructions @@ -8338,6 +8393,7 @@ impl CodeInfo { let mut cursor = self.blocks[seed.idx()].next; while cursor != BlockIdx::NULL && !block_is_exceptional(&self.blocks[cursor.idx()]) { if !handler_continues + && !nested_protected_tail && block_has_protected_instructions(&self.blocks[cursor.idx()]) { break; @@ -8362,16 +8418,44 @@ impl CodeInfo { cursor = self.blocks[cursor.idx()].next; } + if nested_protected_tail { + let mut stack = Vec::new(); + for (block_idx, _) in &segment { + push_normal_successors(&mut stack, &self.blocks[block_idx.idx()]); + } + while let Some(candidate) = stack.pop() { + if candidate == BlockIdx::NULL + || in_segment[candidate.idx()] + || block_is_exceptional(&self.blocks[candidate.idx()]) + || !block_has_protected_instructions(&self.blocks[candidate.idx()]) + { + continue; + } + if predecessors[candidate.idx()].iter().any(|pred| { + !in_segment[pred.idx()] + && block_order[pred.idx()] < block_order[candidate.idx()] + && !is_handler_resume_predecessor(&self.blocks[pred.idx()], candidate) + }) { + continue; + } + in_segment[candidate.idx()] = true; + segment.push((candidate, 0)); + push_normal_successors(&mut stack, &self.blocks[candidate.idx()]); + } + } + for (block_idx, start) in segment { if visited[block_idx.idx()] { continue; } visited[block_idx.idx()] = true; if block_idx == seed { + let protected_store_locals = + (!nested_protected_tail).then_some(protected_store_locals.as_slice()); deoptimize_protected_block_borrows_from( &mut self.blocks[block_idx.idx()], start, - &protected_store_locals, + protected_store_locals, ); } else { deoptimize_block_borrows_from(&mut self.blocks[block_idx.idx()], start); @@ -11550,6 +11634,15 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { && target_ins.target != BlockIdx::NULL && target_ins.target != target { + if !include_conditional + && blocks[target.idx()] + .instructions + .iter() + .take_while(|info| matches!(info.instr.real(), Some(Instruction::Nop))) + .any(instruction_has_lineno) + { + continue; + } let source_pos = block_order[bi]; let target_pos = block_order.get(target.idx()).copied().unwrap_or(u32::MAX); let final_target = target_ins.target; From f7d82408be0d4824d928665f67b03250df3ff735 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 10:24:44 +0900 Subject: [PATCH 44/76] Align conditional loop backedge layout --- crates/codegen/src/compile.rs | 44 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 18 ++++++++++++-- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 8bc9325598..9788e980c2 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -21442,6 +21442,50 @@ def while_not_chained(a, b, c): ); } + #[test] + fn test_nested_while_chained_compare_break_keeps_break_jump_block() { + let code = compile_exec( + "\ +def f(start, self, stop, size): + while size > 0: + while True: + if start <= self.position < stop: + break + else: + self.map_index += 1 + if self.map_index == len(self.map): + self.map_index = 0 + length = min(size, stop - self.position) + size -= length + return size +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpForward { .. }, + Instruction::PopTop, + Instruction::JumpForward { .. }, + Instruction::JumpForward { .. }, + ] + ) + }), + "expected CPython-style chained-compare success hop into the break jump block, got ops={ops:?}" + ); + } + #[test] fn test_while_break_else_keeps_true_edge_into_forward_break_body() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 784bee9c6c..ad39d03373 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -11643,6 +11643,9 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { { continue; } + if !include_conditional && instruction_has_lineno(&target_ins) { + continue; + } let source_pos = block_order[bi]; let target_pos = block_order.get(target.idx()).copied().unwrap_or(u32::MAX); let final_target = target_ins.target; @@ -14601,7 +14604,17 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); let after_jump_target = next_nonempty_block(blocks, after_jump); let after_jump_continues_conditional_chain = after_jump_target != BlockIdx::NULL - && trailing_conditional_jump_index(&blocks[after_jump_target.idx()]).is_some(); + && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); + let simple_single_block_can_reorder = body_is_single_block + && !body_tail_is_conditional + && !block_has_call(&blocks[body.idx()]) + && !body_has_scope_exit + && !blocks[body.idx()] + .instructions + .iter() + .any(|info| info.instr.is_unconditional_jump()) + && !after_jump_continues_conditional_chain + && matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })); let body_starts_with_conditional_test = block_is_pure_conditional_test(&blocks[body.idx()]); let trailing_implicit_continue_can_reorder = after_jump != BlockIdx::NULL @@ -14613,7 +14626,8 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && body_is_single_block && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) - || body_has_for_iter)) + || body_has_for_iter + || simple_single_block_can_reorder)) || (trailing_implicit_continue_can_reorder && (body_has_scope_exit || body_has_loop_backedge) && (!after_jump_continues_conditional_chain From 5001df7781ee85eaed7ef14a7afd8c9e33ca04ca Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 10:52:08 +0900 Subject: [PATCH 45/76] Align protected loop exit duplication --- crates/codegen/src/compile.rs | 49 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 36 +++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 9788e980c2..b2333ec2e3 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20286,6 +20286,55 @@ def f(new, old): ); } + #[test] + fn test_try_loop_inner_if_keeps_duplicate_jump_back_paths() { + let code = compile_exec( + "\ +def f(config, logging): + handlers = config.get('handlers', {}) + for name in handlers: + if name not in logging._handlers: + raise ValueError('missing') + else: + try: + handler = logging._handlers[name] + handler_config = handlers[name] + level = handler_config.get('level', None) + if level: + handler.setLevel(logging._checkLevel(level)) + except Exception as e: + raise ValueError('bad') from e + loggers = config.get('loggers', {}) + for name in loggers: + pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::EndFor, + ] + ) + }), + "expected CPython-style separate body and false-path jump-back blocks, got ops={ops:?}" + ); + } + #[test] fn test_line_bearing_loop_if_false_backedge_keeps_body_before_jump_back() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index ad39d03373..12d226f3ab 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -12861,6 +12861,30 @@ fn is_exit_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { && has_non_exception_loop_backedge_to(blocks, block_idx, last.target) } +fn is_eval_break_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { + let block = &blocks[block_idx.idx()]; + let Some(first) = block.instructions.first() else { + return false; + }; + !instruction_has_lineno(first) && block_has_no_lineno(block) && block_has_eval_break(block) +} + +fn block_has_eval_break(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr, + AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) + | AnyInstruction::Real( + Instruction::Call { .. } + | Instruction::CallFunctionEx + | Instruction::CallKw { .. } + | Instruction::JumpBackward { .. } + | Instruction::Resume { .. } + ) + ) + }) +} + fn block_has_no_lineno(block: &Block) -> bool { block .instructions @@ -14963,7 +14987,14 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve }; let target = next_nonempty_block(blocks, last.target); - if target == BlockIdx::NULL || !is_exit_without_lineno(blocks, target) { + if target == BlockIdx::NULL { + current = blocks[current.idx()].next; + continue; + } + let target_is_exit_without_lineno = is_exit_without_lineno(blocks, target); + let target_is_protected_eval_break = + is_eval_break_without_lineno(blocks, target) && last.except_handler.is_some(); + if !target_is_exit_without_lineno && !target_is_protected_eval_break { current = blocks[current.idx()].next; continue; } @@ -15012,7 +15043,8 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve current, target, )) - && is_exit_without_lineno(blocks, target) + && (is_exit_without_lineno(blocks, target) + || is_eval_break_without_lineno(blocks, target)) && let Some((location, end_location)) = propagation_location(last) && let Some(first) = blocks[target.idx()].instructions.first_mut() { From 6b604df853ce9b70b3db01997eb0fa182f955f6b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 11:15:57 +0900 Subject: [PATCH 46/76] Preserve protected jump-back duplicates --- crates/codegen/src/compile.rs | 50 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 18 +++++++++++++ 2 files changed, 68 insertions(+) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index b2333ec2e3..145048873a 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20335,6 +20335,56 @@ def f(config, logging): ); } + #[test] + fn test_try_loop_nested_bool_tail_keeps_duplicate_jump_back_paths() { + let code = compile_exec( + "\ +def f(obj, flags, writer, value, Error): + while obj.running: + try: + if value == 0: + return + elif obj.ready and obj.active and value == 1: + obj.work() + if flags.verbose and obj.chatty: + writer.write('trace') + elif value == 2: + obj.other() + else: + obj.default() + except Error as e: + obj.running = False +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }, + ] + ) + }), + "expected CPython-style body and both bool false-path jump-back blocks, got ops={ops:?}" + ); + } + #[test] fn test_line_bearing_loop_if_false_backedge_keeps_body_before_jump_back() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 12d226f3ab..1937aa86d9 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14301,6 +14301,22 @@ fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { .then(|| instruction_lineno(info)) } + fn has_protected_conditional_predecessor( + blocks: &[Block], + incoming_origins: &[Vec], + target: BlockIdx, + ) -> bool { + incoming_origins[target.idx()].iter().copied().any(|origin| { + blocks[origin.idx()].instructions.iter().any(|info| { + info.except_handler.is_some() + && is_conditional_jump(&info.instr) + && next_nonempty_block(blocks, info.target) == target + }) + }) + } + + let reachable = compute_reachable_blocks(blocks); + let incoming_origins = compute_incoming_origins(blocks, &reachable); let mut current = BlockIdx(0); while current != BlockIdx::NULL { let Some(target) = jump_back_target(&blocks[current.idx()]) else { @@ -14322,6 +14338,8 @@ fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { || jump_back_target(&blocks[duplicate.idx()]) != Some(target) || jump_back_lineno(&blocks[duplicate.idx()]) != jump_back_lineno(&blocks[current.idx()]) + || has_protected_conditional_predecessor(blocks, &incoming_origins, current) + || has_protected_conditional_predecessor(blocks, &incoming_origins, duplicate) { current = blocks[current.idx()].next; continue; From 83f109e053cc9cb0287fe15f116ddc6c8abad642 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 11:40:02 +0900 Subject: [PATCH 47/76] Align loop call-body backedge layout --- crates/codegen/src/compile.rs | 44 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 26 ++++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 145048873a..a299879bc6 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -25880,6 +25880,50 @@ def f(_config_vars, _INITPRE): ); } + #[test] + fn test_loop_if_call_body_implicit_continue_places_body_after_jumpback() { + let code = compile_exec( + "\ +def f(seq, db): + count = 0 + for c in seq: + dec = db.value(c, -1) + if dec != -1: + db.check(dec, c) + count += 1 + return count +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::Call { .. }, + ] + ) + }), + "no-else loop call body should use CPython body-after-jumpback layout, got ops={ops:?}" + ); + } + #[test] fn test_loop_nested_if_delete_slice_places_body_after_jumpback() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 1937aa86d9..8392ac131a 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14597,6 +14597,23 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec }) } + fn has_exceptional_duplicate_lineno( + blocks: &[Block], + source: BlockIdx, + lineno: i32, + ) -> bool { + blocks.iter().enumerate().any(|(idx, block)| { + BlockIdx(idx as u32) != source + && (block.cold + || block_is_exceptional(block) + || block_is_protected(block)) + && block + .instructions + .iter() + .any(|info| instruction_lineno(info) == lineno) + }) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -14649,7 +14666,14 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); let simple_single_block_can_reorder = body_is_single_block && !body_tail_is_conditional - && !block_has_call(&blocks[body.idx()]) + && (!block_has_call(&blocks[body.idx()]) + || (block_starts_loop_cleanup(blocks, after_jump_target) + && !block_is_protected(&blocks[body.idx()]) + && !has_exceptional_duplicate_lineno( + blocks, + current, + instruction_lineno(&cond), + ))) && !body_has_scope_exit && !blocks[body.idx()] .instructions From 3e11e1fb31131a10a211a08ceca265315d0cb80e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 12:03:49 +0900 Subject: [PATCH 48/76] Align future annotation setup ordering --- crates/codegen/src/compile.rs | 170 +++++++++++++++++++++++++----- crates/codegen/src/ir.rs | 27 +++-- crates/codegen/src/symboltable.rs | 16 +-- 3 files changed, 161 insertions(+), 52 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index a299879bc6..97bc344e01 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -2137,16 +2137,8 @@ impl Compiler { emit!(self, PseudoInstruction::AnnotationsPlaceholder); let (doc, statements) = split_doc(&body.body, &self.opts); - if let Some(value) = doc { - self.emit_load_const(ConstantData::Str { - value: value.into(), - }); - let doc = self.name("__doc__"); - emit!(self, Instruction::StoreName { namei: doc }) - } - - // Handle annotation bookkeeping in CPython order: initialize the - // conditional annotation set first, then materialize __annotations__. + // Handle annotation bookkeeping before the docstring assignment, as + // codegen_body() does after _PyCodegen_Module() inserts the prefix set. if Self::find_ann(statements) { if Self::scope_needs_conditional_annotations_cell(self.current_symbol_table()) { emit!(self, Instruction::BuildSet { count: 0 }); @@ -2158,6 +2150,14 @@ impl Compiler { } } + if let Some(value) = doc { + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + let doc = self.name("__doc__"); + emit!(self, Instruction::StoreName { namei: doc }) + } + // Compile all statements self.compile_statements(statements)?; @@ -2300,13 +2300,8 @@ impl Compiler { fn scope_needs_conditional_annotations_cell(symbol_table: &SymbolTable) -> bool { match symbol_table.typ { - CompilerScope::Module => { - symbol_table.has_conditional_annotations - || (symbol_table.future_annotations && symbol_table.annotation_block.is_some()) - } - CompilerScope::Class => { + CompilerScope::Module | CompilerScope::Class => { symbol_table.has_conditional_annotations - || symbol_table.lookup("__conditional_annotations__").is_some() } _ => false, } @@ -5343,13 +5338,6 @@ impl Compiler { emit!(self, Instruction::StoreDeref { i: classdict_idx }); } - // Store __doc__ only if there's an explicit docstring. - if let Some(doc) = doc_str { - self.emit_load_const(ConstantData::Str { value: doc.into() }); - let doc_name = self.name("__doc__"); - emit!(self, Instruction::StoreName { namei: doc_name }); - } - // Handle class annotation bookkeeping in CPython order. if Self::find_ann(body) { if Self::scope_needs_conditional_annotations_cell(self.current_symbol_table()) { @@ -5362,6 +5350,13 @@ impl Compiler { } } + // Store __doc__ only if there's an explicit docstring. + if let Some(doc) = doc_str { + self.emit_load_const(ConstantData::Str { value: doc.into() }); + let doc_name = self.name("__doc__"); + emit!(self, Instruction::StoreName { namei: doc_name }); + } + // 3. Compile the class body self.compile_statements(body)?; @@ -16650,7 +16645,7 @@ class C: } #[test] - fn test_future_annotations_class_keeps_conditional_annotations_cell() { + fn test_future_annotations_class_uses_direct_annotation_store() { let code = compile_exec( "\ from __future__ import annotations @@ -16660,16 +16655,141 @@ class C: ); let class_code = find_code(&code, "C").expect("missing class code"); + assert!( + !class_code + .cellvars + .iter() + .any(|name| name.as_str() == "__conditional_annotations__"), + "future annotations should not create __conditional_annotations__ cellvar, got cellvars={:?}", + class_code.cellvars + ); + let ops: Vec<_> = class_code + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + assert!( + ops.iter() + .any(|op| matches!(op, Instruction::SetupAnnotations)), + "future annotations should emit SETUP_ANNOTATIONS, got ops={ops:?}" + ); + assert!( + ops.iter().any(|op| matches!(op, Instruction::StoreSubscr)), + "future annotations should store directly into __annotations__, got ops={ops:?}" + ); + assert!( + !ops.iter() + .any(|op| matches!(op, Instruction::BuildSet { .. })), + "future annotations should not initialize __conditional_annotations__, got ops={ops:?}" + ); + } + + #[test] + fn test_future_annotations_module_keeps_conditional_annotations_cell() { + let code = compile_exec( + "\ +from __future__ import annotations +x: int = 1 +", + ); + + assert!( + code.cellvars + .iter() + .any(|name| name.as_str() == "__conditional_annotations__"), + "module annotations should create __conditional_annotations__ cellvar, got cellvars={:?}", + code.cellvars + ); + } + + #[test] + fn test_future_annotations_conditional_class_keeps_conditional_annotations_cell() { + let code = compile_exec( + "\ +from __future__ import annotations +class C: + if True: + x: int = 1 +", + ); + let class_code = find_code(&code, "C").expect("missing class code"); + assert!( class_code .cellvars .iter() .any(|name| name.as_str() == "__conditional_annotations__"), - "expected __conditional_annotations__ cellvar, got cellvars={:?}", + "conditional class annotations should create __conditional_annotations__ cellvar, got cellvars={:?}", class_code.cellvars ); } + #[test] + fn test_future_annotations_setup_precedes_docstring() { + let code = compile_exec( + "\ +\"module doc\" +from __future__ import annotations +x: int = 1 + +class C: + \"class doc\" + x: int = 1 +", + ); + let module_setup = code + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::SetupAnnotations)) + .expect("missing module SETUP_ANNOTATIONS"); + let module_doc = code + .instructions + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::StoreName { namei } + if code.names + [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] + .as_str() + == "__doc__" + ) + }) + .expect("missing module doc store"); + assert!( + module_setup < module_doc, + "module SETUP_ANNOTATIONS should precede docstring store, got instructions={:?}", + code.instructions + ); + + let class_code = find_code(&code, "C").expect("missing class code"); + let class_setup = class_code + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::SetupAnnotations)) + .expect("missing class SETUP_ANNOTATIONS"); + let class_doc = class_code + .instructions + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::StoreName { namei } + if class_code.names + [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] + .as_str() + == "__doc__" + ) + }) + .expect("missing class doc store"); + assert!( + class_setup < class_doc, + "class SETUP_ANNOTATIONS should precede docstring store, got instructions={:?}", + class_code.instructions + ); + } + #[test] fn test_plain_super_call_keeps_class_freevar() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8392ac131a..3e438119b0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -12881,7 +12881,7 @@ fn block_has_eval_break(block: &Block) -> bool { | Instruction::JumpBackward { .. } | Instruction::Resume { .. } ) - ) + ) }) } @@ -14306,13 +14306,16 @@ fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { incoming_origins: &[Vec], target: BlockIdx, ) -> bool { - incoming_origins[target.idx()].iter().copied().any(|origin| { - blocks[origin.idx()].instructions.iter().any(|info| { - info.except_handler.is_some() - && is_conditional_jump(&info.instr) - && next_nonempty_block(blocks, info.target) == target + incoming_origins[target.idx()] + .iter() + .copied() + .any(|origin| { + blocks[origin.idx()].instructions.iter().any(|info| { + info.except_handler.is_some() + && is_conditional_jump(&info.instr) + && next_nonempty_block(blocks, info.target) == target + }) }) - }) } let reachable = compute_reachable_blocks(blocks); @@ -14597,16 +14600,10 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec }) } - fn has_exceptional_duplicate_lineno( - blocks: &[Block], - source: BlockIdx, - lineno: i32, - ) -> bool { + fn has_exceptional_duplicate_lineno(blocks: &[Block], source: BlockIdx, lineno: i32) -> bool { blocks.iter().enumerate().any(|(idx, block)| { BlockIdx(idx as u32) != source - && (block.cold - || block_is_exceptional(block) - || block_is_protected(block)) + && (block.cold || block_is_exceptional(block) || block_is_protected(block)) && block .instructions .iter() diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 23313b1a59..f1c087332f 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -1324,24 +1324,16 @@ impl SymbolTableBuilder { is_ann_assign: bool, ) -> SymbolTableResult { let current_scope = self.tables.last().map(|t| t.typ); - let needs_future_annotation_bookkeeping = is_ann_assign - && self.future_annotations - && matches!( - current_scope, - Some(CompilerScope::Module | CompilerScope::Class) - ); - let needs_non_future_conditional_annotations = is_ann_assign - && !self.future_annotations + let needs_conditional_annotations = is_ann_assign && (matches!(current_scope, Some(CompilerScope::Module)) || (matches!(current_scope, Some(CompilerScope::Class)) && self.in_conditional_block)); - let should_register_conditional_annotations = needs_future_annotation_bookkeeping - || (needs_non_future_conditional_annotations - && !self.tables.last().unwrap().has_conditional_annotations); + let should_register_conditional_annotations = needs_conditional_annotations + && !self.tables.last().unwrap().has_conditional_annotations; // PEP 649: Only AnnAssign annotations can be conditional. // Function parameter/return annotations are never conditional. - if needs_non_future_conditional_annotations { + if needs_conditional_annotations { self.tables.last_mut().unwrap().has_conditional_annotations = true; } From 88ec62547b5ca70893a57bc5fba58c7588bdd6ae Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 12:28:08 +0900 Subject: [PATCH 49/76] Align named-except cleanup and borrow parity --- crates/codegen/src/compile.rs | 102 +++++++++++++++++++++++++++++++--- crates/codegen/src/ir.rs | 55 +++++------------- 2 files changed, 109 insertions(+), 48 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 97bc344e01..4fa9d92585 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -17208,6 +17208,46 @@ def f(self): ); } + #[test] + fn test_named_except_with_suppress_does_not_duplicate_following_with() { + let code = compile_exec( + "\ +def f(StringIO, captured_output, print): + try: + raise KeyError + except KeyError as e: + with captured_output('stderr') as tbstderr: + print('x') + with captured_output('stderr') as tbstderr: + print('y') + else: + print('else') + s = StringIO() + return s +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let load_y_count = f + .instructions + .iter() + .filter(|unit| match unit.op { + Instruction::LoadConst { consti } => { + matches!( + &f.constants + [consti.get(OpArg::new(u32::from(u8::from(unit.arg))))], + ConstantData::Str { value } if value.as_str() == Ok("y") + ) + } + _ => false, + }) + .count(); + + assert_eq!( + load_y_count, 1, + "following with body should not be duplicated into the previous with suppress path" + ); + } + #[test] fn test_bare_except_deopts_post_handler_load_fast_borrow() { let code = compile_exec( @@ -17498,7 +17538,7 @@ def f(self, typ, dat): } #[test] - fn test_multi_protected_method_call_terminal_handler_deopts_block() { + fn test_multi_protected_method_call_terminal_handler_keeps_try_body_borrows() { let code = compile_exec( "\ def f(self, literal): @@ -17541,22 +17581,22 @@ def f(self, literal): assert!( matches!( instructions[first_send - 1].op, - Instruction::LoadFast { .. } + Instruction::LoadFastBorrow { .. } ), - "CPython uses strong LOAD_FAST for first protected send receiver, got ops={:?}", + "CPython keeps first protected send receiver borrowed, got ops={:?}", instructions.iter().map(|unit| unit.op).collect::>() ); assert!( - matches!(first_literal, Instruction::LoadFast { .. }), - "CPython uses strong LOAD_FAST for first protected send arg, got ops={:?}", + matches!(first_literal, Instruction::LoadFastBorrow { .. }), + "CPython keeps first protected send arg borrowed, got ops={:?}", instructions.iter().map(|unit| unit.op).collect::>() ); assert!( matches!( instructions[second_send - 1].op, - Instruction::LoadFast { .. } + Instruction::LoadFastBorrow { .. } ), - "CPython uses strong LOAD_FAST for second protected send receiver, got ops={:?}", + "CPython keeps second protected send receiver borrowed, got ops={:?}", instructions.iter().map(|unit| unit.op).collect::>() ); } @@ -25878,6 +25918,54 @@ def f(items, chunk, out, packI, Error): } } + #[test] + fn test_terminal_reraising_handler_keeps_try_body_method_borrows() { + let code = compile_exec( + "\ +def f(self): + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let first_handler = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let try_body = &instructions[..first_handler]; + let self_borrows = try_body + .iter() + .filter(|unit| { + matches!(unit.op, Instruction::LoadFastBorrow { var_num } + if f.varnames[usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))))] + == "self") + }) + .count(); + + assert!( + self_borrows >= 8, + "terminal reraising handler should keep try-body self loads borrowed, got try_body={try_body:?}" + ); + } + #[test] fn test_terminal_except_loop_successor_augassign_uses_strong_load_pair() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 3e438119b0..b6ffccabd8 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -183,6 +183,19 @@ fn is_named_except_cleanup_normal_exit_block(block: &Block) -> bool { && tail[4].instr.is_unconditional_jump() } +fn is_standalone_named_except_cleanup_normal_exit_block(block: &Block) -> bool { + let len = block.instructions.len(); + if len < 5 || !is_named_except_cleanup_normal_exit_block(block) { + return false; + } + block.instructions[..len - 5].iter().all(|info| { + matches!( + info.instr.real(), + Some(Instruction::Nop | Instruction::NotTaken) + ) + }) +} + // spell-checker:ignore petgraph // TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it // might enable more analysis/optimizations @@ -6708,36 +6721,6 @@ impl CodeInfo { handlers } - fn protected_method_call_count(blocks: &[Block], block: &Block) -> usize { - block - .instructions - .iter() - .filter(|info| { - let is_method_load = matches!( - info.instr.real(), - Some(Instruction::LoadAttr { namei }) if namei.get(info.arg).is_method() - ); - is_method_load - && info.except_handler.is_some_and(|handler| { - handler_chain_has_exception_match(blocks, handler.handler_block) - }) - }) - .count() - } - - fn block_stores_fast(block: &Block) -> bool { - block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some( - Instruction::StoreFast { .. } - | Instruction::StoreFastLoadFast { .. } - | Instruction::StoreFastStoreFast { .. } - ) - ) - }) - } - fn block_shares_handler(block: &Block, handlers: &[BlockIdx]) -> bool { block .instructions @@ -6951,16 +6934,6 @@ impl CodeInfo { let mut to_deopt = Vec::new(); for (idx, block) in self.blocks.iter().enumerate() { - if !block_is_exceptional(block) - && !block.cold - && !block_stores_fast(block) - && protected_method_call_count(&self.blocks, block) >= 2 - && protected_block_has_terminal_exception_handler(&self.blocks, block) - && protected_block_has_raising_exception_handler(&self.blocks, block) - { - to_deopt.push(BlockIdx::new(idx as u32)); - continue; - } if block_is_exceptional(block) || block.cold || starts_with_inlined_comprehension_restore(block) @@ -15756,7 +15729,7 @@ fn inline_named_except_cleanup_normal_exit_jumps(blocks: &mut [Block]) { let target = next_nonempty_block(blocks, jump.target); if target == BlockIdx::NULL || target == BlockIdx(block_idx as u32) - || !is_named_except_cleanup_normal_exit_block(&blocks[target.idx()]) + || !is_standalone_named_except_cleanup_normal_exit_block(&blocks[target.idx()]) { continue; } From e24a9b770c948ca10c11d7bb8b70b02a9a3a1c8d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 12:51:21 +0900 Subject: [PATCH 50/76] Align redundant jump removal with CPython --- crates/codegen/src/ir.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index b6ffccabd8..98ee5d99a3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -12359,25 +12359,6 @@ fn remove_redundant_jumps_in_blocks(blocks: &mut [Block]) -> usize { && last_instr.target != BlockIdx::NULL && next_nonempty_block(blocks, last_instr.target) == next { - let preserve_as_nop = if last_instr.preserve_redundant_jump_as_nop { - let line = instruction_lineno(&last_instr); - let next_line = blocks[next.idx()].instructions.iter().find_map(|instr| { - let line = instruction_lineno(instr); - (!matches!(instr.instr.real(), Some(Instruction::Nop)) || line >= 0) - .then_some(line) - }); - line > 0 && next_line.is_some_and(|next_line| next_line < line) - } else { - false - }; - if preserve_as_nop { - current = blocks[idx].next; - continue; - } - if last_instr.preserve_redundant_jump_as_nop { - let last_instr = blocks[idx].instructions.last_mut().unwrap(); - last_instr.preserve_redundant_jump_as_nop = false; - } let last_instr = blocks[idx].instructions.last_mut().unwrap(); let remove_no_location_nop = last_instr.remove_no_location_nop; let folded_operand_nop = last_instr.folded_operand_nop; From 7f1376be70efadf4b22d093fe47260c38b91c095 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 13:14:22 +0900 Subject: [PATCH 51/76] Preserve finally cleanup jump NOPs --- crates/codegen/src/compile.rs | 41 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 18 +++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 4fa9d92585..563931b236 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -14928,6 +14928,47 @@ def f(a, b, d): ); } + #[test] + fn test_with_try_finally_normal_cleanup_keeps_redundant_jump_nop() { + let code = compile_exec( + "\ +def f(cm): + with cm: + try: + x = 1 + finally: + del x + return x +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops_lines: Vec<_> = f + .instructions + .iter() + .zip(&f.locations) + .filter_map(|(unit, (location, _))| { + (!matches!(unit.op, Instruction::Cache)).then_some((unit.op, location.line.get())) + }) + .collect(); + + assert!( + ops_lines.windows(6).any(|window| { + matches!( + window, + [ + (Instruction::DeleteFast { .. }, 6), + (Instruction::Nop, 6), + (Instruction::LoadConst { .. }, 2), + (Instruction::LoadConst { .. }, 2), + (Instruction::LoadConst { .. }, 2), + (Instruction::Call { .. }, 2), + ] + ) + }), + "expected CPython-style redundant finally cleanup jump to become a line NOP before with-exit cleanup, got ops_lines={ops_lines:?}", + ); + } + #[test] fn test_try_except_finally_normal_cleanup_keeps_body_exit_nop() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 98ee5d99a3..77b1d7f1d0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -496,7 +496,9 @@ impl CodeInfo { let mut remove = false; if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if lineno < 0 + if instr.preserve_redundant_jump_as_nop { + remove = false; + } else if lineno < 0 || prev_lineno == lineno || (src > 0 && matches!( @@ -12359,16 +12361,28 @@ fn remove_redundant_jumps_in_blocks(blocks: &mut [Block]) -> usize { && last_instr.target != BlockIdx::NULL && next_nonempty_block(blocks, last_instr.target) == next { + let preserve_redundant_jump_nop = if last_instr.preserve_redundant_jump_as_nop { + let line = instruction_lineno(&last_instr); + let next_line = blocks[next.idx()].instructions.iter().find_map(|instr| { + let line = instruction_lineno(instr); + (!matches!(instr.instr.real(), Some(Instruction::Nop)) || line >= 0) + .then_some(line) + }); + line > 0 && next_line.is_some_and(|next_line| next_line < line) + } else { + false + }; let last_instr = blocks[idx].instructions.last_mut().unwrap(); let remove_no_location_nop = last_instr.remove_no_location_nop; let folded_operand_nop = last_instr.folded_operand_nop; let preserve_block_start_no_location_nop = last_instr.preserve_block_start_no_location_nop; set_to_nop(last_instr); + last_instr.preserve_redundant_jump_as_nop = preserve_redundant_jump_nop; last_instr.remove_no_location_nop = remove_no_location_nop; last_instr.folded_operand_nop = folded_operand_nop; last_instr.preserve_block_start_no_location_nop = - preserve_block_start_no_location_nop; + preserve_block_start_no_location_nop || preserve_redundant_jump_nop; changes += 1; current = blocks[idx].next; continue; From 6f95452390bfa9b0d30890926174f4437160856f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 13:24:08 +0900 Subject: [PATCH 52/76] Align finally cleanup CFG with CPython --- crates/codegen/src/compile.rs | 4 ++-- crates/codegen/src/ir.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 563931b236..03967ab049 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3479,11 +3479,11 @@ impl Compiler { } // Jump to end (skip exception path blocks) + self.set_no_location(); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } ); - self.set_no_location(); self.preserve_last_redundant_jump_as_nop(); if let Some(finally_except) = finally_except_block { @@ -3718,11 +3718,11 @@ impl Compiler { self.compile_statements(finalbody)?; // Jump to end_block to skip exception path blocks // This prevents fall-through to finally_except_block + self.set_no_location(); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } ); - self.set_no_location(); self.preserve_last_redundant_jump_as_nop(); // finally (exception path) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 77b1d7f1d0..473480171f 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14631,6 +14631,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); let simple_single_block_can_reorder = body_is_single_block && !body_tail_is_conditional + && !has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&cond)) && (!block_has_call(&blocks[body.idx()]) || (block_starts_loop_cleanup(blocks, after_jump_target) && !block_is_protected(&blocks[body.idx()]) From 1d078e383573266e742f39dd84299a6e518d776b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 13:46:44 +0900 Subject: [PATCH 53/76] Fix finally cleanup CFG regression --- crates/codegen/src/compile.rs | 4 +- crates/codegen/src/ir.rs | 94 ++++++++++++++++++++--------------- 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 03967ab049..563931b236 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3479,11 +3479,11 @@ impl Compiler { } // Jump to end (skip exception path blocks) - self.set_no_location(); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } ); + self.set_no_location(); self.preserve_last_redundant_jump_as_nop(); if let Some(finally_except) = finally_except_block { @@ -3718,11 +3718,11 @@ impl Compiler { self.compile_statements(finalbody)?; // Jump to end_block to skip exception path blocks // This prevents fall-through to finally_except_block - self.set_no_location(); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } ); + self.set_no_location(); self.preserve_last_redundant_jump_as_nop(); // finally (exception path) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 473480171f..4b8969a874 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -12882,26 +12882,33 @@ fn shared_jump_back_target(block: &Block) -> Option { Some(last.target) } -fn has_non_exception_loop_backedge_to( +fn block_has_non_exception_loop_backedge_to( blocks: &[Block], - cleanup_block: BlockIdx, + source: BlockIdx, target: BlockIdx, ) -> bool { let target = next_nonempty_block(blocks, target); - if target == BlockIdx::NULL { - return false; - } + source != BlockIdx::NULL + && target != BlockIdx::NULL + && !block_is_exceptional(&blocks[source.idx()]) + && comes_before(blocks, target, source) + && blocks[source.idx()].instructions.iter().any(|info| { + info.instr.is_unconditional_jump() + && info.target != BlockIdx::NULL + && next_nonempty_block(blocks, info.target) == target + }) +} +fn has_non_exception_loop_backedge_to( + blocks: &[Block], + cleanup_block: BlockIdx, + target: BlockIdx, +) -> bool { blocks.iter().enumerate().any(|(source_idx, block)| { let source = BlockIdx(source_idx as u32); source != cleanup_block && !block_is_exceptional(block) - && comes_before(blocks, target, source) - && block.instructions.iter().any(|info| { - info.instr.is_unconditional_jump() - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) + && block_has_non_exception_loop_backedge_to(blocks, source, target) }) } @@ -13166,6 +13173,17 @@ fn block_is_exceptional(block: &Block) -> bool { block.except_handler || block.preserve_lasti || is_exception_cleanup_block(block) } +fn has_exceptional_duplicate_lineno(blocks: &[Block], source: BlockIdx, lineno: i32) -> bool { + blocks.iter().enumerate().any(|(idx, block)| { + BlockIdx(idx as u32) != source + && (block.cold || block_is_exceptional(block) || block_is_protected(block)) + && block + .instructions + .iter() + .any(|info| instruction_lineno(info) == lineno) + }) +} + fn trailing_conditional_jump_index(block: &Block) -> Option { let last_idx = block.instructions.len().checked_sub(1)?; if is_conditional_jump(&block.instructions[last_idx].instr) @@ -13583,6 +13601,12 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; } + if is_generic_false_path_reorder + && has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&last)) + { + current = next; + continue; + } if block_is_protected(&blocks[idx]) && block_contains_suspension_point(&blocks[idx]) { current = next; continue; @@ -14568,17 +14592,6 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec }) } - fn has_exceptional_duplicate_lineno(blocks: &[Block], source: BlockIdx, lineno: i32) -> bool { - blocks.iter().enumerate().any(|(idx, block)| { - BlockIdx(idx as u32) != source - && (block.cold || block_is_exceptional(block) || block_is_protected(block)) - && block - .instructions - .iter() - .any(|info| instruction_lineno(info) == lineno) - }) - } - let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -14626,20 +14639,18 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec body_segment_contains_jump_back_to(blocks, body, body_tail, loop_target); let normalized_single_block_can_reorder = !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); + let has_exceptional_duplicate_condition_line = + has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&cond)); let after_jump_target = next_nonempty_block(blocks, after_jump); let after_jump_continues_conditional_chain = after_jump_target != BlockIdx::NULL && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); let simple_single_block_can_reorder = body_is_single_block && !body_tail_is_conditional - && !has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&cond)) + && !has_exceptional_duplicate_condition_line && (!block_has_call(&blocks[body.idx()]) || (block_starts_loop_cleanup(blocks, after_jump_target) && !block_is_protected(&blocks[body.idx()]) - && !has_exceptional_duplicate_lineno( - blocks, - current, - instruction_lineno(&cond), - ))) + && !has_exceptional_duplicate_condition_line)) && !body_has_scope_exit && !blocks[body.idx()] .instructions @@ -14653,17 +14664,19 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && next_nonempty_block(blocks, after_jump) != body && !block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_jump)) && !is_scope_exit_block(&blocks[body.idx()]); - let can_reorder = (!body_tail_is_conditional - && ((normalized_single_block_can_reorder - && body_is_single_block - && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) - || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) - || body_has_for_iter - || simple_single_block_can_reorder)) - || (trailing_implicit_continue_can_reorder - && (body_has_scope_exit || body_has_loop_backedge) - && (!after_jump_continues_conditional_chain - || body_starts_with_conditional_test)); + let can_reorder = !has_exceptional_duplicate_condition_line + && ((!body_tail_is_conditional + && ((normalized_single_block_can_reorder + && body_is_single_block + && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) + || (body == body_tail + && is_single_delete_subscr_body(&blocks[body.idx()])) + || body_has_for_iter + || simple_single_block_can_reorder)) + || (trailing_implicit_continue_can_reorder + && (body_has_scope_exit || body_has_loop_backedge) + && (!after_jump_continues_conditional_chain + || body_starts_with_conditional_test))); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { let cloned_jump_idx = BlockIdx(blocks.len() as u32); let mut cloned_jump = blocks[true_jump.idx()].clone(); @@ -15347,7 +15360,8 @@ fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { let jump_target = next_nonempty_block(blocks, blocks[target.idx()].instructions[0].target); if jump_target == BlockIdx::NULL - || !has_non_exception_loop_backedge_to(blocks, target, jump_target) + || (!has_non_exception_loop_backedge_to(blocks, target, jump_target) + && !block_has_non_exception_loop_backedge_to(blocks, target, jump_target)) { layout_pred = blocks[layout_pred.idx()].next; continue; From 85c70caa524c19ebbe1c96a49395cf4cc1c9e6a1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 14:20:27 +0900 Subject: [PATCH 54/76] Align async cleanup CFG marker handling --- crates/codegen/src/compile.rs | 41 +++++++--------- crates/codegen/src/ir.rs | 90 +++++++++++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 27 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 563931b236..c799920980 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -29,9 +29,9 @@ use rustpython_compiler_core::{ Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, bytecode::{ self, AnyInstruction, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, CodeObject, - ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, InstructionMetadata, - IntrinsicFunction1, Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, - SpecialMethod, UnpackExArgs, oparg, + ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, IntrinsicFunction1, + Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, SpecialMethod, + UnpackExArgs, oparg, }, }; use rustpython_wtf8::Wtf8Buf; @@ -10068,7 +10068,6 @@ impl Compiler { let code = self.code_stack.last().expect("no code on stack"); let target = code.current_block; let mut has_suppress_exit = false; - let mut has_normal_exit = false; for block in &code.blocks { let Some((last, prefix)) = block.instructions.split_last() else { @@ -10077,29 +10076,23 @@ impl Compiler { if last.target != target { continue; } - match last.instr.pseudo() { - Some(PseudoInstruction::JumpNoInterrupt { .. }) => { - let real_instrs: Vec<_> = - prefix.iter().filter_map(|info| info.instr.real()).collect(); - has_suppress_exit |= matches!( - real_instrs.as_slice(), - [ - Instruction::PopTop, - Instruction::PopExcept, - Instruction::PopTop, - Instruction::PopTop, - Instruction::PopTop, - ] - ); - } - Some(PseudoInstruction::Jump { .. }) => { - has_normal_exit |= !prefix.iter().any(|info| info.instr.is_scope_exit()); - } - _ => {} + if let Some(PseudoInstruction::JumpNoInterrupt { .. }) = last.instr.pseudo() { + let real_instrs: Vec<_> = + prefix.iter().filter_map(|info| info.instr.real()).collect(); + has_suppress_exit |= matches!( + real_instrs.as_slice(), + [ + Instruction::PopTop, + Instruction::PopExcept, + Instruction::PopTop, + Instruction::PopTop, + Instruction::PopTop, + ] + ); } } - has_suppress_exit && !has_normal_exit + has_suppress_exit } fn remove_last_no_location_nop(&mut self) { diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4b8969a874..a5bea29118 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -12257,9 +12257,12 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { if matches!(instr.instr.real(), Some(Instruction::Nop)) { if src == 0 - && !keep_target_start_nop && lineno > 0 - && follows_same_line_pop_iter == Some(lineno) + && ((!keep_target_start_nop && follows_same_line_pop_iter == Some(lineno)) + || (instr.preserve_block_start_no_location_nop + && block_tail_starts_with_async_with_normal_exit( + &src_instructions[src + 1..], + ))) { remove = true; } else if instr.preserve_block_start_no_location_nop { @@ -12368,7 +12371,13 @@ fn remove_redundant_jumps_in_blocks(blocks: &mut [Block]) -> usize { (!matches!(instr.instr.real(), Some(Instruction::Nop)) || line >= 0) .then_some(line) }); - line > 0 && next_line.is_some_and(|next_line| next_line < line) + line > 0 + && !block_jump_follows_async_send_pop(&blocks[idx]) + && !(block_jump_follows_with_normal_exit(&blocks[idx]) + && block_tail_starts_with_async_with_normal_exit( + &blocks[next.idx()].instructions, + )) + && next_line.is_some_and(|next_line| next_line < line) } else { false }; @@ -12764,6 +12773,35 @@ fn is_load_const_none(instr: &InstructionInfo, metadata: &CodeUnitMetadata) -> b ) } +fn block_tail_starts_with_async_with_normal_exit(instructions: &[InstructionInfo]) -> bool { + matches!( + instructions, + [ + InstructionInfo { + instr: AnyInstruction::Real(Instruction::LoadConst { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::LoadConst { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::LoadConst { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::Call { .. }), + .. + }, + InstructionInfo { + instr: AnyInstruction::Real(Instruction::GetAwaitable { .. }), + .. + }, + .. + ] + ) +} + fn instruction_lineno(instr: &InstructionInfo) -> i32 { instr .lineno_override @@ -13104,6 +13142,52 @@ fn block_contains_suspension_point(block: &Block) -> bool { }) } +fn block_jump_follows_async_send_pop(block: &Block) -> bool { + let mut before_jump = + block + .instructions + .iter() + .rev() + .skip(1) + .filter_map(|info| match info.instr.real() { + Some(Instruction::Nop) => None, + instr => instr, + }); + matches!( + (before_jump.next(), before_jump.next()), + (Some(Instruction::PopTop), Some(Instruction::EndSend)) + ) +} + +fn block_jump_follows_with_normal_exit(block: &Block) -> bool { + let mut before_jump = + block + .instructions + .iter() + .rev() + .skip(1) + .filter_map(|info| match info.instr.real() { + Some(Instruction::Nop) => None, + instr => instr, + }); + matches!( + ( + before_jump.next(), + before_jump.next(), + before_jump.next(), + before_jump.next(), + before_jump.next(), + ), + ( + Some(Instruction::PopTop), + Some(Instruction::Call { .. }), + Some(Instruction::LoadConst { .. }), + Some(Instruction::LoadConst { .. }), + Some(Instruction::LoadConst { .. }), + ) + ) +} + fn is_stop_iteration_error_handler_block(block: &Block) -> bool { matches!( block.instructions.as_slice(), From 606e81615bb9bdc8a777040f36c819d5ae2f369c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 14:39:55 +0900 Subject: [PATCH 55/76] Preserve CPython continue CFG layout before conditional bodies --- crates/codegen/src/compile.rs | 42 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 8 ++----- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c799920980..75b1966dd6 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -24085,6 +24085,48 @@ def f(b, curr, curr_append, decoded_append, packI, curr_clear): ); } + #[test] + fn test_if_not_continue_before_conditional_listcomp_body_keeps_cpython_layout() { + let code = compile_exec( + "\ +def f(data, use): + for line in data: + line = line.strip() + if not line: + continue + if line.startswith('@'): + continue + values = [use(x) for x in line] + use(values) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "if-not continue should keep CPython's forward true edge over the continue backedge, got ops={ops:?}" + ); + } + #[test] fn test_try_else_loop_if_body_keeps_cpython_fallthrough_before_backedge() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a5bea29118..d9493b6c11 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14717,7 +14717,6 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec let body_tail_is_conditional = trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); let body_is_single_block = body == body_tail; - let body_has_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); let body_has_scope_exit = body_segment_contains_scope_exit(blocks, body, body_tail); let body_has_loop_backedge = body_segment_contains_jump_back_to(blocks, body, body_tail, loop_target); @@ -14742,8 +14741,6 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec .any(|info| info.instr.is_unconditional_jump()) && !after_jump_continues_conditional_chain && matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })); - let body_starts_with_conditional_test = - block_is_pure_conditional_test(&blocks[body.idx()]); let trailing_implicit_continue_can_reorder = after_jump != BlockIdx::NULL && next_nonempty_block(blocks, after_jump) != body && !block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_jump)) @@ -14755,12 +14752,10 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) || (body == body_tail && is_single_delete_subscr_body(&blocks[body.idx()])) - || body_has_for_iter || simple_single_block_can_reorder)) || (trailing_implicit_continue_can_reorder && (body_has_scope_exit || body_has_loop_backedge) - && (!after_jump_continues_conditional_chain - || body_starts_with_conditional_test))); + && !after_jump_continues_conditional_chain)); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { let cloned_jump_idx = BlockIdx(blocks.len() as u32); let mut cloned_jump = blocks[true_jump.idx()].clone(); @@ -14816,6 +14811,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec continue; } if !body_segment_contains_for_iter(blocks, body, body_tail) + || trailing_conditional_jump_index(&blocks[body.idx()]).is_some() || block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_body)) { current = next; From b340ca8c529bf454fdbb45fa4a78a586c32472e1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 15:19:19 +0900 Subject: [PATCH 56/76] Refine protected CFG bytecode parity --- crates/codegen/src/compile.rs | 112 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 18 +++++- 2 files changed, 128 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 75b1966dd6..848e0faeef 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -15393,6 +15393,55 @@ def f(): ); } + #[test] + fn test_try_import_return_handler_deopts_later_protected_tail_borrow() { + let code = compile_exec( + "\ +def f(info_add): + try: + import pwd + except ImportError: + return + import os + uid = os.getuid() + try: + entry = pwd.getpwuid(uid) + except KeyError: + entry = None + info_add(uid, entry) + if entry is None: + return + if hasattr(os, 'getgrouplist'): + groups = os.getgrouplist(entry.pw_name, entry.pw_gid) + groups = ', '.join(map(str, groups)) + info_add('os.getgrouplist', groups) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let import_idx = ops + .iter() + .position(|op| matches!(op, Instruction::ImportName { .. })) + .expect("missing IMPORT_NAME"); + let protected_tail = &ops[import_idx + 1..]; + + assert!( + !protected_tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "CPython keeps strong LOAD_FAST ops after protected import with return handler, even across later protected tails, got tail={protected_tail:?}" + ); + } + #[test] fn test_try_import_continue_handler_deopts_loop_tail_borrow() { let code = compile_exec( @@ -24127,6 +24176,69 @@ def f(data, use): ); } + #[test] + fn test_chained_compare_continue_does_not_duplicate_cleanup_backedge() { + let code = compile_exec( + "\ +def f(items): + offsets = [] + ranges = [(0, 10), (20, 30)] + for item in items: + trans_time, offset_before, offset_after = item + for dt_min, dt_max in ranges: + if trans_time is not None and not (dt_min <= trans_time <= dt_max): + continue + if offset_before not in offsets: + offsets.append(offset_before) + if offset_after not in offsets: + offsets.append(offset_after) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::ContainsOp { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFast { .. }, + ] + ) + }), + "chained-compare continue cleanup should fall through to the following body after one backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "chained-compare continue cleanup should not duplicate the loop backedge, got ops={ops:?}" + ); + } + #[test] fn test_try_else_loop_if_body_keeps_cpython_fallthrough_before_backedge() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d9493b6c11..4629f4d579 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -8337,6 +8337,7 @@ impl CodeInfo { Some(( BlockIdx::new(idx as u32), import_idx, + handler_returns, handler_continues, nested_protected_tail, )) @@ -8344,7 +8345,7 @@ impl CodeInfo { .collect(); let mut visited = vec![false; self.blocks.len()]; - for (seed, import_idx, handler_continues, nested_protected_tail) in seeds { + for (seed, import_idx, handler_returns, handler_continues, nested_protected_tail) in seeds { let mut protected_store_locals = vec![false; self.metadata.varnames.len()]; for info in self.blocks[seed.idx()] .instructions @@ -8368,6 +8369,7 @@ impl CodeInfo { let mut cursor = self.blocks[seed.idx()].next; while cursor != BlockIdx::NULL && !block_is_exceptional(&self.blocks[cursor.idx()]) { if !handler_continues + && !handler_returns && !nested_protected_tail && block_has_protected_instructions(&self.blocks[cursor.idx()]) { @@ -8386,6 +8388,7 @@ impl CodeInfo { .instructions .iter() .any(|info| info.instr.real().is_some_and(|instr| instr.is_scope_exit())) + && !handler_returns && !handler_continues { break; @@ -15409,12 +15412,23 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { } fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { + fn block_has_real_fallthrough_body(block: &Block) -> bool { + block.instructions.iter().any(|info| { + !matches!( + info.instr.real(), + Some(Instruction::Nop | Instruction::NotTaken | Instruction::PopTop) + ) + }) + } + let predecessors = compute_predecessors(blocks); let mut clones = Vec::new(); let mut layout_pred = BlockIdx(0); while layout_pred != BlockIdx::NULL { - if !block_has_fallthrough(&blocks[layout_pred.idx()]) { + if !block_has_fallthrough(&blocks[layout_pred.idx()]) + || !block_has_real_fallthrough_body(&blocks[layout_pred.idx()]) + { layout_pred = blocks[layout_pred.idx()].next; continue; } From 330c44c0aed104091007aea99a369967eaac2688 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 15:56:01 +0900 Subject: [PATCH 57/76] Align protected CFG cleanup layout --- crates/codegen/src/compile.rs | 277 +++++++++++++++++++++++++++++++++- crates/codegen/src/ir.rs | 49 ++++-- 2 files changed, 311 insertions(+), 15 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 848e0faeef..1165e4e15b 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -662,6 +662,11 @@ impl Compiler { }) } + fn statements_end_with_loop_fallthrough(body: &[ast::Stmt]) -> bool { + body.last() + .is_some_and(|stmt| matches!(stmt, ast::Stmt::For(_) | ast::Stmt::While(_))) + } + fn has_resuming_bare_except(handlers: &[ast::ExceptHandler]) -> bool { handlers.iter().any(|handler| { let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -3819,7 +3824,11 @@ impl Compiler { self.pop_fblock(FBlockType::TryExcept); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.remove_last_no_location_nop(); + if !orelse.is_empty() && Self::statements_end_with_conditional_scope_exit(body) { + self.preserve_last_redundant_nop(); + } else { + self.remove_last_no_location_nop(); + } if !orelse.is_empty() && has_terminal_raise_handlers { let orelse_block = self.new_block(); self.switch_to_block(orelse_block); @@ -3831,7 +3840,11 @@ impl Compiler { PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - self.remove_last_no_location_nop(); + if !orelse.is_empty() && Self::statements_end_with_loop_fallthrough(orelse) { + self.preserve_last_redundant_jump_as_nop(); + } else { + self.remove_last_no_location_nop(); + } self.switch_to_block(handler_block); emit!( @@ -5884,6 +5897,7 @@ impl Compiler { let preserve_outer_cleanup_target_nop = !is_async && (Self::statements_end_with_with_cleanup_scope_exit(body) || Self::statements_end_with_conditional_scope_exit(body) + || Self::statements_end_with_loop_fallthrough(body) || self.current_block_has_terminal_with_suppress_exit_predecessor()); // Pop fblock before normal exit. CPython emits this POP_BLOCK with @@ -13651,6 +13665,104 @@ def f(msg): ); } + #[test] + fn test_try_else_conditional_scope_exit_keeps_pop_block_nop() { + let code = compile_exec( + "\ +def f(values, check): + found = '' + for value in values: + try: + if check(value): + raise UnicodeError + except UnicodeError: + pass + else: + found = value + break + return found +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::RaiseVarargs { .. }, + Instruction::Nop, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::StoreFast { .. }, + Instruction::PopTop, + ] + ) + }), + "try-else after conditional scope exit should keep CPython's POP_BLOCK NOP anchor, got ops={ops:?}" + ); + } + + #[test] + fn test_try_else_loop_fallthrough_keeps_end_jump_nop_before_finally() { + let code = compile_exec( + "\ +def f(locale, category, locales): + try: + orig_locale = locale.setlocale(category) + except AttributeError: + raise + except Exception: + locale = orig_locale = None + if '' not in locales: + raise SkipTest('no locales') + else: + for loc in locales: + try: + locale.setlocale(category, loc) + break + except locale.Error: + pass + else: + if '' not in locales: + raise SkipTest(locales) + try: + yield + finally: + if locale and orig_locale: + locale.setlocale(category, orig_locale) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::RaiseVarargs { .. }, + Instruction::Nop, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::YieldValue { .. }, + ] + ) + }), + "try-else loop fallthrough should keep CPython's end-label NOP before following try/finally, got ops={ops:?}" + ); + } + #[test] fn test_conditional_compare_uses_bool_compare_oparg() { let code = compile_exec( @@ -21607,6 +21719,70 @@ def f(s, size, encodeSetO, encodeWhiteSpace): ); } + #[test] + fn test_exception_cleanup_backedge_target_is_shared() { + let code = compile_exec( + "\ +def f(enum_class, value, Flag, int_type, is_single_bit): + try: + try: + enum_member = enum_class[value] + except TypeError: + raise KeyError + except KeyError: + if Flag is None or not issubclass(enum_class, Flag): + enum_class.names.append(value) + elif ( + Flag is not None + and issubclass(enum_class, Flag) + and isinstance(value, int_type) + and is_single_bit(value) + ): + enum_class.names.append(value) + enum_class.add(enum_member) + return enum_member +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::PopExcept, + Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::JumpBackward { .. }, + Instruction::Reraise { .. }, + ] + ) + }), + "expected CPython-style shared exception cleanup backedge before reraise, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::PopExcept, + Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::JumpBackward { .. }, + Instruction::PopExcept, + Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::JumpBackward { .. }, + ] + ) + }), + "exception cleanup backedge should be shared, not duplicated per conditional edge, got ops={ops:?}" + ); + } + #[test] fn test_protected_loop_conditional_keeps_forward_body_entry() { let code = compile_exec( @@ -24074,6 +24250,64 @@ def f(chunk, dec, i): ); } + #[test] + fn test_exception_handler_loop_conditional_return_orders_backedge_before_return() { + let code = compile_exec( + "\ +def f(cls, value): + try: + return cls[value] + except TypeError: + for name, values in cls.items(): + if value in values: + return cls[name] + for name, member in cls.items(): + if value == member.value: + return cls[name] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. }, + ] + ) + }), + "expected exception-handler loop false path to jump back before return body, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. }, + ] + ) + }), + "unexpected exception-handler loop return body before backedge, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_body_keeps_fallthrough_before_implicit_continue_backedge() { let code = compile_exec( @@ -25067,6 +25301,45 @@ def f(self): ); } + #[test] + fn test_with_while_fallthrough_preserves_cleanup_nop() { + let code = compile_exec( + "\ +def f(cm, source): + with cm as out: + s = source.read() + while s: + out.write(s) + s = source.read() + source.close() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .map(|unit| unit.op) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "with cleanup after while fallthrough should preserve the CPython POP_BLOCK NOP, got ops={ops:?}" + ); + } + #[test] fn test_named_except_conditional_reraise_final_store_attr_keeps_borrow() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4629f4d579..8dc45537d9 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -342,7 +342,7 @@ impl CodeInfo { reorder_conditional_explicit_continue_scope_exit_blocks(&mut self.blocks); reorder_conditional_implicit_continue_scope_exit_blocks(&mut self.blocks); reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); - reorder_exception_handler_conditional_continue_raise_blocks(&mut self.blocks); + reorder_exception_handler_conditional_continue_scope_exit_blocks(&mut self.blocks); deduplicate_adjacent_jump_back_blocks(&mut self.blocks); reorder_conditional_body_and_implicit_continue_blocks(&mut self.blocks); reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); @@ -12374,13 +12374,14 @@ fn remove_redundant_jumps_in_blocks(blocks: &mut [Block]) -> usize { (!matches!(instr.instr.real(), Some(Instruction::Nop)) || line >= 0) .then_some(line) }); - line > 0 - && !block_jump_follows_async_send_pop(&blocks[idx]) - && !(block_jump_follows_with_normal_exit(&blocks[idx]) - && block_tail_starts_with_async_with_normal_exit( - &blocks[next.idx()].instructions, - )) - && next_line.is_some_and(|next_line| next_line < line) + line < 0 + || line > 0 + && !block_jump_follows_async_send_pop(&blocks[idx]) + && !(block_jump_follows_with_normal_exit(&blocks[idx]) + && block_tail_starts_with_async_with_normal_exit( + &blocks[next.idx()].instructions, + )) + && next_line.is_some_and(|next_line| next_line < line) } else { false }; @@ -14297,7 +14298,18 @@ fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) } } -fn reorder_exception_handler_conditional_continue_raise_blocks(blocks: &mut [Block]) { +fn reorder_exception_handler_conditional_continue_scope_exit_blocks(blocks: &mut [Block]) { + fn handler_scope_exit_returns(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + && block + .instructions + .last() + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -14325,6 +14337,13 @@ fn reorder_exception_handler_conditional_continue_raise_blocks(blocks: &mut [Blo } else { BlockIdx::NULL }; + let exit_raises = exit_block != BlockIdx::NULL + && blocks[exit_block.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))); + let exit_returns = + exit_block != BlockIdx::NULL && handler_scope_exit_returns(&blocks[exit_block.idx()]); if exit_start == BlockIdx::NULL || exit_block == BlockIdx::NULL || jump_start == BlockIdx::NULL @@ -14333,10 +14352,7 @@ fn reorder_exception_handler_conditional_continue_raise_blocks(blocks: &mut [Blo || exit_block == jump_block || !is_scope_exit_block(&blocks[exit_block.idx()]) || !is_jump_back_only_block(blocks, jump_block) - || !blocks[exit_block.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))) + || !(exit_raises || exit_returns) || !blocks[jump_target.idx()] .instructions .first() @@ -15322,6 +15338,13 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { let Some(jump_target) = shared_jump_back_target(&blocks[target.idx()]) else { continue; }; + if blocks[target.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + { + continue; + } let jump_target = next_nonempty_block(blocks, jump_target); if jump_target == BlockIdx::NULL || !comes_before(blocks, jump_target, target) { From 9c737eb9d325d39f6f87e043238aead9b1a128f4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 16:30:20 +0900 Subject: [PATCH 58/76] Align CFG cleanup and peephole parity --- crates/codegen/src/compile.rs | 138 +++++++++++++++++++++++++++++++--- crates/codegen/src/ir.rs | 12 +++ 2 files changed, 140 insertions(+), 10 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 1165e4e15b..3988f17868 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -646,8 +646,18 @@ impl Compiler { }) } - fn statements_end_with_conditional_scope_exit(body: &[ast::Stmt]) -> bool { + fn statements_end_with_try_finally(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| { + matches!( + stmt, + ast::Stmt::Try(ast::StmtTry { finalbody, .. }) if !finalbody.is_empty() + ) + }) + } + + fn statements_end_with_conditional_scope_exit(&self, body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| match stmt { + ast::Stmt::Assert(_) => self.opts.optimize == 0, ast::Stmt::If(ast::StmtIf { body, elif_else_clauses, @@ -662,9 +672,14 @@ impl Compiler { }) } - fn statements_end_with_loop_fallthrough(body: &[ast::Stmt]) -> bool { - body.last() - .is_some_and(|stmt| matches!(stmt, ast::Stmt::For(_) | ast::Stmt::While(_))) + fn statements_end_with_loop_fallthrough(&mut self, body: &[ast::Stmt]) -> CompileResult { + match body.last() { + Some(ast::Stmt::For(_)) => Ok(true), + Some(ast::Stmt::While(ast::StmtWhile { test, .. })) => { + Ok(!matches!(self.constant_expr_truthiness(test)?, Some(true))) + } + _ => Ok(false), + } } fn has_resuming_bare_except(handlers: &[ast::ExceptHandler]) -> bool { @@ -3489,7 +3504,6 @@ impl Compiler { PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - self.preserve_last_redundant_jump_as_nop(); if let Some(finally_except) = finally_except_block { // Restore sub_tables for exception path compilation @@ -3728,7 +3742,6 @@ impl Compiler { PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - self.preserve_last_redundant_jump_as_nop(); // finally (exception path) // This is where exceptions go to run finally before reraise @@ -3824,7 +3837,7 @@ impl Compiler { self.pop_fblock(FBlockType::TryExcept); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - if !orelse.is_empty() && Self::statements_end_with_conditional_scope_exit(body) { + if !orelse.is_empty() && self.statements_end_with_conditional_scope_exit(body) { self.preserve_last_redundant_nop(); } else { self.remove_last_no_location_nop(); @@ -3840,7 +3853,7 @@ impl Compiler { PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - if !orelse.is_empty() && Self::statements_end_with_loop_fallthrough(orelse) { + if !orelse.is_empty() && self.statements_end_with_loop_fallthrough(orelse)? { self.preserve_last_redundant_jump_as_nop(); } else { self.remove_last_no_location_nop(); @@ -5896,8 +5909,9 @@ impl Compiler { let preserve_outer_cleanup_target_nop = !is_async && (Self::statements_end_with_with_cleanup_scope_exit(body) - || Self::statements_end_with_conditional_scope_exit(body) - || Self::statements_end_with_loop_fallthrough(body) + || self.statements_end_with_conditional_scope_exit(body) + || Self::statements_end_with_try_finally(body) + || self.statements_end_with_loop_fallthrough(body)? || self.current_block_has_terminal_with_suppress_exit_predecessor()); // Pop fblock before normal exit. CPython emits this POP_BLOCK with @@ -19648,6 +19662,36 @@ def f(a, b, c, x): ); } + #[test] + fn test_unary_not_membership_and_identity_invert_compare_op() { + let code = compile_exec( + "\ +def f(a, b, d): + x = not (a in d) + y = not (a is b) + return x, y +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); + + assert!( + !ops.iter().any(|op| matches!(op, Instruction::UnaryNot)), + "CPython folds CONTAINS_OP/IS_OP + UNARY_NOT into inverted op, got ops={ops:?}" + ); + assert!(instructions.iter().any(|unit| { + matches!(unit.op, Instruction::ContainsOp { invert } if invert.get(OpArg::new(unit.arg.as_u32())) == Invert::Yes) + })); + assert!(instructions.iter().any(|unit| { + matches!(unit.op, Instruction::IsOp { invert } if invert.get(OpArg::new(unit.arg.as_u32())) == Invert::Yes) + })); + } + #[test] fn test_starred_tuple_iterable_drops_list_to_tuple_before_get_iter() { let code = compile_exec( @@ -25340,6 +25384,80 @@ def f(cm, source): ); } + #[test] + fn test_with_while_true_break_drops_cleanup_nop() { + let code = compile_exec( + "\ +def f(cm, source): + with cm as out: + while True: + data = source.read() + if not data: + break + out.write(data) + source.close() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .map(|unit| unit.op) + .collect(); + + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "with cleanup after while True break should not preserve a POP_BLOCK NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_with_final_assert_preserves_cleanup_nop() { + let code = compile_exec( + "\ +def f(cm, dst): + with cm: + assert not dst.closed + return dst +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .map(|unit| unit.op) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "with cleanup after a final assert should preserve CPython's POP_BLOCK NOP anchor, got ops={ops:?}" + ); + } + #[test] fn test_named_except_conditional_reraise_final_store_attr_keeps_borrow() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8dc45537d9..be5f13239c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -2667,6 +2667,18 @@ impl CodeInfo { continue; } + if matches!( + curr_instr, + Instruction::ContainsOp { .. } | Instruction::IsOp { .. } + ) && matches!(next_instr, Instruction::UnaryNot) + { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = curr_instr.into(); + block.instructions[i + 1].arg = OpArg::new(u32::from(curr_arg) ^ 1); + i += 1; + continue; + } + if let Some(is_true) = const_truthiness(curr_instr, curr.arg, &self.metadata) { let jump_if_true = match next_instr { Instruction::PopJumpIfTrue { .. } => Some(true), From d35c13d6cccf347cc57c0360dda5924f95cc7292 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 17:04:27 +0900 Subject: [PATCH 59/76] Align nested loop conditional CFG layout --- crates/codegen/src/compile.rs | 47 +++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 53 +++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 3988f17868..3d8e3dde05 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -14638,6 +14638,53 @@ def gen(fields): ); } + #[test] + fn test_loop_filter_with_nested_loop_body_uses_cpython_implicit_continue_layout() { + let code = compile_exec( + "\ +def f(values): + for fmt, items in values: + nd = make(items, fmt) + for i in range(-5, 5): + check(nd[i], items[i]) + check(nd[-6]) + check(nd[5]) + if is_memoryview_format(fmt): + mv = memoryview(nd) + check(mv, nd) + for i in range(-5, 5): + check(mv[i], items[i]) + check(mv[-6]) + check(mv[5]) + return None +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected CPython-style nested loop filter to fall through into the implicit continue and jump on true into the body, got ops={ops:?}" + ); + } + #[test] fn test_multi_with_header_uses_store_fast_load_fast() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index be5f13239c..fdd1bdd648 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14616,6 +14616,48 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec false } + fn body_segment_contains_any_jump_back( + blocks: &[Block], + body_start: BlockIdx, + body_tail: BlockIdx, + ) -> bool { + fn jump_back_or_self_target(blocks: &[Block], block_idx: BlockIdx) -> Option { + if block_idx == BlockIdx::NULL { + return None; + } + let jump = blocks[block_idx.idx()].instructions.last()?; + if !jump.instr.is_unconditional_jump() { + return None; + } + if jump.target == BlockIdx::NULL { + return None; + } + let target = next_nonempty_block(blocks, jump.target); + if target == block_idx || comes_before(blocks, target, block_idx) { + Some(jump.target) + } else { + None + } + } + + let mut cursor = body_start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL { + if visited[cursor.idx()] { + return false; + } + visited[cursor.idx()] = true; + if jump_back_or_self_target(blocks, cursor).is_some() { + return true; + } + if cursor == body_tail { + return false; + } + cursor = blocks[cursor.idx()].next; + } + false + } + fn empty_chain_reaches(blocks: &[Block], start: BlockIdx, target: BlockIdx) -> bool { let mut cursor = start; let mut visited = vec![false; blocks.len()]; @@ -14751,11 +14793,16 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec let body_has_scope_exit = body_segment_contains_scope_exit(blocks, body, body_tail); let body_has_loop_backedge = body_segment_contains_jump_back_to(blocks, body, body_tail, loop_target); + let body_has_inner_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); + let body_has_any_loop_backedge = body_has_inner_for_iter + && body_segment_contains_any_jump_back(blocks, body, body_tail); let normalized_single_block_can_reorder = !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); let has_exceptional_duplicate_condition_line = has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&cond)); let after_jump_target = next_nonempty_block(blocks, after_jump); + let after_jump_starts_loop_cleanup = + block_starts_loop_cleanup(blocks, after_jump_target); let after_jump_continues_conditional_chain = after_jump_target != BlockIdx::NULL && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); let simple_single_block_can_reorder = body_is_single_block @@ -14774,7 +14821,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })); let trailing_implicit_continue_can_reorder = after_jump != BlockIdx::NULL && next_nonempty_block(blocks, after_jump) != body - && !block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_jump)) + && (!after_jump_starts_loop_cleanup || body_has_any_loop_backedge) && !is_scope_exit_block(&blocks[body.idx()]); let can_reorder = !has_exceptional_duplicate_condition_line && ((!body_tail_is_conditional @@ -14785,7 +14832,9 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && is_single_delete_subscr_body(&blocks[body.idx()])) || simple_single_block_can_reorder)) || (trailing_implicit_continue_can_reorder - && (body_has_scope_exit || body_has_loop_backedge) + && (body_has_scope_exit + || body_has_loop_backedge + || (after_jump_starts_loop_cleanup && body_has_any_loop_backedge)) && !after_jump_continues_conditional_chain)); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { let cloned_jump_idx = BlockIdx(blocks.len() as u32); From bba340b6ab57e4534c686c2e143faa023c8f94f9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 17:37:58 +0900 Subject: [PATCH 60/76] Align named expression comprehension scope --- crates/codegen/src/compile.rs | 73 ++++++++++++++++-- crates/codegen/src/symboltable.rs | 122 ++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 6 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 3d8e3dde05..ca3db2dc23 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -9719,15 +9719,10 @@ impl Compiler { if sym.flags.contains(SymbolFlags::PARAMETER) { continue; // skip .0 } - // Walrus operator targets (ASSIGNED_IN_COMPREHENSION without ITER) - // are not local to the comprehension; they leak to the outer scope. - let is_walrus = sym.flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) - && !sym.flags.contains(SymbolFlags::ITER); let is_local = sym .flags .intersects(SymbolFlags::ASSIGNED | SymbolFlags::ITER) - && !sym.flags.contains(SymbolFlags::NONLOCAL) - && !is_walrus; + && !sym.flags.contains(SymbolFlags::NONLOCAL); if is_local { pushed_locals.push(name); } @@ -20366,6 +20361,72 @@ def f(): ); } + #[test] + fn test_inlined_comprehension_namedexpr_target_stays_parent_fast_local() { + let code = compile_exec( + "\ +def f(seq, emit): + return [(x, y) for x in seq if (y := emit(x))] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + + assert!(f.varnames.iter().any(|name| name == "y")); + assert!(!f.cellvars.iter().any(|name| name == "y")); + assert!(!f.freevars.iter().any(|name| name == "y")); + assert!( + !f.names.iter().any(|name| name == "y"), + "inlined comprehension namedexpr target should not use NAME ops, got names={:?}", + f.names + ); + } + + #[test] + fn test_global_namedexpr_in_inlined_comprehension_saves_fast_slot() { + let code = compile_exec( + "\ +def f(seq, value): + global G + [G := value for _ in seq] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + + assert!(f.varnames.iter().any(|name| name == "G")); + assert!(f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFastAndClear { var_num } => { + let idx = var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.varnames[usize::from(idx)] == "G" + } + _ => false, + })); + assert!(f.instructions.iter().any(|unit| match unit.op { + Instruction::StoreGlobal { namei } => { + let idx = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(idx).unwrap()] == "G" + } + _ => false, + })); + } + + #[test] + fn test_genexpr_namedexpr_target_is_cell_not_fast_local() { + let code = compile_exec( + "\ +def f(seq): + a = 1 + return (c := x + a for x in seq) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + + assert!(!f.varnames.iter().any(|name| name == "c")); + assert_eq!( + f.cellvars.iter().map(String::as_str).collect::>(), + ["a", "c"] + ); + } + #[test] fn test_inlined_comprehension_restore_does_not_form_store_fast_load_fast() { let code = compile_exec( diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index f1c087332f..fabdc40bf9 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -2219,6 +2219,7 @@ impl SymbolTableBuilder { self.check_name(id, ExpressionContext::Store, *range)?; let table = self.tables.last().unwrap(); if table.typ == CompilerScope::Comprehension { + self.extend_namedexpr_scope(id, *range)?; self.register_name( id, SymbolUsage::AssignedNamedExprInComprehension, @@ -2655,6 +2656,127 @@ impl SymbolTableBuilder { Ok(()) } + fn add_varname_to_scope(&mut self, table_idx: usize, name: &str) { + let varnames = if table_idx + 1 == self.tables.len() { + &mut self.current_varnames + } else { + &mut self.varnames_stack[table_idx + 1] + }; + if !varnames.iter().any(|existing| existing == name) { + varnames.push(name.to_owned()); + } + } + + // Mirrors CPython symtable_extend_namedexpr_scope(): assignment expressions + // inside comprehensions bind in the nearest function/module-like scope, not + // in the synthetic comprehension scope itself. + fn extend_namedexpr_scope(&mut self, name: &str, range: TextRange) -> SymbolTableResult { + let location = Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ); + + for table_idx in (0..self.tables.len()).rev() { + let table_type = self.tables[table_idx].typ; + let mangled = maybe_mangle_name( + self.class_name.as_deref(), + self.tables[table_idx].mangled_names.as_ref(), + name, + ) + .into_owned(); + + if table_type == CompilerScope::Comprehension { + if self.tables[table_idx] + .symbols + .get(mangled.as_str()) + .is_some_and(|symbol| symbol.flags.contains(SymbolFlags::ITER)) + { + return Err(SymbolTableError { + error: format!( + "assignment expression cannot rebind comprehension iteration variable '{}'", + mangled + ), + location, + }); + } + continue; + } + + match table_type { + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda => { + let current_comp_inlined = self.tables.last().is_some_and(|table| { + table.typ == CompilerScope::Comprehension && table.comp_inlined + }); + let parent_is_global = self.tables[table_idx] + .symbols + .get(mangled.as_str()) + .is_some_and(|symbol| symbol.flags.contains(SymbolFlags::GLOBAL)); + let current = self.tables.last_mut().unwrap(); + let current_symbol = current + .symbols + .entry(mangled.clone()) + .or_insert_with(|| Symbol::new(mangled.as_str())); + if parent_is_global { + current_symbol.flags.insert(SymbolFlags::GLOBAL); + current_symbol.scope = SymbolScope::GlobalExplicit; + } else { + current_symbol.flags.insert(SymbolFlags::NONLOCAL); + current_symbol.scope = SymbolScope::Free; + } + + let symbol = self.tables[table_idx] + .symbols + .entry(mangled.clone()) + .or_insert_with(|| Symbol::new(mangled.as_str())); + symbol.flags.insert(SymbolFlags::ASSIGNED); + if !parent_is_global && current_comp_inlined { + self.add_varname_to_scope(table_idx, mangled.as_str()); + } + return Ok(()); + } + CompilerScope::Module => { + let current = self.tables.last_mut().unwrap(); + let current_symbol = current + .symbols + .entry(mangled.clone()) + .or_insert_with(|| Symbol::new(mangled.as_str())); + current_symbol.flags.insert(SymbolFlags::GLOBAL); + current_symbol.scope = SymbolScope::GlobalExplicit; + + let symbol = self.tables[table_idx] + .symbols + .entry(mangled.clone()) + .or_insert_with(|| Symbol::new(mangled.as_str())); + symbol.flags.insert(SymbolFlags::GLOBAL); + symbol.scope = SymbolScope::GlobalExplicit; + return Ok(()); + } + CompilerScope::Class => { + return Err(SymbolTableError { + error: "assignment expression within a comprehension cannot be used in a class body".to_string(), + location, + }); + } + CompilerScope::TypeParams => { + return Err(SymbolTableError { + error: "assignment expression within a comprehension cannot be used within the definition of a generic".to_string(), + location, + }); + } + CompilerScope::Annotation => { + return Err(SymbolTableError { + error: "named expression cannot be used within an annotation".to_string(), + location, + }); + } + CompilerScope::Comprehension => unreachable!(), + } + } + + unreachable!("named expression scope extension requires an enclosing scope") + } + fn register_name( &mut self, name: &str, From ea757cf5e47ca240f602c8db20f437d4e27dead3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 18:01:19 +0900 Subject: [PATCH 61/76] Align CFG cleanup with CPython line markers --- crates/codegen/src/compile.rs | 93 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 24 +++++++-- 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index ca3db2dc23..a3e3a9589b 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -23483,6 +23483,99 @@ def f(self): ); } + #[test] + fn test_reraising_except_loop_break_tail_keeps_post_loop_borrows() { + let code = compile_exec( + "\ +def f(flag=1, count=0): + value = 2 + while value: + count += 1 + try: + if flag and value == 1: + flag -= 1 + break + value -= 1 + continue + except: + raise + return count > 2 or value != 1 +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let return_idx = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::ReturnValue)) + .expect("missing return"); + let tail = &instructions[return_idx.saturating_sub(12)..return_idx]; + + let load_name = |unit: &CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + for name in ["count", "value"] { + assert!( + tail.iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. }) + && load_name(unit) == Some(name)), + "post-loop condition should borrow {name}, got tail={tail:?}" + ); + assert!( + !tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. }) + && load_name(unit) == Some(name)), + "post-loop condition should not deopt {name} to LOAD_FAST, got tail={tail:?}" + ); + } + } + + #[test] + fn test_try_except_continue_keeps_try_line_nop_before_continue_jump() { + let code = compile_exec( + "\ +def f(done=False): + while not done: + done = True + try: + continue + except: + done = False + return done +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + assert!( + instructions.windows(2).any(|window| matches!( + window, + [ + CodeUnit { + op: Instruction::Nop, + .. + }, + CodeUnit { + op: Instruction::JumpBackward { .. }, + .. + } + ] + )), + "try/except continue should keep CPython-style try-line NOP before continue jump, got instructions={instructions:?}" + ); + } + #[test] fn test_handler_resume_before_later_loop_keeps_borrowed_tail_loads() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index fdd1bdd648..8222f6bef7 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5184,6 +5184,13 @@ impl CodeInfo { }) } + fn block_jumps_unconditionally_to(block: &Block, target: BlockIdx) -> bool { + block + .instructions + .iter() + .any(|info| info.target == target && info.instr.is_unconditional_jump()) + } + fn handler_chain_has_explicit_reraise(blocks: &[Block], handler: BlockIdx) -> bool { let mut cursor = handler; let mut visited = vec![false; blocks.len()]; @@ -5409,6 +5416,9 @@ impl CodeInfo { if block_jumps_backward_to(pred_block, BlockIdx::new(idx as u32)) { continue; } + if block_jumps_unconditionally_to(pred_block, BlockIdx::new(idx as u32)) { + continue; + } let handlers = nonresuming_reraise_handlers(&self.blocks, pred_block); follows_protected_body[idx] = !handlers.is_empty() && !handlers.iter().any(|handler| { @@ -7866,6 +7876,7 @@ impl CodeInfo { let mut predecessors = vec![Vec::new(); self.blocks.len()]; let mut is_handler_resume_block = vec![false; self.blocks.len()]; for (pred_idx, block) in self.blocks.iter().enumerate() { + let block_idx = BlockIdx::new(pred_idx as u32); if block .instructions .iter() @@ -7876,12 +7887,12 @@ impl CodeInfo { { is_handler_resume_block[pred_idx] = true; } - if block.next != BlockIdx::NULL { - predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(block_idx); } for info in &block.instructions { if info.target != BlockIdx::NULL { - predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + predecessors[info.target.idx()].push(block_idx); } } } @@ -12309,8 +12320,11 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { } else if src_instructions[src + 1].instr.is_unconditional_jump() && src_instructions[src + 1].target != block_idx { - src_instructions[src + 1].lineno_override = Some(lineno); - remove = true; + let next_lineno = instruction_lineno(&src_instructions[src + 1]); + if next_lineno == lineno || next_lineno < 0 { + src_instructions[src + 1].lineno_override = Some(lineno); + remove = true; + } } else if src_instructions[src + 1].folded_from_nonliteral_expr { remove = true; } else { From 555e8fca93bab4d9fbdcaa162395896bef4a799b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 18:23:02 +0900 Subject: [PATCH 62/76] Align CFG line marker cleanup with CPython --- crates/codegen/src/compile.rs | 103 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 13 ++--- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index a3e3a9589b..d7eb87bae3 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -12126,6 +12126,43 @@ mod tests { stack_top.debug_late_cfg_trace().unwrap() } + #[test] + #[ignore = "debug helper"] + fn debug_trace_nested_continue_after_optional_body() { + let trace = compile_single_function_late_cfg_trace( + "\ +def f(names, show_empty, keywords, args_buffer, args, cls, object, level): + for name in names: + value = getattr(cls, name) + if not show_empty: + if value == []: + field_type = cls._field_types.get(name, object) + if getattr(field_type, '__origin__', ...) is list: + if not keywords: + args_buffer.append(repr(value)) + continue + if not keywords: + args.extend(args_buffer) + args_buffer = [] + value, simple = _format(value, level) + if keywords: + args.append('%s=%s' % (name, value)) + else: + args.append(value) +", + "f", + ); + for (label, dump) in trace { + if label == "after_reorder" + || label == "after_remove_redundant_nops_and_jumps" + || label == "after_final_cfg_cleanup" + || label == "after_borrow_deopts" + { + eprintln!("=== {label} ===\n{dump}"); + } + } + } + #[test] #[ignore = "debug helper"] fn debug_trace_make_dataclass_borrow_tail() { @@ -23576,6 +23613,72 @@ def f(done=False): ); } + #[test] + fn test_for_else_pass_keeps_line_marker_after_pop_iter() { + let code = compile_exec( + "\ +def f(): + for item in (): + pass + else: + pass + marker = 1 + return marker +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + + assert!( + instructions.windows(2).any(|window| matches!( + window, + [ + CodeUnit { + op: Instruction::PopIter, + .. + }, + CodeUnit { + op: Instruction::Nop, + .. + } + ] + )), + "for-else pass should keep CPython-style else-line NOP after POP_ITER, got instructions={instructions:?}" + ); + } + + #[test] + fn test_folded_if_chain_after_previous_chain_keeps_final_elif_line_marker() { + let code = compile_exec( + "\ +def f(): + if 0: pass + elif 0: pass + if 0: pass + elif 0: pass + elif 0: pass + elif 0: pass + else: pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let nop_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::Nop)) + .count(); + + assert_eq!( + nop_count, 6, + "folded if chains should preserve CPython-style line-marker NOPs, got instructions={:?}", + f.instructions + ); + } + #[test] fn test_handler_resume_before_later_loop_keeps_borrowed_tail_loads() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8222f6bef7..b338a7e6aa 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -498,14 +498,7 @@ impl CodeInfo { if matches!(instr.instr.real(), Some(Instruction::Nop)) { if instr.preserve_redundant_jump_as_nop { remove = false; - } else if lineno < 0 - || prev_lineno == lineno - || (src > 0 - && matches!( - src_instructions[src - 1].instr.real(), - Some(Instruction::PopIter) - )) - { + } else if lineno < 0 || prev_lineno == lineno { remove = true; } else if src < src_instructions.len() - 1 { if src_instructions[src + 1].instr.is_block_push() { @@ -15700,7 +15693,9 @@ fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { .last() .map(|instr| (instr.location, instr.end_location)); let mut cloned_return = return_insts.clone(); - if let Some((location, end_location)) = propagated_location { + if !instruction_has_lineno(&cloned_return[0]) + && let Some((location, end_location)) = propagated_location + { for instr in &mut cloned_return { overwrite_location(instr, location, end_location); } From 8abe32a9a890a87cbb5b3e62e1891e9ccaee9640 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 18:58:28 +0900 Subject: [PATCH 63/76] Align loop CFG fallthrough with CPython --- crates/codegen/src/compile.rs | 129 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 13 ++++ 2 files changed, 142 insertions(+) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index d7eb87bae3..2e36f22a8f 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -20929,6 +20929,69 @@ def f(obj, flags, writer, value, Error): ); } + #[test] + fn test_nested_continue_shares_backedge_with_fallthrough_body() { + let code = compile_exec( + "\ +def f(names, show_empty, keywords, args_buffer, args, cls, object, level): + for name in names: + value = getattr(cls, name) + if not show_empty: + if value == []: + field_type = cls._field_types.get(name, object) + if getattr(field_type, '__origin__', ...) is list: + if not keywords: + args_buffer.append(repr(value)) + continue + if not keywords: + args.extend(args_buffer) + args_buffer = [] + value, simple = _format(value, level) + if keywords: + args.append('%s=%s' % (name, value)) + else: + args.append(value) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache | Instruction::NotTaken)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + ] + ) + }), + "expected CPython-style shared continue backedge before outer condition, got ops={ops:?}" + ); + assert!( + !ops.windows(2).any(|window| { + matches!( + window, + [ + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "unexpected duplicated continue/backedge jumps, got ops={ops:?}" + ); + } + #[test] fn test_line_bearing_loop_if_false_backedge_keeps_body_before_jump_back() { let code = compile_exec( @@ -21189,6 +21252,72 @@ def f(keys, parse_int, d, ampm, AM, PM): ); } + #[test] + fn test_loop_nested_if_before_elif_keeps_body_before_false_backedge() { + let code = compile_exec( + "\ +def f(keys, parse_int, found_dict, locale_time): + hour = minute = 0 + for group_key in keys: + if group_key == 'I': + hour = parse_int(found_dict['I']) + ampm = found_dict.get('p', '').lower() + if ampm in ('', locale_time.am_pm[0]): + if hour == 12: + hour = 0 + elif ampm == locale_time.am_pm[1]: + if hour != 12: + hour += 12 + elif group_key == 'M': + minute = parse_int(found_dict['M']) + return hour, minute +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadSmallInt { .. }, + Instruction::StoreFast { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style nested if body before false backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadSmallInt { .. }, + ] + ) + }), + "unexpected inverted nested if body after false backedge, got ops={ops:?}" + ); + } + #[test] fn test_loop_multiblock_conditional_body_keeps_body_before_jump_back() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index b338a7e6aa..79b9c55ea7 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14745,9 +14745,11 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec | Instruction::LoadFastBorrow { .. } | Instruction::LoadFastLoadFast { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadDeref { .. } | Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } | Instruction::LoadAttr { .. } + | Instruction::BinaryOp { .. } | Instruction::ContainsOp { .. } | Instruction::CompareOp { .. } | Instruction::ToBool @@ -14812,6 +14814,10 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec block_starts_loop_cleanup(blocks, after_jump_target); let after_jump_continues_conditional_chain = after_jump_target != BlockIdx::NULL && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); + if after_jump_continues_conditional_chain { + current = next; + continue; + } let simple_single_block_can_reorder = body_is_single_block && !body_tail_is_conditional && !has_exceptional_duplicate_condition_line @@ -15532,6 +15538,13 @@ fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { layout_pred = blocks[layout_pred.idx()].next; continue; } + if blocks[target.idx()].instructions[0] + .lineno_override + .is_some_and(|lineno| lineno >= 0) + { + layout_pred = blocks[layout_pred.idx()].next; + continue; + } if !block_has_no_lineno(&blocks[target.idx()]) && trailing_conditional_jump_index(&blocks[layout_pred.idx()]).is_some() { From 05d96e4dd98c95f956dd488c394ddf28e3d9522d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 19:32:46 +0900 Subject: [PATCH 64/76] Align conditional CFG fallthrough cases with CPython --- crates/codegen/src/compile.rs | 172 ++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 64 +++++++------ 2 files changed, 208 insertions(+), 28 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 2e36f22a8f..b5474633f6 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -655,6 +655,17 @@ impl Compiler { }) } + fn statements_end_with_open_conditional_fallthrough(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| match stmt { + ast::Stmt::If(ast::StmtIf { + elif_else_clauses, .. + }) => !elif_else_clauses + .last() + .is_some_and(|clause| clause.test.is_none()), + _ => false, + }) + } + fn statements_end_with_conditional_scope_exit(&self, body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| match stmt { ast::Stmt::Assert(_) => self.opts.optimize == 0, @@ -702,6 +713,7 @@ impl Compiler { .. }) => { !finalbody.is_empty() + && !Self::statements_end_with_open_conditional_fallthrough(finalbody) || (!handlers.is_empty() && Self::statements_end_with_scope_exit(body)) } ast::Stmt::If(ast::StmtIf { @@ -15126,6 +15138,60 @@ def f(a, b, d): ); } + #[test] + fn test_nested_finally_open_conditional_falls_through_without_entry_nop() { + let code = compile_exec( + "\ +def f(self, f, closed, new_key): + try: + try: + work() + finally: + if self._locked: + _unlock_file(f) + finally: + if not closed: + _sync_close(f) + return new_key +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache | Instruction::NotTaken)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + ] + ) + }), + "expected CPython-style fallthrough from inner finally body into outer finalbody condition, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + ] + ) + }), + "unexpected preserved inner-finally entry NOP before outer finalbody condition, got ops={ops:?}" + ); + } + #[test] fn test_with_try_finally_normal_cleanup_keeps_redundant_jump_nop() { let code = compile_exec( @@ -21318,6 +21384,59 @@ def f(keys, parse_int, found_dict, locale_time): ); } + #[test] + fn test_elif_pass_before_raise_keeps_line_bearing_forward_jump() { + let code = compile_exec( + "\ +def f(entries, path, self): + if entries == ['.mh_sequences']: + os.remove(os.path.join(path, '.mh_sequences')) + elif entries == []: + pass + else: + raise NotEmptyError('Folder not empty: %s' % self._path) + os.rmdir(path) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpForward { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected CPython-style pass branch forward jump before raise body, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "unexpected inverted pass branch before raise body, got ops={ops:?}" + ); + } + #[test] fn test_loop_multiblock_conditional_body_keeps_body_before_jump_back() { let code = compile_exec( @@ -24632,6 +24751,59 @@ def f(kw): ); } + #[test] + fn test_loop_conditional_raise_before_elif_keeps_raise_before_backedge() { + let code = compile_exec( + "\ +def f(checks, missing, named): + for check in checks: + if check == 1: + if missing: + raise ValueError('x') + elif check is named: + pass + return checks +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadGlobal { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "expected CPython-style false edge into raise body before following elif chain, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. }, + ] + ) + }), + "unexpected loop backedge before conditional raise body in if/elif chain, got ops={ops:?}" + ); + } + #[test] fn test_protected_for_is_none_raise_threads_backedge_before_raise() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 79b9c55ea7..121c6f15ac 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -13312,6 +13312,32 @@ fn trailing_conditional_jump_index(block: &Block) -> Option { } } +fn block_is_pure_conditional_test(block: &Block) -> bool { + let Some(cond_idx) = trailing_conditional_jump_index(block) else { + return false; + }; + block.instructions[..cond_idx].iter().all(|info| { + matches!( + info.instr.real(), + Some( + Instruction::Nop + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadDeref { .. } + | Instruction::LoadConst { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::LoadAttr { .. } + | Instruction::BinaryOp { .. } + | Instruction::ContainsOp { .. } + | Instruction::CompareOp { .. } + | Instruction::ToBool + ) + ) + }) +} + fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { let mut current = BlockIdx(0); while current != BlockIdx::NULL { @@ -13522,6 +13548,12 @@ fn reorder_conditional_jump_and_exit_blocks(blocks: &mut [Block]) { current = next; continue; } + if jump_instr.lineno_override.is_some_and(|line| line >= 0) + && instruction_lineno(&jump_instr) != instruction_lineno(&last) + { + current = next; + continue; + } let mut exit_end = BlockIdx::NULL; let mut exit_block = BlockIdx::NULL; @@ -13963,11 +13995,14 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( } else { BlockIdx::NULL }; + let after_jump_continues_conditional_chain = after_jump != BlockIdx::NULL + && block_is_pure_conditional_test(&blocks[after_jump.idx()]); if exit_start == BlockIdx::NULL || exit_block == BlockIdx::NULL || jump_start == BlockIdx::NULL || jump_block == BlockIdx::NULL || jump_block == exit_block + || after_jump_continues_conditional_chain || block_is_exceptional(&blocks[idx]) || block_is_exceptional(&blocks[jump_block.idx()]) || block_is_exceptional(&blocks[exit_block.idx()]) @@ -13983,8 +14018,7 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( }) || (!allow_for_iter_jump_targets && is_explicit_non_for_jump_back(blocks, jump_block)) - || (!allow_for_iter_jump_targets - && after_jump != BlockIdx::NULL + || (after_jump != BlockIdx::NULL && !blocks[after_jump.idx()].cold && !is_scope_exit_block(&blocks[after_jump.idx()]) && !is_loop_cleanup_block(&blocks[after_jump.idx()])) @@ -14732,32 +14766,6 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) } - fn block_is_pure_conditional_test(block: &Block) -> bool { - let Some(cond_idx) = trailing_conditional_jump_index(block) else { - return false; - }; - block.instructions[..cond_idx].iter().all(|info| { - matches!( - info.instr.real(), - Some( - Instruction::Nop - | Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::LoadDeref { .. } - | Instruction::LoadConst { .. } - | Instruction::LoadSmallInt { .. } - | Instruction::LoadAttr { .. } - | Instruction::BinaryOp { .. } - | Instruction::ContainsOp { .. } - | Instruction::CompareOp { .. } - | Instruction::ToBool - ) - ) - }) - } - let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); From a891818d028976173b5f8f2e07ce55b286fd9446 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 19:57:29 +0900 Subject: [PATCH 65/76] Align protected CFG layout with CPython --- crates/codegen/src/compile.rs | 119 +++++++++++++++++++++++++++++++++- crates/codegen/src/ir.rs | 27 +++++++- 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index b5474633f6..304f9e5dbb 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3849,6 +3849,10 @@ impl Compiler { self.pop_fblock(FBlockType::TryExcept); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); + let exits_to_with_cleanup = + self.current_code_info().fblock.last().is_some_and(|info| { + matches!(info.fb_type, FBlockType::With | FBlockType::AsyncWith) + }); if !orelse.is_empty() && self.statements_end_with_conditional_scope_exit(body) { self.preserve_last_redundant_nop(); } else { @@ -3865,7 +3869,9 @@ impl Compiler { PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - if !orelse.is_empty() && self.statements_end_with_loop_fallthrough(orelse)? { + if (!orelse.is_empty() && self.statements_end_with_loop_fallthrough(orelse)?) + || (orelse.is_empty() && exits_to_with_cleanup) + { self.preserve_last_redundant_jump_as_nop(); } else { self.remove_last_no_location_nop(); @@ -14729,6 +14735,77 @@ def f(values): ); } + #[test] + fn test_final_elif_with_inlined_comprehensions_threads_backedge_before_body() { + let code = compile_exec( + "\ +def f(checks, enumeration, named): + for check in checks: + if check == 1: + pass + elif check is named: + member_names = enumeration._member_names_ + member_values = [m.value for m in enumeration] + missing_names = [] + missing_value = 0 + for name, alias in enumeration._member_map_.items(): + if name in member_names: + continue + if alias.value < 0: + continue + values = list(_iter_bits_lsb(alias.value)) + missed = [v for v in values if v not in member_values] + if missed: + missing_names.append(name) + for val in missed: + missing_value |= val + if missing_names: + raise ValueError('x') + return enumeration +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::IsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "expected CPython-style final elif to put loop backedge before the inlined-comprehension body, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::IsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "unexpected final elif body before loop backedge, got ops={ops:?}" + ); + } + #[test] fn test_multi_with_header_uses_store_fast_load_fast() { let code = compile_exec( @@ -15233,6 +15310,46 @@ def f(cm): ); } + #[test] + fn test_with_try_except_normal_cleanup_keeps_body_exit_nop() { + let code = compile_exec( + "\ +def f(cm, names, modname): + with cm: + try: + exec('import %s' % modname, names) + except: + raise FailedImport(modname) + return names +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "try/except inside with should preserve the CPython body-exit NOP before with cleanup, got ops={ops:?}" + ); + } + #[test] fn test_try_except_finally_normal_cleanup_keeps_body_exit_nop() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 121c6f15ac..64aade7bb3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14569,7 +14569,8 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec } visited[cursor.idx()] = true; if block_is_exceptional(&blocks[cursor.idx()]) - || block_is_protected(&blocks[cursor.idx()]) + || (block_is_protected(&blocks[cursor.idx()]) + && !block_has_only_stop_iteration_error_handlers(&blocks[cursor.idx()], blocks)) { return None; } @@ -14606,6 +14607,29 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec false } + fn body_segment_contains_protected_block( + blocks: &[Block], + body_start: BlockIdx, + body_tail: BlockIdx, + ) -> bool { + let mut cursor = body_start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL { + if visited[cursor.idx()] { + return false; + } + visited[cursor.idx()] = true; + if block_is_protected(&blocks[cursor.idx()]) { + return true; + } + if cursor == body_tail { + return false; + } + cursor = blocks[cursor.idx()].next; + } + false + } + fn body_segment_contains_scope_exit( blocks: &[Block], body_start: BlockIdx, @@ -14912,6 +14936,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec continue; } if !body_segment_contains_for_iter(blocks, body, body_tail) + || body_segment_contains_protected_block(blocks, body, body_tail) || trailing_conditional_jump_index(&blocks[body.idx()]).is_some() || block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_body)) { From 24dde16255ff63c28002fb1e5e48f6da3d9302d5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 20:48:27 +0900 Subject: [PATCH 66/76] Align CFG cleanup and type-param calls with CPython --- crates/codegen/src/compile.rs | 499 ++++++++++++++++++++++++---------- crates/codegen/src/ir.rs | 4 + 2 files changed, 366 insertions(+), 137 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 304f9e5dbb..113017de8d 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -19,7 +19,6 @@ use crate::{ }; use alloc::borrow::Cow; use core::mem; -use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex; use num_traits::{Num, ToPrimitive, Zero}; @@ -541,6 +540,7 @@ impl Compiler { fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: 0, // Module is always the first symbol table in_conditional_block: 0, + in_final_with_cleanup_statement: 0, next_conditional_annotation_index: 0, }; Self { @@ -659,9 +659,9 @@ impl Compiler { body.last().is_some_and(|stmt| match stmt { ast::Stmt::If(ast::StmtIf { elif_else_clauses, .. - }) => !elif_else_clauses + }) => elif_else_clauses .last() - .is_some_and(|clause| clause.test.is_none()), + .is_none_or(|clause| clause.test.is_some()), _ => false, }) } @@ -1521,6 +1521,7 @@ impl Compiler { fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: key, in_conditional_block: 0, + in_final_with_cleanup_statement: 0, next_conditional_annotation_index: 0, }; @@ -2330,6 +2331,20 @@ impl Compiler { Ok(()) } + fn compile_with_body_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { + for (idx, statement) in statements.iter().enumerate() { + if idx + 1 == statements.len() && matches!(statement, ast::Stmt::Try(_)) { + self.current_code_info().in_final_with_cleanup_statement += 1; + let result = self.compile_statement(statement); + self.current_code_info().in_final_with_cleanup_statement -= 1; + result?; + } else { + self.compile_statement(statement)?; + } + } + Ok(()) + } + fn scope_needs_conditional_annotations_cell(symbol_table: &SymbolTable) -> bool { match symbol_table.typ { CompilerScope::Module | CompilerScope::Class => { @@ -3476,7 +3491,8 @@ impl Compiler { // if handlers is empty, compile body directly // without wrapping in TryExcept (only FinallyTry is needed) if handlers.is_empty() { - let preserve_finally_entry_nop = Self::preserves_finally_entry_nop(body); + let preserve_finally_entry_nop = Self::preserves_finally_entry_nop(body) + || self.statements_end_with_loop_fallthrough(body)?; // Just compile body with FinallyTry fblock active (if finalbody exists) self.compile_statements(body)?; @@ -3833,6 +3849,11 @@ impl Compiler { body.last() .is_some_and(|stmt| matches!(stmt, ast::Stmt::Raise(_))) }); + let handlers_end_with_scope_exit = handlers.iter().all(|handler| { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = + handler; + Self::statements_end_with_scope_exit(body) + }); if Self::has_resuming_bare_except(handlers) { self.disable_load_fast_borrow_for_block(end_block); } @@ -3849,10 +3870,13 @@ impl Compiler { self.pop_fblock(FBlockType::TryExcept); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - let exits_to_with_cleanup = - self.current_code_info().fblock.last().is_some_and(|info| { - matches!(info.fb_type, FBlockType::With | FBlockType::AsyncWith) - }); + let exits_directly_to_with_cleanup = { + let code_info = self.current_code_info(); + code_info.in_final_with_cleanup_statement > 0 + && code_info.fblock.last().is_some_and(|info| { + matches!(info.fb_type, FBlockType::With | FBlockType::AsyncWith) + }) + }; if !orelse.is_empty() && self.statements_end_with_conditional_scope_exit(body) { self.preserve_last_redundant_nop(); } else { @@ -3870,7 +3894,7 @@ impl Compiler { ); self.set_no_location(); if (!orelse.is_empty() && self.statements_end_with_loop_fallthrough(orelse)?) - || (orelse.is_empty() && exits_to_with_cleanup) + || (orelse.is_empty() && exits_directly_to_with_cleanup && handlers_end_with_scope_exit) { self.preserve_last_redundant_jump_as_nop(); } else { @@ -4935,6 +4959,9 @@ impl Compiler { if funcflags.contains(&bytecode::MakeFunctionFlag::KwOnlyDefaults) { num_typeparam_args += 1; } + if num_typeparam_args == 2 { + emit!(self, Instruction::Swap { i: 2 }); + } // Enter type params scope let type_params_name = format!(""); @@ -5017,33 +5044,17 @@ impl Compiler { // Make closure for type params code self.make_closure(type_params_code, bytecode::MakeFunctionFlags::new())?; - // Call the type params closure with defaults/kwdefaults as arguments. - // Call protocol: [callable, self_or_null, arg1, ..., argN] - // We need to reorder: [args..., closure] -> [closure, NULL, args...] - // Using Swap operations to move closure down and insert NULL. - // Note: num_typeparam_args is at most 2 (defaults tuple, kwdefaults dict). if num_typeparam_args > 0 { - match num_typeparam_args { - 1 => { - // Stack: [arg1, closure] - emit!(self, Instruction::Swap { i: 2 }); // [closure, arg1] - emit!(self, Instruction::PushNull); // [closure, arg1, NULL] - emit!(self, Instruction::Swap { i: 2 }); // [closure, NULL, arg1] - } - 2 => { - // Stack: [arg1, arg2, closure] - emit!(self, Instruction::Swap { i: 3 }); // [closure, arg2, arg1] - emit!(self, Instruction::Swap { i: 2 }); // [closure, arg1, arg2] - emit!(self, Instruction::PushNull); // [closure, arg1, arg2, NULL] - emit!(self, Instruction::Swap { i: 3 }); // [closure, NULL, arg2, arg1] - emit!(self, Instruction::Swap { i: 2 }); // [closure, NULL, arg1, arg2] + emit!( + self, + Instruction::Swap { + i: num_typeparam_args as u32 + 1 } - _ => unreachable!("only defaults and kwdefaults are supported"), - } + ); emit!( self, Instruction::Call { - argc: num_typeparam_args as u32 + argc: num_typeparam_args as u32 - 1 } ); } else { @@ -5584,7 +5595,7 @@ impl Compiler { let has_double_star = arguments.is_some_and(|args| args.keywords.iter().any(|kw| kw.arg.is_none())); - if has_starred || has_double_star { + if has_starred { // Use CallFunctionEx for *bases or **kwargs // Stack has: [__build_class__, NULL, class_func, name] // Need to build: args tuple = (class_func, name, *bases, .generic_base) @@ -5619,12 +5630,25 @@ impl Compiler { } ); - // Build kwargs if needed - if arguments.is_some_and(|args| !args.keywords.is_empty()) { - self.compile_keywords(&arguments.unwrap().keywords)?; - } else { - emit!(self, Instruction::PushNull); + self.compile_call_function_ex_keywords( + arguments.map_or(&[][..], |args| &args.keywords[..]), + )?; + emit!(self, Instruction::CallFunctionEx); + } else if has_double_star { + if let Some(arguments) = arguments { + for arg in &arguments.args { + self.compile_expression(arg)?; + } } + self.load_name(".generic_base")?; + emit!( + self, + Instruction::BuildTuple { + count: 3 + arguments + .map_or(0, |args| u32::try_from(args.args.len()).unwrap()) + } + ); + self.compile_call_function_ex_keywords(&arguments.unwrap().keywords[..])?; emit!(self, Instruction::CallFunctionEx); } else { // Simple case: no starred bases, no **kwargs @@ -5919,7 +5943,7 @@ impl Compiler { if body.is_empty() { return Err(self.error(CodegenErrorType::EmptyWithBody)); } - self.compile_statements(body)?; + self.compile_with_body_statements(body)?; } else { self.set_source_range(with_range); self.compile_with(items, body, is_async)?; @@ -5929,8 +5953,7 @@ impl Compiler { && (Self::statements_end_with_with_cleanup_scope_exit(body) || self.statements_end_with_conditional_scope_exit(body) || Self::statements_end_with_try_finally(body) - || self.statements_end_with_loop_fallthrough(body)? - || self.current_block_has_terminal_with_suppress_exit_predecessor()); + || self.statements_end_with_loop_fallthrough(body)?); // Pop fblock before normal exit. CPython emits this POP_BLOCK with // no location for sync with, but with the with-item location for @@ -8781,39 +8804,6 @@ impl Compiler { Ok(()) } - fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> { - let mut size = 0; - let groupby = keywords.iter().chunk_by(|e| e.arg.is_none()); - for (is_unpacking, sub_keywords) in &groupby { - if is_unpacking { - for keyword in sub_keywords { - self.compile_expression(&keyword.value)?; - size += 1; - } - } else { - let mut sub_size = 0; - for keyword in sub_keywords { - if let Some(name) = &keyword.arg { - self.emit_load_const(ConstantData::Str { - value: name.as_str().into(), - }); - self.compile_expression(&keyword.value)?; - sub_size += 1; - } - } - emit!(self, Instruction::BuildMap { count: sub_size }); - size += 1; - } - } - if size > 1 { - // Merge all dicts: first dict is accumulator, merge rest into it - for _ in 1..size { - emit!(self, Instruction::DictMerge { i: 1 }); - } - } - Ok(()) - } - fn detect_builtin_generator_call( &self, func: &ast::Expr, @@ -9156,54 +9146,59 @@ impl Compiler { ); } - // Compile keyword arguments - if nkwelts > 0 { - let mut have_dict = false; - let mut nseen = 0usize; - - for (i, keyword) in arguments.keywords.iter().enumerate() { - if keyword.arg.is_none() { - // **kwargs unpacking - if nseen > 0 { - // Pack up preceding keywords using codegen_subkwargs - self.codegen_subkwargs(&arguments.keywords, i - nseen, i)?; - if have_dict { - emit!(self, Instruction::DictMerge { i: 1 }); - } - have_dict = true; - nseen = 0; - } + self.compile_call_function_ex_keywords(&arguments.keywords)?; - if !have_dict { - emit!(self, Instruction::BuildMap { count: 0 }); - have_dict = true; - } + self.set_source_range(call_range); + emit!(self, Instruction::CallFunctionEx); + } - self.compile_expression_without_const_boolop_folding(&keyword.value)?; - emit!(self, Instruction::DictMerge { i: 1 }); - } else { - nseen += 1; - } - } + Ok(()) + } + + fn compile_call_function_ex_keywords( + &mut self, + keywords: &[ast::Keyword], + ) -> CompileResult<()> { + if keywords.is_empty() { + emit!(self, Instruction::PushNull); + return Ok(()); + } + + let mut have_dict = false; + let mut nseen = 0usize; - // Pack up any trailing keyword arguments + for (i, keyword) in keywords.iter().enumerate() { + if keyword.arg.is_none() { if nseen > 0 { - self.codegen_subkwargs(&arguments.keywords, nkwelts - nseen, nkwelts)?; + self.codegen_subkwargs(keywords, i - nseen, i)?; if have_dict { emit!(self, Instruction::DictMerge { i: 1 }); } have_dict = true; + nseen = 0; + } + + if !have_dict { + emit!(self, Instruction::BuildMap { count: 0 }); + have_dict = true; } - assert!(have_dict); + self.compile_expression_without_const_boolop_folding(&keyword.value)?; + emit!(self, Instruction::DictMerge { i: 1 }); } else { - emit!(self, Instruction::PushNull); + nseen += 1; } + } - self.set_source_range(call_range); - emit!(self, Instruction::CallFunctionEx); + if nseen > 0 { + self.codegen_subkwargs(keywords, keywords.len() - nseen, keywords.len())?; + if have_dict { + emit!(self, Instruction::DictMerge { i: 1 }); + } + have_dict = true; } + debug_assert!(have_dict); Ok(()) } @@ -10105,37 +10100,6 @@ impl Compiler { } } - fn current_block_has_terminal_with_suppress_exit_predecessor(&self) -> bool { - let code = self.code_stack.last().expect("no code on stack"); - let target = code.current_block; - let mut has_suppress_exit = false; - - for block in &code.blocks { - let Some((last, prefix)) = block.instructions.split_last() else { - continue; - }; - if last.target != target { - continue; - } - if let Some(PseudoInstruction::JumpNoInterrupt { .. }) = last.instr.pseudo() { - let real_instrs: Vec<_> = - prefix.iter().filter_map(|info| info.instr.real()).collect(); - has_suppress_exit |= matches!( - real_instrs.as_slice(), - [ - Instruction::PopTop, - Instruction::PopExcept, - Instruction::PopTop, - Instruction::PopTop, - Instruction::PopTop, - ] - ); - } - } - - has_suppress_exit - } - fn remove_last_no_location_nop(&mut self) { if let Some(info) = self.current_block().instructions.last_mut() { info.remove_no_location_nop = true; @@ -15350,6 +15314,139 @@ def f(cm, names, modname): ); } + #[test] + fn test_with_try_except_return_handler_keeps_body_exit_nop() { + let code = compile_exec( + "\ +def f(cm): + with cm: + try: + x = 1 + except OSError: + return False + return True +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::StoreFast { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "scope-exiting except handler inside with should preserve the CPython body-exit NOP before with cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_with_nonterminal_try_except_normal_cleanup_drops_body_exit_nop() { + let code = compile_exec( + "\ +def f(cm): + with cm: + try: + x = 1 + except Exception: + pass + return x +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::StoreFast { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "non-terminal except inside with should not preserve a body-exit NOP before with cleanup, got ops={ops:?}" + ); + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::StoreFast { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "expected CPython-style direct fallthrough into with cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_with_nested_if_try_except_normal_cleanup_drops_body_exit_nop() { + let code = compile_exec( + "\ +def f(cm, root): + with cm: + if root: + try: + x = 1 + except Exception as e: + raise ValueError from e + return root +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::StoreFast { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "nested try/except should not preserve a body-exit NOP before with cleanup, got ops={ops:?}" + ); + } + #[test] fn test_try_except_finally_normal_cleanup_keeps_body_exit_nop() { let code = compile_exec( @@ -15395,6 +15492,48 @@ def f(self, x): ); } + #[test] + fn test_try_finally_loop_fallthrough_keeps_finalbody_entry_nop() { + let code = compile_exec( + "\ +def f(close, dup, first, second): + try: + retries = 0 + while second != first + 1: + close(first) + retries += 1 + if retries > 10: + raise RuntimeError + first, second = second, dup(second) + finally: + close(second) + close(first) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::JumpBackward { .. }, + Instruction::Nop, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::PushNull, + ] + ) + }), + "try/finally loop fallthrough should preserve CPython finalbody-entry NOP, got ops={ops:?}" + ); + } + #[test] fn test_try_except_finally_suppressing_handler_drops_body_exit_nop() { let code = compile_exec( @@ -19704,6 +19843,92 @@ class C[T: int]: assert_eq!(varnames, vec![".format"]); } + #[test] + fn test_generic_class_double_star_bases_use_tuple_ex_call_path() { + let code = compile_exec( + "\ +def f(Base, kwargs): + class C[T](Base, **kwargs): + pass + return C +", + ); + let type_params = + find_code(&code, "").expect("missing type params code"); + let ops: Vec<_> = type_params + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !has_intrinsic_1(type_params, IntrinsicFunction1::ListToTuple), + "generic class call with **kwargs but no starred bases should not use list-to-tuple, got ops={ops:?}" + ); + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::BuildTuple { .. }, + Instruction::BuildMap { .. }, + Instruction::LoadDeref { .. } | Instruction::LoadFast { .. }, + Instruction::DictMerge { .. }, + ] + ) + }), + "expected CPython-style BUILD_TUPLE/BUILD_MAP/DICT_MERGE ex-call path, got ops={ops:?}" + ); + } + + #[test] + fn test_generic_function_defaults_call_type_params_like_cpython() { + let code = compile_exec( + "\ +def func[T](a: T = 'a', *, b: T = 'b'): + return a, b +", + ); + let ops: Vec<_> = code + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::Swap { .. }, + Instruction::LoadConst { .. }, + Instruction::MakeFunction, + Instruction::Swap { .. }, + Instruction::Call { .. }, + ] + ) + }), + "expected CPython generic defaults call pattern SWAP/MAKE_FUNCTION/SWAP/CALL, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::MakeFunction, + Instruction::Swap { .. }, + Instruction::Swap { .. }, + Instruction::PushNull, + Instruction::Swap { .. }, + ] + ) + }), + "generic defaults call should not use RustPython-specific PUSH_NULL reshuffle, got ops={ops:?}" + ); + } + #[test] fn test_class_annotation_global_resolution_matches_cpython() { let class_global = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 64aade7bb3..ecf18b1e1d 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -260,6 +260,9 @@ pub struct CodeInfo { // u_in_conditional_block pub in_conditional_block: u32, + // Track when compiling the final direct statement in a sync with body. + pub in_final_with_cleanup_statement: u32, + // PEP 649: Next index for conditional annotation tracking // u_next_conditional_annotation_index pub next_conditional_annotation_index: u32, @@ -444,6 +447,7 @@ impl CodeInfo { fblock: _, symbol_table_index: _, in_conditional_block: _, + in_final_with_cleanup_statement: _, next_conditional_annotation_index: _, } = self; From 93e8bc40fee1a6b18bb8d031be9e0cb5c86f7056 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 21:10:56 +0900 Subject: [PATCH 67/76] Align annotation and super call bytecode parity --- crates/codegen/src/compile.rs | 91 +++++++++++++++++++++++++++++++ crates/codegen/src/symboltable.rs | 64 ++++++++++++++-------- 2 files changed, 131 insertions(+), 24 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 113017de8d..63bf31c65c 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -8948,6 +8948,12 @@ impl Compiler { self.emit_load_zero_super_attr(idx); } } + // CPython's Attribute_kind super path emits an attr-line + // NOP after LOAD_SUPER_ATTR, even when the call later uses + // CALL_FUNCTION_EX for starred arguments. + self.set_source_range(attr.range()); + emit!(self, Instruction::Nop); + self.set_source_range(super_range); emit!(self, Instruction::PushNull); self.codegen_call_helper(0, args, call_range)?; } else { @@ -12914,6 +12920,42 @@ def f(obj, arg): ); } + #[test] + fn test_starred_super_call_keeps_attr_line_nop() { + let code = compile_exec( + "\ +def outer(log): + class DelegatingHTTPRequestHandler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + if log: + super(DelegatingHTTPRequestHandler, + self).log_message(format, *args) +", + ); + let log_message = find_code(&code, "log_message").expect("missing log_message code"); + let ops: Vec<_> = log_message + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadSuperAttr { .. }, + Instruction::Nop, + Instruction::PushNull, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "starred super call should keep CPython's attr-line NOP after LOAD_SUPER_ATTR, got ops={ops:?}" + ); + } + #[test] fn test_builtin_any_genexpr_call_is_optimized() { let code = compile_exec( @@ -14480,6 +14522,55 @@ def f(expected_ns, namespace): } } + #[test] + fn test_bare_function_annotations_check_attribute_and_subscript_expressions() { + assert_dis_snapshot!(compile_exec( + "\ +def f(one: int): + int.new_attr: int + [list][0].new_attr: [int, str] + my_lst = [1] + my_lst[one]: int + return my_lst +" + )); + } + + #[test] + fn test_function_local_annassign_annotation_does_not_capture_outer_local() { + let code = compile_exec( + "\ +def f(): + from collections import namedtuple + + class MyHandler: + def __init__(self, resource): + self.resource: namedtuple = resource + + return namedtuple +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let class_code = find_code(&code, "MyHandler").expect("missing MyHandler code"); + let init = find_code(&code, "__init__").expect("missing __init__ code"); + + assert!( + !f.cellvars.iter().any(|name| name == "namedtuple"), + "function-local AnnAssign annotation must not make namedtuple a cell, got cellvars={:?}", + f.cellvars + ); + assert!( + !class_code.freevars.iter().any(|name| name == "namedtuple"), + "class body must not close over function-local annotation-only name, got freevars={:?}", + class_code.freevars + ); + assert!( + !init.freevars.iter().any(|name| name == "namedtuple"), + "method body must not close over function-local annotation-only name, got freevars={:?}", + init.freevars + ); + } + #[test] fn test_non_simple_bare_name_annotation_does_not_create_local_binding() { let code = compile_exec( diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index fabdc40bf9..a15b26fe4a 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -1015,6 +1015,9 @@ struct SymbolTableBuilder { in_iter_def_exp: bool, // Track if we're inside an annotation (yield/await/named expr not allowed) in_annotation: bool, + // CPython's ste_in_unevaluated_annotation: function-local AnnAssign + // annotations are not executed and do not contribute name bindings. + in_unevaluated_annotation: bool, // Track if we're inside a type alias (yield/await/named expr not allowed) in_type_alias: bool, // Track if we're scanning an inner loop iteration target (not the first generator) @@ -1049,6 +1052,7 @@ impl SymbolTableBuilder { varnames_stack: Vec::new(), in_iter_def_exp: false, in_annotation: false, + in_unevaluated_annotation: false, in_type_alias: false, in_comp_inner_loop_target: false, scope_info: None, @@ -1324,6 +1328,13 @@ impl SymbolTableBuilder { is_ann_assign: bool, ) -> SymbolTableResult { let current_scope = self.tables.last().map(|t| t.typ); + let is_unevaluated = is_ann_assign + && current_scope.is_some_and(|scope| { + matches!( + scope, + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda + ) + }); let needs_conditional_annotations = is_ann_assign && (matches!(current_scope, Some(CompilerScope::Module)) || (matches!(current_scope, Some(CompilerScope::Class)) @@ -1365,9 +1376,12 @@ impl SymbolTableBuilder { // PEP 649: scan expression for symbol references // Class annotations are evaluated in class locals (not module globals) let was_in_annotation = self.in_annotation; + let was_in_unevaluated_annotation = self.in_unevaluated_annotation; self.in_annotation = true; + self.in_unevaluated_annotation = is_unevaluated; let result = self.scan_expression(annotation, ExpressionContext::Load); self.in_annotation = was_in_annotation; + self.in_unevaluated_annotation = was_in_unevaluated_annotation; self.leave_annotation_scope(); @@ -2088,33 +2102,35 @@ impl SymbolTableBuilder { self.check_name(id, context, *range)?; - // Determine the contextual usage of this symbol: - match context { - ExpressionContext::Delete => { - self.register_name(id, SymbolUsage::Assigned, *range)?; - self.register_name(id, SymbolUsage::Used, *range)?; - } - ExpressionContext::Load | ExpressionContext::IterDefinitionExp => { - self.register_name(id, SymbolUsage::Used, *range)?; - } - ExpressionContext::Store => { - self.register_name(id, SymbolUsage::Assigned, *range)?; + if !self.in_unevaluated_annotation { + // Determine the contextual usage of this symbol: + match context { + ExpressionContext::Delete => { + self.register_name(id, SymbolUsage::Assigned, *range)?; + self.register_name(id, SymbolUsage::Used, *range)?; + } + ExpressionContext::Load | ExpressionContext::IterDefinitionExp => { + self.register_name(id, SymbolUsage::Used, *range)?; + } + ExpressionContext::Store => { + self.register_name(id, SymbolUsage::Assigned, *range)?; + } + ExpressionContext::Iter => { + self.register_name(id, SymbolUsage::Iter, *range)?; + } } - ExpressionContext::Iter => { - self.register_name(id, SymbolUsage::Iter, *range)?; + // Interesting stuff about the __class__ variable: + // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object + if context == ExpressionContext::Load + && matches!( + self.tables.last().unwrap().typ, + CompilerScope::Function | CompilerScope::AsyncFunction + ) + && id == "super" + { + self.register_name("__class__", SymbolUsage::Used, *range)?; } } - // Interesting stuff about the __class__ variable: - // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object - if context == ExpressionContext::Load - && matches!( - self.tables.last().unwrap().typ, - CompilerScope::Function | CompilerScope::AsyncFunction - ) - && id == "super" - { - self.register_name("__class__", SymbolUsage::Used, *range)?; - } } Expr::Lambda(ExprLambda { body, From 1b414fec830a0010eada1ebd17fd976b5e6be892 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 21:43:57 +0900 Subject: [PATCH 68/76] Align finally reraise tail inlining --- crates/codegen/src/compile.rs | 48 +++++++++++++++++++++++++++++++++++ crates/codegen/src/ir.rs | 30 ++++++++++++---------- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 63bf31c65c..39e5b1708b 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -14571,6 +14571,54 @@ def f(): ); } + #[test] + fn test_finally_exception_path_inlines_except_pass_reraise_tail() { + let source = "\ +def f(self, file, backupfilename): + try: + if file: + file.close() + finally: + backupfilename = self._backupfilename + self._backupfilename = None + if backupfilename and not self._backup: + try: + os.unlink(backupfilename) + except OSError: + pass + self._isstdin = False +"; + let code = compile_exec(source); + let f = find_code(&code, "f").expect("missing f code"); + assert!( + f.instructions.windows(9).any(|window| { + matches!( + [ + window[0].op, + window[1].op, + window[2].op, + window[3].op, + window[8].op, + ], + [ + Instruction::PopExcept, + Instruction::LoadConst { .. }, + Instruction::LoadFast { .. }, + Instruction::StoreAttr { .. }, + Instruction::Reraise { .. }, + ] + ) && match window[8].op { + Instruction::Reraise { depth } => { + depth.get(OpArg::new(u32::from(u8::from(window[8].arg)))) == 0 + } + _ => false, + } + }), + "except-pass normal exit in a finally exception path should inline the CPython reraise tail, got instructions={:?}", + f.instructions + ); + } + #[test] fn test_non_simple_bare_name_annotation_does_not_create_local_binding() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index ecf18b1e1d..c76280f470 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -12075,9 +12075,8 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { } let target = last.target; - if block_is_exceptional(&blocks[target.idx()]) - || (is_named_except_cleanup_normal_exit_block(&blocks[current.idx()]) - && target_pushes_handler(&blocks[target.idx()])) + if is_named_except_cleanup_normal_exit_block(&blocks[current.idx()]) + && target_pushes_handler(&blocks[target.idx()]) { current = next; continue; @@ -12467,20 +12466,23 @@ fn redirect_empty_block_targets(blocks: &mut [Block]) { } fn redirect_empty_unconditional_jump_targets(blocks: &mut [Block]) { - let block_exits_to_reraise = |block_idx: BlockIdx| { + const MAX_COPY_SIZE: usize = 4; + + let block_exits_to_large_reraise = |block_idx: BlockIdx| { let block = &blocks[block_idx.idx()]; let Some(last) = block.instructions.last() else { return false; }; - if matches!(last.instr.real(), Some(Instruction::Reraise { .. })) { - return true; - } - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { - return false; - } - let target = next_nonempty_block(blocks, last.target); - target != BlockIdx::NULL - && blocks[target.idx()] + let reraise_block = if matches!(last.instr.real(), Some(Instruction::Reraise { .. })) { + block_idx + } else if last.instr.is_unconditional_jump() && last.target != BlockIdx::NULL { + next_nonempty_block(blocks, last.target) + } else { + BlockIdx::NULL + }; + reraise_block != BlockIdx::NULL + && blocks[reraise_block.idx()].instructions.len() > MAX_COPY_SIZE + && blocks[reraise_block.idx()] .instructions .last() .is_some_and(|instr| { @@ -12514,7 +12516,7 @@ fn redirect_empty_unconditional_jump_targets(blocks: &mut [Block]) { && raw_predecessors[instr.target.idx()] > 1 && { let target = next_nonempty_block(blocks, instr.target); - target != BlockIdx::NULL && block_exits_to_reraise(target) + target != BlockIdx::NULL && block_exits_to_large_reraise(target) } { return instr.target; From 72689836a30eb08c8ae14833eca03648adee328d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Fri, 8 May 2026 22:15:01 +0900 Subject: [PATCH 69/76] Align protected loop CFG cleanup --- crates/codegen/src/compile.rs | 135 +++++++++++++++++++++++++++++++++- crates/codegen/src/ir.rs | 31 ++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 39e5b1708b..e79266350a 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -646,6 +646,16 @@ impl Compiler { }) } + fn statements_end_with_nonterminal_with_cleanup(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| match stmt { + ast::Stmt::With(ast::StmtWith { body, .. }) => { + !Self::statements_end_with_scope_exit(body) + && !Self::statements_end_with_with_cleanup_scope_exit(body) + } + _ => false, + }) + } + fn statements_end_with_try_finally(body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| { matches!( @@ -3894,7 +3904,10 @@ impl Compiler { ); self.set_no_location(); if (!orelse.is_empty() && self.statements_end_with_loop_fallthrough(orelse)?) - || (orelse.is_empty() && exits_directly_to_with_cleanup && handlers_end_with_scope_exit) + || (orelse.is_empty() + && exits_directly_to_with_cleanup + && handlers_end_with_scope_exit + && !Self::statements_end_with_nonterminal_with_cleanup(body)) { self.preserve_last_redundant_jump_as_nop(); } else { @@ -15546,6 +15559,65 @@ def f(cm): ); } + #[test] + fn test_with_try_except_nested_with_normal_cleanup_drops_body_exit_nop() { + let code = compile_exec( + "\ +def f(open, src, dst, copyfileobj): + with open(src) as fsrc: + try: + with open(dst) as fdst: + copyfileobj(fsrc, fdst) + except IsADirectoryError: + raise + return dst +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "nested with normal cleanup in try/except should fall directly into outer with cleanup, got ops={ops:?}" + ); + assert!( + !ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "nested with normal cleanup in try/except should not preserve a body-exit NOP before outer with cleanup, got ops={ops:?}" + ); + } + #[test] fn test_with_nested_if_try_except_normal_cleanup_drops_body_exit_nop() { let code = compile_exec( @@ -25666,6 +25738,67 @@ def f(self, ready, selector, key, input_view, os, BrokenPipeError): ); } + #[test] + fn test_try_else_after_conditional_raise_keeps_loop_if_body_before_backedge() { + let code = compile_exec( + "\ +def f(seq, flag, stat, OSError, pred, SpecialFileError): + for i in seq: + try: + st = stat(i) + except OSError: + pass + else: + if pred(st.mode): + raise SpecialFileError(i) + if flag and i == 0: + x = st.real +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::StoreFast { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "try-else tail after conditional raise should keep CPython body-before-backedge layout, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "try-else tail should not invert the inner condition before its body, got ops={ops:?}" + ); + } + #[test] fn test_explicit_continue_after_return_orders_return_before_backedge() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index c76280f470..dc9c20ddff 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14796,6 +14796,34 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) } + fn protected_region_has_prior_scope_exit( + blocks: &[Block], + loop_target: BlockIdx, + current: BlockIdx, + ) -> bool { + let mut cursor = next_nonempty_block(blocks, loop_target); + let mut visited = vec![false; blocks.len()]; + let mut saw_protected = false; + while cursor != BlockIdx::NULL && cursor != current { + if visited[cursor.idx()] { + return false; + } + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + saw_protected |= block_is_protected(block); + if saw_protected + && block + .instructions + .iter() + .any(|info| info.instr.is_scope_exit()) + { + return true; + } + cursor = block.next; + } + false + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -14847,6 +14875,8 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); let has_exceptional_duplicate_condition_line = has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&cond)); + let has_prior_protected_scope_exit = + protected_region_has_prior_scope_exit(blocks, loop_target, current); let after_jump_target = next_nonempty_block(blocks, after_jump); let after_jump_starts_loop_cleanup = block_starts_loop_cleanup(blocks, after_jump_target); @@ -14875,6 +14905,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && (!after_jump_starts_loop_cleanup || body_has_any_loop_backedge) && !is_scope_exit_block(&blocks[body.idx()]); let can_reorder = !has_exceptional_duplicate_condition_line + && !has_prior_protected_scope_exit && ((!body_tail_is_conditional && ((normalized_single_block_can_reorder && body_is_single_block From a24a791477c42d7512855b08d782038eb9145f8e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 9 May 2026 12:44:43 +0900 Subject: [PATCH 70/76] Align loop CFG bytecode layout with CPython --- crates/codegen/src/compile.rs | 438 +++++++++++++++++++++++++++++++++- crates/codegen/src/ir.rs | 130 ++++++++-- 2 files changed, 536 insertions(+), 32 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index e79266350a..4be3dd1825 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3904,8 +3904,7 @@ impl Compiler { ); self.set_no_location(); if (!orelse.is_empty() && self.statements_end_with_loop_fallthrough(orelse)?) - || (orelse.is_empty() - && exits_directly_to_with_cleanup + || (exits_directly_to_with_cleanup && handlers_end_with_scope_exit && !Self::statements_end_with_nonterminal_with_cleanup(body)) { @@ -15505,6 +15504,48 @@ def f(cm): ); } + #[test] + fn test_with_try_except_else_return_handler_keeps_body_exit_nop() { + let code = compile_exec( + "\ +def f(cm, func, check): + with cm: + try: + func() + except ValueError: + return False + else: + check() + return True +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "try/except/else inside with with scope-exiting handler should preserve the CPython body-exit NOP before with cleanup, got ops={ops:?}" + ); + } + #[test] fn test_with_nonterminal_try_except_normal_cleanup_drops_body_exit_nop() { let code = compile_exec( @@ -25140,6 +25181,52 @@ def f(self, tag): ); } + #[test] + fn test_while_boolop_conditional_return_splits_backedges_before_return_body() { + let code = compile_exec( + "\ +def f(flags, A, B, stop): + while True: + if stop: + break + if flags & A and flags & B: + return None + return flags +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(12).any(|window| { + matches!( + window, + [ + Instruction::BinaryOp { .. }, + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::BinaryOp { .. }, + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "boolop conditional return in a while tail should split CPython-style false backedges before the return body, got ops={ops:?}" + ); + } + #[test] fn test_for_break_to_return_orders_backedge_before_return() { let code = compile_exec( @@ -27930,6 +28017,338 @@ def f(seq, db): ); } + #[test] + fn test_nested_loop_if_try_body_implicit_continue_places_body_after_jumpback() { + let code = compile_exec( + "\ +def f(seq, broken, codecs, LookupError, s, Queue, bytes): + for encoding in seq: + if encoding not in broken: + q = Queue(b'') + writer = codecs.getwriter(encoding)(q) + encodedresult = b'' + for c in s: + writer.write(c) + chunk = q.read() + encodedresult += chunk + q = Queue(b'') + reader = codecs.getreader(encoding)(q) + decodedresult = '' + for c in encodedresult: + q.write(bytes([c])) + decodedresult += reader.read() + if encoding not in broken: + try: + encoder = codecs.getincrementalencoder(encoding)() + except LookupError: + pass + else: + encoder.encode('x') + if encoding not in ('idna', 'mbcs'): + try: + encoder = codecs.getincrementalencoder(encoding)('ignore') + except LookupError: + pass + else: + encodedresult = b''.join(encoder.encode(c) for c in s) + decoder = codecs.getincrementaldecoder(encoding)('ignore') + decodedresult = ''.join(decoder.decode(bytes([c])) for c in encodedresult) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "nested final if with try body should put false backedge before body, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "nested final if with try body should not leave body before false backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_branch_raise_before_elif_keeps_body_before_backedge() { + let code = compile_exec( + "\ +def f(checks, UNIQUE, CONTINUOUS, ValueError): + for check in checks: + if check is UNIQUE: + duplicates = [] + for name in checks: + if name: + duplicates.append(name) + if duplicates: + detail = ', '.join(str(name) for name in duplicates) + raise ValueError('aliases: %s' % detail) + elif check is CONTINUOUS: + value = 1 + return value +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadConst { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "raise body before an elif chain should stay before the branch backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "raise body before an elif chain should not be moved after the branch backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_nested_raise_then_append_places_body_after_false_backedge() { + let code = compile_exec( + "\ +def f(args, parameters, enforce_default_ordering, type_var_tuple_encountered, default_encountered, TypeError): + for t in args: + if t not in parameters: + if enforce_default_ordering: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('a') + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError('b') + parameters.append(t) + return parameters +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + ] + ) + }), + "CPython places the nested raise/append body after the false backedge for this loop tail, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + ] + ) + }), + "this case should not keep the body before the false backedge; CPython emits the split backedge first, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_break_before_adjacent_break_keeps_body_before_backedge() { + let code = compile_exec( + "\ +def f(pattern, prefix, get_prefix): + for op, av in pattern: + if op == 1: + prefix.append(av) + elif op == 2: + prefix1, got_all = get_prefix(av) + prefix.extend(prefix1) + if not got_all: + break + else: + break + else: + return prefix, True + return prefix, False +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::PopTop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadConst { .. }, + Instruction::BuildTuple { .. }, + ] + ) + }), + "break before an adjacent break exit should stay before the loop backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::PopTop, + ] + ) + }), + "break before an adjacent break exit should not be moved after the loop backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_elif_and_pass_keeps_shared_false_backedge_after_body() { + let code = compile_exec( + "\ +def f(methods, simple_keys, checked_keys, checked_enum, simple_enum, failed): + for method in methods: + if method in simple_keys and method in checked_keys: + continue + elif method not in simple_keys and method not in checked_keys: + checked_method = getattr(checked_enum, method, None) + simple_method = getattr(simple_enum, method, None) + if hasattr(checked_method, '__func__'): + checked_method = checked_method.__func__ + simple_method = simple_method.__func__ + if checked_method != simple_method: + failed.append(method) + else: + pass +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(9).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadGlobal { .. }, + _, + ] + ) + }), + "elif-and body should stay before the shared false backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + ] + ) + }), + "elif-and shared false backedge should not be split before the body, got ops={ops:?}" + ); + } + #[test] fn test_loop_nested_if_delete_slice_places_body_after_jumpback() { let code = compile_exec( @@ -28062,18 +28481,17 @@ def f(state, nextchar, whitespace, token, posix, quoted, debug): .collect(); assert!( - ops.windows(7).any(|window| { + ops.windows(5).any(|window| { matches!( window, [ - Instruction::LoadConst { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, Instruction::ContainsOp { .. }, Instruction::PopJumpIfTrue { .. }, Instruction::NotTaken, Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::ToBool, ] ) }), @@ -28111,18 +28529,18 @@ def f(self, nextchar, quoted): .collect(); assert!( - ops.windows(7).any(|window| { + ops.windows(6).any(|window| { matches!( window, [ - Instruction::LoadConst { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::LoadAttr { .. }, Instruction::ContainsOp { .. }, Instruction::PopJumpIfTrue { .. }, Instruction::NotTaken, Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::ToBool, ] ) }), diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index dc9c20ddff..925b31dcf2 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -13036,6 +13036,21 @@ fn is_scope_exit_block(block: &Block) -> bool { .is_some_and(|instr| instr.instr.is_scope_exit()) } +fn is_pop_top_scope_exit_block(block: &Block) -> bool { + is_scope_exit_block(block) + && matches!( + block + .instructions + .first() + .and_then(|info| info.instr.real()), + Some(Instruction::PopTop) + ) +} + +fn is_pop_top_exit_like_block(block: &Block) -> bool { + is_pop_top_scope_exit_block(block) || is_pop_top_jump_block(block) +} + fn is_loop_cleanup_block(block: &Block) -> bool { block .instructions @@ -13332,11 +13347,13 @@ fn block_is_pure_conditional_test(block: &Block) -> bool { | Instruction::LoadFastLoadFast { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } | Instruction::LoadDeref { .. } + | Instruction::LoadGlobal { .. } | Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } | Instruction::LoadAttr { .. } | Instruction::BinaryOp { .. } | Instruction::ContainsOp { .. } + | Instruction::IsOp { .. } | Instruction::CompareOp { .. } | Instruction::ToBool ) @@ -13459,7 +13476,9 @@ fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { .instructions .iter() .any(|info| matches!(info.instr.real(), Some(Instruction::EndAsyncFor))); - if after_jump_is_end_async_for { + let after_jump_is_adjacent_scope_exit = after_jump != BlockIdx::NULL + && is_pop_top_exit_like_block(&blocks[after_jump.idx()]); + if after_jump_is_end_async_for || after_jump_is_adjacent_scope_exit { current = next; continue; } @@ -13850,6 +13869,15 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { .instructions .first() .is_some_and(|info| matches!(info.lineno_override, Some(line) if line < 0)); + let after_jump_is_adjacent_scope_exit = + after_jump != BlockIdx::NULL && is_pop_top_exit_like_block(&blocks[after_jump.idx()]); + if !is_generic_false_path_reorder + && chain_is_single_exit_block + && after_jump_is_adjacent_scope_exit + { + current = next; + continue; + } if is_generic_false_path_reorder && jump_is_artificial && after_jump != BlockIdx::NULL @@ -14024,10 +14052,13 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( }) || (!allow_for_iter_jump_targets && is_explicit_non_for_jump_back(blocks, jump_block)) + || (after_jump != BlockIdx::NULL + && is_pop_top_exit_like_block(&blocks[after_jump.idx()])) || (after_jump != BlockIdx::NULL && !blocks[after_jump.idx()].cold && !is_scope_exit_block(&blocks[after_jump.idx()]) - && !is_loop_cleanup_block(&blocks[after_jump.idx()])) + && !is_loop_cleanup_block(&blocks[after_jump.idx()]) + && jump_targets_for_iter(blocks, jump_block)) || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block { current = next; @@ -14574,10 +14605,7 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec return None; } visited[cursor.idx()] = true; - if block_is_exceptional(&blocks[cursor.idx()]) - || (block_is_protected(&blocks[cursor.idx()]) - && !block_has_only_stop_iteration_error_handlers(&blocks[cursor.idx()], blocks)) - { + if block_is_exceptional(&blocks[cursor.idx()]) { return None; } tail = cursor; @@ -14729,6 +14757,33 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec false } + fn block_ends_with_jump_back_to( + blocks: &[Block], + block_idx: BlockIdx, + target: BlockIdx, + ) -> bool { + let Some(last) = blocks[block_idx.idx()].instructions.last() else { + return false; + }; + last.instr.is_unconditional_jump() + && last.target != BlockIdx::NULL + && next_nonempty_block(blocks, last.target) == next_nonempty_block(blocks, target) + && comes_before(blocks, next_nonempty_block(blocks, target), block_idx) + } + + fn conditional_jump_target_count(blocks: &[Block], target: BlockIdx) -> usize { + let target = next_nonempty_block(blocks, target); + blocks + .iter() + .flat_map(|block| &block.instructions) + .filter(|info| { + is_conditional_jump(&info.instr) + && info.target != BlockIdx::NULL + && next_nonempty_block(blocks, info.target) == target + }) + .count() + } + fn empty_chain_reaches(blocks: &[Block], start: BlockIdx, target: BlockIdx) -> bool { let mut cursor = start; let mut visited = vec![false; blocks.len()]; @@ -14866,8 +14921,16 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); let body_is_single_block = body == body_tail; let body_has_scope_exit = body_segment_contains_scope_exit(blocks, body, body_tail); - let body_has_loop_backedge = - body_segment_contains_jump_back_to(blocks, body, body_tail, loop_target); + let body_tail_has_scope_exit = is_scope_exit_block(&blocks[body_tail.idx()]); + let body_starts_conditional_chain = block_is_pure_conditional_test(&blocks[body.idx()]); + let body_tail_ends_with_loop_backedge = + block_ends_with_jump_back_to(blocks, body_tail, loop_target); + let body_has_loop_backedge = body_tail_ends_with_loop_backedge + || body_segment_contains_jump_back_to(blocks, body, body_tail, loop_target); + if body_has_scope_exit && !body_tail_has_scope_exit { + current = next; + continue; + } let body_has_inner_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); let body_has_any_loop_backedge = body_has_inner_for_iter && body_segment_contains_any_jump_back(blocks, body, body_tail); @@ -14882,30 +14945,47 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec block_starts_loop_cleanup(blocks, after_jump_target); let after_jump_continues_conditional_chain = after_jump_target != BlockIdx::NULL && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); + let body_is_pop_top_exit_like = + body_is_single_block && is_pop_top_exit_like_block(&blocks[body.idx()]); + if (body_has_scope_exit || body_is_pop_top_exit_like) + && after_jump_target != BlockIdx::NULL + && is_pop_top_exit_like_block(&blocks[after_jump_target.idx()]) + { + current = next; + continue; + } if after_jump_continues_conditional_chain { current = next; continue; } + let jump_target_has_multiple_conditional_predecessors = + conditional_jump_target_count(blocks, true_jump_start) > 1; + let body_has_call = block_has_call(&blocks[body.idx()]); let simple_single_block_can_reorder = body_is_single_block && !body_tail_is_conditional && !has_exceptional_duplicate_condition_line - && (!block_has_call(&blocks[body.idx()]) - || (block_starts_loop_cleanup(blocks, after_jump_target) - && !block_is_protected(&blocks[body.idx()]) - && !has_exceptional_duplicate_condition_line)) + && (!body_has_call || block_starts_loop_cleanup(blocks, after_jump_target)) && !body_has_scope_exit - && !blocks[body.idx()] + && (!blocks[body.idx()] .instructions .iter() .any(|info| info.instr.is_unconditional_jump()) + || body_tail_ends_with_loop_backedge) && !after_jump_continues_conditional_chain && matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })); let trailing_implicit_continue_can_reorder = after_jump != BlockIdx::NULL && next_nonempty_block(blocks, after_jump) != body - && (!after_jump_starts_loop_cleanup || body_has_any_loop_backedge) + && (!after_jump_starts_loop_cleanup + || body_has_loop_backedge + || body_has_any_loop_backedge) + && !(body_has_scope_exit + && after_jump_target != BlockIdx::NULL + && is_scope_exit_block(&blocks[after_jump_target.idx()]) + && !body_starts_conditional_chain) && !is_scope_exit_block(&blocks[body.idx()]); let can_reorder = !has_exceptional_duplicate_condition_line && !has_prior_protected_scope_exit + && (!jump_target_has_multiple_conditional_predecessors || body_tail_has_scope_exit) && ((!body_tail_is_conditional && ((normalized_single_block_can_reorder && body_is_single_block @@ -14914,20 +14994,26 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec && is_single_delete_subscr_body(&blocks[body.idx()])) || simple_single_block_can_reorder)) || (trailing_implicit_continue_can_reorder - && (body_has_scope_exit + && ((body_has_scope_exit && body_tail_has_scope_exit) || body_has_loop_backedge || (after_jump_starts_loop_cleanup && body_has_any_loop_backedge)) && !after_jump_continues_conditional_chain)); if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { - let cloned_jump_idx = BlockIdx(blocks.len() as u32); - let mut cloned_jump = blocks[true_jump.idx()].clone(); - cloned_jump.next = body_start; - blocks.push(cloned_jump); - blocks[idx].instructions[cond_idx].instr = reversed_cond; blocks[idx].instructions[cond_idx].target = body_start; - blocks[idx].next = cloned_jump_idx; - blocks[body_tail.idx()].next = true_jump_start; + if body_tail_ends_with_loop_backedge && true_jump_start == true_jump { + blocks[idx].next = true_jump_start; + blocks[true_jump.idx()].next = body_start; + blocks[body_tail.idx()].next = after_jump; + } else { + let cloned_jump_idx = BlockIdx(blocks.len() as u32); + let mut cloned_jump = blocks[true_jump.idx()].clone(); + cloned_jump.next = body_start; + blocks.push(cloned_jump); + + blocks[idx].next = cloned_jump_idx; + blocks[body_tail.idx()].next = true_jump_start; + } current = blocks[idx].next; continue; } From 0941b7a189a4dc5259e5eaf2c0fc4f221db4f795 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 9 May 2026 13:22:31 +0900 Subject: [PATCH 71/76] Align nested loop jump-back layout --- crates/codegen/src/compile.rs | 63 ++++++++++++++++++ crates/codegen/src/ir.rs | 117 +++++++++++++++++++++++++++++++++- 2 files changed, 178 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 4be3dd1825..83cfd1fe13 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -25391,6 +25391,69 @@ def f(kw): ); } + #[test] + fn test_loop_nested_boolop_exit_keeps_cpython_backedge_line_order() { + let code = compile_exec( + "\ +def f(found, value, m, done, name, renamed_variables, keep_unresolved, variables): + for _ in [0]: + if m is not None: + if found: + if '$' in value: + done[name] = value + else: + try: + value = int(value) + except ValueError: + done[name] = value.strip() + else: + done[name] = value + variables.remove(name) + if name.startswith('PY_') \\ + and name[3:] in renamed_variables: + name = name[3:] + if name not in done: + done[name] = value + else: + if keep_unresolved: + done[name] = value + variables.remove(name) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops_lines: Vec<_> = f + .instructions + .iter() + .zip(&f.locations) + .filter_map(|(unit, (location, _))| { + (!matches!(unit.op, Instruction::Cache)).then_some((unit.op, location.line.get())) + }) + .collect(); + + assert!( + ops_lines.windows(9).any(|window| { + matches!( + window, + [ + (Instruction::StoreSubscr, 19), + (Instruction::JumpBackward { .. }, 19), + (Instruction::JumpBackward { .. }, 18), + (Instruction::JumpBackward { .. }, 16), + (Instruction::JumpBackward { .. }, 15), + (Instruction::JumpBackward { .. }, 4), + ( + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + 21 + ), + (Instruction::ToBool, 21), + (Instruction::PopJumpIfFalse { .. }, 21), + ] + ) + }), + "expected CPython-style nested boolop backedge line order before enclosing else, got ops_lines={ops_lines:?}" + ); + } + #[test] fn test_loop_conditional_raise_before_elif_keeps_raise_before_backedge() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 925b31dcf2..61fb2789d4 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -10653,6 +10653,7 @@ impl CodeInfo { reorder_conditional_explicit_continue_scope_exit_blocks(&mut self.blocks); reorder_conditional_implicit_continue_scope_exit_blocks(&mut self.blocks); reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); + reorder_exception_handler_conditional_continue_scope_exit_blocks(&mut self.blocks); deduplicate_adjacent_jump_back_blocks(&mut self.blocks); reorder_conditional_body_and_implicit_continue_blocks(&mut self.blocks); reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); @@ -10684,9 +10685,10 @@ impl CodeInfo { )); duplicate_end_returns(&mut self.blocks, &self.metadata); + duplicate_fallthrough_jump_back_targets(&mut self.blocks); duplicate_shared_jump_back_targets(&mut self.blocks); trace.push(( - "after_duplicate_end_returns".to_owned(), + "after_duplicate_jump_back_targets".to_owned(), self.debug_block_dump(), )); @@ -12609,6 +12611,7 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { let mut jump_back_inserts = Vec::new(); let mut inserts = Vec::new(); let mut target_start_inserts = Vec::new(); + let mut jump_back_target_locations = Vec::new(); for (block_idx, block) in blocks.iter().enumerate() { let source = BlockIdx(block_idx as u32); let (last, allow_scope_exit_target) = if let Some(last) = block @@ -12627,6 +12630,10 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { } let target = last.target; if !blocks[target.idx()].instructions.is_empty() { + if is_jump_back_only_block(blocks, target) && block_has_no_lineno(&blocks[target.idx()]) + { + jump_back_target_locations.push((*last, target)); + } if with_normal_exit_is_followed_by_try(blocks, target) && has_loop_backedge_to(blocks, source) && !matches!( @@ -12664,6 +12671,15 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { inserts.push((*last, target)); } + for (source, target) in jump_back_target_locations { + if !is_jump_back_only_block(blocks, target) || !block_has_no_lineno(&blocks[target.idx()]) { + continue; + } + if let Some(first) = blocks[target.idx()].instructions.first_mut() { + overwrite_location(first, source.location, source.end_location); + } + } + for (source, target, next) in jump_back_inserts { if !blocks[target.idx()].instructions.is_empty() { continue; @@ -15456,7 +15472,14 @@ fn propagate_line_numbers(blocks: &mut [Block], predecessors: &[u32]) { target = blocks[target.idx()].next; } if target != BlockIdx::NULL - && predecessors[target.idx()] == 1 + && (predecessors[target.idx()] == 1 + || has_unique_jump_origin( + blocks, + &reachable, + &incoming_origins, + current, + target, + )) && let Some((location, end_location)) = prev_location && let Some(first) = blocks[target.idx()].instructions.first_mut() { @@ -15537,6 +15560,31 @@ fn has_unique_fallthrough_origin( .all(|origin| allowed[origin.idx()]) } +fn has_unique_jump_origin( + blocks: &[Block], + reachable: &[bool], + incoming_origins: &[Vec], + source: BlockIdx, + target: BlockIdx, +) -> bool { + if source == BlockIdx::NULL + || target == BlockIdx::NULL + || !reachable[source.idx()] + || !blocks[source.idx()] + .instructions + .last() + .is_some_and(|instr| is_jump_instruction(instr) && instr.target != BlockIdx::NULL) + { + return false; + } + + incoming_origins[target.idx()].iter().all(|&origin| { + origin == source + || (blocks[origin.idx()].instructions.is_empty() + && next_nonempty_block(blocks, blocks[origin.idx()].next) == target) + }) +} + fn comes_before(blocks: &[Block], first: BlockIdx, second: BlockIdx) -> bool { let mut current = BlockIdx(0); while current != BlockIdx::NULL { @@ -15554,6 +15602,7 @@ fn comes_before(blocks: &[Block], first: BlockIdx, second: BlockIdx) -> bool { fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { let predecessors = compute_predecessors(blocks); let mut clones = Vec::new(); + let mut lineful_clones_before_target = Vec::new(); let mut block_order = vec![usize::MAX; blocks.len()]; let mut current = BlockIdx(0); let mut pos = 0usize; @@ -15565,6 +15614,39 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { for target in 0..blocks.len() { let target = BlockIdx(target as u32); + if is_jump_back_only_block(blocks, target) + && instruction_lineno(&blocks[target.idx()].instructions[0]) >= 0 + { + let jump_target = + next_nonempty_block(blocks, blocks[target.idx()].instructions[0].target); + let layout_pred = find_layout_predecessor(blocks, target); + if jump_target != BlockIdx::NULL + && comes_before(blocks, jump_target, target) + && layout_pred != BlockIdx::NULL + && !block_has_fallthrough(&blocks[layout_pred.idx()]) + && predecessors[target.idx()] >= 2 + { + let target_lineno = instruction_lineno(&blocks[target.idx()].instructions[0]); + for block_idx in 0..blocks.len() { + let block_idx = BlockIdx(block_idx as u32); + for (instr_idx, info) in blocks[block_idx.idx()].instructions.iter().enumerate() + { + if !is_conditional_jump(&info.instr) + || info.target == BlockIdx::NULL + || next_nonempty_block(blocks, info.target) != target + || block_order[block_idx.idx()] >= block_order[target.idx()] + { + continue; + } + let jump_lineno = instruction_lineno(info); + if jump_lineno >= 0 && jump_lineno != target_lineno { + lineful_clones_before_target.push((target, block_idx, instr_idx)); + } + } + } + } + } + let Some(jump_target) = shared_jump_back_target(&blocks[target.idx()]) else { continue; }; @@ -15648,6 +15730,37 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { } } + lineful_clones_before_target.sort_by_key(|(target, block_idx, _)| { + ( + block_order[target.idx()], + usize::MAX - block_order[block_idx.idx()], + ) + }); + for (target, block_idx, instr_idx) in lineful_clones_before_target { + if next_nonempty_block( + blocks, + blocks[block_idx.idx()].instructions[instr_idx].target, + ) != target + { + continue; + } + let jump = blocks[block_idx.idx()].instructions[instr_idx]; + let layout_pred = find_layout_predecessor(blocks, target); + if layout_pred == BlockIdx::NULL { + continue; + } + + let mut cloned = blocks[target.idx()].clone(); + if let Some(first) = cloned.instructions.first_mut() { + overwrite_location(first, jump.location, jump.end_location); + } + let new_idx = BlockIdx(blocks.len() as u32); + cloned.next = target; + blocks.push(cloned); + blocks[layout_pred.idx()].next = new_idx; + blocks[block_idx.idx()].instructions[instr_idx].target = new_idx; + } + for (target, block_idx, instr_idx) in clones.into_iter().rev() { let jump = blocks[block_idx.idx()].instructions[instr_idx]; let mut cloned = blocks[target.idx()].clone(); From 448e1f28a8d17006107ccd353f40e708788c9aae Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 9 May 2026 13:58:17 +0900 Subject: [PATCH 72/76] Align conditional jump threading with CPython --- .cspell.dict/cpython.txt | 10 + crates/codegen/src/compile.rs | 1638 ++++++++++++++++++++++++-- crates/codegen/src/ir.rs | 1777 ++++++++++++++++++++++++----- crates/codegen/src/symboltable.rs | 23 +- 4 files changed, 3079 insertions(+), 369 deletions(-) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 3bbe7426c7..4530ac9546 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -4,6 +4,7 @@ argdefs argtypes asdl asname +atopen attro augassign badcert @@ -47,6 +48,7 @@ datastack defaultdict denom deopt +deopts dictbytype DICTFLAG dictoffset @@ -62,6 +64,7 @@ fastlocals fblock fblocks fdescr +fdst ffi_argtypes fielddesc fieldlist @@ -75,6 +78,7 @@ freelist freevar freevars fromlist +fsrc getdict getfunc getiter @@ -94,8 +98,10 @@ IMMUTABLETYPE INCREF inlinedepth inplace +inpos ismine ISPOINTER +isoctal iteminfo Itertool keeped @@ -105,6 +111,7 @@ kwonlyargs lasti libffi linearise +lineful lineiterator linetable loadfast @@ -169,6 +176,7 @@ PYTHREAD_NAME releasebuffer repr resinfo +retarget Rshift SA_ONSTACK saveall @@ -180,6 +188,7 @@ SETREF setresult setslice settraceallthreads +sget SLOTDEFINED SMALLBUF SOABI @@ -197,6 +206,7 @@ subscr sval swappedbytes sysdict +tbstderr templatelib testconsole threadstate diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 83cfd1fe13..3496b14801 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -173,6 +173,9 @@ struct Compiler { /// Disable constant tuple/list/set collection folding in contexts where /// CPython keeps the builder form for later assignment lowering. disable_const_collection_folding: bool, + split_next_for_normal_exit_from_break: bool, + fallthrough_has_statement_successor: bool, + fallthrough_successor_stack: Vec, } #[derive(Clone, Copy)] @@ -563,6 +566,9 @@ impl Compiler { do_not_emit_bytecode: 0, disable_const_boolop_folding: false, disable_const_collection_folding: false, + split_next_for_normal_exit_from_break: false, + fallthrough_has_statement_successor: false, + fallthrough_successor_stack: Vec::new(), } } @@ -665,6 +671,70 @@ impl Compiler { }) } + fn statements_end_with_nested_finalbody_try_finally(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| match stmt { + ast::Stmt::Try(ast::StmtTry { finalbody, .. }) if !finalbody.is_empty() => { + Self::statements_end_with_try_finally(finalbody) + } + _ => false, + }) + } + + fn statements_end_with_try_star_except(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| { + matches!( + stmt, + ast::Stmt::Try(ast::StmtTry { + handlers, + finalbody, + is_star: true, + .. + }) if !handlers.is_empty() && finalbody.is_empty() + ) + }) + } + + fn statements_end_with_try_except_handler_fallthrough(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| match stmt { + ast::Stmt::Try(ast::StmtTry { + body, + handlers, + finalbody, + is_star: false, + .. + }) => { + finalbody.is_empty() + && !handlers.is_empty() + && Self::statements_end_with_scope_exit(body) + && handlers.iter().any(|handler| { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + !Self::statements_end_with_scope_exit(&handler.body) + }) + } + _ => false, + }) + } + + fn statements_end_with_try_except_else_handler_scope_exit(body: &[ast::Stmt]) -> bool { + body.last().is_some_and(|stmt| match stmt { + ast::Stmt::Try(ast::StmtTry { + handlers, + orelse, + finalbody, + is_star: false, + .. + }) => { + !orelse.is_empty() + && finalbody.is_empty() + && handlers.iter().any(|handler| { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + Self::statements_end_with_finally_entry_scope_exit(&handler.body) + }) + } + _ => false, + }) + } + fn statements_end_with_open_conditional_fallthrough(body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| match stmt { ast::Stmt::If(ast::StmtIf { @@ -695,14 +765,68 @@ impl Compiler { fn statements_end_with_loop_fallthrough(&mut self, body: &[ast::Stmt]) -> CompileResult { match body.last() { - Some(ast::Stmt::For(_)) => Ok(true), - Some(ast::Stmt::While(ast::StmtWhile { test, .. })) => { - Ok(!matches!(self.constant_expr_truthiness(test)?, Some(true))) + Some(ast::Stmt::For(ast::StmtFor { body, .. })) => { + Ok(!Self::statements_contain_direct_break(body)) + } + Some(ast::Stmt::While(ast::StmtWhile { test, body, .. })) => { + Ok(!matches!(self.constant_expr_truthiness(test)?, Some(true)) + && !Self::statements_contain_direct_break(body)) } _ => Ok(false), } } + fn statements_contain_direct_break(body: &[ast::Stmt]) -> bool { + body.iter().any(Self::statement_contains_direct_break) + } + + fn statements_are_single_for_direct_break(body: &[ast::Stmt]) -> bool { + matches!( + body, + [ast::Stmt::For(ast::StmtFor { body, .. })] + if Self::statements_contain_direct_break(body) + ) + } + + fn statement_contains_direct_break(stmt: &ast::Stmt) -> bool { + match stmt { + ast::Stmt::Break(_) => true, + ast::Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) => { + Self::statements_contain_direct_break(body) + || elif_else_clauses + .iter() + .any(|clause| Self::statements_contain_direct_break(&clause.body)) + } + ast::Stmt::With(ast::StmtWith { body, .. }) => { + Self::statements_contain_direct_break(body) + } + ast::Stmt::Try(ast::StmtTry { + body, + handlers, + orelse, + finalbody, + .. + }) => { + Self::statements_contain_direct_break(body) + || handlers.iter().any(|handler| { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + Self::statements_contain_direct_break(&handler.body) + }) + || Self::statements_contain_direct_break(orelse) + || Self::statements_contain_direct_break(finalbody) + } + ast::Stmt::Match(ast::StmtMatch { cases, .. }) => cases + .iter() + .any(|case| Self::statements_contain_direct_break(&case.body)), + ast::Stmt::For(_) | ast::Stmt::While(_) => false, + _ => false, + } + } + fn has_resuming_bare_except(handlers: &[ast::ExceptHandler]) -> bool { handlers.iter().any(|handler| { let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -714,7 +838,22 @@ impl Compiler { }) } - fn preserves_finally_entry_nop(body: &[ast::Stmt]) -> bool { + fn statements_end_with_optimized_finally_entry_scope_exit(&self, body: &[ast::Stmt]) -> bool { + body.last() + .is_some_and(|stmt| self.statement_ends_with_optimized_finally_entry_scope_exit(stmt)) + } + + fn statement_ends_with_optimized_finally_entry_scope_exit(&self, stmt: &ast::Stmt) -> bool { + match stmt { + ast::Stmt::Assert(_) => self.opts.optimize == 0, + ast::Stmt::If(ast::StmtIf { body, .. }) => { + self.statements_end_with_optimized_finally_entry_scope_exit(body) + } + _ => Self::statement_ends_with_finally_entry_scope_exit(stmt), + } + } + + fn preserves_finally_entry_nop(&self, body: &[ast::Stmt]) -> bool { body.last().is_some_and(|stmt| match stmt { ast::Stmt::Try(ast::StmtTry { body, @@ -732,8 +871,9 @@ impl Compiler { .. }) => { elif_else_clauses.is_empty() - && Self::statements_end_with_finally_entry_scope_exit(body) + && self.statements_end_with_optimized_finally_entry_scope_exit(body) } + ast::Stmt::Assert(_) => self.opts.optimize == 0, _ => false, }) } @@ -1537,6 +1677,9 @@ impl Compiler { // Push the old compiler unit on the stack (like PyCapsule) // This happens before setting qualname + self.fallthrough_successor_stack + .push(self.fallthrough_has_statement_successor); + self.fallthrough_has_statement_successor = false; self.code_stack.push(code_info); // Set qualname after pushing (uses compiler_set_qualname logic) @@ -1668,6 +1811,9 @@ impl Compiler { // compiler_exit_scope fn exit_scope(&mut self) -> CodeObject { let _table = self.pop_symbol_table(); + if let Some(previous) = self.fallthrough_successor_stack.pop() { + self.fallthrough_has_statement_successor = previous; + } // Various scopes can have sub_tables: // - ast::TypeParams scope can have sub_tables (the function body's symbol table) @@ -1685,6 +1831,9 @@ impl Compiler { fn exit_annotation_scope(&mut self, saved_ctx: CompileContext) -> CodeObject { self.pop_annotation_symbol_table(); self.ctx = saved_ctx; + if let Some(previous) = self.fallthrough_successor_stack.pop() { + self.fallthrough_has_statement_successor = previous; + } let pop = self.code_stack.pop(); let stack_top = compiler_unwrap_option(self, pop); @@ -2335,26 +2484,47 @@ impl Compiler { } fn compile_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { - for statement in statements { - self.compile_statement(statement)? + let inherited_successor = self.fallthrough_has_statement_successor; + for (idx, statement) in statements.iter().enumerate() { + let previous_successor = self.fallthrough_has_statement_successor; + self.fallthrough_has_statement_successor = + inherited_successor || idx + 1 < statements.len(); + let result = self.compile_statement(statement); + self.fallthrough_has_statement_successor = previous_successor; + result?; } Ok(()) } fn compile_with_body_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { + let inherited_successor = self.fallthrough_has_statement_successor; for (idx, statement) in statements.iter().enumerate() { + let previous_successor = self.fallthrough_has_statement_successor; + self.fallthrough_has_statement_successor = + inherited_successor || idx + 1 < statements.len(); if idx + 1 == statements.len() && matches!(statement, ast::Stmt::Try(_)) { self.current_code_info().in_final_with_cleanup_statement += 1; let result = self.compile_statement(statement); self.current_code_info().in_final_with_cleanup_statement -= 1; + self.fallthrough_has_statement_successor = previous_successor; result?; } else { - self.compile_statement(statement)?; + let result = self.compile_statement(statement); + self.fallthrough_has_statement_successor = previous_successor; + result?; } } Ok(()) } + fn compile_loop_body_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { + let previous_successor = self.fallthrough_has_statement_successor; + self.fallthrough_has_statement_successor = false; + let result = self.compile_statements(statements); + self.fallthrough_has_statement_successor = previous_successor; + result + } + fn scope_needs_conditional_annotations_cell(symbol_table: &SymbolTable) -> bool { match symbol_table.typ { CompilerScope::Module | CompilerScope::Class => { @@ -2422,9 +2592,6 @@ impl Compiler { let is_typeparams = current_table.typ == CompilerScope::TypeParams; let is_annotation = current_table.typ == CompilerScope::Annotation; let can_see_class = current_table.can_see_class_scope; - let parent_table = current_idx - .checked_sub(1) - .and_then(|idx| self.symbol_table_stack.get(idx)); // First try to find in current table let symbol = current_table.lookup(name.as_ref()); @@ -2439,8 +2606,10 @@ impl Compiler { symbol }; let class_declared_global = can_see_class - && parent_table.is_some_and(|table| table.typ == CompilerScope::Class) - && parent_table + && self.symbol_table_stack[..current_idx] + .iter() + .rev() + .find(|table| table.typ == CompilerScope::Class) .and_then(|table| table.lookup(name.as_ref())) .is_some_and(|symbol| symbol.flags.contains(SymbolFlags::GLOBAL)); @@ -3501,7 +3670,7 @@ impl Compiler { // if handlers is empty, compile body directly // without wrapping in TryExcept (only FinallyTry is needed) if handlers.is_empty() { - let preserve_finally_entry_nop = Self::preserves_finally_entry_nop(body) + let preserve_finally_entry_nop = self.preserves_finally_entry_nop(body) || self.statements_end_with_loop_fallthrough(body)?; // Just compile body with FinallyTry fblock active (if finalbody exists) @@ -3749,6 +3918,8 @@ impl Compiler { if !finalbody.is_empty() { let preserve_finally_normal_pop_block_nop = orelse.is_empty() && !Self::statements_end_with_scope_exit(body) + && (!Self::statements_end_with_open_conditional_fallthrough(body) + || Self::statements_end_with_finally_entry_scope_exit(body)) && handlers.iter().all(|handler| match handler { ast::ExceptHandler::ExceptHandler(handler) => { Self::statements_end_with_scope_exit(&handler.body) @@ -3876,7 +4047,20 @@ impl Compiler { ); self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; - self.compile_statements(body)?; + let split_for_normal_exit_from_break = orelse.is_empty() + && self.fallthrough_has_statement_successor + && Self::statements_are_single_for_direct_break(body) + && !self + .current_code_info() + .fblock + .iter() + .any(|info| matches!(info.fb_type, FBlockType::With | FBlockType::AsyncWith)); + let previous_split_for_normal_exit_from_break = self.split_next_for_normal_exit_from_break; + self.split_next_for_normal_exit_from_break = + previous_split_for_normal_exit_from_break || split_for_normal_exit_from_break; + let compile_body_result = self.compile_statements(body); + self.split_next_for_normal_exit_from_break = previous_split_for_normal_exit_from_break; + compile_body_result?; self.pop_fblock(FBlockType::TryExcept); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); @@ -3952,9 +4136,11 @@ impl Compiler { let cleanup_end = self.new_block(); emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup_end }); + let cleanup_body = self.new_block(); + self.switch_to_block(cleanup_body); self.push_fblock_full( FBlockType::HandlerCleanup, - cleanup_end, + cleanup_body, cleanup_end, FBlockDatum::ExceptionName(alias.as_str().to_owned()), )?; @@ -3994,7 +4180,9 @@ impl Compiler { self.set_no_location(); } else { emit!(self, Instruction::PopTop); - self.push_fblock(FBlockType::HandlerCleanup, end_block, end_block)?; + let cleanup_body = self.new_block(); + self.switch_to_block(cleanup_body); + self.push_fblock(FBlockType::HandlerCleanup, cleanup_body, end_block)?; self.compile_statements(body)?; @@ -5759,6 +5947,7 @@ impl Compiler { if matches!(self.constant_expr_truthiness(test)?, Some(false)) { self.disable_load_fast_borrow_for_block(next_block); + self.disable_load_fast_borrow_for_block(end_block); } self.compile_jump_if(test, false, next_block)?; self.compile_statements(body)?; @@ -5810,7 +5999,7 @@ impl Compiler { self.compile_jump_if(test, false, else_block)?; let was_in_loop = self.ctx.loop_data.replace((while_block, after_block)); - self.compile_statements(body)?; + self.compile_loop_body_statements(body)?; self.ctx.loop_data = was_in_loop; emit!(self, PseudoInstruction::Jump { delta: while_block }); self.set_no_location(); @@ -5964,8 +6153,17 @@ impl Compiler { let preserve_outer_cleanup_target_nop = !is_async && (Self::statements_end_with_with_cleanup_scope_exit(body) || self.statements_end_with_conditional_scope_exit(body) + || Self::statements_end_with_try_except_handler_fallthrough(body) + || Self::statements_end_with_try_except_else_handler_scope_exit(body) || Self::statements_end_with_try_finally(body) || self.statements_end_with_loop_fallthrough(body)?); + let materialize_async_with_outer_cleanup_target_nop = is_async + && Self::statements_end_with_nested_finalbody_try_finally(body) + && self + .current_code_info() + .fblock + .iter() + .any(|info| matches!(info.fb_type, FBlockType::With)); // Pop fblock before normal exit. CPython emits this POP_BLOCK with // no location for sync with, but with the with-item location for @@ -5978,6 +6176,8 @@ impl Compiler { self.set_no_location(); if preserve_outer_cleanup_target_nop { self.preserve_last_redundant_nop(); + } else if Self::statements_end_with_try_star_except(body) { + self.force_remove_last_no_location_nop(); } else { self.remove_last_no_location_nop(); } @@ -6075,6 +6275,10 @@ impl Compiler { // ===== After block ===== self.switch_to_block(after_block); + if materialize_async_with_outer_cleanup_target_nop { + self.set_source_range(with_range); + emit!(self, Instruction::Nop); + } self.leave_conditional_block(); Ok(()) @@ -6094,6 +6298,19 @@ impl Compiler { let for_block = self.new_block(); let else_block = self.new_block(); let after_block = self.new_block(); + let split_normal_exit_from_break = !is_async + && self.split_next_for_normal_exit_from_break + && Self::statements_contain_direct_break(body) + && self + .current_code_info() + .fblock + .iter() + .any(|info| matches!(info.fb_type, FBlockType::TryExcept)); + let normal_exit_block = if split_normal_exit_from_break { + self.new_block() + } else { + after_block + }; let mut end_async_for_target = BlockIdx::NULL; // The thing iterated: @@ -6142,7 +6359,7 @@ impl Compiler { }; let was_in_loop = self.ctx.loop_data.replace((for_block, after_block)); - self.compile_statements(body)?; + self.compile_loop_body_statements(body)?; self.ctx.loop_data = was_in_loop; emit!(self, PseudoInstruction::Jump { delta: for_block }); self.set_no_location(); @@ -6165,10 +6382,25 @@ impl Compiler { self.set_source_range(saved_range); self.compile_statements(orelse)?; - self.switch_to_block(after_block); + self.switch_to_block(normal_exit_block); // Implicit return after for-loop should be attributed to the `for` line self.set_source_range(iter.range()); + if split_normal_exit_from_break { + emit!(self, Instruction::Nop); + self.set_no_location(); + self.preserve_last_redundant_nop(); + self.preserve_last_redundant_jump_as_nop(); + self.mark_last_no_location_exit(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: after_block } + ); + self.set_no_location(); + self.remove_last_no_location_nop(); + self.switch_to_block(after_block); + self.set_source_range(iter.range()); + } self.leave_conditional_block(); Ok(()) @@ -6185,6 +6417,7 @@ impl Compiler { // the original object semantics. if !is_async && let ast::Expr::List(ast::ExprList { elts, .. }) = iter + && elts.len() <= usize::try_from(STACK_USE_GUIDELINE).unwrap() && !elts.iter().any(|e| matches!(e, ast::Expr::Starred(_))) { if let Some(folded) = self.try_fold_constant_collection(elts, CollectionType::List)? { @@ -10124,6 +10357,13 @@ impl Compiler { } } + fn force_remove_last_no_location_nop(&mut self) { + if let Some(info) = self.current_block().instructions.last_mut() { + info.remove_no_location_nop = true; + info.folded_operand_nop = true; + } + } + /// Mark the last emitted instruction as having no source location. /// Prevents it from triggering LINE events in sys.monitoring. fn set_no_location(&mut self) { @@ -12281,6 +12521,122 @@ def f(sys, os, file): } } + #[test] + fn test_boolop_or_shared_body_keeps_false_jump_before_loop_backedge() { + let code = compile_exec( + r#" +def f(value): + digits = [] + for digit in value: + if isinstance(digit, int) and 0 <= digit <= 9: + if digits or digit != 0: + digits.append(digit) + else: + raise ValueError + return digits +"#, + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(11).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadSmallInt { .. }, + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "OR-shared body should keep CPython last-condition false jump before the loop backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + ] + ) + }), + "OR-shared body should not be moved after the implicit loop backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_single_if_loop_backedge_keeps_true_body_fallthrough_backedge_shape() { + let code = compile_exec( + r##" +def f(buffer, pos, last_char): + while pos > 0: + pos -= 1 + if buffer[pos] == "#": + last_char = None + return last_char +"##, + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadConst { .. }, + Instruction::StoreFast { .. }, + ] + ) + }), + "single-if loop tail should keep CPython true-body plus fallthrough-backedge shape, got ops={ops:?}", + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadConst { .. }, + ] + ) + }), + "single-if loop tail should not be inverted away from CPython shape, got ops={ops:?}", + ); + } + fn find_code<'a>(code: &'a CodeObject, name: &str) -> Option<&'a CodeObject> { if code.obj_name == name { return Some(code); @@ -13553,19 +13909,14 @@ def f(xs, env): } #[test] - fn test_try_finally_if_break_false_edge_keeps_finalbody_entry_nop() { + fn test_try_finally_assert_keeps_finalbody_entry_nop() { let code = compile_exec( "\ -def f(self, pid): - while True: - try: - if pid == self.pid: - self.h() - break - finally: - self.r() - self.g() - return self.x +def f(x): + try: + assert x + finally: + g() ", ); let f = find_code(&code, "f").expect("missing function code"); @@ -13577,31 +13928,87 @@ def f(self, pid): .collect(); assert!( - ops.windows(6).any(|window| { + ops.windows(4).any(|window| { matches!( window, [ - Instruction::ReturnValue, + Instruction::RaiseVarargs { .. }, Instruction::Nop, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::LoadAttr { .. }, + Instruction::LoadGlobal { .. }, Instruction::Call { .. }, - Instruction::PopTop, ] ) }), - "expected CPython-style if-line NOP before fallthrough finally body, got ops={ops:?}" + "assert in try/finally should preserve CPython finalbody-entry NOP after the raise edge, got ops={ops:?}" ); - } - - #[test] - fn test_try_percent_format_preprocess_removes_redundant_try_nop() { - let code = compile_exec( - "\ -def f(self, signal): - if self.returncode and self.returncode < 0: - try: - return \"Command '%s' died with %r.\" % ( + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::LoadCommonConstant { .. }, + Instruction::RaiseVarargs { .. }, + Instruction::Nop, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "assert true edge should land on a distinct finalbody-entry NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_try_finally_if_break_false_edge_keeps_finalbody_entry_nop() { + let code = compile_exec( + "\ +def f(self, pid): + while True: + try: + if pid == self.pid: + self.h() + break + finally: + self.r() + self.g() + return self.x +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::ReturnValue, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "expected CPython-style if-line NOP before fallthrough finally body, got ops={ops:?}" + ); + } + + #[test] + fn test_try_percent_format_preprocess_removes_redundant_try_nop() { + let code = compile_exec( + "\ +def f(self, signal): + if self.returncode and self.returncode < 0: + try: + return \"Command '%s' died with %r.\" % ( self.cmd, signal.Signals(-self.returncode)) except ValueError: return \"Command '%s' died with unknown signal %d.\" % ( @@ -14536,7 +14943,7 @@ def f(expected_ns, namespace): #[test] fn test_bare_function_annotations_check_attribute_and_subscript_expressions() { - assert_dis_snapshot!(compile_exec( + let code = compile_exec( "\ def f(one: int): int.new_attr: int @@ -14544,8 +14951,28 @@ def f(one: int): my_lst = [1] my_lst[one]: int return my_lst -" - )); +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let loads_global = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadGlobal { namei } => { + let namei = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) >> 1; + f.names[usize::try_from(namei).unwrap()].as_str() == name + } + _ => false, + }) + }; + assert!( + loads_global("int"), + "bare attribute annotations should evaluate int, got ops={:?}", + f.instructions + ); + assert!( + loads_global("list"), + "bare subscript annotations should evaluate list, got ops={:?}", + f.instructions + ); } #[test] @@ -15546,6 +15973,101 @@ def f(cm, func, check): ); } + #[test] + fn test_with_try_except_else_continue_handler_keeps_body_exit_nop() { + let code = compile_exec( + "\ +def f(meta_path, cm): + for finder in meta_path: + with cm: + try: + find_spec = finder.find_spec + except AttributeError: + continue + else: + spec = find_spec() + if spec is not None: + return spec + return None +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops_lines: Vec<_> = f + .instructions + .iter() + .zip(&f.locations) + .filter_map(|(unit, (location, _))| { + (!matches!(unit.op, Instruction::Cache)).then_some((unit.op, location.line.get())) + }) + .collect(); + + assert!( + ops_lines.windows(6).any(|window| { + matches!( + window, + [ + (Instruction::Call { .. }, 9), + (Instruction::StoreFast { .. }, 9), + (Instruction::Nop, 9), + (Instruction::LoadConst { .. }, 3), + (Instruction::LoadConst { .. }, 3), + (Instruction::LoadConst { .. }, 3), + ] + ) + }), + "try/except/else with continue handler should preserve the CPython body-exit NOP before with cleanup, got ops_lines={ops_lines:?}", + ); + } + + #[test] + fn test_elif_boolop_skips_following_elif_with_forward_jumpback_block() { + let code = compile_exec( + r#" +def f(module, fromlist, import_, recursive=False): + for x in fromlist: + if not isinstance(x, str): + raise TypeError + elif x == '*': + if not recursive and hasattr(module, '__all__'): + _handle_fromlist(module, module.__all__, import_, recursive=True) + elif not hasattr(module, x): + pass +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let is_load_global = + |unit: &&rustpython_compiler_core::bytecode::CodeUnit, name: &str| match unit.op { + Instruction::LoadGlobal { namei } => { + let namei = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) >> 1; + f.names[usize::try_from(namei).unwrap()].as_str() == name + } + _ => false, + }; + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + unit0, + unit1, + unit2, + unit3, + ] if matches!(unit0.op, Instruction::ToBool) + && matches!(unit1.op, Instruction::PopJumpIfTrue { .. }) + && matches!(unit2.op, Instruction::NotTaken) + && is_load_global(unit3, "hasattr") + ) + }), + "boolop branch in elif should keep CPython's forward jump-back block before hasattr, got ops={ops:?}", + ); + } + #[test] fn test_with_nonterminal_try_except_normal_cleanup_drops_body_exit_nop() { let code = compile_exec( @@ -15600,6 +16122,45 @@ def f(cm): ); } + #[test] + fn test_with_try_except_scope_exit_body_handler_fallthrough_keeps_body_exit_nop() { + let code = compile_exec( + "\ +def f(cm, ValueError): + with cm: + try: + raise ValueError + except 42: + pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops_lines: Vec<_> = f + .instructions + .iter() + .zip(&f.locations) + .filter_map(|(unit, (location, _))| { + (!matches!(unit.op, Instruction::Cache)).then_some((unit.op, location.line.get())) + }) + .collect(); + + assert!( + ops_lines.windows(5).any(|window| { + matches!( + window, + [ + (Instruction::Nop, 6), + (Instruction::LoadConst { .. }, 2), + (Instruction::LoadConst { .. }, 2), + (Instruction::LoadConst { .. }, 2), + (Instruction::Call { .. }, 2), + ] + ) + }), + "handler fallthrough target should preserve the CPython NOP before with cleanup, got ops_lines={ops_lines:?}", + ); + } + #[test] fn test_with_try_except_nested_with_normal_cleanup_drops_body_exit_nop() { let code = compile_exec( @@ -15744,6 +16305,39 @@ def f(self, x): ); } + #[test] + fn test_try_except_finally_open_conditional_fallthrough_drops_body_exit_nop() { + let code = compile_exec( + "\ +def f(err, ov, self): + try: + if err: + assert ov + except: + ov.cancel() + raise + finally: + self.cleanup() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let assert_raise = ops + .iter() + .position(|op| matches!(op, Instruction::RaiseVarargs { .. })) + .expect("missing assertion raise"); + + assert!( + !matches!(ops.get(assert_raise + 1), Some(Instruction::Nop)), + "open conditional fallthrough should go directly into finally cleanup, got ops={ops:?}", + ); + } + #[test] fn test_try_finally_loop_fallthrough_keeps_finalbody_entry_nop() { let code = compile_exec( @@ -15786,6 +16380,60 @@ def f(close, dup, first, second): ); } + #[test] + fn test_try_finally_loop_direct_break_drops_finalbody_entry_nop() { + let code = compile_exec( + "\ +def f(lines, close): + try: + while lines: + if lines[0]: + break + close(1) + finally: + close(2) + close(3) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::PushNull, + ] + ) + }), + "direct loop break should enter CPython finalbody without a NOP, got ops={ops:?}", + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::Nop, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::PushNull, + ] + ) + }), + "direct loop break should not preserve finalbody-entry NOP, got ops={ops:?}", + ); + } + #[test] fn test_try_except_finally_suppressing_handler_drops_body_exit_nop() { let code = compile_exec( @@ -16273,19 +16921,82 @@ def f(size): } #[test] - fn test_try_import_pass_else_keeps_borrow() { + fn test_try_import_continue_inside_loop_keeps_earlier_loop_body_borrows() { let code = compile_exec( - "\ -def f(self): - try: - from _ctypes import set_conversion_mode - except ImportError: - pass - else: - self.prev_conv_mode = set_conversion_mode('ascii', 'strict') -", - ); - let f = find_code(&code, "f").expect("missing f code"); + r#" +def f(s, size, errors): + p = [] + pos = 0 + while pos < size: + if s[pos] != ord("\\"): + p.append(chr(s[pos])) + pos += 1 + continue + if pos < size: + try: + import unicodedata + except ImportError: + errors(pos, size) + continue + if pos < size: + p.append(chr(s[pos])) + pos += 1 + return p, pos +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let import_idx = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::ImportName { .. })) + .expect("missing IMPORT_NAME"); + let is_pair = |unit: &&CodeUnit, left_name: &str, right_name: &str, borrowed: bool| { + let Some(var_nums) = (match (unit.op, borrowed) { + (Instruction::LoadFastBorrowLoadFastBorrow { var_nums }, true) + | (Instruction::LoadFastLoadFast { var_nums }, false) => Some(var_nums), + _ => None, + }) else { + return false; + }; + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == left_name + && f.varnames[usize::from(right)] == right_name + }; + let before_import = &instructions[..import_idx]; + + assert!( + before_import + .iter() + .any(|unit| is_pair(unit, "s", "pos", true)), + "loop body before protected import should keep CPython-style borrowed s/pos pair, got instructions={instructions:?}" + ); + assert!( + !before_import + .iter() + .any(|unit| is_pair(unit, "s", "pos", false)), + "protected import later in the loop should not deopt earlier s/pos pair to strong LOAD_FAST_LOAD_FAST, got instructions={instructions:?}" + ); + } + + #[test] + fn test_try_import_pass_else_keeps_borrow() { + let code = compile_exec( + "\ +def f(self): + try: + from _ctypes import set_conversion_mode + except ImportError: + pass + else: + self.prev_conv_mode = set_conversion_mode('ascii', 'strict') +", + ); + let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() @@ -17895,6 +18606,123 @@ def f(x, E): ); } + #[test] + fn test_try_except_for_direct_break_preserves_normal_exhaustion_nop() { + let code = compile_exec( + "\ +def f(xs, g, E): + try: + for x in xs: + if x: + break + g() + except E: + pass + g() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::Nop, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + ] + ) + }), + "try/except for-body with direct break should keep CPython normal-exhaustion NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_try_except_for_without_direct_break_drops_normal_exhaustion_nop() { + let code = compile_exec( + "\ +def f(xs, g, E): + try: + for x in xs: + g() + except E: + pass + g() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::Nop, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + ] + ) + }), + "try/except for-body without direct break should not keep a redundant normal-exhaustion NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_terminal_except_before_conditional_tail_uses_strong_load() { + let code = compile_exec( + "\ +def f(self, Exception): + try: + tree = self.g() + except Exception: + return False + if tree.body: + return True + return False +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let tree_load_before_body_attr = instructions.windows(2).find(|window| { + let loads_tree = match window[0].op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + f.varnames + [usize::from(var_num.get(OpArg::new(u32::from(u8::from(window[0].arg)))))] + == "tree" + } + _ => false, + }; + loads_tree && matches!(window[1].op, Instruction::LoadAttr { .. }) + }); + + assert!( + matches!( + tree_load_before_body_attr.map(|window| window[0].op), + Some(Instruction::LoadFast { .. }) + ), + "conditional tail after terminal except should match CPython's strong LOAD_FAST, got instructions={instructions:?}", + ); + } + #[test] fn test_try_except_continuation_folded_tuple_drops_operand_nop() { let code = compile_exec( @@ -18229,6 +19057,54 @@ def f(x, os, self, pid, exitcode): ); } + #[test] + fn test_reraising_outer_handler_keeps_explicit_raise_call_arg_borrow() { + let code = compile_exec( + "\ +def f(file, os, stat, errno, self, fd): + try: + self._stat_atopen = os.fstat(fd) + try: + if stat.S_ISDIR(self._stat_atopen.st_mode): + raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) + except AttributeError: + pass + self.name = file + try: + os.lseek(fd, 0, SEEK_END) + except OSError as e: + if e.errno != errno.ESPIPE: + raise + except OSError: + raise +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let file_borrow_before_raise_call = instructions.windows(3).any(|window| { + let is_file_borrow = match window[0].op { + Instruction::LoadFastBorrow { var_num } => { + f.varnames + [usize::from(var_num.get(OpArg::new(u32::from(u8::from(window[0].arg)))))] + == "file" + } + _ => false, + }; + is_file_borrow + && matches!(window[1].op, Instruction::Call { .. }) + && matches!(window[2].op, Instruction::RaiseVarargs { .. }) + }); + + assert!( + file_borrow_before_raise_call, + "outer reraising handler should not deopt explicit raise call args; CPython keeps file as LOAD_FAST_BORROW, got instructions={instructions:?}" + ); + } + #[test] fn test_reraising_except_loop_backedge_keeps_loop_header_borrow() { let code = compile_exec( @@ -20181,6 +21057,110 @@ def func[T](a: T = 'a', *, b: T = 'b'): ); } + #[test] + fn test_class_type_param_bound_prefers_classdict_over_outer_function_local() { + let code = compile_exec( + "\ +def f(self): + class X: + T = int + def foo[U: T](self): ... + T, U = X.foo.__type_params__ + return T.__bound__, U.__bound__ +", + ); + let f = find_code(&code, "f").expect("missing f code"); + assert!( + !f.cellvars.iter().any(|name| name == "T"), + "class-local type-param bound must not force outer function T cell, got cellvars={:?}", + f.cellvars + ); + + let class_code = find_code(f, "X").expect("missing X class code"); + assert!( + !class_code.freevars.iter().any(|name| name == "T"), + "class body must not close over outer T for class-local bound, got freevars={:?}", + class_code.freevars + ); + + let type_params = + find_code(class_code, "").expect("missing type params code"); + assert_eq!( + type_params + .freevars + .iter() + .map(String::as_str) + .collect::>(), + vec!["__classdict__"], + "type params scope should close only over __classdict__, got freevars={:?}", + type_params.freevars + ); + + let bound = find_code(type_params, "U").expect("missing U bound code"); + assert_eq!( + bound + .freevars + .iter() + .map(String::as_str) + .collect::>(), + vec!["__classdict__"], + "bound evaluator should close only over __classdict__, got freevars={:?}", + bound.freevars + ); + assert!( + bound + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFromDictOrGlobals { .. })), + "bound evaluator should use LOAD_FROM_DICT_OR_GLOBALS for class-local T, got instructions={:?}", + bound.instructions + ); + } + + #[test] + fn test_class_type_param_bound_respects_class_global_over_outer_function_local() { + let code = compile_exec( + "\ +def f(self): + T = int + class X: + global T + def foo[U: T](self): ... + return X.foo.__type_params__ +", + ); + let f = find_code(&code, "f").expect("missing f code"); + assert!( + f.cellvars.is_empty(), + "class global type-param bound must not force outer function cells, got cellvars={:?}", + f.cellvars + ); + + let class_code = find_code(f, "X").expect("missing X class code"); + let type_params = + find_code(class_code, "").expect("missing type params code"); + let bound = find_code(type_params, "U").expect("missing U bound code"); + assert!( + bound + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadGlobal { .. })), + "explicit class global should resolve as LOAD_GLOBAL in bound evaluator, got instructions={:?}", + bound.instructions + ); + assert!( + !bound.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFromDictOrGlobals { .. } + | Instruction::LoadFromDictOrDeref { .. } + ) + }), + "explicit class global should not use classdict/deref lookup, got instructions={:?}", + bound.instructions + ); + } + #[test] fn test_class_annotation_global_resolution_matches_cpython() { let class_global = compile_exec( @@ -20289,26 +21269,54 @@ def f(): } #[test] - fn test_constant_set_iterable_uses_frozenset_const() { - let code = compile_exec( - "\ -def f(): - return [x for x in {1, 2, 3}] -", + fn test_large_constant_list_iterable_keeps_streaming_list_build() { + let source = format!( + "def f():\n for x in [{}]:\n pass\n", + (0..=STACK_USE_GUIDELINE) + .map(|i| format!("'v{i}'")) + .collect::>() + .join(", ") ); + let code = compile_exec(&source); let f = find_code(&code, "f").expect("missing function code"); assert!( - !f.instructions + f.instructions .iter() - .any(|unit| matches!(unit.op, Instruction::BuildSet { .. })), - "constant set iterable should avoid BUILD_SET before GET_ITER" + .any(|unit| matches!(unit.op, Instruction::BuildList { .. })), + "large list iterable should keep CPython streaming BUILD_LIST form, got instructions={:?}", + f.instructions ); - assert!(f.constants.iter().any(|constant| matches!( - constant, - ConstantData::Frozenset { elements } - if matches!( - elements.as_slice(), + assert!( + f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListAppend { .. })), + "large list iterable should use LIST_APPEND streaming form, got instructions={:?}", + f.instructions + ); + } + + #[test] + fn test_constant_set_iterable_uses_frozenset_const() { + let code = compile_exec( + "\ +def f(): + return [x for x in {1, 2, 3}] +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildSet { .. })), + "constant set iterable should avoid BUILD_SET before GET_ITER" + ); + assert!(f.constants.iter().any(|constant| matches!( + constant, + ConstantData::Frozenset { elements } + if matches!( + elements.as_slice(), [ ConstantData::Integer { .. }, ConstantData::Integer { .. }, @@ -22627,6 +23635,58 @@ def f(xs): ); } + #[test] + fn test_targeted_nop_after_return_uses_strong_pair_call_args() { + let code = compile_exec( + "\ +def f(x, d, count, inner, hi, w1, lo, w2): + if x: + return + if 0: + pass + d[count] += 1 + inner(hi, w1) + del hi + inner(lo, w2) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::PushNull, + Instruction::LoadFastLoadFast { .. }, + ] + ) + }), + "targeted NOP after return should keep CPython-style strong pair call args, got ops={ops:?}" + ); + let nop_idx = ops + .iter() + .position(|op| matches!(op, Instruction::Nop)) + .expect("missing targeted NOP"); + assert!( + !ops[nop_idx..].iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "targeted NOP tail should not reintroduce borrowed fast loads, got ops={ops:?}" + ); + } + #[test] fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { let code = compile_exec( @@ -23330,6 +24390,52 @@ async def foo(): ); } + #[test] + fn test_async_with_try_finally_before_outer_sync_with_cleanup_keeps_anchor_nop() { + let code = compile_exec( + "\ +async def foo(self): + with self.assertRaises(ZeroDivisionError): + async with timeout(): + try: + try: + raise ValueError + finally: + await sleep(1) + finally: + try: + raise KeyError + finally: + 1 / 0 + after() +", + ); + let f = find_code(&code, "foo").expect("missing foo code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::Copy { .. }, + Instruction::PopExcept, + Instruction::Reraise { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "expected CPython-style async-with after-block NOP before outer sync-with cleanup, got ops={ops:?}" + ); + } + #[test] fn test_nested_terminal_with_keeps_outer_cleanup_target_nop() { let code = compile_exec( @@ -24362,6 +25468,315 @@ def f(self): ); } + #[test] + fn test_handler_resume_while_conditional_tail_keeps_borrow_load_fast() { + let code = compile_exec( + "\ +def f(value): + items = [] + while value: + try: + token, value = parse(value) + items.append(token) + except Error: + token, value = recover(value) + items.append(token) + if value and value[0] != ',': + item = items[-1] + token, value = recover(value) + item.extend(token) + if value and value[0] == ',': + items.append(',') + value = value[1:] + return items, value +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let handler_start = f + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_region = &f.instructions[..handler_start]; + let is_strong_local_load = |unit: &CodeUnit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + matches!( + f.varnames[usize::from(var_num.get(arg))].as_str(), + "value" | "items" | "token" | "item" + ) + } + Instruction::LoadFastLoadFast { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + let left = f.varnames[usize::from(left)].as_str(); + let right = f.varnames[usize::from(right)].as_str(); + matches!(left, "value" | "items" | "token" | "item") + || matches!(right, "value" | "items" | "token" | "item") + } + _ => false, + }; + + assert!( + !normal_region.iter().any(is_strong_local_load), + "while-loop handler resume tail should keep CPython-style borrowed normal loads, got instructions={:?}", + f.instructions + ); + } + + #[test] + fn test_multi_handler_resume_while_tail_keeps_borrow_load_fast() { + let code = compile_exec( + "\ +def f(value): + items = [] + while value and value[0] != ';': + try: + token, value = parse(value) + items.append(token) + except Error: + leader = None + if value[0] in leaders: + leader, value = recover(value) + if not value or value[0] in ',;': + items.append(leader) + else: + token, value = recover_invalid(value) + if leader is not None: + token[:0] = [leader] + items.append(token) + elif value[0] == ',': + items.append(',') + else: + token, value = recover_invalid(value) + if leader is not None: + token[:0] = [leader] + items.append(token) + if value and value[0] not in ',;': + item = items[-1] + token, value = recover_invalid(value) + item.extend(token) + if value and value[0] == ',': + items.append(',') + value = value[1:] + return items, value +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let handler_start = f + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_region = &f.instructions[..handler_start]; + + assert!( + normal_region.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected multi-handler while tail to keep borrowed normal loads, got instructions={:?}", + f.instructions + ); + assert!( + !normal_region + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "multi-handler while tail should not be deoptimized to strong LOAD_FAST, got instructions={:?}", + f.instructions + ); + } + + #[test] + fn test_suppressing_with_and_typed_except_resume_loop_method_tail_keeps_strong_load_fast() { + let code = compile_exec( + "\ +def f(proc, text): + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + pass + except OSError: + pass + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let load_name = |unit: &CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + + let wait_receiver_is_strong = f.instructions.windows(2).any(|window| { + matches!(window[0].op, Instruction::LoadFast { .. }) + && load_name(&window[0]) == Some("proc") + && matches!(window[1].op, Instruction::LoadAttr { .. }) + }); + assert!( + wait_receiver_is_strong, + "CPython keeps proc.wait() receiver strong after suppressing-with and typed-except loop resumes, got instructions={:?}", + f.instructions + ); + } + + #[test] + fn test_handler_break_join_loop_body_and_tail_keep_strong_load_fast() { + let code = compile_exec( + "\ +def f(function, stem): + result = [] + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + result.sort() + return result +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let load_name = |unit: &CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + + let body_method_call_uses_strong_loads = instructions.windows(4).any(|window| { + matches!(window[0].op, Instruction::LoadFast { .. }) + && load_name(window[0]) == Some("result") + && matches!(window[1].op, Instruction::LoadAttr { .. }) + && matches!(window[2].op, Instruction::LoadFast { .. }) + && load_name(window[2]) == Some("next") + && matches!(window[3].op, Instruction::Call { .. }) + }); + assert!( + body_method_call_uses_strong_loads, + "CPython keeps result.append(next) loads strong after an except-break join, got instructions={:?}", + f.instructions + ); + + let body_update_uses_strong_load = instructions.windows(3).any(|window| { + matches!(window[0].op, Instruction::LoadFast { .. }) + && load_name(window[0]) == Some("state") + && matches!(window[1].op, Instruction::LoadSmallInt { .. }) + && matches!(window[2].op, Instruction::BinaryOp { .. }) + }); + assert!( + body_update_uses_strong_load, + "CPython keeps state += 1 load strong after an except-break join, got instructions={:?}", + f.instructions + ); + + let tail_method_call_uses_strong_load = instructions.windows(2).any(|window| { + matches!(window[0].op, Instruction::LoadFast { .. }) + && load_name(window[0]) == Some("result") + && matches!(window[1].op, Instruction::LoadAttr { .. }) + }); + assert!( + tail_method_call_uses_strong_load, + "CPython keeps result.sort() receiver strong after an except-break join, got instructions={:?}", + f.instructions + ); + + let return_uses_strong_load = instructions.windows(2).any(|window| { + matches!(window[0].op, Instruction::LoadFast { .. }) + && load_name(window[0]) == Some("result") + && matches!(window[1].op, Instruction::ReturnValue) + }); + assert!( + return_uses_strong_load, + "CPython keeps post-loop result return strong after an except-break join, got instructions={:?}", + f.instructions + ); + } + + #[test] + fn test_handler_resume_to_loop_header_keeps_loop_header_borrows() { + let code = compile_exec( + "\ +def f(value, Phrase, get_word, errors, ENDS, DOT): + phrase = Phrase() + try: + token, value = get_word(value) + phrase.append(token) + except errors.HeaderParseError: + phrase.defects.append(errors.InvalidHeaderDefect('bad')) + while value and value[0] not in ENDS: + if value[0] == '.': + phrase.append(DOT) + phrase.defects.append(errors.ObsoleteHeaderDefect('dot')) + value = value[1:] + else: + token, value = get_word(value) + phrase.append(token) + return phrase, value +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let load_name = |unit: &CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + + let loop_header_uses_borrow = instructions.windows(3).any(|window| { + matches!(window[0].op, Instruction::LoadFastBorrow { .. }) + && load_name(window[0]) == Some("value") + && matches!(window[1].op, Instruction::ToBool) + && matches!(window[2].op, Instruction::PopJumpIfFalse { .. }) + }); + assert!( + loop_header_uses_borrow, + "CPython keeps loop-header value test borrowed after a pre-loop handler resumes to that header, got instructions={:?}", + f.instructions + ); + + let return_pair_uses_borrow = instructions.windows(3).any(|window| { + matches!( + window[0].op, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) && matches!(window[1].op, Instruction::BuildTuple { .. }) + && matches!(window[2].op, Instruction::ReturnValue) + }); + assert!( + return_pair_uses_borrow, + "CPython keeps phrase/value return pair borrowed after a pre-loop handler resumes to the loop header, got instructions={:?}", + f.instructions + ); + } + #[test] fn test_reraising_except_loop_break_tail_keeps_post_loop_borrows() { let code = compile_exec( @@ -24704,6 +26119,77 @@ class C: } } + #[test] + fn test_async_with_return_await_after_early_return_keeps_borrow_load_fast() { + let code = compile_exec( + "\ +async def wait_for(fut, timeout): + if timeout is not None and timeout <= 0: + fut = ensure_future(fut) + if fut.done(): + return fut.result() + await cancel_and_wait(fut) + try: + return fut.result() + except CancelledError as exc: + raise TimeoutError from exc + async with timeout_cm(timeout): + return await fut +", + ); + let f = find_code(&code, "wait_for").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let loaded_names = |unit: &bytecode::CodeUnit| -> Vec { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + match unit.op { + Instruction::LoadFastBorrow { var_num } => { + vec![f.varnames[usize::from(var_num.get(arg))].to_string()] + } + Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let (left, right) = var_nums.get(arg).indexes(); + vec![ + f.varnames[usize::from(left)].to_string(), + f.varnames[usize::from(right)].to_string(), + ] + } + _ => Vec::new(), + } + }; + let borrowed_names: Vec<_> = f + .instructions + .iter() + .filter_map(|unit| { + let names = loaded_names(unit); + (!names.is_empty()).then_some(names) + }) + .collect(); + + assert!( + borrowed_names + .iter() + .flatten() + .any(|name| name == "timeout"), + "CPython preserves borrowed timeout loads in async-with setup, got instructions={:?}", + f.instructions + ); + assert!( + ops.windows(4).any(|window| { + matches!(window[0].op, Instruction::LoadFastBorrow { .. }) + && loaded_names(window[0]).iter().any(|name| name == "fut") + && matches!(window[1].op, Instruction::LoadAttr { namei } + if f.names[usize::try_from(namei.get(OpArg::new(u32::from(u8::from(window[1].arg)))).name_idx()).unwrap()] == "result") + && matches!(window[2].op, Instruction::Call { .. }) + && matches!(window[3].op, Instruction::ReturnValue) + }), + "CPython preserves borrowed fut loads for direct try-return method calls, got instructions={:?}", + f.instructions + ); + } + #[test] fn test_protected_import_tail_keeps_strong_load_fast() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 61fb2789d4..75e2f677fd 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -196,6 +196,24 @@ fn is_standalone_named_except_cleanup_normal_exit_block(block: &Block) -> bool { }) } +fn named_except_cleanup_body_is_fast_local_only(block: &Block) -> bool { + let len = block.instructions.len(); + if len < 5 || !is_named_except_cleanup_normal_exit_block(block) { + return false; + } + block.instructions[..len - 5].iter().all(|info| { + matches!( + info.instr.real(), + Some( + Instruction::Nop + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::StoreFast { .. } + ) + ) + }) +} + // spell-checker:ignore petgraph // TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it // might enable more analysis/optimizations @@ -358,6 +376,7 @@ impl CodeInfo { resolve_line_numbers(&mut self.blocks); materialize_empty_conditional_exit_targets(&mut self.blocks); redirect_empty_block_targets(&mut self.blocks); + inline_small_fast_return_blocks(&mut self.blocks); duplicate_end_returns(&mut self.blocks, &self.metadata); duplicate_fallthrough_jump_back_targets(&mut self.blocks); duplicate_shared_jump_back_targets(&mut self.blocks); @@ -421,7 +440,6 @@ impl CodeInfo { self.deoptimize_borrow_before_import_after_join_store(); self.deoptimize_borrow_after_protected_store_tail(); self.deoptimize_borrow_after_deoptimized_async_with_enter(); - self.deoptimize_borrow_after_push_exc_info(); self.deoptimize_borrow_for_handler_return_paths(); self.deoptimize_borrow_for_match_keys_attr(); self.deoptimize_borrow_in_protected_attr_chain_tail(); @@ -477,7 +495,6 @@ impl CodeInfo { // Convert pseudo ops (LoadClosure uses cellfixedoffsets) and fixup DEREF opargs convert_pseudo_ops(&mut blocks, &cellfixedoffsets); fixup_deref_opargs(&mut blocks, &cellfixedoffsets); - deoptimize_borrow_after_push_exc_info_in_blocks(&mut blocks); // Remove redundant NOPs, keeping line-marker NOPs only when // they are needed to preserve tracing. let mut block_order = Vec::new(); @@ -4618,8 +4635,6 @@ impl CodeInfo { info.instr.real(), Some( Instruction::ForIter { .. } - | Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } | Instruction::EndFor | Instruction::PopIter | Instruction::LoadFastAndClear { .. } @@ -5131,28 +5146,55 @@ impl CodeInfo { } } - fn starts_with_fast_load(block: &Block) -> bool { - block - .instructions - .iter() - .find(|info| { - info.instr.real().is_some_and(|instr| { - !matches!(instr, Instruction::Nop | Instruction::NotTaken) - }) - }) - .is_some_and(|info| { + fn block_has_fast_load(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) + }) + } + + fn block_requires_post_reraise_strong_loads(block: &Block) -> bool { + block_has_protected_instructions(block) + || block.instructions.iter().any(|info| { matches!( info.instr.real(), Some( - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::DeleteFast { .. } + | Instruction::StoreAttr { .. } + | Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + | Instruction::StoreSubscr ) ) }) } + fn block_ends_with_explicit_raise(block: &Block) -> bool { + block + .instructions + .iter() + .rev() + .filter_map(|info| info.instr.real().map(|instr| (instr, info.arg))) + .find(|(instr, _)| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + .is_some_and(|(instr, arg)| { + matches!( + instr, + Instruction::RaiseVarargs { argc } + if argc.get(arg) != oparg::RaiseKind::BareRaise + ) + }) + } + fn block_has_protected_instructions(block: &Block) -> bool { block .instructions @@ -5277,6 +5319,66 @@ impl CodeInfo { false } + fn handler_chain_has_named_cleanup_or_explicit_reraise( + blocks: &[Block], + handler: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block + .instructions + .iter() + .any(|info| match info.instr.real() { + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. }, + ) => true, + Some(Instruction::RaiseVarargs { argc }) => { + argc.get(info.arg) == oparg::RaiseKind::BareRaise + } + _ => false, + }) + { + return true; + } + if block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::PopExcept | Instruction::Reraise { .. }) + ) + }) { + continue; + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + stack.push(block.next); + } + } + false + } + + fn protected_block_handler_has_named_cleanup_or_explicit_reraise( + blocks: &[Block], + block: &Block, + ) -> bool { + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| handler_chain_has_named_cleanup_or_explicit_reraise(blocks, handler)) + } + fn nonresuming_reraise_handlers(blocks: &[Block], block: &Block) -> Vec { let mut handlers = Vec::new(); for handler in block @@ -5389,10 +5491,11 @@ impl CodeInfo { let mut follows_protected_body = vec![false; self.blocks.len()]; for (idx, block) in self.blocks.iter().enumerate() { - if block_is_exceptional(block) || block.cold || !starts_with_fast_load(block) { - continue; - } - if block_has_protected_instructions(block) { + if block_is_exceptional(block) + || block.cold + || !block_has_fast_load(block) + || !block_requires_post_reraise_strong_loads(block) + { continue; } if predecessors[idx] @@ -5436,34 +5539,47 @@ impl CodeInfo { } } - let mut visited = vec![false; self.blocks.len()]; for (idx, follows_protected_body) in follows_protected_body.iter().enumerate() { if !*follows_protected_body { continue; } - let mut cursor = BlockIdx::new(idx as u32); - while cursor != BlockIdx::NULL && !visited[cursor.idx()] { - let block = &self.blocks[cursor.idx()]; + let mut visited = vec![false; self.blocks.len()]; + let mut stack = vec![BlockIdx::new(idx as u32)]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + let block = &self.blocks[block_idx.idx()]; if block_is_exceptional(block) || block.cold { - break; + continue; } - if block_has_protected_instructions(block) - || predecessors[cursor.idx()].iter().any(|pred| { - is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()]) - }) + if protected_block_handler_has_named_cleanup_or_explicit_reraise( + &self.blocks, + block, + ) { + visited[block_idx.idx()] = true; + stack.extend(normal_successors(block)); + continue; + } + if predecessors[block_idx.idx()] + .iter() + .any(|pred| is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()])) { - break; + continue; } - visited[cursor.idx()] = true; - deoptimize_block_borrows(&mut self.blocks[cursor.idx()]); - if self.blocks[cursor.idx()] + if block_ends_with_explicit_raise(block) { + continue; + } + visited[block_idx.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + if self.blocks[block_idx.idx()] .instructions .last() .is_some_and(|info| info.instr.is_scope_exit()) { - break; + continue; } - cursor = self.blocks[cursor.idx()].next; + stack.extend(normal_successors(&self.blocks[block_idx.idx()])); } } } @@ -6313,6 +6429,88 @@ impl CodeInfo { && !handler_chain_resumes_normally(blocks, handler) } + fn handler_chain_has_handled_return_or_bare_reraise( + blocks: &[Block], + handler: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let has_return = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))); + if has_pop_except && has_return { + return true; + } + if block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::RaiseVarargs { argc }) + if argc.get(info.arg) == oparg::RaiseKind::BareRaise + ) + }) { + return true; + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + stack.push(block.next); + } + } + false + } + + fn handler_body_has_conditional_after_match(blocks: &[Block], handler: BlockIdx) -> bool { + let mut cursor = handler; + let mut visited = vec![false; blocks.len()]; + let mut in_body = false; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + for info in &block.instructions { + match info.instr.real() { + Some(Instruction::PopTop) if !in_body => { + in_body = true; + } + Some(instr) + if in_body && is_conditional_jump(&AnyInstruction::Real(instr)) => + { + return true; + } + Some( + Instruction::PopExcept + | Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue, + ) => return false, + _ => {} + } + } + cursor = block.next; + } + false + } + + fn handler_is_terminal_for_conditional_tail(blocks: &[Block], handler: BlockIdx) -> bool { + handler_reaches_match_before_terminal(blocks, handler) + && handler_chain_has_handled_return_or_bare_reraise(blocks, handler) + && !handler_body_has_conditional_after_match(blocks, handler) + && !handler_chain_resumes_normally(blocks, handler) + } + fn trailing_protected_tail_terminal_exception_handler( blocks: &[Block], block: &Block, @@ -6331,6 +6529,24 @@ impl CodeInfo { None } + fn trailing_protected_tail_conditional_exception_handler( + blocks: &[Block], + block: &Block, + ) -> Option { + for info in block.instructions.iter().rev() { + match info.instr.real() { + Some(Instruction::Nop | Instruction::NotTaken | Instruction::PopTop) => {} + Some(_) => { + let handler = info.except_handler.map(|handler| handler.handler_block)?; + return handler_is_terminal_for_conditional_tail(blocks, handler) + .then_some(handler); + } + None => {} + } + } + None + } + fn protected_tail_ends_with_conditional(block: &Block) -> bool { block .instructions @@ -6418,19 +6634,21 @@ impl CodeInfo { let Some(cond_idx) = trailing_conditional_jump_index(block) else { return false; }; - block.instructions[..cond_idx] - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) - && block.instructions[..cond_idx].iter().any(|info| { - matches!( - info.instr.real(), - Some( - Instruction::StoreFast { .. } - | Instruction::StoreFastLoadFast { .. } - | Instruction::StoreFastStoreFast { .. } - ) + block.instructions[..cond_idx].iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::Call { .. } | Instruction::CallKw { .. }) + ) + }) && block.instructions[..cond_idx].iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } ) - }) + ) + }) } fn has_call_and_store(block: &Block) -> bool { @@ -6438,7 +6656,7 @@ impl CodeInfo { let mut has_store_fast = false; for info in &block.instructions { match info.instr.real() { - Some(Instruction::Call { .. }) => has_call = true, + Some(Instruction::Call { .. } | Instruction::CallKw { .. }) => has_call = true, Some( Instruction::StoreFast { .. } | Instruction::StoreFastLoadFast { .. } @@ -6450,23 +6668,54 @@ impl CodeInfo { has_call && has_store_fast } - fn has_load_fast_pair(block: &Block) -> bool { - block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some( - Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) + fn normal_tail_reaches_conditional(blocks: &[Block], start: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + if trailing_conditional_jump_index(block).is_some() + || has_call_store_before_trailing_conditional(block) + { + return true; + } + if block + .instructions + .last() + .is_some_and(|info| info.instr.is_scope_exit()) + { + continue; + } + stack.extend(normal_successors(block)); + } + false + } + + fn has_load_fast_pair(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) ) }) } fn has_call(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::Call { .. } | Instruction::CallKw { .. }) + ) + }) } fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { @@ -6497,6 +6746,22 @@ impl CodeInfo { && normal_successors(block).contains(&successor) } + fn is_simple_fast_return_block(block: &Block) -> bool { + let reals: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + .collect(); + matches!( + reals.as_slice(), + [ + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::ReturnValue, + ] + ) + } + let mut predecessors = vec![Vec::new(); self.blocks.len()]; for (pred_idx, block) in self.blocks.iter().enumerate() { if block.next != BlockIdx::NULL { @@ -6582,6 +6847,7 @@ impl CodeInfo { seed, (has_protected_call_predecessor || has_call_store_tail) && !has_structured_terminal_tail_shape, + false, )); } break; @@ -6599,8 +6865,37 @@ impl CodeInfo { } } + for (pred_idx, pred_block) in self.blocks.iter().enumerate() { + if block_is_exceptional(pred_block) || pred_block.cold { + continue; + } + let Some(handler) = + trailing_protected_tail_conditional_exception_handler(&self.blocks, pred_block) + else { + continue; + }; + for successor in normal_successors(pred_block) { + if successor == BlockIdx::NULL { + continue; + } + let successor_block = &self.blocks[successor.idx()]; + if block_is_exceptional(successor_block) + || successor_block.cold + || !normal_tail_reaches_conditional(&self.blocks, successor) + || normal_path_reaches_handler(&self.blocks, successor, handler) + || predecessors[successor.idx()].iter().any(|pred| { + *pred != BlockIdx::new(pred_idx as u32) + && is_handler_resume_predecessor(&self.blocks[pred.idx()], successor) + }) + { + continue; + } + seeds.push((successor, false, true)); + } + } + let mut visited = vec![false; self.blocks.len()]; - for (seed, direct_only) in seeds { + for (seed, direct_only, skip_simple_fast_returns) in seeds { let mut reachable_from_seed = vec![false; self.blocks.len()]; let mut reachability_stack = vec![seed]; while let Some(block_idx) = reachability_stack.pop() { @@ -6637,7 +6932,10 @@ impl CodeInfo { } visited[block_idx.idx()] = true; let successors = normal_successors(&self.blocks[block_idx.idx()]); - if !self.blocks[block_idx.idx()].try_else_orelse_entry { + if !(self.blocks[block_idx.idx()].try_else_orelse_entry + || skip_simple_fast_returns + && is_simple_fast_return_block(&self.blocks[block_idx.idx()])) + { deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); } if direct_only { @@ -6686,6 +6984,15 @@ impl CodeInfo { }) } + fn block_ends_with_return_value(block: &Block) -> bool { + block + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()) + .is_some_and(|instr| matches!(instr, Instruction::ReturnValue)) + } + fn handler_chain_has_exception_match(blocks: &[Block], handler: BlockIdx) -> bool { let mut cursor = handler; let mut visited = vec![false; blocks.len()]; @@ -6959,6 +7266,7 @@ impl CodeInfo { || block.cold || starts_with_inlined_comprehension_restore(block) || !block_has_protected_method_call(&self.blocks, block) + || block_ends_with_return_value(block) || predecessors[idx].iter().any(|pred| { let pred_block = &self.blocks[pred.idx()]; !block_is_exceptional(pred_block) @@ -7466,6 +7774,19 @@ impl CodeInfo { .any(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))) } + fn block_has_outer_jump_back(block: &Block, block_idx: BlockIdx) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) && info.target != BlockIdx::NULL + && info.target != block_idx + }) + } + fn normal_successors(block: &Block) -> Vec { let Some(last) = block.instructions.last() else { return (block.next != BlockIdx::NULL) @@ -7499,6 +7820,31 @@ impl CodeInfo { .collect() } + fn has_outer_loop_after_send(blocks: &[Block], send_block: BlockIdx) -> bool { + let mut stack = blocks[send_block.idx()] + .instructions + .iter() + .filter(|info| matches!(info.instr.real(), Some(Instruction::Send { .. }))) + .filter_map(|info| (info.target != BlockIdx::NULL).then_some(info.target)) + .collect::>(); + let mut visited = vec![false; blocks.len()]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block.cold || block_is_exceptional(block) { + continue; + } + if block_has_outer_jump_back(block, block_idx) { + return true; + } + stack.extend(normal_successors(block)); + } + false + } + let mut predecessors = vec![Vec::new(); self.blocks.len()]; for (idx, block) in self.blocks.iter().enumerate() { for successor in normal_successors(block) { @@ -7520,6 +7866,15 @@ impl CodeInfo { if !relevant_send_blocks.iter().any(|has_send| *has_send) { return; } + if !relevant_send_blocks + .iter() + .enumerate() + .any(|(idx, has_send)| { + *has_send && has_outer_loop_after_send(&self.blocks, BlockIdx::new(idx as u32)) + }) + { + return; + } fn has_early_return_before_send(blocks: &[Block], relevant_send_blocks: &[bool]) -> bool { let mut visited = vec![false; blocks.len()]; @@ -7597,6 +7952,58 @@ impl CodeInfo { has_pop_except && jumps_to_target } + fn handler_chain_resumes_to_loop_header( + blocks: &[Block], + handler: BlockIdx, + loop_header: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + if has_pop_except + && block.instructions.iter().any(|info| { + info.target == loop_header + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) + { + return true; + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + stack.push(block.next); + } + } + false + } + + fn protected_block_handler_resumes_to_self(blocks: &[Block], block_idx: BlockIdx) -> bool { + let block = &blocks[block_idx.idx()]; + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| handler_chain_resumes_to_loop_header(blocks, handler, block_idx)) + } + fn is_suppressing_with_resume_predecessor(block: &Block, target: BlockIdx) -> bool { if !block .instructions @@ -7631,6 +8038,21 @@ impl CodeInfo { }) } + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_non_nop_real_instructions(block: &Block) -> bool { + block.instructions.iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + }) + } + fn predecessor_chain_has_check_exc_match( blocks: &[Block], predecessors: &[Vec], @@ -7667,6 +8089,29 @@ impl CodeInfo { }) } + fn predecessor_chain_has_protected_instructions( + blocks: &[Block], + predecessors: &[Vec], + start: BlockIdx, + stop: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || cursor == stop || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + if block_has_protected_instructions(&blocks[cursor.idx()]) { + return true; + } + for pred in &predecessors[cursor.idx()] { + stack.push(*pred); + } + } + false + } + fn starts_with_bool_guard(block: &Block) -> bool { let infos: Vec<_> = block .instructions @@ -7758,6 +8203,18 @@ impl CodeInfo { }) } + fn block_has_fast_load_pair(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) + }) + } + fn block_has_method_load(block: &Block) -> bool { block.instructions.iter().any(|info| { matches!( @@ -7767,8 +8224,21 @@ impl CodeInfo { }) } - fn block_is_calling_finally_cleanup(block: &Block) -> bool { - let has_call = block.instructions.iter().any(|info| { + fn block_has_store_fast(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + ) + ) + }) + } + + fn block_has_call(block: &Block) -> bool { + block.instructions.iter().any(|info| { matches!( info.instr.real(), Some( @@ -7777,29 +8247,220 @@ impl CodeInfo { | Instruction::CallFunctionEx ) ) - }); - has_call - && block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::Reraise { .. }))) - && !block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + }) + } + + fn block_has_conditional_jump_to(block: &Block, target: BlockIdx) -> bool { + block + .instructions + .iter() + .any(|info| info.target == target && is_conditional_jump(&info.instr)) + } + + fn loop_back_target(block: &Block) -> Option { + block.instructions.iter().find_map(|info| { + matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } ) - }) + ) + .then_some(info.target) + }) } - fn has_jump_back_predecessor_to( + fn conditional_fallthrough_loop_header( blocks: &[Block], - predecessors: &[Vec], + block: &Block, target: BlockIdx, - ) -> bool { - predecessors[target.idx()].iter().any(|pred| { - let pred_block = &blocks[pred.idx()]; - if pred_block.cold || block_is_exceptional(pred_block) { - return false; + ) -> Option { + if !block_has_conditional_jump_to(block, target) { + return None; + } + loop_back_target(block).or_else(|| { + (block.next != BlockIdx::NULL) + .then(|| loop_back_target(&blocks[block.next.idx()]))? + }) + } + + fn any_protected_handler_resumes_to_loop_header( + blocks: &[Block], + loop_header: BlockIdx, + ) -> bool { + blocks.iter().any(|block| { + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| { + handler_chain_resumes_to_loop_header(blocks, handler, loop_header) + }) + }) + } + + fn any_exception_cleanup_jumps_to(blocks: &[Block], target: BlockIdx) -> bool { + blocks.iter().any(|block| { + block.cold + && block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + && block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) + }) + } + + fn fallthrough_chain_reaches_target( + blocks: &[Block], + start: BlockIdx, + target: BlockIdx, + ) -> bool { + let mut cursor = start; + let mut seen = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !seen[cursor.idx()] { + if cursor == target { + return true; + } + seen[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + if block_has_non_nop_real_instructions(block) || !block_has_fallthrough(block) { + return false; + } + cursor = block.next; + } + false + } + + fn block_is_normal_finally_cleanup_call(block: &Block) -> bool { + block_has_call(block) + && !block_has_store_fast(block) + && block_has_fallthrough(block) + && block + .instructions + .last() + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::PopTop))) + } + + fn first_fast_load_local(block: &Block) -> Option { + block + .instructions + .iter() + .find_map(|info| match info.instr.real() { + Some( + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num }, + ) => Some(usize::from(var_num.get(info.arg))), + _ => None, + }) + } + + fn trailing_conditional_guard_local(block: &Block, target: BlockIdx) -> Option { + let infos: Vec<_> = block + .instructions + .iter() + .filter(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .collect(); + let jump = infos.last()?; + if jump.target != target || !is_conditional_jump(&jump.instr) { + return None; + } + let load = if infos + .get(infos.len().wrapping_sub(2)) + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ToBool))) + { + infos.get(infos.len().wrapping_sub(3))? + } else { + infos.get(infos.len().wrapping_sub(2))? + }; + match load.instr.real() { + Some( + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num }, + ) => Some(usize::from(var_num.get(load.arg))), + _ => None, + } + } + + fn block_is_calling_finally_cleanup(block: &Block) -> bool { + let has_call = block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx + ) + ) + }); + has_call + && block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Reraise { .. }))) + && !block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + } + + fn handler_chain_calls_finally_cleanup(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_calling_finally_cleanup(block) { + return true; + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + stack.push(block.next); + } + } + false + } + + fn protected_block_handler_calls_finally_cleanup( + blocks: &[Block], + block_idx: BlockIdx, + ) -> bool { + let block = &blocks[block_idx.idx()]; + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| handler_chain_calls_finally_cleanup(blocks, handler)) + } + + fn has_jump_back_predecessor_to( + blocks: &[Block], + predecessors: &[Vec], + target: BlockIdx, + ) -> bool { + predecessors[target.idx()].iter().any(|pred| { + let pred_block = &blocks[pred.idx()]; + if pred_block.cold || block_is_exceptional(pred_block) { + return false; } blocks[pred.idx()].instructions.iter().any(|info| { info.target == target @@ -7870,6 +8531,39 @@ impl CodeInfo { .collect() } + fn tail_has_for_loop_back( + blocks: &[Block], + predecessors: &[Vec], + seed: BlockIdx, + ) -> bool { + let mut seen = vec![false; blocks.len()]; + let mut stack = vec![seed]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || seen[cursor.idx()] { + continue; + } + seen[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + for info in &block.instructions { + if matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) && info.target != BlockIdx::NULL + && block_has_for_iter(&blocks[info.target.idx()]) + { + return true; + } + } + for successor in tail_successors(blocks, predecessors, cursor) { + stack.push(successor); + } + } + false + } + let mut predecessors = vec![Vec::new(); self.blocks.len()]; let mut is_handler_resume_block = vec![false; self.blocks.len()]; for (pred_idx, block) in self.blocks.iter().enumerate() { @@ -7894,7 +8588,6 @@ impl CodeInfo { } } - let has_calling_finally_cleanup = self.blocks.iter().any(block_is_calling_finally_cleanup); let has_exception_match_handler = self.blocks.iter().any(|block| { block.instructions.iter().any(|info| { matches!( @@ -7916,27 +8609,299 @@ impl CodeInfo { .enumerate() .filter_map(|(idx, block)| { if block_is_exceptional(block) - || !has_exception_group_match_handler + || block_has_for_iter(block) || !block_has_fast_load(block) || !block_has_method_load(block) { return None; } + let target = BlockIdx::new(idx as u32); + let has_suppressing_with_resume_predecessor = + predecessors[idx].iter().any(|pred| { + is_suppressing_with_resume_predecessor(&self.blocks[pred.idx()], target) + }); + (has_suppressing_with_resume_predecessor + && (has_exception_group_match_handler + || has_exception_match_resume_predecessor( + &self.blocks, + &predecessors, + target, + ))) + .then_some(target) + }) + .collect(); + for block_idx in suppressing_exception_match_method_tails { + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + + let handler_resume_loop_successor_tails: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + if block_is_exceptional(block) + || block.cold + || !block_has_fast_load_pair(block) + || !block_has_call(block) + { + return None; + } predecessors[idx] .iter() .any(|pred| { - is_suppressing_with_resume_predecessor( - &self.blocks[pred.idx()], - BlockIdx::new(idx as u32), - ) + !block_is_exceptional(&self.blocks[pred.idx()]) + && !self.blocks[pred.idx()].cold + && !block_has_store_fast(&self.blocks[pred.idx()]) + && protected_block_handler_resumes_to_self(&self.blocks, *pred) }) .then_some(BlockIdx::new(idx as u32)) }) .collect(); - for block_idx in suppressing_exception_match_method_tails { + for block_idx in handler_resume_loop_successor_tails { deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); } + let finally_cleanup_successor_tails: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + if block_is_exceptional(block) + || block.cold + || block_has_protected_instructions(block) + || block_has_call(block) + || !starts_with_bool_guard(block) + || !block_has_fast_load(block) + { + return None; + } + let target = BlockIdx::new(idx as u32); + predecessors[idx] + .iter() + .any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + block_has_conditional_jump_to(pred_block, target) + && pred_block.next != BlockIdx::NULL + && { + let cleanup_block = &self.blocks[pred_block.next.idx()]; + block_is_normal_finally_cleanup_call(cleanup_block) + && trailing_conditional_guard_local(pred_block, target) + == first_fast_load_local(cleanup_block) + && fallthrough_chain_reaches_target( + &self.blocks, + cleanup_block.next, + target, + ) + } + }) + .then_some(BlockIdx::new(idx as u32)) + }) + .collect(); + let mut visited_finally_tail = vec![false; self.blocks.len()]; + for seed in finally_cleanup_successor_tails { + let mut segment = Vec::new(); + let mut seen = vec![false; self.blocks.len()]; + let mut stack = vec![seed]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || seen[cursor.idx()] { + continue; + } + seen[cursor.idx()] = true; + let block = &self.blocks[cursor.idx()]; + if block_is_exceptional(block) || block.cold || block_has_loop_back(block) { + continue; + } + segment.push(cursor); + for successor in tail_successors(&self.blocks, &predecessors, cursor) { + stack.push(successor); + } + } + let segment_ops: Vec<_> = segment + .iter() + .flat_map(|block_idx| { + self.blocks[block_idx.idx()] + .instructions + .iter() + .filter_map(|info| info.instr.real()) + }) + .collect(); + let has_call = segment_ops.iter().any(|instr| { + matches!(instr, Instruction::Call { .. } | Instruction::CallKw { .. }) + }); + let has_store_fast = segment_ops.iter().any(|instr| { + matches!( + instr, + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + ) + }); + if !has_call || !has_store_fast { + continue; + } + for block_idx in segment { + if visited_finally_tail[block_idx.idx()] { + continue; + } + visited_finally_tail[block_idx.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + + let handler_break_join_tails: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + if block_is_exceptional(block) + || block.cold + || !block_has_fast_load(block) + || !block_has_method_load(block) + || !block_has_call(block) + || !has_exception_match_resume_predecessor( + &self.blocks, + &predecessors, + BlockIdx::new(idx as u32), + ) + { + return None; + } + let join = BlockIdx::new(idx as u32); + let body = predecessors[idx].iter().find_map(|jump_pred| { + let jump_block = &self.blocks[jump_pred.idx()]; + if block_is_exceptional(jump_block) + || jump_block.cold + || !jump_block.instructions.last().is_some_and(|info| { + info.target == join && info.instr.is_unconditional_jump() + }) + { + return None; + } + predecessors[jump_pred.idx()].iter().find_map(|cond_pred| { + let cond_block = &self.blocks[cond_pred.idx()]; + if block_is_exceptional(cond_block) || cond_block.cold { + return None; + } + let cond_idx = trailing_conditional_jump_index(cond_block)?; + let cond = cond_block.instructions[cond_idx]; + if cond.target == *jump_pred + || cond.target == BlockIdx::NULL + || cond_block.next != *jump_pred + { + return None; + } + let body = next_nonempty_block(&self.blocks, cond.target); + if body == BlockIdx::NULL { + return None; + } + let body_block = &self.blocks[body.idx()]; + (!block_is_exceptional(body_block) + && !body_block.cold + && block_has_loop_back(body_block) + && block_has_fast_load(body_block) + && block_has_method_load(body_block) + && block_has_call(body_block) + && block_has_store_fast(body_block)) + .then_some(body) + }) + })?; + Some((join, body)) + }) + .collect(); + for (join, body) in handler_break_join_tails { + deoptimize_block_borrows(&mut self.blocks[join.idx()]); + deoptimize_block_borrows(&mut self.blocks[body.idx()]); + } + + let conditional_loop_bypass_tails: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + if block_is_exceptional(block) + || block.cold + || !block_has_fast_load(block) + || !has_exception_match_handler + { + return None; + } + let target = BlockIdx::new(idx as u32); + predecessors[idx] + .iter() + .any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + conditional_fallthrough_loop_header(&self.blocks, pred_block, target) + .is_some_and(|loop_header| { + block_has_for_iter(&self.blocks[loop_header.idx()]) + && predecessor_chain_has_protected_instructions( + &self.blocks, + &predecessors, + *pred, + loop_header, + ) + && (any_protected_handler_resumes_to_loop_header( + &self.blocks, + loop_header, + ) || any_protected_handler_resumes_to_loop_header( + &self.blocks, + *pred, + ) || any_exception_cleanup_jumps_to( + &self.blocks, + loop_header, + ) || any_exception_cleanup_jumps_to(&self.blocks, *pred)) + }) + }) + .then_some(target) + }) + .collect(); + let mut visited_conditional_tail = vec![false; self.blocks.len()]; + for seed in conditional_loop_bypass_tails { + let mut segment = Vec::new(); + let mut seen = vec![false; self.blocks.len()]; + let mut stack = vec![seed]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || seen[cursor.idx()] { + continue; + } + seen[cursor.idx()] = true; + let block = &self.blocks[cursor.idx()]; + if block_is_exceptional(block) || block.cold || block_has_loop_back(block) { + continue; + } + segment.push(cursor); + for successor in tail_successors(&self.blocks, &predecessors, cursor) { + stack.push(successor); + } + } + let segment_ops: Vec<_> = segment + .iter() + .flat_map(|block_idx| { + self.blocks[block_idx.idx()] + .instructions + .iter() + .filter_map(|info| info.instr.real()) + }) + .collect(); + let has_store_fast = segment_ops.iter().any(|instr| { + matches!( + instr, + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + ) + }); + if !has_store_fast { + continue; + } + for block_idx in segment { + if visited_conditional_tail[block_idx.idx()] { + continue; + } + visited_conditional_tail[block_idx.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + let seeds: Vec<_> = self .blocks .iter() @@ -7944,14 +8909,18 @@ impl CodeInfo { .filter_map(|(idx, block)| { let has_bool_guard_tail = starts_with_bool_guard(block); let has_loop_tail = block_has_for_iter(block) || block_has_get_iter(block); - let has_finally_except_loop_tail = - has_calling_finally_cleanup && has_exception_match_handler && has_loop_tail; let has_protected_predecessor = predecessors[idx].iter().any(|pred| { self.blocks[pred.idx()] .instructions .iter() .any(|info| info.except_handler.is_some()) }); + let has_protected_finally_cleanup_predecessor = predecessors[idx] + .iter() + .any(|pred| protected_block_handler_calls_finally_cleanup(&self.blocks, *pred)); + let has_finally_except_loop_tail = has_exception_match_handler + && has_loop_tail + && has_protected_finally_cleanup_predecessor; let has_handler_resume_predecessor = predecessors[idx].iter().any(|pred| { let pred_block = &self.blocks[pred.idx()]; !is_named_except_cleanup_normal_exit_block(pred_block) @@ -7970,6 +8939,11 @@ impl CodeInfo { &predecessors, BlockIdx::new(idx as u32), ); + let is_loop_header = has_jump_back_predecessor_to( + &self.blocks, + &predecessors, + BlockIdx::new(idx as u32), + ); let has_handler_resume_loop_tail = block_has_get_iter(block) && has_suppressing_with_resume_predecessor && has_exception_match_resume_predecessor; @@ -7979,8 +8953,17 @@ impl CodeInfo { if block_is_exceptional(block) || !has_supported_tail { return None; } - let should_seed = (has_protected_predecessor && has_finally_except_loop_tail) - || (has_bool_guard_tail && has_handler_resume_predecessor) + let should_seed = (has_protected_predecessor + && has_finally_except_loop_tail + && !has_suppressing_with_resume_predecessor) + || (has_bool_guard_tail + && has_handler_resume_predecessor + && !is_loop_header + && tail_has_for_loop_back( + &self.blocks, + &predecessors, + BlockIdx::new(idx as u32), + )) || has_handler_resume_loop_tail; let allow_any_loop_back = has_finally_except_loop_tail || has_handler_resume_loop_tail; @@ -8062,43 +9045,18 @@ impl CodeInfo { { continue; } + if has_exception_group_match_handler + && block_has_for_iter(&self.blocks[block_idx.idx()]) + && block_has_protected_instructions(&self.blocks[block_idx.idx()]) + { + continue; + } visited[block_idx.idx()] = true; deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); } } } - fn deoptimize_borrow_after_push_exc_info(&mut self) { - for block in &mut self.blocks { - let mut in_exception_state = false; - for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::PushExcInfo) => { - in_exception_state = true; - } - Some(Instruction::PopExcept | Instruction::Reraise { .. }) => { - in_exception_state = false; - } - Some(Instruction::LoadFastBorrow { .. }) if in_exception_state => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) - if in_exception_state => - { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} - } - } - } - } - fn deoptimize_borrow_after_protected_import(&mut self) { fn deoptimize_borrow(info: &mut InstructionInfo) { match info.instr.real() { @@ -8580,6 +9538,50 @@ impl CodeInfo { false } + fn handler_chain_exits_loop_after_pop_except(blocks: &[Block], block: &Block) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .collect(); + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + let handler = &blocks[cursor.idx()]; + let mut after_pop_except = false; + for info in &handler.instructions { + if matches!(info.instr.real(), Some(Instruction::PopExcept)) { + after_pop_except = true; + continue; + } + if after_pop_except + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + { + return true; + } + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + if info.instr.is_unconditional_jump() && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if handler.next != BlockIdx::NULL { + stack.push(handler.next); + } + } + false + } + fn handler_chain_has_nested_exception_match(blocks: &[Block], block: &Block) -> bool { let mut visited = vec![false; blocks.len()]; let mut stack: Vec<_> = block @@ -8788,10 +9790,15 @@ impl CodeInfo { None } - fn starts_with_borrowed_local_bool_guard(block: &Block, locals: &[usize]) -> bool { + fn starts_with_borrowed_local_bool_guard_from( + block: &Block, + locals: &[usize], + start: usize, + ) -> bool { let mut reals = block .instructions .iter() + .skip(start) .filter(|info| { info.instr.real().is_some_and(|instr| { !matches!(instr, Instruction::Nop | Instruction::NotTaken) @@ -8811,14 +9818,79 @@ impl CodeInfo { Some(Instruction::LoadFastBorrow { var_num }) => { locals.contains(&usize::from(var_num.get(first.arg))) } - _ => false, - }; - borrows_stored_local - && matches!(second.instr.real(), Some(Instruction::ToBool)) - && matches!( - third.instr.real(), - Some(Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. }) - ) + _ => false, + }; + borrows_stored_local + && matches!(second.instr.real(), Some(Instruction::ToBool)) + && matches!( + third.instr.real(), + Some(Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. }) + ) + } + + fn starts_with_borrowed_local_bool_guard(block: &Block, locals: &[usize]) -> bool { + starts_with_borrowed_local_bool_guard_from(block, locals, 0) + } + + fn protected_store_bool_guard_start(block: &Block) -> Option<(usize, Vec)> { + let mut saw_call = false; + for store_idx in 0..block.instructions.len() { + saw_call |= matches!( + block.instructions[store_idx].instr.real(), + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx + ) + ); + if !saw_call { + continue; + } + let local = match block.instructions[store_idx].instr.real() { + Some(Instruction::StoreFast { var_num }) => { + usize::from(var_num.get(block.instructions[store_idx].arg)) + } + _ => continue, + }; + let mut reals = block + .instructions + .iter() + .enumerate() + .skip(store_idx + 1) + .filter(|(_, info)| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }); + let Some((load_idx, load)) = reals.next() else { + continue; + }; + let borrows_stored_local = matches!( + load.instr.real(), + Some(Instruction::LoadFastBorrow { var_num }) + if usize::from(var_num.get(load.arg)) == local + ); + if !borrows_stored_local { + continue; + } + let Some((_, second)) = reals.next() else { + continue; + }; + let Some((_, third)) = reals.next() else { + continue; + }; + if matches!(second.instr.real(), Some(Instruction::ToBool)) + && matches!( + third.instr.real(), + Some( + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + ) + { + return Some((load_idx, vec![local])); + } + } + None } fn conditional_target(block: &Block) -> Option { @@ -8989,8 +10061,68 @@ impl CodeInfo { to_deopt.push((BlockIdx::new(block_idx as u32), start)); continue; } + if let Some((start, stored_locals)) = protected_store_bool_guard_start(block) { + if !handler_chain_exits_loop_after_pop_except(&self.blocks, block) { + continue; + } + let handler_has_explicit_raise = + handler_chain_has_explicit_raise(&self.blocks, block); + let jump_target = conditional_target(block); + let fallthrough = next_nonempty_block(&self.blocks, block.next); + if !handler_has_explicit_raise && let Some(jump_target) = jump_target { + let branches = [(jump_target, fallthrough), (fallthrough, jump_target)]; + for (work, exit) in branches { + if work == BlockIdx::NULL || exit == BlockIdx::NULL { + continue; + } + let work_block = &self.blocks[work.idx()]; + let exit_block = &self.blocks[exit.idx()]; + if !block_is_exceptional(work_block) + && !block_has_protected_instructions(work_block) + && block_has_tail_deopt_trigger_from(work_block, 0) + && block_is_simple_exit_branch(exit_block) + { + to_deopt.push((BlockIdx::new(block_idx as u32), start)); + if borrows_any_local_from(work_block, &stored_locals, 0) { + to_deopt.push((work, 0)); + } + } + } + } + } let same_block_tail_start = first_unprotected_suffix(block); - if same_block_tail_start.is_some() { + if let Some(start) = same_block_tail_start { + let stored_locals = collect_stored_fast_locals_until(block, start); + let handler_has_explicit_raise = + handler_chain_has_explicit_raise(&self.blocks, block); + if stored_locals.is_empty() + || handler_has_explicit_raise + || !starts_with_borrowed_local_bool_guard_from(block, &stored_locals, start) + { + continue; + } + let jump_target = conditional_target(block); + let fallthrough = next_nonempty_block(&self.blocks, block.next); + if let Some(jump_target) = jump_target { + let branches = [(jump_target, fallthrough), (fallthrough, jump_target)]; + for (work, exit) in branches { + if work == BlockIdx::NULL || exit == BlockIdx::NULL { + continue; + } + let work_block = &self.blocks[work.idx()]; + let exit_block = &self.blocks[exit.idx()]; + if !block_is_exceptional(work_block) + && !block_has_protected_instructions(work_block) + && block_has_tail_deopt_trigger_from(work_block, 0) + && block_is_simple_exit_branch(exit_block) + { + to_deopt.push((BlockIdx::new(block_idx as u32), start)); + if borrows_any_local_from(work_block, &stored_locals, 0) { + to_deopt.push((work, 0)); + } + } + } + } continue; } let tail = next_nonempty_block(&self.blocks, block.next); @@ -9030,11 +10162,12 @@ impl CodeInfo { if !block_is_exceptional(work_block) && !block_has_protected_instructions(work_block) && block_has_tail_deopt_trigger_from(work_block, 0) - && borrows_any_local_from(work_block, &stored_locals, 0) && block_is_simple_exit_branch(exit_block) { to_deopt.push((tail, 0)); - to_deopt.push((work, 0)); + if borrows_any_local_from(work_block, &stored_locals, 0) { + to_deopt.push((work, 0)); + } } } } @@ -9271,16 +10404,32 @@ impl CodeInfo { if starts_with_named_except_value_load { continue; } - if let Some(info) = self.blocks[block_idx].instructions.iter_mut().find(|info| { + let first_real = self.blocks[block_idx].instructions.iter().position(|info| { info.instr .real() .is_some_and(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) - }) && matches!(info.instr.real(), Some(Instruction::LoadFast { .. })) - { - info.instr = Instruction::LoadFastBorrow { - var_num: Arg::marker(), + }); + if let Some(first_real) = first_real { + let starts_with_receiver_load = matches!( + ( + self.blocks[block_idx].instructions[first_real].instr.real(), + self.blocks[block_idx] + .instructions + .get(first_real + 1) + .and_then(|info| info.instr.real()), + ), + ( + Some(Instruction::LoadFast { .. }), + Some(Instruction::LoadAttr { .. } | Instruction::LoadSuperAttr { .. }) + ) + ); + if starts_with_receiver_load { + self.blocks[block_idx].instructions[first_real].instr = + Instruction::LoadFastBorrow { + var_num: Arg::marker(), + } + .into(); } - .into(); } } } @@ -10846,7 +11995,6 @@ impl CodeInfo { "after_optimize_load_fast_borrow".to_owned(), self.debug_block_dump(), )); - self.deoptimize_borrow_after_push_exc_info(); self.deoptimize_borrow_for_handler_return_paths(); self.deoptimize_borrow_for_match_keys_attr(); self.deoptimize_borrow_in_protected_attr_chain_tail(); @@ -10858,7 +12006,6 @@ impl CodeInfo { self.optimize_load_global_push_null(); self.reorder_entry_prefix_cell_setup(); self.remove_unused_consts(); - deoptimize_borrow_after_push_exc_info_in_blocks(&mut self.blocks); Ok(trace) } @@ -11654,83 +12801,6 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { .copied() .unwrap_or(u32::MAX); let conditional = is_conditional_jump(&ins.instr); - if conditional && final_target_pos <= source_pos { - let mut scan = blocks[bi].next; - let mut chain_has_delete_subscr = false; - let mut chain_has_for_iter = false; - let mut seen = vec![false; blocks.len()]; - while scan != BlockIdx::NULL && scan != target && !seen[scan.idx()] { - seen[scan.idx()] = true; - chain_has_delete_subscr |= - blocks[scan.idx()].instructions.iter().any(|info| { - matches!(info.instr.real(), Some(Instruction::DeleteSubscr)) - }); - chain_has_for_iter |= blocks[scan.idx()].instructions.iter().any(|info| { - matches!(info.instr.real(), Some(Instruction::ForIter { .. })) - }); - scan = blocks[scan.idx()].next; - } - if chain_has_delete_subscr { - continue; - } - if target_ins.lineno_override.is_some_and(|lineno| lineno < 0) - && blocks[bi].instructions[..last_idx] - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - { - continue; - } - let after_target = next_nonempty_block(blocks, blocks[target.idx()].next); - let final_target_has_for_iter = blocks[final_target.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))); - let target_exits_current_loop = trailing_conditional_jump_index( - &blocks[final_target.idx()], - ) - .is_some_and(|cond_idx| { - !final_target_has_for_iter - && next_nonempty_block( - blocks, - blocks[final_target.idx()].instructions[cond_idx].target, - ) == after_target - }); - let mut scan = blocks[bi].next; - let mut first_nonempty_between = BlockIdx::NULL; - let mut reaches_target = false; - let mut seen = vec![false; blocks.len()]; - while scan != BlockIdx::NULL && !seen[scan.idx()] { - if scan == target { - reaches_target = true; - break; - } - seen[scan.idx()] = true; - if !blocks[scan.idx()].instructions.is_empty() - && first_nonempty_between == BlockIdx::NULL - { - first_nonempty_between = scan; - } - scan = blocks[scan.idx()].next; - } - if reaches_target - && !chain_has_for_iter - && first_nonempty_between != BlockIdx::NULL - && !is_marker_jump_only_block(&blocks[first_nonempty_between.idx()]) - && !is_pop_top_jump_block(&blocks[first_nonempty_between.idx()]) - && !is_scope_exit_block(&blocks[first_nonempty_between.idx()]) - && !is_loop_cleanup_block(&blocks[first_nonempty_between.idx()]) - && after_target != BlockIdx::NULL - && !blocks[after_target.idx()].cold - && !block_is_exceptional(&blocks[after_target.idx()]) - && !is_marker_jump_only_block(&blocks[after_target.idx()]) - && !is_pop_top_jump_block(&blocks[after_target.idx()]) - && !is_scope_exit_block(&blocks[after_target.idx()]) - && !is_loop_cleanup_block(&blocks[after_target.idx()]) - && !target_exits_current_loop - { - continue; - } - } if !include_conditional && source_pos < target_pos && final_target_pos < target_pos { // Keep the forward hop when threading would turn it into a @@ -11780,6 +12850,9 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { changed = true; } } + if include_conditional { + break; + } } } @@ -12079,6 +13152,7 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { let target = last.target; if is_named_except_cleanup_normal_exit_block(&blocks[current.idx()]) && target_pushes_handler(&blocks[target.idx()]) + && !named_except_cleanup_body_is_fast_local_only(&blocks[current.idx()]) { current = next; continue; @@ -12102,6 +13176,7 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { && (small_exit_block || no_lineno_no_fallthrough) { let removed_jump_kind = jump_thread_kind(last.instr); + let preserve_removed_jump_nop = last.preserve_redundant_jump_as_nop; let keep_removed_jump_nop = removed_jump_kind == Some(JumpThreadKind::NoInterrupt) || blocks[current.idx()] .instructions @@ -12118,6 +13193,8 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { let lineno_override = last_instr.lineno_override; set_to_nop(last_instr); last_instr.lineno_override = lineno_override; + last_instr.preserve_block_start_no_location_nop |= + preserve_removed_jump_nop; if preserve_empty_end_label_nop { last_instr.lineno_override = None; last_instr.preserve_block_start_no_location_nop = true; @@ -12280,7 +13357,9 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { let mut remove = false; if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if src == 0 + if instr.no_location_exit && instr.preserve_redundant_jump_as_nop { + remove = false; + } else if src == 0 && lineno > 0 && ((!keep_target_start_nop && follows_same_line_pop_iter == Some(lineno)) || (instr.preserve_block_start_no_location_nop @@ -12289,7 +13368,9 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { ))) { remove = true; - } else if instr.preserve_block_start_no_location_nop { + } else if instr.preserve_redundant_jump_as_nop + || instr.preserve_block_start_no_location_nop + { remove = false; } else if lineno < 0 { remove = true; @@ -12767,38 +13848,6 @@ fn merge_unsafe_mask(slot: &mut Option>, incoming: &[bool]) -> bool { } } -fn deoptimize_borrow_after_push_exc_info_in_blocks(blocks: &mut [Block]) { - let mut in_exception_state = false; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let block = &mut blocks[current.idx()]; - for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::PushExcInfo) => { - in_exception_state = true; - } - Some(Instruction::PopExcept | Instruction::Reraise { .. }) => { - in_exception_state = false; - } - Some(Instruction::LoadFastBorrow { .. }) if in_exception_state => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) if in_exception_state => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} - } - } - current = block.next; - } -} - /// Follow chain of empty blocks to find first non-empty block. fn next_nonempty_block(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { while idx != BlockIdx::NULL @@ -13002,21 +14051,6 @@ fn is_jump_only_block(block: &Block) -> bool { instr.instr.is_unconditional_jump() && instr.target != BlockIdx::NULL } -fn is_marker_jump_only_block(block: &Block) -> bool { - let mut real_instrs = block.instructions.iter().filter(|info| { - !matches!( - info.instr.real(), - Some(Instruction::Nop | Instruction::NotTaken) - ) - }); - let Some(instr) = real_instrs.next() else { - return false; - }; - real_instrs.next().is_none() - && instr.instr.is_unconditional_jump() - && instr.target != BlockIdx::NULL -} - fn is_jump_back_only_block(blocks: &[Block], block_idx: BlockIdx) -> bool { if block_idx == BlockIdx::NULL || !is_jump_only_block(&blocks[block_idx.idx()]) { return false; @@ -13098,6 +14132,13 @@ fn is_exception_cleanup_block(block: &Block) -> bool { .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::Reraise { .. }))) } +fn is_reraise_scope_exit_block(block: &Block) -> bool { + block + .instructions + .last() + .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::Reraise { .. }))) +} + fn block_starts_with_with_exit_none_call(block: &Block) -> bool { let real_instrs: Vec<_> = block .instructions @@ -13710,6 +14751,76 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { && is_single_delete_subscr_body(&blocks[body.idx()]) } + fn jump_targets_for_iter(blocks: &[Block], jump_block: BlockIdx) -> bool { + if jump_block == BlockIdx::NULL { + return false; + } + let Some(info) = blocks[jump_block.idx()].instructions.first() else { + return false; + }; + let target = next_nonempty_block(blocks, info.target); + target != BlockIdx::NULL + && blocks[target.idx()] + .instructions + .first() + .is_some_and(|target_info| { + matches!(target_info.instr.real(), Some(Instruction::ForIter { .. })) + }) + } + + fn true_body_false_backedge_is_already_normalized( + blocks: &[Block], + conditional: InstructionInfo, + false_backedge: BlockIdx, + true_body: BlockIdx, + ) -> bool { + fn comes_before(blocks: &[Block], target: BlockIdx, block: BlockIdx) -> bool { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + if current == target { + return true; + } + if current == block { + return false; + } + current = blocks[current.idx()].next; + } + false + } + + if !matches!( + conditional.instr.real(), + Some(Instruction::PopJumpIfTrue { .. }) + ) || false_backedge == BlockIdx::NULL + || true_body == BlockIdx::NULL + || !is_jump_only_block(&blocks[false_backedge.idx()]) + { + return false; + } + let false_target = blocks[false_backedge.idx()].instructions[0].target; + if false_target == BlockIdx::NULL || !comes_before(blocks, false_target, false_backedge) { + return false; + } + let true_tail = next_nonempty_block(blocks, blocks[true_body.idx()].next); + true_tail != BlockIdx::NULL + && is_jump_only_block(&blocks[true_tail.idx()]) + && blocks[true_tail.idx()].instructions[0].target == false_target + } + + fn has_other_conditional_predecessor_to( + blocks: &[Block], + target: BlockIdx, + current: BlockIdx, + ) -> bool { + blocks.iter().enumerate().any(|(idx, block)| { + BlockIdx::new(idx as u32) != current + && block + .instructions + .iter() + .any(|info| info.target == target && is_conditional_jump(&info.instr)) + }) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -13734,6 +14845,10 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; } + if true_body_false_backedge_is_already_normalized(blocks, last, chain_start, jump_start) { + current = next; + continue; + } let mut chain_has_suspension_point = false; let mut scan = chain_start; while scan != BlockIdx::NULL && scan != jump_start { @@ -13787,6 +14902,12 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; } + if is_generic_false_path_reorder + && has_other_conditional_predecessor_to(blocks, chain_start, current) + { + current = next; + continue; + } if block_is_protected(&blocks[idx]) && block_contains_suspension_point(&blocks[idx]) { current = next; continue; @@ -13903,6 +15024,13 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; } + if !is_generic_false_path_reorder + && jump_is_artificial + && jump_targets_for_iter(blocks, jump_block) + { + current = next; + continue; + } if nonempty_blocks == 1 && !is_jump_only_block(&blocks[chain_start.idx()]) && after_jump != BlockIdx::NULL @@ -14117,7 +15245,6 @@ fn reorder_conditional_scope_exit_and_jump_back_blocks( current = next; continue; } - let after_exit = blocks[exit_block.idx()].next; retarget_empty_chain_targets(blocks, next, jump_block, jump_block); blocks[idx].instructions[cond_idx].instr = @@ -14976,11 +16103,10 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec } let jump_target_has_multiple_conditional_predecessors = conditional_jump_target_count(blocks, true_jump_start) > 1; - let body_has_call = block_has_call(&blocks[body.idx()]); let simple_single_block_can_reorder = body_is_single_block && !body_tail_is_conditional && !has_exceptional_duplicate_condition_line - && (!body_has_call || block_starts_loop_cleanup(blocks, after_jump_target)) + && after_jump_starts_loop_cleanup && !body_has_scope_exit && (!blocks[body.idx()] .instructions @@ -15074,7 +16200,18 @@ fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec current = next; continue; } - if !body_segment_contains_for_iter(blocks, body, body_tail) + let body_contains_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); + let simple_body_can_reorder = !block_has_call(&blocks[body.idx()]) + && jump_back_target(blocks, body_tail) == Some(loop_target); + let false_jump_starts_with_not_taken = blocks[false_jump.idx()] + .instructions + .first() + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::NotTaken))); + if simple_body_can_reorder && false_jump_starts_with_not_taken && !body_contains_for_iter { + current = next; + continue; + } + if (!body_contains_for_iter && !simple_body_can_reorder) || body_segment_contains_protected_block(blocks, body, body_tail) || trailing_conditional_jump_index(&blocks[body.idx()]).is_some() || block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_body)) @@ -15153,8 +16290,9 @@ fn reorder_jump_over_exception_cleanup_blocks(blocks: &mut [Block]) { cursor = blocks[cursor.idx()].next; continue; } - if !block_is_exceptional(&blocks[cursor.idx()]) - && !is_exception_cleanup_block(&blocks[cursor.idx()]) + if !(block_is_exceptional(&blocks[cursor.idx()]) + || is_exception_cleanup_block(&blocks[cursor.idx()]) + || blocks[cursor.idx()].cold && is_reraise_scope_exit_block(&blocks[cursor.idx()])) { cleanup_end = BlockIdx::NULL; break; @@ -15626,7 +16764,14 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { && !block_has_fallthrough(&blocks[layout_pred.idx()]) && predecessors[target.idx()] >= 2 { - let target_lineno = instruction_lineno(&blocks[target.idx()].instructions[0]); + let target_location = blocks[target.idx()].instructions[0].location; + let target_end_location = blocks[target.idx()].instructions[0].end_location; + let target_follows_forward_jump = blocks[layout_pred.idx()] + .instructions + .last() + .is_some_and(|info| { + matches!(info.instr.real(), Some(Instruction::JumpForward { .. })) + }); for block_idx in 0..blocks.len() { let block_idx = BlockIdx(block_idx as u32); for (instr_idx, info) in blocks[block_idx.idx()].instructions.iter().enumerate() @@ -15639,7 +16784,14 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { continue; } let jump_lineno = instruction_lineno(info); - if jump_lineno >= 0 && jump_lineno != target_lineno { + if target_follows_forward_jump && target_location.line >= info.location.line + { + continue; + } + if jump_lineno >= 0 + && (info.location != target_location + || info.end_location != target_end_location) + { lineful_clones_before_target.push((target, block_idx, instr_idx)); } } @@ -15820,11 +16972,6 @@ fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { layout_pred = blocks[layout_pred.idx()].next; continue; } - if !block_has_no_lineno(&blocks[target.idx()]) && blocks[layout_pred.idx()].next != target { - layout_pred = blocks[layout_pred.idx()].next; - continue; - } - let jump_target = next_nonempty_block(blocks, blocks[target.idx()].instructions[0].target); if jump_target == BlockIdx::NULL || (!has_non_exception_loop_backedge_to(blocks, target, jump_target) @@ -16020,6 +17167,66 @@ fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { } } +fn inline_small_fast_return_blocks(blocks: &mut [Block]) { + fn block_is_small_fast_return(block: &Block) -> bool { + if block.instructions.len() > 3 { + return false; + } + let real: Vec<_> = block + .instructions + .iter() + .filter(|info| !matches!(info.instr.real(), Some(Instruction::Nop))) + .collect(); + matches!( + real.as_slice(), + [load, ret] + if matches!( + load.instr.real(), + Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) + ) && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) + ) + } + + loop { + let mut changed = false; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let next = blocks[current.idx()].next; + let Some(last) = blocks[current.idx()].instructions.last().copied() else { + current = next; + continue; + }; + if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { + current = next; + continue; + } + let target = next_nonempty_block(blocks, last.target); + if target == BlockIdx::NULL || !block_is_small_fast_return(&blocks[target.idx()]) { + current = next; + continue; + } + + if jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) + || instruction_has_lineno(&last) + { + let last_instr = blocks[current.idx()].instructions.last_mut().unwrap(); + let lineno_override = last_instr.lineno_override; + set_to_nop(last_instr); + last_instr.lineno_override = lineno_override; + } else { + blocks[current.idx()].instructions.pop(); + } + let cloned = blocks[target.idx()].instructions.clone(); + blocks[current.idx()].instructions.extend(cloned); + changed = true; + current = next; + } + if !changed { + break; + } + } +} + fn inline_with_suppress_return_blocks(blocks: &mut [Block]) { fn has_with_suppress_prefix(block: &Block, jump_idx: usize) -> bool { let tail: Vec<_> = block.instructions[..jump_idx] diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index a15b26fe4a..92f53d3aac 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -724,6 +724,21 @@ impl SymbolTableAnalyzer { self.found_in_inner_scope(sub_tables, &symbol.name, st_typ) .unwrap_or(SymbolScope::Local) } + } else if let Some(scope) = class_entry + .and_then(|class_symbols| class_symbols.get(&symbol.name)) + .and_then(|class_sym| { + if class_sym.flags.contains(SymbolFlags::GLOBAL) { + Some(SymbolScope::GlobalExplicit) + } else if class_sym.is_bound() && class_sym.scope != SymbolScope::Free { + // If name is bound in enclosing class, use GlobalImplicit + // so it can be accessed via __classdict__ + Some(SymbolScope::GlobalImplicit) + } else { + None + } + }) + { + scope } else if let Some(scope) = self.found_in_outer_scope( &symbol.name, st_typ, @@ -731,14 +746,6 @@ impl SymbolTableAnalyzer { ) { // If found in enclosing scope (function/TypeParams), use that scope - } else if let Some(class_symbols) = class_entry - && let Some(class_sym) = class_symbols.get(&symbol.name) - && class_sym.is_bound() - && class_sym.scope != SymbolScope::Free - { - // If name is bound in enclosing class, use GlobalImplicit - // so it can be accessed via __classdict__ - SymbolScope::GlobalImplicit } else if self.tables.is_empty() { // Don't make assumptions when we don't know. SymbolScope::Unknown From a448fb6a54b91002f7fd141b1fe4a4e97774cf62 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 10 May 2026 01:00:02 +0900 Subject: [PATCH 73/76] fix --- Lib/test/test_named_expressions.py | 3 - Lib/test/test_peepholer.py | 1 - Lib/test/test_positional_only_arg.py | 1 - Lib/test/test_type_params.py | 2 - crates/codegen/src/compile.rs | 2243 ++++++++++++++++++++++++-- crates/codegen/src/ir.rs | 2113 +++++++++++++++++++----- crates/codegen/src/preprocess.rs | 2 +- crates/codegen/src/symboltable.rs | 28 +- 8 files changed, 3842 insertions(+), 551 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 2a3ce809fb..adf774f102 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -187,7 +187,6 @@ def test_named_expression_invalid_rebinding_iteration_variable(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope - @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_expression_invalid_rebinding_list_comprehension_iteration_variable(self): cases = [ ("Local reuse", 'i', "[i := 0 for i in range(5)]"), @@ -246,7 +245,6 @@ def test_named_expression_invalid_list_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope - @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_expression_invalid_rebinding_set_comprehension_iteration_variable(self): cases = [ ("Local reuse", 'i', "{i := 0 for i in range(5)}"), @@ -310,7 +308,6 @@ def test_named_expression_invalid_set_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope - @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_named_expression_invalid_rebinding_dict_comprehension_iteration_variable(self): cases = [ ("Key reuse", 'i', "{(i := 0): 1 for i in range(5)}"), diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 6c2e4c0811..c9d79de30b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -92,7 +92,6 @@ def unot(x): self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') self.check_lnotab(unot) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_elim_inversion_of_is_or_in(self): for line, cmp_op, invert in ( ('not a is b', 'IS_OP', 1,), diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index e002babab4..1817592ca2 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -437,7 +437,6 @@ def method(self, /): self.assertEqual(C().method(), sentinel) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_annotations_constant_fold(self): def g(): def f(x: not (int is int), /): ... diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index af26a6a0a0..ffafedc775 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -713,7 +713,6 @@ def meth[T: x](self, arg: x): ... self.assertEqual(cls.bound, "nonlocal") self.assertEqual(cls.meth.__annotations__["arg"], "nonlocal") - @unittest.expectedFailure # TODO: RUSTPYTHON; + global def test_explicit_global(self): ns = run_code(""" x = "global" @@ -741,7 +740,6 @@ class Cls: cls = ns["outer"]() self.assertEqual(cls.Alias.__value__, "global") - @unittest.expectedFailure # TODO: RUSTPYTHON; + global from class def test_explicit_global_with_assignment(self): ns = run_code(""" x = "global" diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 3496b14801..029edcb8ed 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -175,7 +175,9 @@ struct Compiler { disable_const_collection_folding: bool, split_next_for_normal_exit_from_break: bool, fallthrough_has_statement_successor: bool, - fallthrough_successor_stack: Vec, + fallthrough_has_local_statement_successor: bool, + fallthrough_successor_stack: Vec<(bool, bool)>, + try_else_orelse_conditional_base_stack: Vec, } #[derive(Clone, Copy)] @@ -495,6 +497,38 @@ impl Compiler { .map(|constant| Self::constant_truthiness(&constant))) } + fn has_always_taken_jump_in_test( + &mut self, + expr: &ast::Expr, + condition: bool, + ) -> CompileResult { + Ok(match expr { + ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) => { + let (last_value, prefix_values) = values.split_last().unwrap(); + let cond2 = matches!(op, ast::BoolOp::Or); + for value in prefix_values { + if self.has_always_taken_jump_in_test(value, cond2)? { + return Ok(true); + } + } + self.has_always_taken_jump_in_test(last_value, condition)? + } + ast::Expr::UnaryOp(ast::ExprUnaryOp { + op: ast::UnaryOp::Not, + operand, + .. + }) => self.has_always_taken_jump_in_test(operand, !condition)?, + ast::Expr::If(ast::ExprIf { + test, body, orelse, .. + }) => { + self.has_always_taken_jump_in_test(test, false)? + || self.has_always_taken_jump_in_test(body, condition)? + || self.has_always_taken_jump_in_test(orelse, condition)? + } + _ => matches!(self.constant_expr_truthiness(expr)?, Some(value) if value == condition), + }) + } + fn disable_load_fast_borrow_for_block(&mut self, block: BlockIdx) { if block != BlockIdx::NULL { self.current_code_info().blocks[block.idx()].disable_load_fast_borrow = true; @@ -544,6 +578,7 @@ impl Compiler { symbol_table_index: 0, // Module is always the first symbol table in_conditional_block: 0, in_final_with_cleanup_statement: 0, + in_try_else_orelse: 0, next_conditional_annotation_index: 0, }; Self { @@ -568,7 +603,9 @@ impl Compiler { disable_const_collection_folding: false, split_next_for_normal_exit_from_break: false, fallthrough_has_statement_successor: false, + fallthrough_has_local_statement_successor: false, fallthrough_successor_stack: Vec::new(), + try_else_orelse_conditional_base_stack: Vec::new(), } } @@ -1179,6 +1216,10 @@ impl Compiler { .expect("symbol_table_stack is empty! This is a compiler bug.") } + fn has_enclosing_non_module_code_scope(&self) -> bool { + self.code_stack.len() > 1 + } + /// Match CPython's `is_import_originated()`: only imports recorded in the /// module-level symbol table suppress method-call optimization. fn is_name_imported(&self, name: &str) -> bool { @@ -1565,9 +1606,7 @@ impl Compiler { annotation .symbols .iter() - .filter(|(_, s)| { - s.scope == SymbolScope::Free || s.flags.contains(SymbolFlags::FREE_CLASS) - }) + .filter(|(_, s)| s.scope == SymbolScope::Free) .map(|(name, _)| name.clone()) .collect() }) @@ -1576,7 +1615,12 @@ impl Compiler { .symbols .iter() .filter(|(_, s)| { - s.scope == SymbolScope::Free || s.flags.contains(SymbolFlags::FREE_CLASS) + s.scope == SymbolScope::Free + || (scope_type != CompilerScope::Class + && s.flags.contains(SymbolFlags::FREE_CLASS)) + || (scope_type == CompilerScope::Class + && s.flags.contains(SymbolFlags::FREE_CLASS) + && self.has_enclosing_non_module_code_scope()) }) .filter(|(name, symbol)| { if !matches!( @@ -1672,14 +1716,18 @@ impl Compiler { symbol_table_index: key, in_conditional_block: 0, in_final_with_cleanup_statement: 0, + in_try_else_orelse: 0, next_conditional_annotation_index: 0, }; // Push the old compiler unit on the stack (like PyCapsule) // This happens before setting qualname - self.fallthrough_successor_stack - .push(self.fallthrough_has_statement_successor); + self.fallthrough_successor_stack.push(( + self.fallthrough_has_statement_successor, + self.fallthrough_has_local_statement_successor, + )); self.fallthrough_has_statement_successor = false; + self.fallthrough_has_local_statement_successor = false; self.code_stack.push(code_info); // Set qualname after pushing (uses compiler_set_qualname logic) @@ -1741,6 +1789,7 @@ impl Compiler { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }); } @@ -1811,8 +1860,9 @@ impl Compiler { // compiler_exit_scope fn exit_scope(&mut self) -> CodeObject { let _table = self.pop_symbol_table(); - if let Some(previous) = self.fallthrough_successor_stack.pop() { + if let Some((previous, previous_local)) = self.fallthrough_successor_stack.pop() { self.fallthrough_has_statement_successor = previous; + self.fallthrough_has_local_statement_successor = previous_local; } // Various scopes can have sub_tables: @@ -1831,8 +1881,9 @@ impl Compiler { fn exit_annotation_scope(&mut self, saved_ctx: CompileContext) -> CodeObject { self.pop_annotation_symbol_table(); self.ctx = saved_ctx; - if let Some(previous) = self.fallthrough_successor_stack.pop() { + if let Some((previous, previous_local)) = self.fallthrough_successor_stack.pop() { self.fallthrough_has_statement_successor = previous; + self.fallthrough_has_local_statement_successor = previous_local; } let pop = self.code_stack.pop(); @@ -2487,10 +2538,13 @@ impl Compiler { let inherited_successor = self.fallthrough_has_statement_successor; for (idx, statement) in statements.iter().enumerate() { let previous_successor = self.fallthrough_has_statement_successor; + let previous_local_successor = self.fallthrough_has_local_statement_successor; self.fallthrough_has_statement_successor = inherited_successor || idx + 1 < statements.len(); + self.fallthrough_has_local_statement_successor = idx + 1 < statements.len(); let result = self.compile_statement(statement); self.fallthrough_has_statement_successor = previous_successor; + self.fallthrough_has_local_statement_successor = previous_local_successor; result?; } Ok(()) @@ -2500,17 +2554,21 @@ impl Compiler { let inherited_successor = self.fallthrough_has_statement_successor; for (idx, statement) in statements.iter().enumerate() { let previous_successor = self.fallthrough_has_statement_successor; + let previous_local_successor = self.fallthrough_has_local_statement_successor; self.fallthrough_has_statement_successor = inherited_successor || idx + 1 < statements.len(); + self.fallthrough_has_local_statement_successor = idx + 1 < statements.len(); if idx + 1 == statements.len() && matches!(statement, ast::Stmt::Try(_)) { self.current_code_info().in_final_with_cleanup_statement += 1; let result = self.compile_statement(statement); self.current_code_info().in_final_with_cleanup_statement -= 1; self.fallthrough_has_statement_successor = previous_successor; + self.fallthrough_has_local_statement_successor = previous_local_successor; result?; } else { let result = self.compile_statement(statement); self.fallthrough_has_statement_successor = previous_successor; + self.fallthrough_has_local_statement_successor = previous_local_successor; result?; } } @@ -2519,9 +2577,12 @@ impl Compiler { fn compile_loop_body_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { let previous_successor = self.fallthrough_has_statement_successor; + let previous_local_successor = self.fallthrough_has_local_statement_successor; self.fallthrough_has_statement_successor = false; + self.fallthrough_has_local_statement_successor = false; let result = self.compile_statements(statements); self.fallthrough_has_statement_successor = previous_successor; + self.fallthrough_has_local_statement_successor = previous_local_successor; result } @@ -4035,9 +4096,20 @@ impl Compiler { handler; Self::statements_end_with_scope_exit(body) }); + let typed_handlers_end_with_scope_exit = handlers.iter().all(|handler| { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + body, + .. + }) = handler; + type_.is_some() && Self::statements_end_with_scope_exit(body) + }); if Self::has_resuming_bare_except(handlers) { self.disable_load_fast_borrow_for_block(end_block); } + if typed_handlers_end_with_scope_exit { + self.disable_load_fast_borrow_for_block(end_block); + } emit!( self, @@ -4076,12 +4148,22 @@ impl Compiler { } else { self.remove_last_no_location_nop(); } - if !orelse.is_empty() && has_terminal_raise_handlers { - let orelse_block = self.new_block(); - self.switch_to_block(orelse_block); - self.mark_try_else_orelse_entry_block(orelse_block); - } - self.compile_statements(orelse)?; + if !orelse.is_empty() { + if has_terminal_raise_handlers { + let orelse_block = self.new_block(); + self.switch_to_block(orelse_block); + } + let current = self.current_code_info().current_block; + self.mark_try_else_orelse_entry_block(current); + } + let try_else_orelse_conditional_base = self.current_code_info().in_conditional_block; + self.current_code_info().in_try_else_orelse += 1; + self.try_else_orelse_conditional_base_stack + .push(try_else_orelse_conditional_base); + let compile_orelse_result = self.compile_statements(orelse); + self.try_else_orelse_conditional_base_stack.pop(); + self.current_code_info().in_try_else_orelse -= 1; + compile_orelse_result?; emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } @@ -4119,6 +4201,8 @@ impl Compiler { }) = handler; self.set_source_range(*handler_range); let next_handler = self.new_block(); + let handler_body_exits = Self::statements_end_with_scope_exit(body); + let mut exception_handler_was_popped = false; if let Some(exc_type) = type_ { self.compile_expression(exc_type)?; @@ -4148,26 +4232,29 @@ impl Compiler { self.compile_statements(body)?; self.pop_fblock(FBlockType::HandlerCleanup); - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.pop_fblock(FBlockType::ExceptionHandler); - emit!(self, Instruction::PopExcept); - self.set_no_location(); + if !handler_body_exits { + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + self.pop_fblock(FBlockType::ExceptionHandler); + exception_handler_was_popped = true; + emit!(self, Instruction::PopExcept); + self.set_no_location(); - self.emit_load_const(ConstantData::None); - self.set_no_location(); - self.store_name(alias.as_str())?; - self.set_no_location(); - self.compile_name(alias.as_str(), NameUsage::Delete)?; - self.set_no_location(); + self.emit_load_const(ConstantData::None); + self.set_no_location(); + self.store_name(alias.as_str())?; + self.set_no_location(); + self.compile_name(alias.as_str(), NameUsage::Delete)?; + self.set_no_location(); - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: end_block } - ); - self.set_no_location(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: end_block } + ); + self.set_no_location(); + } self.switch_to_block(cleanup_end); self.emit_load_const(ConstantData::None); @@ -4187,19 +4274,24 @@ impl Compiler { self.compile_statements(body)?; self.pop_fblock(FBlockType::HandlerCleanup); - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.pop_fblock(FBlockType::ExceptionHandler); - emit!(self, Instruction::PopExcept); - self.set_no_location(); - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: end_block } - ); - self.set_no_location(); + if !handler_body_exits { + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + self.pop_fblock(FBlockType::ExceptionHandler); + exception_handler_was_popped = true; + emit!(self, Instruction::PopExcept); + self.set_no_location(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: end_block } + ); + self.set_no_location(); + } } - self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + if exception_handler_was_popped { + self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + } self.switch_to_block(next_handler); } @@ -5945,15 +6037,29 @@ impl Compiler { self.new_block() }; - if matches!(self.constant_expr_truthiness(test)?, Some(false)) { + if self.has_always_taken_jump_in_test(test, false)? { self.disable_load_fast_borrow_for_block(next_block); self.disable_load_fast_borrow_for_block(end_block); } + let in_direct_try_else_orelse_conditional = { + let base = self.try_else_orelse_conditional_base_stack.last().copied(); + let code_info = self.current_code_info(); + code_info.in_try_else_orelse > 0 + && base.is_some_and(|base| code_info.in_conditional_block == base + 1) + }; + let preserve_try_else_scope_exit_target_nop = elif_else_clauses.is_empty() + && in_direct_try_else_orelse_conditional + && !self.fallthrough_has_local_statement_successor + && Self::statements_end_with_scope_exit(body); self.compile_jump_if(test, false, next_block)?; self.compile_statements(body)?; let Some((clause, rest)) = elif_else_clauses.split_first() else { self.switch_to_block(end_block); + if preserve_try_else_scope_exit_target_nop { + self.set_source_range(test.range()); + emit!(self, Instruction::Nop); + } return Ok(()); }; @@ -6050,11 +6156,10 @@ impl Compiler { // RERAISE 1 // after: ... - let with_range = self.current_source_range; - let Some((item, items)) = items.split_first() else { return Err(self.error(CodegenErrorType::EmptyWithItems)); }; + let with_range = item.context_expr.range(); let exc_handler_block = self.new_block(); let after_block = self.new_block(); @@ -6146,10 +6251,28 @@ impl Compiler { } self.compile_with_body_statements(body)?; } else { - self.set_source_range(with_range); + self.set_source_range(items[0].context_expr.range()); self.compile_with(items, body, is_async)?; } + let nested_multiline_with_cleanup_target_nop = + !is_async && Self::statements_end_with_scope_exit(body) && { + let parent_with_ranges: Vec<_> = self + .current_code_info() + .fblock + .iter() + .rev() + .skip(1) + .filter_map(|info| { + matches!(info.fb_type, FBlockType::With).then_some(info.fb_range) + }) + .collect(); + let source = self.source_file.to_source_code(); + let current_line = source.line_index(with_range.start()); + parent_with_ranges + .iter() + .any(|range| source.line_index(range.start()) != current_line) + }; let preserve_outer_cleanup_target_nop = !is_async && (Self::statements_end_with_with_cleanup_scope_exit(body) || self.statements_end_with_conditional_scope_exit(body) @@ -6275,7 +6398,9 @@ impl Compiler { // ===== After block ===== self.switch_to_block(after_block); - if materialize_async_with_outer_cleanup_target_nop { + if materialize_async_with_outer_cleanup_target_nop + || nested_multiline_with_cleanup_target_nop + { self.set_source_range(with_range); emit!(self, Instruction::Nop); } @@ -7436,10 +7561,17 @@ impl Compiler { emit!(self, Instruction::Nop); } emit!(self, Instruction::PopTop); + if matches!(m.body.first(), Some(ast::Stmt::Try(_))) { + let body_block = self.new_block(); + self.switch_to_block(body_block); + } } self.compile_statements(&m.body)?; emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); + if let Some(last) = self.current_block().instructions.last_mut() { + last.match_success_jump = true; + } self.set_source_range(m.pattern.range()); self.emit_and_reset_fail_pop(pattern_context)?; } @@ -8112,6 +8244,10 @@ impl Compiler { } _ => { // Fall back case which always will work! + if matches!(self.constant_expr_truthiness(expression)?, Some(value) if value == condition) + { + self.disable_load_fast_borrow_for_block(target_block); + } self.compile_expression(expression)?; emit!(self, Instruction::ToBool); if condition { @@ -10330,6 +10466,7 @@ impl Compiler { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }); } @@ -12521,6 +12658,70 @@ def f(sys, os, file): } } + #[test] + fn test_named_except_continue_resume_try_body_keeps_method_borrows() { + let code = compile_exec( + r#" +def f(self, block=True): + if not block and not self.wait(timeout=0): + return None + while self.event_queue.empty(): + while True: + try: + self.push_char(self.read(1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + return self.event_queue.get() +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + + let attr_idx = |name: &str| { + instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == name + } + _ => false, + }) + .unwrap_or_else(|| panic!("missing {name} LOAD_ATTR")) + }; + let push_char_idx = attr_idx("push_char"); + let read_idx = attr_idx("read"); + + assert!( + matches!( + instructions[push_char_idx - 1].op, + Instruction::LoadFastBorrow { .. } + ), + "named-except cleanup continue backedge should not deopt the protected try-body method receiver, got instructions={:?}", + instructions.iter().map(|unit| unit.op).collect::>() + ); + assert!( + matches!( + instructions[read_idx - 1].op, + Instruction::LoadFastBorrow { .. } + ), + "nested protected try-body method receiver should remain borrowed like CPython, got instructions={:?}", + instructions.iter().map(|unit| unit.op).collect::>() + ); + } + #[test] fn test_boolop_or_shared_body_keeps_false_jump_before_loop_backedge() { let code = compile_exec( @@ -13023,6 +13224,77 @@ def outer(null): ); } + #[test] + fn test_taken_constant_boolop_jump_disables_following_borrows() { + for source in [ + "\ +def f(self): + if 0 and self.h: + self.x = self.y + elif self.a and self.b: + self.x = self.y + self.z = self.w +", + "\ +def f(self): + if 1 or self.h: + self.x = self.y + self.z = self.w +", + ] { + let code = compile_exec(source); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + assert!( + ops.iter() + .any(|op| matches!(op, Instruction::LoadFast { .. })), + "CPython keeps plain LOAD_FAST after an always-taken constant bool-op jump, got ops={ops:?}" + ); + assert!( + !ops.iter() + .any(|op| matches!(op, Instruction::LoadFastBorrow { .. })), + "always-taken constant bool-op jump should suppress later LOAD_FAST_BORROW, got ops={ops:?}" + ); + } + } + + #[test] + fn test_untaken_constant_boolop_jump_keeps_following_borrows() { + for source in [ + "\ +def f(self): + if 1 and self.h: + self.x = self.y + self.z = self.w +", + "\ +def f(self): + if 0 or self.h: + self.x = self.y + self.z = self.w +", + ] { + let code = compile_exec(source); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + assert!( + ops.iter() + .any(|op| matches!(op, Instruction::LoadFastBorrow { .. })), + "constant bool-op jump that is not taken should keep CPython-style borrows, got ops={ops:?}" + ); + } + } + #[test] fn test_nonliteral_constant_bool_op_preserves_short_circuit_shape() { let code = compile_exec( @@ -13455,6 +13727,93 @@ def f(xs): ); } + #[test] + fn test_builtin_tuple_genexpr_unprotected_assignment_return_duplicates_tail() { + let code = compile_exec( + "\ +def f(arg): + if isinstance(arg, (list, tuple)): + arg = tuple(a for a in arg) + elif not p(arg): + raise TypeError(f'bad {arg}') + return arg +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let intrinsic = ops + .iter() + .position(|op| matches!(op, Instruction::CallIntrinsic1 { .. })) + .expect("tuple(genexpr) fast path should emit LIST_TO_TUPLE"); + + assert!( + matches!(ops[intrinsic + 1], Instruction::StoreFast { .. }) + && matches!( + ops[intrinsic + 2], + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. } + ) + && matches!(ops[intrinsic + 3], Instruction::ReturnValue), + "unprotected tuple(genexpr) assignment before return should inline CPython's assignment-return tail, got ops={ops:?}" + ); + } + + #[test] + fn test_unprotected_builtin_any_prefix_before_returning_try_keeps_borrow() { + let code = compile_exec( + "\ +def f(template): + if any(part.expression.strip() == '' for part in template.interpolations): + return ctor(template) + try: + parsed = tuple( + ast.parse(f'({part.expression})', mode='eval').body + for part in template.interpolations + ) + except SyntaxError: + return ctor(template) + return lit(template, parsed) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let first_try_nop = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::Nop)) + .expect("missing try entry NOP"); + let mut saw_interpolations = false; + for window in instructions[..first_try_nop].windows(2) { + let [receiver, attr] = window else { + continue; + }; + let Instruction::LoadAttr { namei } = attr.op else { + continue; + }; + let load_attr = namei.get(OpArg::new(u32::from(u8::from(attr.arg)))); + if f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() != "interpolations" + { + continue; + } + saw_interpolations = true; + assert!( + matches!(receiver.op, Instruction::LoadFastBorrow { .. }), + "unprotected builtin any(genexpr) prefix before a later returning try should keep CPython-style borrowed receiver, got instructions={instructions:?}" + ); + } + assert!( + saw_interpolations, + "missing interpolations attr load in builtin any prefix, got instructions={instructions:?}" + ); + } + #[test] fn test_module_store_uses_store_global_when_nested_scope_declares_global() { let code = compile_exec( @@ -15506,7 +15865,43 @@ def http_error(status): } #[test] - fn test_match_mapping_attribute_key_keeps_plain_load_fast() { + fn test_match_try_body_keeps_setup_nop_after_success_pop() { + let code = compile_exec( + "\ +def f(x): + match x: + case 1: + try: + y = 2 + except Exception: + pass + case 2: + y = 3 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| matches!( + window, + [ + Instruction::PopTop, + Instruction::Nop, + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. } + ] + )), + "expected CPython-style match success POP_TOP followed by try-entry NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_match_mapping_attribute_key_keeps_plain_load_fast() { let code = compile_exec( "\ def f(self): @@ -17021,6 +17416,110 @@ def f(self): ); } + #[test] + fn test_try_import_broad_handler_implicit_return_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, record): + try: + import smtplib + port = self.mailport + if not port: + port = smtplib.SMTP_PORT + smtp = smtplib.SMTP(self.mailhost, port, timeout=self.timeout) + smtp.quit() + except Exception: + self.handleError(record) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &instructions[..handler_start]; + let local_name = |unit: &&CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + + for name in ["self", "smtplib", "smtp"] { + assert!( + normal_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. }) + && local_name(unit) == Some(name)), + "broad except handler with implicit return should keep CPython-style borrowed {name} loads, got tail={normal_tail:?}" + ); + assert!( + !normal_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. }) + && local_name(unit) == Some(name)), + "broad except handler with implicit return should not force strong {name} loads, got tail={normal_tail:?}" + ); + } + } + + #[test] + fn test_try_import_handler_assignment_resume_tail_keeps_borrow() { + let code = compile_exec( + "\ +def f(): + try: + import subprocess + out = subprocess.check_output(['/usr/bin/lslpp', '-Lqc', 'bos.rte']) + except ImportError: + out = _read_cmd_output('/usr/bin/lslpp -Lqc bos.rte') + out = out.decode('utf-8') + out = out.strip().split(':') + _bd = int(out[-1]) if out[-1] != '' else 9988 + return (str(out[2]), _bd) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &instructions[..handler_start]; + let out_idx = f + .varnames + .iter() + .position(|name| name == "out") + .expect("missing out local"); + let load_out_is = |unit: &&CodeUnit, borrowed: bool| match (unit.op, borrowed) { + (Instruction::LoadFastBorrow { var_num }, true) + | (Instruction::LoadFast { var_num }, false) => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + usize::from(var_num.get(arg)) == out_idx + } + _ => false, + }; + + assert!( + normal_tail.iter().any(|unit| load_out_is(unit, true)), + "handler assignment resume tail should keep CPython-style borrowed out loads, got tail={normal_tail:?}" + ); + assert!( + !normal_tail.iter().any(|unit| load_out_is(unit, false)), + "handler assignment resume tail should not force strong out loads, got tail={normal_tail:?}" + ); + } + #[test] fn test_protected_attr_direct_return_keeps_borrow() { let code = compile_exec( @@ -17082,23 +17581,541 @@ def f(tarfile, tarinfo, self): .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let filtered_store = ops - .iter() - .position(|op| matches!(op, Instruction::StoreFast { .. })) - .expect("missing filtered store"); - let handler_start = ops - .iter() - .position(|op| matches!(op, Instruction::PushExcInfo)) - .expect("missing handler entry"); - let normal_tail = &ops[filtered_store + 1..handler_start]; + let filtered_store = ops + .iter() + .position(|op| matches!(op, Instruction::StoreFast { .. })) + .expect("missing filtered store"); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &ops[filtered_store + 1..handler_start]; + + assert!( + !normal_tail.iter().any(|op| matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + )), + "expected CPython-style strong LOAD_FAST in protected store normal tail, got tail={normal_tail:?}", + ); + } + + #[test] + fn test_protected_call_arm_final_store_return_uses_strong_load() { + let code = compile_exec( + "\ +def f(self, action, default_metavar): + get_metavar = self._metavar_formatter(action, default_metavar) + if action.nargs is None: + result = '%s' % get_metavar(1) + elif action.nargs == OPTIONAL: + result = '[%s]' % get_metavar(1) + elif action.nargs == ZERO_OR_MORE: + metavar = get_metavar(1) + if len(metavar) == 2: + result = '[%s [%s ...]]' % metavar + else: + result = '[%s ...]' % metavar + elif action.nargs == ONE_OR_MORE: + result = '%s [%s ...]' % get_metavar(2) + elif action.nargs == REMAINDER: + result = '...' + elif action.nargs == PARSER: + result = '%s ...' % get_metavar(1) + elif action.nargs == SUPPRESS: + result = '' + else: + try: + formats = ['%s' for _ in range(action.nargs)] + except TypeError: + raise ValueError(\"invalid nargs value\") from None + result = ' '.join(formats) % get_metavar(action.nargs) + return result +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let modulo = ops + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::BinaryOp { op } + if op.get(OpArg::new(u32::from(u8::from(unit.arg)))) + == BinaryOperator::Remainder + ) + }) + .expect("missing final format modulo"); + let tail = &ops[modulo..]; + assert!( + tail.windows(3).any(|window| { + matches!( + window, + [ + bytecode::CodeUnit { + op: Instruction::StoreFast { .. }, + .. + }, + bytecode::CodeUnit { + op: Instruction::LoadFast { .. }, + .. + }, + bytecode::CodeUnit { + op: Instruction::ReturnValue, + .. + }, + ] + ) + }), + "protected call arm final store-return should keep CPython-style strong LOAD_FAST, got tail={tail:?}" + ); + } + + #[test] + fn test_protected_store_try_else_tail_keeps_borrowed_loads() { + let code = compile_exec( + "\ +def f(value): + message_id = MessageID() + try: + token, value = get_msg_id(value) + message_id.append(token) + except HeaderParseError as ex: + token = get_unstructured(value) + message_id = InvalidMessageID(token) + message_id.defects.append(InvalidHeaderDefect('Invalid msg-id: {!r}'.format(ex))) + else: + if value: + message_id.defects.append(InvalidHeaderDefect('Unexpected {!r}'.format(value))) + return message_id +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &ops[..handler_start]; + + assert!( + normal_tail.windows(3).any(|window| { + matches!( + window, + [ + CodeUnit { + op: Instruction::LoadFastBorrow { .. }, + .. + }, + CodeUnit { + op: Instruction::ToBool, + .. + }, + CodeUnit { + op: Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. }, + .. + }, + ] + ) + }), + "try/except/else bool guard should keep CPython-style borrowed load, got tail={normal_tail:?}", + ); + + let defects_idx = normal_tail + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "defects" + } + _ => false, + }) + .expect("missing defects LOAD_ATTR in else tail"); + assert!( + matches!( + normal_tail[defects_idx - 1].op, + Instruction::LoadFastBorrow { .. } + ), + "try/except/else method receiver should stay borrowed like CPython, got tail={normal_tail:?}", + ); + + let format_idx = normal_tail + .iter() + .rposition(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "format" + } + _ => false, + }) + .expect("missing format LOAD_ATTR in else tail"); + assert!( + matches!( + normal_tail[format_idx + 1].op, + Instruction::LoadFastBorrow { .. } + ), + "try/except/else format argument should stay borrowed like CPython, got tail={normal_tail:?}", + ); + } + + #[test] + fn test_nested_try_except_common_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(value): + address = Address() + try: + token, value = get_group(value) + except HeaderParseError: + try: + token, value = get_mailbox(value) + except HeaderParseError: + raise HeaderParseError('expected {}'.format(value)) + address.append(token) + return address, value +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFast { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::LoadFastLoadFast { .. }, + ] + ) + }), + "nested try/except common tail should use CPython-style strong loads, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_try_except_branch_tail_with_following_try_uses_strong_loads() { + let code = compile_exec( + r#" +def f(value): + msg_id = MsgID() + try: + token, value = get_dot_atom_text(value) + except HeaderParseError: + try: + token, value = get_obs_local_part(value) + msg_id.defects.append(ObsoleteHeaderDefect("obsolete id-left in msg-id")) + except HeaderParseError: + raise HeaderParseError("expected {}".format(value)) + msg_id.append(token) + if not value or value[0] != "@": + msg_id.defects.append(InvalidHeaderDefect("msg-id with no id-right")) + if value and value[0] == ">": + msg_id.append(ValueTerminal(">", "msg-id-end")) + value = value[1:] + return msg_id, value + msg_id.append(ValueTerminal("@", "address-at-symbol")) + value = value[1:] + try: + token, value = get_dot_atom_text(value) + except HeaderParseError: + pass + return msg_id, value +"#, + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(16).any(|window| { + matches!( + window, + [ + Instruction::ReturnValue, + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadGlobal { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::LoadFast { .. }, + Instruction::LoadConst { .. }, + Instruction::BinaryOp { .. }, + Instruction::StoreFast { .. }, + Instruction::Nop, + Instruction::LoadGlobal { .. }, + Instruction::LoadFast { .. }, + ] + ) + }), + "nested try branch tail before a following try should use CPython-style strong loads, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_try_store_subscr_following_try_tail_uses_strong_loads() { + let code = compile_exec( + r#" +def f(value): + local_part = LocalPart() + try: + token, value = get_dot_atom(value) + except HeaderParseError: + try: + token, value = get_word(value) + except HeaderParseError: + token = TokenList() + if value: + obs_local_part, value = get_obs_local_part(str(local_part) + value) + if obs_local_part.token_type == "invalid-obs-local-part": + local_part.defects.append(InvalidHeaderDefect("invalid")) + else: + local_part.defects.append(ObsoleteHeaderDefect("obsolete")) + local_part[0] = obs_local_part + try: + local_part.value.encode("ascii") + except UnicodeEncodeError: + local_part.defects.append(NonASCIILocalPartDefect("non-ascii")) + return local_part, value +"#, + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(10).any(|window| { + matches!( + window, + [ + Instruction::StoreSubscr, + Instruction::Nop, + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::LoadFastLoadFast { .. }, + Instruction::BuildTuple { .. }, + ] + ) + }), + "nested try STORE_SUBSCR tail before a following try should use CPython-style strong loads, got ops={ops:?}" + ); + } + + #[test] + fn test_resuming_except_in_loop_keeps_post_try_store_tail_borrowed() { + let code = compile_exec( + "\ +def f(part, lines, maxlen, encoding): + for name, value in part.params: + charset = encoding + error_handler = 'strict' + try: + value.encode(encoding) + encoding_required = False + except UnicodeEncodeError: + encoding_required = True + charset = 'utf-8' + if encoding_required: + encoded_value = quote(value, safe='', errors=error_handler) + tstr = \"{}*={}''{}\".format(name, charset, encoded_value) + else: + tstr = '{}={}'.format(name, quote_string(value)) + if len(lines[-1]) + len(tstr) + 1 < maxlen: + lines[-1] = lines[-1] + ' ' + tstr + continue +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. }, + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. }, + ] + ) + }), + "resuming except handler should keep CPython-style borrowed bool guard, got ops={ops:?}" + ); + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. }, + Instruction::LoadConst { .. }, + Instruction::StoreSubscr, + ] + ) + }), + "resuming except handler should not deopt the post-try STORE_SUBSCR tail, got ops={ops:?}" + ); + } + + #[test] + fn test_handler_resume_loop_latch_method_call_uses_strong_loads() { + let code = compile_exec( + "\ +def f(phrase, value): + while value and value[0] not in PHRASE_ENDS: + if value[0] == '.': + phrase.append(DOT) + phrase.defects.append(ObsoleteHeaderDefect('period in phrase')) + value = value[1:] + else: + try: + token, value = get_word(value) + except HeaderParseError: + if value[0] in CFWS_LEADER: + token, value = get_cfws(value) + phrase.defects.append(ObsoleteHeaderDefect('comment found without atom')) + else: + raise + phrase.append(token) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFast { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::JumpBackward { .. }, + ] + ) + }) || ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFast { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::JumpBackward { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "exception-handler resume loop latch should match CPython's strong method-call loads, got ops={ops:?}" + ); + } + + #[test] + fn test_single_handler_multiple_resume_branches_keep_post_try_tail_borrowed() { + let code = compile_exec( + "\ +def f(part, lines, maxlen, encoding): + for name, value in part.params: + charset = encoding + error_handler = 'strict' + try: + value.encode(encoding) + encoding_required = False + except UnicodeEncodeError: + encoding_required = True + if utils._has_surrogates(value): + charset = 'unknown-8bit' + error_handler = 'surrogateescape' + else: + charset = 'utf-8' + if encoding_required: + encoded_value = quote(value, safe='', errors=error_handler) + tstr = \"{}*={}''{}\".format(name, charset, encoded_value) + else: + tstr = '{}={}'.format(name, quote_string(value)) + if len(lines[-1]) + len(tstr) + 1 < maxlen: + lines[-1] = lines[-1] + ' ' + tstr + continue +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); assert!( - !normal_tail.iter().any(|op| matches!( - op, - Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - )), - "expected CPython-style strong LOAD_FAST in protected store normal tail, got tail={normal_tail:?}", + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. }, + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. }, + ] + ) + }), + "single except handler with multiple resume branches should keep the post-try bool guard borrowed, got ops={ops:?}" + ); + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. }, + Instruction::LoadConst { .. }, + Instruction::StoreSubscr, + ] + ) + }), + "single except handler with multiple resume branches should keep the post-try STORE_SUBSCR tail borrowed, got ops={ops:?}" ); } @@ -17536,6 +18553,52 @@ def f(tp, parent=None): ); } + #[test] + fn test_generator_returning_except_keeps_yield_from_resume_tail_borrow() { + let code = compile_exec( + "\ +def f(self, action): + try: + get_subactions = action._get_subactions + except AttributeError: + pass + else: + self._indent() + yield from get_subactions() + self._dedent() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let end_send = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::EndSend)) + .expect("missing END_SEND"); + let dedent_attr = instructions[end_send..] + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "_dedent" + } + _ => false, + }) + .map(|idx| end_send + idx) + .expect("missing _dedent LOAD_ATTR"); + + assert!( + matches!( + instructions[dedent_attr - 1].op, + Instruction::LoadFastBorrow { .. } + ), + "CPython keeps yield-from resume receiver borrowed after END_SEND, got instructions={instructions:?}" + ); + } + #[test] fn test_generator_except_pass_resume_tail_keeps_borrows() { let code = compile_exec( @@ -19003,6 +20066,106 @@ def f(self): ); } + #[test] + fn test_conditional_typed_except_return_join_keeps_borrow() { + let code = compile_exec( + "\ +def f(cond, obj, xs, E): + if cond: + try: + obj.m() + except E: + return 1 + for x in xs: + obj.n(x) + return obj +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &instructions[..handler_start]; + let load_name = |unit: &&CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + + for name in ["xs", "obj", "x"] { + assert!( + normal_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. }) + && load_name(unit) == Some(name)), + "conditional typed except return join should keep CPython-style borrowed {name} loads, got tail={normal_tail:?}" + ); + assert!( + !normal_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. }) + && load_name(unit) == Some(name)), + "conditional typed except return join should not force strong {name} loads, got tail={normal_tail:?}" + ); + } + } + + #[test] + fn test_typed_except_pass_resume_store_subscr_tail_keeps_borrows() { + let code = compile_exec( + "\ +def f(self, sys, KeyError): + mod_name = self.mod_name + try: + self._saved_module.append(sys.modules[mod_name]) + except KeyError: + pass + sys.modules[mod_name] = self.module + return self +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let store_subscr_idx = ops + .iter() + .position(|op| matches!(op, Instruction::StoreSubscr)) + .expect("missing STORE_SUBSCR tail"); + assert!( + matches!( + ops.get(store_subscr_idx - 5), + Some(Instruction::LoadFastBorrow { .. }) + ) && matches!( + ops.get(store_subscr_idx - 3), + Some(Instruction::LoadFastBorrow { .. }) + ) && matches!( + ops.get(store_subscr_idx - 1), + Some(Instruction::LoadFastBorrow { .. }) + ), + "typed except pass tail should keep STORE_SUBSCR operands borrowed, got ops={ops:?}" + ); + assert!( + matches!( + ops.get(store_subscr_idx + 1), + Some(Instruction::LoadFastBorrow { .. }) + ), + "typed except pass return should keep self borrowed, got ops={ops:?}" + ); + } + #[test] fn test_reraising_typed_except_deopts_post_handler_loads() { let code = compile_exec( @@ -19853,45 +21016,173 @@ def f(cond): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let has_cpython_nop_target = ops.windows(5).any(|window| { - matches!( - window, - [ - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, - Instruction::Nop, - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, - ] - ) - }); - let has_direct_fallthrough = ops.windows(4).any(|window| { - matches!( - window, - [ - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, - ] - ) - }); - assert!( - has_cpython_nop_target || has_direct_fallthrough, - "expected adjacent try-else return and final return targets, got ops={ops:?}" + let has_cpython_nop_target = ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, + Instruction::ReturnValue, + Instruction::Nop, + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }); + assert!( + has_cpython_nop_target, + "expected CPython-style try-else conditional target NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_try_else_nested_if_return_drops_inner_conditional_target_nop() { + let code = compile_exec( + "\ +def f(obj, Sig): + try: + sig = obj.__signature__ + except AttributeError: + pass + else: + if sig is not None: + if not isinstance(sig, Sig): + raise TypeError(sig) + return sig + return obj +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::RaiseVarargs { .. }, + Instruction::Nop, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::ReturnValue, + ] + ) + }), + "inner try-else if target should not materialize as a NOP before the return, got ops={ops:?}" + ); + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::RaiseVarargs { .. }, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::ReturnValue, + Instruction::Nop, + ] + ) + }), + "expected only the outer try-else if false target after the return, got ops={ops:?}" + ); + } + + #[test] + fn test_try_else_nested_final_if_return_drops_nested_conditional_target_nop() { + let code = compile_exec( + "\ +def f(cond, outer): + try: + x = cond + except E: + pass + else: + if outer: + if x: + return 1 + return 2 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::ReturnValue, + Instruction::Nop, + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }), + "nested try-else conditional target should fall through directly to following code, got ops={ops:?}" + ); + } + + #[test] + fn test_named_except_conditional_branch_duplicates_cleanup_return() { + let code = compile_exec( + "\ +def f(self): + try: + raise TypeError('x') + except TypeError as e: + if '+' not in str(e): + self.fail('join() ate exception message') +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let cleanup_return_count = ops + .windows(6) + .filter(|window| { + matches!( + window, + [ + Instruction::PopExcept, + Instruction::LoadConst { .. }, + Instruction::StoreFast { .. } | Instruction::StoreName { .. }, + Instruction::DeleteFast { .. } | Instruction::DeleteName { .. }, + Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }) + .count(); + + assert_eq!( + cleanup_return_count, 2, + "expected duplicated named-except cleanup return blocks, got ops={ops:?}" ); } #[test] - fn test_named_except_conditional_branch_duplicates_cleanup_return() { + fn test_named_except_conditional_before_explicit_return_shares_cleanup_return() { let code = compile_exec( "\ -def f(self): +def f(onerror, err, OSError): try: - raise TypeError('x') - except TypeError as e: - if '+' not in str(e): - self.fail('join() ate exception message') + x() + except OSError as err: + if onerror is not None: + onerror(err) + return ", ); let f = find_code(&code, "f").expect("missing function code"); @@ -19920,8 +21211,8 @@ def f(self): .count(); assert_eq!( - cleanup_return_count, 2, - "expected duplicated named-except cleanup return blocks, got ops={ops:?}" + cleanup_return_count, 1, + "explicit return after the conditional should share the named-except cleanup return block, got ops={ops:?}" ); } @@ -20213,6 +21504,49 @@ def f(cm, registry, altkey): ); } + #[test] + fn test_multiline_nested_with_return_finally_keeps_inner_cleanup_anchor_nop() { + let code = compile_exec( + "\ +def f(a, b, path): + try: + with cm(a) as x, \\ + cm(b): + return g(x).copy() + finally: + try: + cleanup(path) + except ValueError: + pass +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::Copy { .. }, + Instruction::PopExcept, + Instruction::Reraise { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + ] + ) + }), + "multi-line nested with return/finally cleanup should keep CPython's inner item anchor NOP before outer __exit__, got ops={ops:?}" + ); + } + #[test] fn test_try_finally_conditional_return_duplicates_finally_exit_return() { let code = compile_exec( @@ -21161,6 +22495,127 @@ def f(self): ); } + #[test] + fn test_generic_type_alias_in_class_does_not_capture_module_name() { + let code = compile_exec( + r#" +T = U = "global" +class C: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) +"#, + ); + assert!( + code.cellvars.is_empty(), + "module T must remain a global name, got cellvars={:?}", + code.cellvars + ); + + let class_code = find_code(&code, "C").expect("missing class code"); + assert!( + class_code.freevars.is_empty(), + "plain module-level class must not close over module T, got freevars={:?}", + class_code.freevars + ); + + let type_params = find_code(class_code, "") + .expect("missing alias type params"); + assert_eq!( + type_params + .cellvars + .iter() + .map(String::as_str) + .collect::>(), + vec!["T"], + "alias type parameter T should be local to the type-params scope, got cellvars={:?}", + type_params.cellvars + ); + assert_eq!( + type_params + .freevars + .iter() + .map(String::as_str) + .collect::>(), + vec!["__classdict__"], + "alias type params should close only over the classdict, got freevars={:?}", + type_params.freevars + ); + + let alias = find_code(type_params, "Alias").expect("missing alias value code"); + assert_eq!( + alias + .freevars + .iter() + .map(String::as_str) + .collect::>(), + vec!["T", "__classdict__"], + "alias value should close over its type parameter and classdict, got freevars={:?}", + alias.freevars + ); + let lambda = find_code(alias, "").expect("missing alias lambda"); + assert_eq!( + lambda + .freevars + .iter() + .map(String::as_str) + .collect::>(), + vec!["T"], + "lambda should close over alias type parameter T only, got freevars={:?}", + lambda.freevars + ); + } + + #[test] + fn test_nested_generic_class_base_child_free_keeps_classdict_lookup() { + for (child_name, base_expr) in [ + ("", "make_base(T for _ in (1,))"), + ("", "make_base([T for _ in (1,)])"), + ("", "make_base(lambda: T)"), + ] { + let code = compile_exec(&format!( + "\ +class C[T]: + T = 'class' + class Inner[U]({base_expr}, make_base(T)): + pass +" + )); + let type_params = find_code(&code, "") + .expect("missing inner type params code"); + assert_eq!( + type_params + .freevars + .iter() + .map(String::as_str) + .collect::>(), + vec!["T", "__classdict__"], + "inner type params should keep T as a child-only freevar for {child_name}, got freevars={:?}", + type_params.freevars + ); + assert!( + type_params + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFromDictOrGlobals { .. })), + "direct class-local T lookup should still use classdict/global path for {child_name}, got instructions={:?}", + type_params.instructions + ); + + let child = find_code(type_params, child_name).expect("missing child code"); + assert_eq!( + child + .freevars + .iter() + .map(String::as_str) + .collect::>(), + vec!["T"], + "{child_name} should close over T from the type-params scope, got freevars={:?}", + child.freevars + ); + } + } + #[test] fn test_class_annotation_global_resolution_matches_cpython() { let class_global = compile_exec( @@ -25590,6 +27045,59 @@ def f(value): ); } + #[test] + fn test_multi_handler_resume_before_with_keeps_with_body_borrows() { + let code = compile_exec( + "\ +def f(self, input, cm): + try: + self.stdin.flush() + except BrokenPipeError: + pass + except ValueError: + if not self.stdin.closed: + raise + if not input: + self.stdin.close() + with cm() as selector: + if self.stdin and self._input: + selector.register(self.stdin, EVENT_WRITE) + while selector.get_map(): + ready = selector.select() + for key in ready: + self._fileobj2output[key].append(key) + return self.stdin +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let handler_start = f + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_path = &f.instructions[..handler_start]; + let self_load_is = |unit: &CodeUnit, borrowed: bool| match unit.op { + Instruction::LoadFast { var_num } if !borrowed => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == "self" + } + Instruction::LoadFastBorrow { var_num } if borrowed => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == "self" + } + _ => false, + }; + + assert!( + warm_path.iter().any(|unit| self_load_is(unit, true)), + "expected multi-handler resume before with-body to keep borrowed self loads, got warm_path={warm_path:?}" + ); + assert!( + !warm_path.iter().any(|unit| self_load_is(unit, false)), + "multi-handler resume must not deopt with-body self loads to strong LOAD_FAST, got warm_path={warm_path:?}" + ); + } + #[test] fn test_suppressing_with_and_typed_except_resume_loop_method_tail_keeps_strong_load_fast() { let code = compile_exec( @@ -26319,32 +27827,113 @@ def f(self, document): .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - - let import_idx = ops + + let import_idx = ops + .iter() + .position(|op| matches!(op, Instruction::ImportName { .. })) + .expect("missing IMPORT_NAME"); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_path = &ops[import_idx + 1..handler_start]; + + assert!( + warm_path.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected unprotected import before with-block to keep LOAD_FAST_BORROW ops, got warm_path={warm_path:?}" + ); + assert!( + warm_path + .iter() + .any(|op| matches!(op, Instruction::LoadFastBorrowLoadFastBorrow { .. })), + "expected with body arguments to keep LOAD_FAST_BORROW_LOAD_FAST_BORROW, got warm_path={warm_path:?}" + ); + } + + #[test] + fn test_from_import_after_conditional_store_join_uses_strong_prefix_loads() { + let code = compile_exec( + "\ +def f(x): + if x is None: + x = 'a' + y = x.rpartition('.')[0] + from pkgutil import get_importer + return y +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let import_from_idx = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::ImportFrom { .. })) + .expect("missing IMPORT_FROM"); + let rpartition_idx = ops[..import_from_idx] + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "rpartition" + } + _ => false, + }) + .expect("missing rpartition LOAD_ATTR"); + + assert!( + matches!(ops[rpartition_idx - 1].op, Instruction::LoadFast { .. }), + "CPython keeps the conditional-store join receiver strong before IMPORT_FROM, got ops={ops:?}" + ); + } + + #[test] + fn test_plain_import_after_conditional_store_join_keeps_borrow_prefix_loads() { + let code = compile_exec( + "\ +def f(x): + if x is None: + x = 'a' + y = x.rpartition('.')[0] + import pkgutil + return y +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let import_name_idx = ops .iter() - .position(|op| matches!(op, Instruction::ImportName { .. })) + .position(|unit| matches!(unit.op, Instruction::ImportName { .. })) .expect("missing IMPORT_NAME"); - let handler_start = ops + let rpartition_idx = ops[..import_name_idx] .iter() - .position(|op| matches!(op, Instruction::PushExcInfo)) - .expect("missing handler entry"); - let warm_path = &ops[import_idx + 1..handler_start]; + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "rpartition" + } + _ => false, + }) + .expect("missing rpartition LOAD_ATTR"); assert!( - warm_path.iter().any(|op| { - matches!( - op, - Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - }), - "expected unprotected import before with-block to keep LOAD_FAST_BORROW ops, got warm_path={warm_path:?}" - ); - assert!( - warm_path - .iter() - .any(|op| matches!(op, Instruction::LoadFastBorrowLoadFastBorrow { .. })), - "expected with body arguments to keep LOAD_FAST_BORROW_LOAD_FAST_BORROW, got warm_path={warm_path:?}" + matches!( + ops[rpartition_idx - 1].op, + Instruction::LoadFastBorrow { .. } + ), + "plain import after conditional-store join should keep CPython-style borrowed receiver, got ops={ops:?}" ); } @@ -29281,6 +30870,124 @@ def f(self, pos=None): ); } + #[test] + fn test_terminal_except_else_final_store_attr_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(self, E, Event): + try: + decoded = bytes(self.buf).decode(self.encoding) + except E: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let store_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::StoreAttr { namei } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.names[usize::try_from(namei.get(arg)).unwrap()].as_str() == "keymap" + } + _ => false, + }) + .expect("missing keymap STORE_ATTR"); + let insert_attr_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "insert" + } + _ => false, + }) + .expect("missing insert LOAD_ATTR"); + + assert!( + matches!( + instructions[insert_attr_idx - 1].op, + Instruction::LoadFastBorrow { .. } + ), + "terminal except should not deopt the try/else method receiver before the final tail, got instructions={instructions:?}" + ); + assert!( + matches!( + ( + instructions[store_idx - 3].op, + instructions[store_idx - 1].op + ), + (Instruction::LoadFast { .. }, Instruction::LoadFast { .. }) + ), + "terminal except final STORE_ATTR tail should use CPython-style strong LOAD_FAST ops, got instructions={instructions:?}" + ); + } + + #[test] + fn test_except_break_try_else_loop_tail_keeps_else_borrows() { + let code = compile_exec( + "\ +def f(self): + self.setup() + prompt = 'Hit Return for more, or q (and Return) to quit: ' + lineno = 0 + while 1: + try: + for i in range(lineno, lineno + self.MAXLINES): + print(self.lines[i]) + except IndexError: + break + else: + lineno += self.MAXLINES + key = None + while key is None: + key = input(prompt) + if key not in ('', 'q'): + key = None + if key == 'q': + break +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let borrowed_loads = instructions + .iter() + .filter(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }) + .count(); + let strong_loads_in_else_tail = instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. } + ) + }); + + assert!( + borrowed_loads >= 7, + "except-break try/else loop tail should keep CPython-style borrowed loads, got instructions={instructions:?}" + ); + assert!( + !strong_loads_in_else_tail, + "except-break try/else loop tail must not be deoptimized as terminal return/raise, got instructions={instructions:?}" + ); + } + #[test] fn test_protected_method_call_after_terminal_except_tail_uses_strong_loads() { let code = compile_exec( @@ -30211,6 +31918,89 @@ def f(formatstr, args, output, overflowok): ); } + #[test] + fn test_typed_except_resume_import_warning_tail_keeps_borrows() { + let code = compile_exec( + r#" +def f(mod_name, error, sys, RuntimeWarning): + pkg_name, _, _ = mod_name.rpartition(".") + if pkg_name: + try: + __import__(pkg_name) + except ImportError as e: + if e.name is None or (e.name != pkg_name and + not pkg_name.startswith(e.name + ".")): + raise + existing = sys.modules.get(mod_name) + if existing is not None and not hasattr(existing, "__path__"): + from warnings import warn + msg = "{mod_name!r} found in sys.modules after import of " \ + "package {pkg_name!r}, but prior to execution of " \ + "{mod_name!r}; this may result in unpredictable " \ + "behaviour".format(mod_name=mod_name, pkg_name=pkg_name) + warn(RuntimeWarning(msg)) + return mod_name +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let arg = |unit: &bytecode::CodeUnit| OpArg::new(u32::from(u8::from(unit.arg))); + let strong_names = |unit: &bytecode::CodeUnit| -> Vec<&str> { + match unit.op { + Instruction::LoadFast { var_num } => { + vec![f.varnames[usize::from(var_num.get(arg(unit)))].as_str()] + } + Instruction::LoadFastLoadFast { var_nums } => { + let (left, right) = var_nums.get(arg(unit)).indexes(); + vec![ + f.varnames[usize::from(left)].as_str(), + f.varnames[usize::from(right)].as_str(), + ] + } + _ => Vec::new(), + } + }; + let borrowed_names = |unit: &bytecode::CodeUnit| -> Vec<&str> { + match unit.op { + Instruction::LoadFastBorrow { var_num } => { + vec![f.varnames[usize::from(var_num.get(arg(unit)))].as_str()] + } + Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let (left, right) = var_nums.get(arg(unit)).indexes(); + vec![ + f.varnames[usize::from(left)].as_str(), + f.varnames[usize::from(right)].as_str(), + ] + } + _ => Vec::new(), + } + }; + + let handler_start = f + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_path = &f.instructions[..handler_start]; + for name in ["mod_name", "existing", "warn", "msg"] { + assert!( + normal_path + .iter() + .flat_map(borrowed_names) + .any(|borrowed| borrowed == name), + "CPython keeps {name} borrowed after typed-except resume, got ops={:?}", + normal_path.iter().map(|unit| unit.op).collect::>() + ); + assert!( + !normal_path + .iter() + .flat_map(strong_names) + .any(|strong| strong == name), + "typed-except resume should not force strong {name} loads, got ops={:?}", + normal_path.iter().map(|unit| unit.op).collect::>() + ); + } + } + #[test] fn test_reraising_except_else_tail_keeps_borrow() { let code = compile_exec( @@ -30598,6 +32388,95 @@ def f(self, xs): ); } + #[test] + fn test_except_pass_resume_loop_branch_keeps_borrows() { + let code = compile_exec( + r#" +def f(self, cls, fns): + for name, fn in zip(self.names, fns): + try: + annotation_fields, return_type = self.method_annotations[name] + except KeyError: + pass + else: + annotate_fn = _make_annotate_function(cls, name, annotation_fields, return_type) + fn.__annotate__ = annotate_fn + if self.unconditional_adds.get(name, False): + setattr(cls, name, fn) + else: + already_exists = _set_new_attribute(cls, name, fn) + if already_exists and (msg_extra := self.overwrite_errors.get(name)): + error_msg = (f'Cannot overwrite attribute {fn.__name__} ' + f'in class {cls.__name__}') + if not msg_extra is True: + error_msg = f'{error_msg} {msg_extra}' + raise TypeError(error_msg) +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let load_global_name = |unit: &&CodeUnit, name: &str| match unit.op { + Instruction::LoadGlobal { namei } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let load_global = namei.get(arg) >> 1; + f.names[usize::try_from(load_global).unwrap()].as_str() == name + } + _ => false, + }; + let load_name = |unit: &&CodeUnit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + let load_pair_names = |unit: &&CodeUnit| match unit.op { + Instruction::LoadFastLoadFast { var_nums } + | Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + Some(( + f.varnames[usize::from(left)].as_str(), + f.varnames[usize::from(right)].as_str(), + )) + } + _ => None, + }; + + let set_new_attribute = instructions + .iter() + .position(|unit| load_global_name(unit, "_set_new_attribute")) + .expect("missing _set_new_attribute call"); + let tail = &instructions[set_new_attribute..instructions.len()]; + assert!( + tail.iter().any(|unit| { + matches!(unit.op, Instruction::LoadFastBorrowLoadFastBorrow { .. }) + && load_pair_names(unit) == Some(("cls", "name")) + }) && tail.iter().any(|unit| { + matches!(unit.op, Instruction::LoadFastBorrow { .. }) + && load_name(unit) == Some("fn") + }), + "except-pass resume should keep CPython-style borrowed loads in loop branch, got tail={tail:?}" + ); + assert!( + !tail.iter().any(|unit| { + matches!(unit.op, Instruction::LoadFastLoadFast { .. }) + && load_pair_names(unit) == Some(("cls", "name")) + }) && !tail.iter().any(|unit| { + matches!(unit.op, Instruction::LoadFast { .. }) + && matches!( + load_name(unit), + Some("fn" | "already_exists" | "msg_extra" | "error_msg") + ) + }), + "except-pass resume must not deopt the independent loop branch, got tail={tail:?}" + ); + } + #[test] fn test_named_except_cleanup_deopts_same_guard_fallbacks_not_outer_tail() { let code = compile_exec( @@ -30787,4 +32666,128 @@ async def name_4(): "expected async comprehension iterator capture to borrow name_5 before GET_AITER, got {prev:?}" ); } + + #[test] + fn test_with_protected_generator_tail_after_cleanup_uses_strong_loads() { + let code = compile_exec( + r#" +def f(scandir, fspath, path, reversed, top, OSError, topdown=True, followlinks=False): + stack = [fspath(top)] + islink, join = path.islink, path.join + while stack: + top = stack.pop() + dirs = [] + nondirs = [] + walk_dirs = [] + try: + with scandir(top) as entries: + for entry in entries: + try: + is_dir = entry.is_dir() + except OSError: + is_dir = False + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + if not topdown and is_dir: + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + is_symlink = False + walk_into = not is_symlink + if walk_into: + walk_dirs.append(entry.path) + except OSError: + continue + if topdown: + yield top, dirs, nondirs + for dirname in reversed(dirs): + new_path = join(top, dirname) + if not followlinks and islink(new_path): + continue + stack.append(new_path) + else: + stack.append((top, dirs, nondirs)) + for new_path in reversed(walk_dirs): + stack.append(new_path) +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let yield_pos = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::YieldValue { .. })) + .expect("missing YIELD_VALUE"); + assert!( + matches!( + instructions[yield_pos - 3].op, + Instruction::LoadFastLoadFast { .. } + ) && matches!(instructions[yield_pos - 2].op, Instruction::LoadFast { .. }), + "CPython keeps the yielded tuple inputs strong after with cleanup, got {:?} {:?}", + instructions[yield_pos - 3], + instructions[yield_pos - 2], + ); + assert!( + instructions[yield_pos + 1..] + .iter() + .take(30) + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "expected post-yield traversal tail to contain strong LOAD_FAST ops" + ); + } + + #[test] + fn test_yield_from_finally_cleanup_keeps_normal_path_borrows() { + let code = compile_exec( + r#" +def f(_fwalk, stack, isbytes, topdown, onerror, follow_symlinks, close): + try: + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) + finally: + while stack: + action, value = stack.pop() + if action == 2: + close(value) +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let send_pos = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::Send { .. })) + .expect("missing SEND"); + assert!( + instructions[..send_pos] + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. })), + "CPython keeps yield-from normal path loads borrowed, got prefix={:?}", + &instructions[..send_pos] + ); + let cleanup_pop = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::StoreFastStoreFast { .. })) + .expect("missing finally cleanup unpack"); + assert!( + instructions[..cleanup_pop] + .iter() + .rev() + .take(12) + .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. })), + "CPython keeps normal finally cleanup loads borrowed, got cleanup prefix={:?}", + &instructions[cleanup_pop.saturating_sub(12)..cleanup_pop] + ); + } } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 75e2f677fd..d01e9c7d3c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -131,6 +131,8 @@ pub struct InstructionInfo { pub no_location_exit: bool, /// Keep this no-location NOP until line propagation when it starts a block. pub preserve_block_start_no_location_nop: bool, + /// This is the success jump emitted after a non-final match case body. + pub match_success_jump: bool, } /// Exception handler information for an instruction. @@ -155,6 +157,7 @@ fn set_to_nop(info: &mut InstructionInfo) { info.folded_operand_nop = false; info.no_location_exit = false; info.preserve_block_start_no_location_nop = false; + info.match_success_jump = false; } fn nop_out_no_location(info: &mut InstructionInfo) { @@ -281,6 +284,9 @@ pub struct CodeInfo { // Track when compiling the final direct statement in a sync with body. pub in_final_with_cleanup_statement: u32, + // Track when compiling the orelse suite of try/except. + pub in_try_else_orelse: u32, + // PEP 649: Next index for conditional annotation tracking // u_next_conditional_annotation_index pub next_conditional_annotation_index: u32, @@ -377,6 +383,7 @@ impl CodeInfo { materialize_empty_conditional_exit_targets(&mut self.blocks); redirect_empty_block_targets(&mut self.blocks); inline_small_fast_return_blocks(&mut self.blocks); + inline_unprotected_tuple_genexpr_assignment_return_blocks(&mut self.blocks); duplicate_end_returns(&mut self.blocks, &self.metadata); duplicate_fallthrough_jump_back_targets(&mut self.blocks); duplicate_shared_jump_back_targets(&mut self.blocks); @@ -419,6 +426,7 @@ impl CodeInfo { self.mark_unprotected_debug_four_tails_borrow_disabled(); self.mark_exception_handler_transition_targets_borrow_disabled(); self.mark_targeted_nop_for_tails_borrow_disabled(); + self.restore_conditional_exception_for_iter_join_borrows(); self.compute_load_fast_start_depths(); // optimize_load_fast: after normalize_jumps self.optimize_load_fast_borrow(); @@ -434,7 +442,6 @@ impl CodeInfo { self.deoptimize_borrow_after_except_star_try_tail(); self.deoptimize_borrow_in_protected_method_call_after_terminal_except_tail(); self.deoptimize_borrow_after_terminal_except_before_with(); - self.deoptimize_borrow_in_async_finally_early_return_tail(); self.deoptimize_borrow_after_handler_resume_loop_tail(); self.deoptimize_borrow_after_protected_import(); self.deoptimize_borrow_before_import_after_join_store(); @@ -466,6 +473,7 @@ impl CodeInfo { symbol_table_index: _, in_conditional_block: _, in_final_with_cleanup_statement: _, + in_try_else_orelse: _, next_conditional_annotation_index: _, } = self; @@ -3245,6 +3253,63 @@ impl CodeInfo { (((packed >> 4) & 0xF) as usize, (packed & 0xF) as usize) } + fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some(Instruction::JumpBackwardNoInterrupt { .. }) + ) + }); + has_pop_except && jumps_to_target + } + + fn block_falls_through_after_store_fast_store_fast( + block: &Block, + target: BlockIdx, + ) -> bool { + block.next == target + && matches!( + block.instructions.last().and_then(|info| info.instr.real()), + Some(Instruction::StoreFastStoreFast { .. }) + ) + } + + fn block_ends_with_backward_jump(block: &Block) -> bool { + matches!( + block.instructions.last().and_then(|info| info.instr.real()), + Some( + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + } + + fn block_is_backward_jump_only(block: &Block) -> bool { + let mut real = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)); + matches!( + real.next(), + Some( + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) && real.next().is_none() + } + + fn block_is_resume_loop_latch(blocks: &[Block], target: BlockIdx) -> bool { + let block = &blocks[target.idx()]; + if block_ends_with_backward_jump(block) { + return true; + } + block.next != BlockIdx::NULL && block_is_backward_jump_only(&blocks[block.next.idx()]) + } + fn push_block( worklist: &mut Vec, visited: &mut [bool], @@ -3277,6 +3342,23 @@ impl CodeInfo { } } + let mut handler_resume_loop_latch = vec![false; self.blocks.len()]; + for block in &self.blocks { + let Some(target) = block.instructions.last().map(|info| info.target) else { + continue; + }; + if target != BlockIdx::NULL + && is_handler_resume_predecessor(block, target) + && block_is_resume_loop_latch(&self.blocks, target) + && self + .blocks + .iter() + .any(|pred| block_falls_through_after_store_fast_store_fast(pred, target)) + { + handler_resume_loop_latch[target.idx()] = true; + } + } + let mut visited = vec![false; self.blocks.len()]; let mut worklist = vec![BlockIdx(0)]; visited[0] = true; @@ -3523,7 +3605,7 @@ impl CodeInfo { } let block = &mut self.blocks[block_idx]; - if block.disable_load_fast_borrow { + if block.disable_load_fast_borrow || handler_resume_loop_latch[block_idx.idx()] { continue; } for (i, info) in block.instructions.iter_mut().enumerate() { @@ -4083,6 +4165,96 @@ impl CodeInfo { } } + fn restore_conditional_exception_for_iter_join_borrows(&mut self) { + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn is_conditional_predecessor_to(block: &Block, target: BlockIdx) -> bool { + block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + ) + ) + }) + } + + fn starts_with_for_iter(block: &Block) -> bool { + let reals: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + .take(3) + .collect(); + matches!( + reals.as_slice(), + [ + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::GetIter, + .. + ] | [ + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + Instruction::LoadAttr { .. }, + Instruction::GetIter, + .. + ] + ) + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (idx, block) in self.blocks.iter().enumerate() { + let block_idx = BlockIdx::new(idx as u32); + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(block_idx); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(block_idx); + } + } + } + + let mut to_restore = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + if !block.disable_load_fast_borrow + || block.cold + || block_is_exceptional(block) + || !starts_with_for_iter(block) + { + continue; + } + let target = BlockIdx::new(idx as u32); + let has_protected_predecessor = predecessors[idx] + .iter() + .any(|pred| block_has_protected_instructions(&self.blocks[pred.idx()])); + let has_conditional_normal_predecessor = predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + !pred_block.disable_load_fast_borrow + && !pred_block.cold + && !block_is_exceptional(pred_block) + && !block_has_protected_instructions(pred_block) + && is_conditional_predecessor_to(pred_block, target) + }); + if has_protected_predecessor && has_conditional_normal_predecessor { + to_restore.push(idx); + } + } + + for idx in to_restore { + self.blocks[idx].disable_load_fast_borrow = false; + } + } + fn deoptimize_borrow_for_handler_return_paths(&mut self) { for block in &mut self.blocks { let len = block.instructions.len(); @@ -4118,7 +4290,15 @@ impl CodeInfo { } fn deoptimize_block_borrows(block: &mut Block) { + let mut after_end_send = false; for info in &mut block.instructions { + if matches!(info.instr.real(), Some(Instruction::EndSend)) { + after_end_send = true; + continue; + } + if after_end_send { + continue; + } match info.instr.real() { Some(Instruction::LoadFastBorrow { .. }) => { info.instr = Instruction::LoadFast { @@ -4225,6 +4405,56 @@ impl CodeInfo { false } + fn normal_path_reaches_returning_handler( + blocks: &[Block], + start: BlockIdx, + returning_handler: &[bool], + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + if block.instructions.iter().any(|info| { + info.except_handler + .is_some_and(|handler| returning_handler[handler.handler_block.idx()]) + }) { + return true; + } + let Some(last) = block.instructions.last() else { + if block.next != BlockIdx::NULL { + stack.push(block.next); + } + continue; + }; + if last.instr.is_scope_exit() { + continue; + } + if last.instr.is_unconditional_jump() { + if last.target != BlockIdx::NULL { + stack.push(last.target); + } + continue; + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + stack.push(target); + } + } + if block.next != BlockIdx::NULL { + stack.push(block.next); + } + } + false + } + let mut returning_handler = vec![false; self.blocks.len()]; for block in &self.blocks { for handler in block @@ -4257,7 +4487,13 @@ impl CodeInfo { }) }) }); - prev_protected_return.then_some(seed) + (prev_protected_return + && !normal_path_reaches_returning_handler( + &self.blocks, + seed, + &returning_handler, + )) + .then_some(seed) }) .collect(); @@ -4466,6 +4702,37 @@ impl CodeInfo { }) } + fn block_has_push_exc_info(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PushExcInfo))) + } + + fn mark_handler_entries( + blocks: &[Block], + predecessors: &[Vec], + start: BlockIdx, + stop: BlockIdx, + entries: &mut [bool], + ) { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || cursor == stop || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + if block_has_push_exc_info(&blocks[cursor.idx()]) { + entries[cursor.idx()] = true; + continue; + } + for pred in &predecessors[cursor.idx()] { + stack.push(*pred); + } + } + } + fn predecessor_chain_has_check_exc_match( blocks: &[Block], predecessors: &[Vec], @@ -4579,9 +4846,19 @@ impl CodeInfo { )) } - let mut handler_resume_predecessors = vec![0usize; self.blocks.len()]; + let mut handler_resume_predecessors = vec![Vec::new(); self.blocks.len()]; let mut is_handler_resume_block = vec![false; self.blocks.len()]; let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } for (block_idx, block) in self.blocks.iter().enumerate() { if !is_handler_resume_jump_block(block) { continue; @@ -4592,25 +4869,34 @@ impl CodeInfo { .last() .expect("resume jump block has a last instruction") .target; - handler_resume_predecessors[target.idx()] += 1; - } - for (pred_idx, block) in self.blocks.iter().enumerate() { - if block.next != BlockIdx::NULL { - predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); - } - for info in &block.instructions { - if info.target != BlockIdx::NULL { - predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); - } - } + handler_resume_predecessors[target.idx()].push(BlockIdx::new(block_idx as u32)); } + let function_has_with_cleanup = self.blocks.iter().any(|block| { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::WithExceptStart))) + }); let mut visited = vec![false; self.blocks.len()]; - for (idx, &count) in handler_resume_predecessors.iter().enumerate() { - if count < 2 { + for (idx, resume_preds) in handler_resume_predecessors.iter().enumerate() { + if resume_preds.len() < 2 { continue; } let seed = BlockIdx::new(idx as u32); + let mut handler_entries = vec![false; self.blocks.len()]; + for pred in resume_preds { + mark_handler_entries( + &self.blocks, + &predecessors, + *pred, + seed, + &mut handler_entries, + ); + } + if handler_entries.iter().filter(|&&is_entry| is_entry).count() < 2 { + continue; + } if matches!( first_real_instr(&self.blocks[seed.idx()]), Some(Instruction::ForIter { .. }) @@ -4646,13 +4932,20 @@ impl CodeInfo { ) }) }); + let tail_enters_stacked_context = segment.iter().any(|block_idx| { + self.blocks[block_idx.idx()] + .start_depth + .is_some_and(|depth| depth > 1) + }); let has_simple_with_except_resume_tail = starts_with_fast_attr_call(&self.blocks[seed.idx()]) && has_exception_match_resume_predecessor(&self.blocks, &predecessors, seed) && predecessors[seed.idx()] .iter() .any(|pred| is_with_suppress_resume_jump_block(&self.blocks[pred.idx()])); - if !(starts_with_conditional_guard(&self.blocks[seed.idx()]) && has_complex_tail + if !(starts_with_conditional_guard(&self.blocks[seed.idx()]) + && has_complex_tail + && !(function_has_with_cleanup && tail_enters_stacked_context) || has_simple_with_except_resume_tail) { continue; @@ -4743,6 +5036,33 @@ impl CodeInfo { } } + fn block_has_simple_scope_exit(block: &Block) -> bool { + for info in &block.instructions { + match info.instr.real() { + Some(instr) if instr.is_scope_exit() => return true, + Some( + Instruction::Nop + | Instruction::NotTaken + | Instruction::LoadConst { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + | Instruction::StoreName { .. } + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastCheck { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::BuildTuple { .. }, + ) => {} + Some(_) => return false, + None => {} + } + } + false + } + fn normal_successors(block: &Block) -> Vec { let Some(last_info) = block.instructions.last() else { return (block.next != BlockIdx::NULL) @@ -4776,33 +5096,6 @@ impl CodeInfo { .collect() } - fn block_has_simple_scope_exit(block: &Block) -> bool { - for info in &block.instructions { - match info.instr.real() { - Some(instr) if instr.is_scope_exit() => return true, - Some( - Instruction::Nop - | Instruction::NotTaken - | Instruction::LoadConst { .. } - | Instruction::LoadSmallInt { .. } - | Instruction::StoreFast { .. } - | Instruction::StoreFastLoadFast { .. } - | Instruction::StoreFastStoreFast { .. } - | Instruction::StoreName { .. } - | Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastCheck { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::BuildTuple { .. }, - ) => {} - Some(_) => return false, - None => {} - } - } - false - } - fn is_return_value_through_with_exit(block: &Block) -> bool { let reals: Vec<_> = block .instructions @@ -4968,6 +5261,34 @@ impl CodeInfo { false } + fn linear_tail_has_with_setup(blocks: &[Block], start: BlockIdx) -> bool { + let mut cursor = start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + if block_is_exceptional(block) { + return false; + } + if block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::LoadSpecial { .. }))) + { + return true; + } + let Some(last_info) = block.instructions.last() else { + cursor = block.next; + continue; + }; + if last_info.instr.is_scope_exit() || last_info.instr.is_unconditional_jump() { + return false; + } + cursor = block.next; + } + false + } + fn is_with_suppress_resume_block(block: &Block) -> bool { let Some(last_info) = block.instructions.last() else { return false; @@ -4990,6 +5311,13 @@ impl CodeInfo { saw_pop_except && pop_top_after_pop_except >= 3 } + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + let mut named_cleanup_predecessors = vec![0usize; self.blocks.len()]; let mut named_cleanup_requires_deopt = vec![false; self.blocks.len()]; let mut has_with_suppress_resume_predecessor = vec![false; self.blocks.len()]; @@ -5022,12 +5350,21 @@ impl CodeInfo { ) { continue; } + if matches!( + last_info.instr.real(), + Some(Instruction::JumpBackward { .. }) + ) && block_has_protected_instructions(&self.blocks[last_info.target.idx()]) + { + continue; + } named_cleanup_predecessors[last_info.target.idx()] += 1; - if named_cleanup_has_conditional_raise_sibling( - &self.blocks, - BlockIdx::new(block_idx as u32), - last_info.target, - ) { + if linear_tail_has_with_setup(&self.blocks, last_info.target) + && named_cleanup_has_conditional_raise_sibling( + &self.blocks, + BlockIdx::new(block_idx as u32), + last_info.target, + ) + { named_cleanup_requires_deopt[last_info.target.idx()] = true; } } @@ -5691,6 +6028,27 @@ impl CodeInfo { has_pop_except && pop_top_count == 0 && jumps_to_target } + fn has_direct_handler_resume_predecessor(blocks: &[Block], target: BlockIdx) -> bool { + blocks.iter().any(|pred_block| { + let has_pop_except = pred_block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let jumps_to_target = pred_block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + has_pop_except && jumps_to_target + }) + } + fn handler_chain_resumes_normally(blocks: &[Block], handler: BlockIdx) -> bool { let mut visited = vec![false; blocks.len()]; let mut stack = vec![handler]; @@ -6116,10 +6474,11 @@ impl CodeInfo { ) }) .count(); - let has_handler_resume_predecessor = predecessors[seed.idx()].iter().any(|pred| { - is_handler_resume_block[pred.idx()] - || is_handler_resume_predecessor(&self.blocks[pred.idx()], seed) - }); + let has_handler_resume_predecessor = + predecessors[seed.idx()].iter().any(|pred| { + is_handler_resume_block[pred.idx()] + || is_handler_resume_predecessor(&self.blocks[pred.idx()], seed) + }) || has_direct_handler_resume_predecessor(&self.blocks, seed); if has_handler_resume_predecessor { continue; } @@ -6210,7 +6569,8 @@ impl CodeInfo { is_handler_resume_block[pred.idx()] || is_handler_resume_predecessor(&self.blocks[pred.idx()], block_idx) || is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()]) - }) { + }) || has_direct_handler_resume_predecessor(&self.blocks, block_idx) + { continue; } if block_has_protected_instructions(&self.blocks[block_idx.idx()]) { @@ -6881,6 +7241,7 @@ impl CodeInfo { let successor_block = &self.blocks[successor.idx()]; if block_is_exceptional(successor_block) || successor_block.cold + || successor_block.try_else_orelse_entry || !normal_tail_reaches_conditional(&self.blocks, successor) || normal_path_reaches_handler(&self.blocks, successor, handler) || predecessors[successor.idx()].iter().any(|pred| { @@ -7732,186 +8093,6 @@ impl CodeInfo { } } - fn deoptimize_borrow_in_async_finally_early_return_tail(&mut self) { - fn deoptimize_block_borrows(block: &mut Block) { - for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} - } - } - } - - fn block_has_send(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::Send { .. }))) - } - - fn block_has_get_anext(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::GetAnext))) - } - - fn block_has_return(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))) - } - - fn block_has_outer_jump_back(block: &Block, block_idx: BlockIdx) -> bool { - block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) && info.target != BlockIdx::NULL - && info.target != block_idx - }) - } - - fn normal_successors(block: &Block) -> Vec { - let Some(last) = block.instructions.last() else { - return (block.next != BlockIdx::NULL) - .then_some(block.next) - .into_iter() - .collect(); - }; - if last.instr.is_scope_exit() { - return Vec::new(); - } - if last.instr.is_unconditional_jump() { - return (last.target != BlockIdx::NULL) - .then_some(last.target) - .into_iter() - .collect(); - } - if let Some(cond_idx) = trailing_conditional_jump_index(block) { - let mut successors = Vec::with_capacity(2); - let target = block.instructions[cond_idx].target; - if target != BlockIdx::NULL { - successors.push(target); - } - if block.next != BlockIdx::NULL { - successors.push(block.next); - } - return successors; - } - (block.next != BlockIdx::NULL) - .then_some(block.next) - .into_iter() - .collect() - } - - fn has_outer_loop_after_send(blocks: &[Block], send_block: BlockIdx) -> bool { - let mut stack = blocks[send_block.idx()] - .instructions - .iter() - .filter(|info| matches!(info.instr.real(), Some(Instruction::Send { .. }))) - .filter_map(|info| (info.target != BlockIdx::NULL).then_some(info.target)) - .collect::>(); - let mut visited = vec![false; blocks.len()]; - while let Some(block_idx) = stack.pop() { - if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { - continue; - } - visited[block_idx.idx()] = true; - let block = &blocks[block_idx.idx()]; - if block.cold || block_is_exceptional(block) { - continue; - } - if block_has_outer_jump_back(block, block_idx) { - return true; - } - stack.extend(normal_successors(block)); - } - false - } - - let mut predecessors = vec![Vec::new(); self.blocks.len()]; - for (idx, block) in self.blocks.iter().enumerate() { - for successor in normal_successors(block) { - predecessors[successor.idx()].push(BlockIdx::new(idx as u32)); - } - } - let relevant_send_blocks: Vec<_> = self - .blocks - .iter() - .enumerate() - .map(|(idx, block)| { - block_has_send(block) - && !block_has_get_anext(block) - && !predecessors[idx] - .iter() - .any(|pred| block_has_get_anext(&self.blocks[pred.idx()])) - }) - .collect(); - if !relevant_send_blocks.iter().any(|has_send| *has_send) { - return; - } - if !relevant_send_blocks - .iter() - .enumerate() - .any(|(idx, has_send)| { - *has_send && has_outer_loop_after_send(&self.blocks, BlockIdx::new(idx as u32)) - }) - { - return; - } - - fn has_early_return_before_send(blocks: &[Block], relevant_send_blocks: &[bool]) -> bool { - let mut visited = vec![false; blocks.len()]; - let mut stack = vec![BlockIdx(0)]; - while let Some(block_idx) = stack.pop() { - if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { - continue; - } - visited[block_idx.idx()] = true; - let block = &blocks[block_idx.idx()]; - if block.cold || block_is_exceptional(block) { - continue; - } - if relevant_send_blocks[block_idx.idx()] { - continue; - } - if block_has_return(block) { - return true; - } - stack.extend(normal_successors(block)); - } - false - } - - let has_early_return = has_early_return_before_send(&self.blocks, &relevant_send_blocks); - if !has_early_return { - return; - } - - for (idx, block) in self.blocks.iter_mut().enumerate() { - if idx == 0 || block.cold || block_is_exceptional(block) { - continue; - } - deoptimize_block_borrows(block); - } - } - fn deoptimize_borrow_after_handler_resume_loop_tail(&mut self) { fn deoptimize_block_borrows(block: &mut Block) { for info in &mut block.instructions { @@ -7995,32 +8176,77 @@ impl CodeInfo { false } - fn protected_block_handler_resumes_to_self(blocks: &[Block], block_idx: BlockIdx) -> bool { - let block = &blocks[block_idx.idx()]; - block - .instructions - .iter() - .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) - .any(|handler| handler_chain_resumes_to_loop_header(blocks, handler, block_idx)) - } - - fn is_suppressing_with_resume_predecessor(block: &Block, target: BlockIdx) -> bool { - if !block - .instructions - .last() - .is_some_and(|info| info.target == target && info.instr.is_unconditional_jump()) - { - return false; - } - let mut reals = block - .instructions - .iter() - .rev() - .filter_map(|info| info.instr.real()); - matches!( - (reals.next(), reals.next(), reals.next(), reals.next(), reals.next()), - ( - Some(instr), + fn handler_chain_stores_and_resumes_to_loop_header( + blocks: &[Block], + handler: BlockIdx, + loop_header: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![(handler, false)]; + while let Some((block_idx, stores_local)) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let stores_local = stores_local || block_has_store_fast(block); + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + if has_pop_except + && stores_local + && block.instructions.iter().any(|info| { + info.target == loop_header + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) + { + return true; + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + stack.push((info.target, stores_local)); + } + } + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + stack.push((block.next, stores_local)); + } + } + false + } + + fn protected_block_handler_resumes_to_self(blocks: &[Block], block_idx: BlockIdx) -> bool { + let block = &blocks[block_idx.idx()]; + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| handler_chain_resumes_to_loop_header(blocks, handler, block_idx)) + } + + fn is_suppressing_with_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + if !block + .instructions + .last() + .is_some_and(|info| info.target == target && info.instr.is_unconditional_jump()) + { + return false; + } + let mut reals = block + .instructions + .iter() + .rev() + .filter_map(|info| info.instr.real()); + matches!( + (reals.next(), reals.next(), reals.next(), reals.next(), reals.next()), + ( + Some(instr), Some(Instruction::PopTop), Some(Instruction::PopTop), Some(Instruction::PopTop), @@ -8089,6 +8315,31 @@ impl CodeInfo { }) } + fn is_plain_protected_resume_successor( + blocks: &[Block], + predecessors: &[Vec], + target: BlockIdx, + ) -> bool { + let mut has_handler_resume_predecessor = false; + let mut has_normal_fallthrough_predecessor = false; + for pred in &predecessors[target.idx()] { + let pred_block = &blocks[pred.idx()]; + if is_handler_resume_predecessor(pred_block, target) { + has_handler_resume_predecessor = true; + continue; + } + if block_is_exceptional(pred_block) || pred_block.cold { + continue; + } + if next_nonempty_block(blocks, pred_block.next) == target { + has_normal_fallthrough_predecessor = true; + continue; + } + return false; + } + has_handler_resume_predecessor && has_normal_fallthrough_predecessor + } + fn predecessor_chain_has_protected_instructions( blocks: &[Block], predecessors: &[Vec], @@ -8112,6 +8363,50 @@ impl CodeInfo { false } + fn block_stores_local(block: &Block, local: usize) -> bool { + block + .instructions + .iter() + .any(|info| match info.instr.real() { + Some(Instruction::StoreFast { var_num }) => { + usize::from(var_num.get(info.arg)) == local + } + Some(Instruction::StoreFastLoadFast { var_nums }) => { + let (store_idx, _) = var_nums.get(info.arg).indexes(); + usize::from(store_idx) == local + } + Some(Instruction::StoreFastStoreFast { var_nums }) => { + let (left, right) = var_nums.get(info.arg).indexes(); + usize::from(left) == local || usize::from(right) == local + } + _ => false, + }) + } + + fn predecessor_chain_stores_local( + blocks: &[Block], + predecessors: &[Vec], + start: BlockIdx, + stop: BlockIdx, + local: usize, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || cursor == stop || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + if block_stores_local(&blocks[cursor.idx()], local) { + return true; + } + for pred in &predecessors[cursor.idx()] { + stack.push(*pred); + } + } + false + } + fn starts_with_bool_guard(block: &Block) -> bool { let infos: Vec<_> = block .instructions @@ -8299,6 +8594,25 @@ impl CodeInfo { }) } + fn any_storing_protected_handler_resumes_to_loop_header( + blocks: &[Block], + loop_header: BlockIdx, + ) -> bool { + blocks.iter().any(|block| { + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| { + handler_chain_stores_and_resumes_to_loop_header( + blocks, + handler, + loop_header, + ) + }) + }) + } + fn any_exception_cleanup_jumps_to(blocks: &[Block], target: BlockIdx) -> bool { blocks.iter().any(|block| { block.cold @@ -8587,7 +8901,6 @@ impl CodeInfo { } } } - let has_exception_match_handler = self.blocks.iter().any(|block| { block.instructions.iter().any(|info| { matches!( @@ -8602,7 +8915,6 @@ impl CodeInfo { .iter() .any(|info| matches!(info.instr.real(), Some(Instruction::CheckEgMatch))) }); - let suppressing_exception_match_method_tails: Vec<_> = self .blocks .iter() @@ -8826,12 +9138,23 @@ impl CodeInfo { return None; } let target = BlockIdx::new(idx as u32); + if is_plain_protected_resume_successor(&self.blocks, &predecessors, target) { + return None; + } predecessors[idx] .iter() .any(|pred| { let pred_block = &self.blocks[pred.idx()]; conditional_fallthrough_loop_header(&self.blocks, pred_block, target) .is_some_and(|loop_header| { + let storing_handler_resumes = + any_storing_protected_handler_resumes_to_loop_header( + &self.blocks, + loop_header, + ) || any_storing_protected_handler_resumes_to_loop_header( + &self.blocks, + *pred, + ); block_has_for_iter(&self.blocks[loop_header.idx()]) && predecessor_chain_has_protected_instructions( &self.blocks, @@ -8839,6 +9162,7 @@ impl CodeInfo { *pred, loop_header, ) + && storing_handler_resumes && (any_protected_handler_resumes_to_loop_header( &self.blocks, loop_header, @@ -8927,6 +9251,11 @@ impl CodeInfo { && (is_handler_resume_block[pred.idx()] || is_handler_resume_predecessor(pred_block, BlockIdx::new(idx as u32))) }); + let is_plain_protected_resume_successor = is_plain_protected_resume_successor( + &self.blocks, + &predecessors, + BlockIdx::new(idx as u32), + ); let has_suppressing_with_resume_predecessor = predecessors[idx].iter().any(|pred| { is_suppressing_with_resume_predecessor( @@ -8939,6 +9268,23 @@ impl CodeInfo { &predecessors, BlockIdx::new(idx as u32), ); + let bool_guard_local = has_bool_guard_tail + .then(|| first_fast_load_local(block)) + .flatten(); + let handler_resume_predecessor_stores_guard = + bool_guard_local.is_some_and(|local| { + predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + is_handler_resume_predecessor(pred_block, BlockIdx::new(idx as u32)) + && predecessor_chain_stores_local( + &self.blocks, + &predecessors, + *pred, + BlockIdx::new(idx as u32), + local, + ) + }) + }); let is_loop_header = has_jump_back_predecessor_to( &self.blocks, &predecessors, @@ -8958,6 +9304,8 @@ impl CodeInfo { && !has_suppressing_with_resume_predecessor) || (has_bool_guard_tail && has_handler_resume_predecessor + && !handler_resume_predecessor_stores_guard + && !is_plain_protected_resume_successor && !is_loop_header && tail_has_for_loop_back( &self.blocks, @@ -9147,7 +9495,7 @@ impl CodeInfo { visited[cursor.idx()] = true; for info in &blocks[cursor.idx()].instructions { match info.instr.real() { - Some(Instruction::ReturnValue) => return true, + Some(Instruction::ReturnValue) if !info.no_location_exit => return true, Some(Instruction::PopExcept) => after_pop_except = true, Some(_) if after_pop_except && is_conditional_jump(&info.instr) => { return false; @@ -9635,110 +9983,483 @@ impl CodeInfo { }) } - fn protected_store_subscr_operand_start(block: &Block) -> Option { - let store_idx = block.instructions.iter().position(|info| { - matches!(info.instr.real(), Some(Instruction::StoreSubscr)) - && info.except_handler.is_some() - })?; - - let mut stack_items = 0; - for start in (0..store_idx).rev() { - let produced = match block.instructions[start].instr.real() { - Some( - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadGlobal { .. } - | Instruction::LoadName { .. } - | Instruction::LoadDeref { .. }, - ) => 1, + fn block_has_generator_delegation(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), Some( - Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. }, - ) => 2, - _ => return None, - }; - stack_items += produced; - if stack_items >= 3 { - return Some(start); - } - } - None + Instruction::GetYieldFromIter + | Instruction::Send { .. } + | Instruction::YieldValue { .. } + ) + ) + }) } - fn block_has_attr_named(block: &Block, names: &IndexSet, attr: &str) -> bool { + fn block_has_external_backward_jump(block: &Block, in_segment: &[bool]) -> bool { block.instructions.iter().any(|info| { - let raw = u32::from(info.arg) as usize; + let target_is_external = + info.target != BlockIdx::NULL && !in_segment[info.target.idx()]; matches!( info.instr.real(), - Some(Instruction::LoadAttr { namei }) - if names[usize::try_from(namei.get(info.arg).name_idx()).unwrap()].as_str() - == attr - || names - .get_index(raw) - .is_some_and(|name| name.as_str() == attr) - || names - .get_index(raw >> 1) - .is_some_and(|name| name.as_str() == attr) - ) + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) && target_is_external }) } - fn block_has_protected_instructions(block: &Block) -> bool { + fn block_has_for_iter(block: &Block) -> bool { block .instructions .iter() - .any(|info| info.except_handler.is_some()) + .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) } - fn block_has_non_nop_real_instructions(block: &Block) -> bool { - block.instructions.iter().any(|info| { - info.instr - .real() - .is_some_and(|instr| !matches!(instr, Instruction::Nop)) + fn block_suffix_has_for_loop_back(block: &Block, blocks: &[Block], start: usize) -> bool { + block.instructions.iter().skip(start).any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) && info.target != BlockIdx::NULL + && block_has_for_iter(&blocks[info.target.idx()]) }) } - fn first_unprotected_suffix(block: &Block) -> Option { - let mut saw_protected = false; - for (idx, info) in block.instructions.iter().enumerate() { - if info.except_handler.is_some() { - saw_protected = true; - } else if saw_protected { - return Some(idx); - } - } - None + fn block_has_for_loop_back(block: &Block, blocks: &[Block]) -> bool { + block_suffix_has_for_loop_back(block, blocks, 0) } - fn collect_stored_fast_locals_until(block: &Block, end: usize) -> Vec { - let mut locals = Vec::new(); - for info in block.instructions.iter().take(end) { - match info.instr.real() { - Some(Instruction::StoreFast { var_num }) => { - locals.push(usize::from(var_num.get(info.arg))); - } - Some(Instruction::StoreFastLoadFast { var_nums }) => { - let (store_idx, _) = var_nums.get(info.arg).indexes(); - locals.push(usize::from(store_idx)); - } - Some(Instruction::StoreFastStoreFast { var_nums }) => { - let (idx1, idx2) = var_nums.get(info.arg).indexes(); - locals.push(usize::from(idx1)); - locals.push(usize::from(idx2)); - } - _ => {} - } - } - locals + fn block_has_backward_jump(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) } - fn borrows_any_local_from(block: &Block, locals: &[usize], start: usize) -> bool { - block - .instructions - .iter() - .skip(start) - .any(|info| match info.instr.real() { - Some(Instruction::LoadFastBorrow { var_num }) => { + fn normal_successors(block: &Block) -> Vec { + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + fn segment_reaches_external_backward_jump( + blocks: &[Block], + segment: &[(BlockIdx, usize)], + in_segment: &[bool], + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = segment + .iter() + .map(|(block_idx, _)| *block_idx) + .collect::>(); + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + if block_has_external_backward_jump(block, in_segment) { + return true; + } + stack.extend(normal_successors(block)); + } + false + } + + fn normal_path_reaches_for_loop_back(blocks: &[Block], start: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block_is_exceptional(block) || block.cold { + continue; + } + if block_has_for_loop_back(block, blocks) { + return true; + } + stack.extend(normal_successors(block)); + } + false + } + + fn segment_has_for_loop_back(blocks: &[Block], segment: &[(BlockIdx, usize)]) -> bool { + segment + .iter() + .any(|(block_idx, _)| block_has_for_loop_back(&blocks[block_idx.idx()], blocks)) + } + + fn segment_has_backward_jump(blocks: &[Block], segment: &[(BlockIdx, usize)]) -> bool { + segment + .iter() + .any(|(block_idx, _)| block_has_backward_jump(&blocks[block_idx.idx()])) + } + + fn segment_has_yield_value(blocks: &[Block], segment: &[(BlockIdx, usize)]) -> bool { + segment.iter().any(|(block_idx, start)| { + blocks[block_idx.idx()] + .instructions + .iter() + .skip(*start) + .any(|info| matches!(info.instr.real(), Some(Instruction::YieldValue { .. }))) + }) + } + + fn block_suffix_starts_with_builtin_any_all_fast_path( + block: &Block, + names: &IndexSet, + start: usize, + ) -> bool { + block + .instructions + .iter() + .skip(start) + .find_map(|info| { + let instr = info.instr.real()?; + if matches!(instr, Instruction::Nop | Instruction::NotTaken) { + return None; + } + Some((instr, info.arg)) + }) + .is_some_and(|(instr, arg)| { + matches!( + instr, + Instruction::LoadGlobal { namei } + if names[usize::try_from(namei.get(arg) >> 1).unwrap()].as_str() + == "any" + || names[usize::try_from(namei.get(arg) >> 1).unwrap()].as_str() + == "all" + ) + }) + } + + fn segment_starts_with_builtin_any_all_fast_path( + blocks: &[Block], + names: &IndexSet, + segment: &[(BlockIdx, usize)], + ) -> bool { + let Some((block_idx, start)) = segment.first() else { + return false; + }; + block_suffix_starts_with_builtin_any_all_fast_path( + &blocks[block_idx.idx()], + names, + *start, + ) + } + + fn segment_has_named_except_cleanup_predecessor( + blocks: &[Block], + predecessors: &[Vec], + segment: &[(BlockIdx, usize)], + ) -> bool { + segment.iter().any(|(block_idx, _)| { + predecessors[block_idx.idx()] + .iter() + .any(|pred| is_named_except_cleanup_normal_exit_block(&blocks[pred.idx()])) + }) + } + + fn protected_store_subscr_operand_start(block: &Block) -> Option { + let store_idx = block.instructions.iter().position(|info| { + matches!(info.instr.real(), Some(Instruction::StoreSubscr)) + && info.except_handler.is_some() + })?; + + let mut stack_items = 0; + for start in (0..store_idx).rev() { + let produced = match block.instructions[start].instr.real() { + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadGlobal { .. } + | Instruction::LoadName { .. } + | Instruction::LoadDeref { .. }, + ) => 1, + Some( + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. }, + ) => 2, + _ => return None, + }; + stack_items += produced; + if stack_items >= 3 { + return Some(start); + } + } + None + } + + fn block_has_attr_named(block: &Block, names: &IndexSet, attr: &str) -> bool { + block.instructions.iter().any(|info| { + let raw = u32::from(info.arg) as usize; + matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) + if names[usize::try_from(namei.get(info.arg).name_idx()).unwrap()].as_str() + == attr + || names + .get_index(raw) + .is_some_and(|name| name.as_str() == attr) + || names + .get_index(raw >> 1) + .is_some_and(|name| name.as_str() == attr) + ) + }) + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_non_nop_real_instructions(block: &Block) -> bool { + block.instructions.iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop)) + }) + } + + fn first_unprotected_suffix(block: &Block) -> Option { + let mut saw_protected = false; + for (idx, info) in block.instructions.iter().enumerate() { + if info.except_handler.is_some() { + saw_protected = true; + } else if saw_protected { + return Some(idx); + } + } + None + } + + fn collect_stored_fast_locals_until(block: &Block, end: usize) -> Vec { + let mut locals = Vec::new(); + for info in block.instructions.iter().take(end) { + collect_stored_fast_local(info, &mut locals); + } + locals + } + + fn collect_protected_stored_fast_locals_until(block: &Block, end: usize) -> Vec { + let mut locals = Vec::new(); + for info in block + .instructions + .iter() + .take(end) + .filter(|info| info.except_handler.is_some()) + { + collect_stored_fast_local(info, &mut locals); + } + locals + } + + fn collect_stored_fast_local(info: &InstructionInfo, locals: &mut Vec) { + match info.instr.real() { + Some(Instruction::StoreFast { var_num }) => { + locals.push(usize::from(var_num.get(info.arg))); + } + Some(Instruction::StoreFastLoadFast { var_nums }) => { + let (store_idx, _) = var_nums.get(info.arg).indexes(); + locals.push(usize::from(store_idx)); + } + Some(Instruction::StoreFastStoreFast { var_nums }) => { + let (idx1, idx2) = var_nums.get(info.arg).indexes(); + locals.push(usize::from(idx1)); + locals.push(usize::from(idx2)); + } + _ => {} + } + } + + fn collect_borrowed_stored_locals_in_segment( + blocks: &[Block], + segment: &[(BlockIdx, usize)], + stored_locals: &[usize], + ) -> Vec { + let mut borrowed = Vec::new(); + for (block_idx, start) in segment { + for info in blocks[block_idx.idx()].instructions.iter().skip(*start) { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { + let local = usize::from(var_num.get(info.arg)); + if stored_locals.contains(&local) { + borrowed.push(local); + } + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { var_nums }) => { + let (left, right) = var_nums.get(info.arg).indexes(); + for local in [usize::from(left), usize::from(right)] { + if stored_locals.contains(&local) { + borrowed.push(local); + } + } + } + _ => {} + } + } + } + borrowed.sort_unstable(); + borrowed.dedup(); + borrowed + } + + fn handler_chain_resumes_after_assigning_locals( + blocks: &[Block], + block: &Block, + in_segment: &[bool], + locals: &[usize], + ) -> bool { + if locals.is_empty() { + return false; + } + + let mut visited = vec![false; blocks.len()]; + let handler_blocks: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .collect(); + for handler_block in handler_blocks { + let mut cursor = handler_block; + let mut assigned = Vec::new(); + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let handler = &blocks[cursor.idx()]; + let mut after_pop_except = false; + for info in &handler.instructions { + if !after_pop_except { + collect_stored_fast_local(info, &mut assigned); + } + if matches!(info.instr.real(), Some(Instruction::PopExcept)) { + after_pop_except = true; + continue; + } + if after_pop_except + && info.target != BlockIdx::NULL + && in_segment[info.target.idx()] + && info.instr.is_unconditional_jump() + { + assigned.sort_unstable(); + assigned.dedup(); + if locals.iter().all(|local| assigned.contains(local)) { + return true; + } + } + } + cursor = handler.next; + } + } + false + } + + fn block_is_normal_cleanup_call(block: &Block, metadata: &CodeUnitMetadata) -> bool { + if !block_has_fallthrough(block) { + return false; + } + let reals: Vec<_> = block + .instructions + .iter() + .filter(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .collect(); + let [.., none1, none2, none3, call, pop_top] = reals.as_slice() else { + return false; + }; + is_load_const_none(none1, metadata) + && is_load_const_none(none2, metadata) + && is_load_const_none(none3, metadata) + && matches!(call.instr.real(), Some(Instruction::Call { .. })) + && matches!(pop_top.instr.real(), Some(Instruction::PopTop)) + && collect_stored_fast_locals_until(block, block.instructions.len()).is_empty() + } + + fn collect_protected_predecessor_stored_fast_locals( + blocks: &[Block], + predecessors: &[Vec], + start: BlockIdx, + ) -> Vec { + let mut locals = Vec::new(); + let mut visited = vec![false; blocks.len()]; + let mut stack = predecessors[start.idx()].clone(); + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block.cold + || block_is_exceptional(block) + || !block_has_protected_instructions(block) + { + continue; + } + locals.extend(collect_protected_stored_fast_locals_until( + block, + block.instructions.len(), + )); + stack.extend(predecessors[block_idx.idx()].iter().copied()); + } + locals.sort_unstable(); + locals.dedup(); + locals + } + + fn borrows_any_local_from(block: &Block, locals: &[usize], start: usize) -> bool { + block + .instructions + .iter() + .skip(start) + .any(|info| match info.instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { locals.contains(&usize::from(var_num.get(info.arg))) } Some(Instruction::LoadFastBorrowLoadFastBorrow { var_nums }) => { @@ -9955,56 +10676,274 @@ impl CodeInfo { }) } - fn predecessor_chain_contains_debug_four_guard( + fn predecessor_chain_contains_debug_four_guard( + blocks: &[Block], + predecessors: &[Vec], + block_idx: BlockIdx, + names: &IndexSet, + ) -> bool { + predecessors[block_idx.idx()].iter().any(|pred| { + contains_debug_four_guard(&blocks[pred.idx()], names) + || (marker_only_block(&blocks[pred.idx()]) + && predecessors[pred.idx()].iter().any(|pred_pred| { + contains_debug_four_guard(&blocks[pred_pred.idx()], names) + })) + }) + } + + fn collect_unprotected_tail_segment( + blocks: &[Block], + tail: BlockIdx, + ) -> (Vec<(BlockIdx, usize)>, Vec) { + let mut in_segment = vec![false; blocks.len()]; + let mut segment = Vec::new(); + let mut cursor = tail; + if cursor == BlockIdx::NULL + || block_is_exceptional(&blocks[cursor.idx()]) + || blocks[cursor.idx()].try_else_orelse_entry + || block_has_protected_instructions(&blocks[cursor.idx()]) + { + return (segment, in_segment); + } + while cursor != BlockIdx::NULL { + let segment_block = &blocks[cursor.idx()]; + if block_is_exceptional(segment_block) + || segment_block.try_else_orelse_entry + || block_has_protected_instructions(segment_block) + { + break; + } + segment.push((cursor, 0)); + in_segment[cursor.idx()] = true; + let last_real = segment_block + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()); + if last_real.is_some_and(|instr| { + instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() + }) { + break; + } + cursor = next_nonempty_block(blocks, segment_block.next); + } + (segment, in_segment) + } + + fn collect_unprotected_tail_region( + blocks: &[Block], + tail: BlockIdx, + ) -> (Vec<(BlockIdx, usize)>, Vec) { + let mut in_segment = vec![false; blocks.len()]; + let mut segment = Vec::new(); + let mut stack = vec![tail]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || in_segment[cursor.idx()] { + continue; + } + let segment_block = &blocks[cursor.idx()]; + if block_is_exceptional(segment_block) || segment_block.try_else_orelse_entry { + continue; + } + segment.push((cursor, 0)); + in_segment[cursor.idx()] = true; + let last_real = segment_block + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()); + if last_real.is_some_and(|instr| { + instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() + }) { + continue; + } + stack.extend(normal_successors(segment_block)); + } + (segment, in_segment) + } + + fn is_plain_protected_resume_successor( + blocks: &[Block], + predecessors: &[Vec], + target: BlockIdx, + ) -> bool { + if target == BlockIdx::NULL { + return false; + } + let mut has_handler_resume_predecessor = false; + let mut has_normal_fallthrough_predecessor = false; + for pred in &predecessors[target.idx()] { + let pred_block = &blocks[pred.idx()]; + if is_handler_resume_predecessor(pred_block, target) { + has_handler_resume_predecessor = true; + continue; + } + if block_is_exceptional(pred_block) || pred_block.cold { + continue; + } + if next_nonempty_block(blocks, pred_block.next) == target { + has_normal_fallthrough_predecessor = true; + continue; + } + return false; + } + has_handler_resume_predecessor && has_normal_fallthrough_predecessor + } + + fn protected_call_store_return_load_start(block: &Block) -> Option { + let mut saw_call = false; + let end = block + .instructions + .iter() + .position(|info| matches!(info.instr.real(), Some(Instruction::PushExcInfo))) + .unwrap_or(block.instructions.len()); + for idx in 0..end.saturating_sub(2) { + match block.instructions[idx].instr.real() { + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx, + ) => { + saw_call = true; + continue; + } + Some(Instruction::StoreFast { var_num }) if saw_call => { + let stored = usize::from(var_num.get(block.instructions[idx].arg)); + let loaded = match block.instructions[idx + 1].instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { + usize::from(var_num.get(block.instructions[idx + 1].arg)) + } + _ => continue, + }; + if stored == loaded + && matches!( + block.instructions[idx + 2].instr.real(), + Some(Instruction::ReturnValue) + ) + { + return Some(idx + 1); + } + } + _ => {} + } + } + None + } + + fn block_has_exception_match_trailer(block: &Block) -> bool { + let mut saw_push_exc_info = false; + for info in &block.instructions { + match info.instr.real() { + Some(Instruction::PushExcInfo) => { + saw_push_exc_info = true; + } + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + if saw_push_exc_info => + { + return true; + } + _ => {} + } + } + false + } + + fn block_starts_with_borrowed_local_return(block: &Block) -> Option<(usize, usize)> { + let mut reals = block.instructions.iter().enumerate().filter(|(_, info)| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + }); + let (load_idx, load) = reals.next()?; + let local = match load.instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => usize::from(var_num.get(load.arg)), + _ => return None, + }; + let (_, ret) = reals.next()?; + if matches!(ret.instr.real(), Some(Instruction::ReturnValue)) { + Some((load_idx, local)) + } else { + None + } + } + + fn block_is_exception_match_entry(block: &Block) -> bool { + block.cold + && block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + } + + fn cold_layout_tail_reaches_exception_match_entry( + blocks: &[Block], + start: BlockIdx, + ) -> bool { + let mut cursor = start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + if !block.cold { + return false; + } + if block_is_exception_match_entry(block) { + return true; + } + cursor = block.next; + } + false + } + + fn protected_call_store_local_predecessor(block: &Block, local: usize) -> bool { + if !block.disable_load_fast_borrow { + return false; + } + let mut saw_call = false; + for info in &block.instructions { + match info.instr.real() { + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx, + ) => { + saw_call = true; + } + Some(Instruction::StoreFast { var_num }) + if saw_call && usize::from(var_num.get(info.arg)) == local => + { + return true; + } + _ => {} + } + } + false + } + + fn predecessor_chain_has_protected_call_store_local( blocks: &[Block], predecessors: &[Vec], - block_idx: BlockIdx, - names: &IndexSet, + target: BlockIdx, + local: usize, ) -> bool { - predecessors[block_idx.idx()].iter().any(|pred| { - contains_debug_four_guard(&blocks[pred.idx()], names) - || (marker_only_block(&blocks[pred.idx()]) - && predecessors[pred.idx()].iter().any(|pred_pred| { - contains_debug_four_guard(&blocks[pred_pred.idx()], names) - })) - }) - } - - fn collect_unprotected_tail_segment( - blocks: &[Block], - tail: BlockIdx, - ) -> (Vec<(BlockIdx, usize)>, Vec) { - let mut in_segment = vec![false; blocks.len()]; - let mut segment = Vec::new(); - let mut cursor = tail; - if cursor == BlockIdx::NULL - || block_is_exceptional(&blocks[cursor.idx()]) - || block_has_protected_instructions(&blocks[cursor.idx()]) - { - return (segment, in_segment); - } - while cursor != BlockIdx::NULL { - let segment_block = &blocks[cursor.idx()]; - if block_is_exceptional(segment_block) - || block_has_protected_instructions(segment_block) - { - break; + let mut visited = vec![false; blocks.len()]; + let mut stack = predecessors[target.idx()].clone(); + while let Some(pred) = stack.pop() { + if pred == BlockIdx::NULL || visited[pred.idx()] { + continue; } - segment.push((cursor, 0)); - in_segment[cursor.idx()] = true; - let last_real = segment_block - .instructions - .iter() - .rev() - .find_map(|info| info.instr.real()); - if last_real.is_some_and(|instr| { - instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() - }) { - break; + visited[pred.idx()] = true; + let block = &blocks[pred.idx()]; + if protected_call_store_local_predecessor(block, local) { + return true; + } + if marker_only_block(block) { + stack.extend(predecessors[pred.idx()].iter().copied()); } - cursor = next_nonempty_block(blocks, segment_block.next); } - (segment, in_segment) + false } let mut predecessors = vec![Vec::new(); self.blocks.len()]; @@ -10037,6 +10976,29 @@ impl CodeInfo { } } } + for (block_idx, block) in self.blocks.iter().enumerate() { + if block_has_exception_match_trailer(block) + && let Some(start) = protected_call_store_return_load_start(block) + { + to_deopt.push((BlockIdx::new(block_idx as u32), start)); + } + } + for (block_idx, block) in self.blocks.iter().enumerate() { + if !cold_layout_tail_reaches_exception_match_entry(&self.blocks, block.next) { + continue; + } + let Some((start, local)) = block_starts_with_borrowed_local_return(block) else { + continue; + }; + if predecessor_chain_has_protected_call_store_local( + &self.blocks, + &predecessors, + BlockIdx::new(block_idx as u32), + local, + ) { + to_deopt.push((BlockIdx::new(block_idx as u32), start)); + } + } for (block_idx, block) in self.blocks.iter().enumerate() { if block_is_exceptional(block) || !block @@ -10053,6 +11015,7 @@ impl CodeInfo { ) ) }) + || block_has_generator_delegation(block) || !block_has_exception_match_handler(&self.blocks, block) { continue; @@ -10062,6 +11025,15 @@ impl CodeInfo { continue; } if let Some((start, stored_locals)) = protected_store_bool_guard_start(block) { + if block_suffix_has_for_loop_back(block, &self.blocks, start) + && !block_suffix_starts_with_builtin_any_all_fast_path( + block, + &self.metadata.names, + start, + ) + { + continue; + } if !handler_chain_exits_loop_after_pop_except(&self.blocks, block) { continue; } @@ -10082,6 +11054,9 @@ impl CodeInfo { && block_has_tail_deopt_trigger_from(work_block, 0) && block_is_simple_exit_branch(exit_block) { + if normal_path_reaches_for_loop_back(&self.blocks, work) { + continue; + } to_deopt.push((BlockIdx::new(block_idx as u32), start)); if borrows_any_local_from(work_block, &stored_locals, 0) { to_deopt.push((work, 0)); @@ -10092,7 +11067,19 @@ impl CodeInfo { } let same_block_tail_start = first_unprotected_suffix(block); if let Some(start) = same_block_tail_start { - let stored_locals = collect_stored_fast_locals_until(block, start); + if block.try_else_orelse_entry { + continue; + } + if block_suffix_has_for_loop_back(block, &self.blocks, start) + && !block_suffix_starts_with_builtin_any_all_fast_path( + block, + &self.metadata.names, + start, + ) + { + continue; + } + let stored_locals = collect_protected_stored_fast_locals_until(block, start); let handler_has_explicit_raise = handler_chain_has_explicit_raise(&self.blocks, block); if stored_locals.is_empty() @@ -10116,6 +11103,9 @@ impl CodeInfo { && block_has_tail_deopt_trigger_from(work_block, 0) && block_is_simple_exit_branch(exit_block) { + if normal_path_reaches_for_loop_back(&self.blocks, work) { + continue; + } to_deopt.push((BlockIdx::new(block_idx as u32), start)); if borrows_any_local_from(work_block, &stored_locals, 0) { to_deopt.push((work, 0)); @@ -10127,7 +11117,21 @@ impl CodeInfo { } let tail = next_nonempty_block(&self.blocks, block.next); let (segment, in_segment) = collect_unprotected_tail_segment(&self.blocks, tail); - if handler_chain_has_nested_exception_match(&self.blocks, block) + let handler_has_nested_exception_match = + handler_chain_has_nested_exception_match(&self.blocks, block); + let linear_segment_reaches_external_backward_jump = + segment_reaches_external_backward_jump(&self.blocks, &segment, &in_segment); + let linear_segment_has_for_loop_back = + segment_has_for_loop_back(&self.blocks, &segment); + let linear_segment_has_backward_jump = + segment_has_backward_jump(&self.blocks, &segment); + let linear_segment_starts_with_builtin_any_all_fast_path = + segment_starts_with_builtin_any_all_fast_path( + &self.blocks, + &self.metadata.names, + &segment, + ); + if handler_has_nested_exception_match && handler_chain_can_resume_to_segment(&self.blocks, block, &in_segment) { for (block_idx, start) in &segment { @@ -10138,14 +11142,52 @@ impl CodeInfo { } } } - let stored_locals = collect_stored_fast_locals_until(block, block.instructions.len()); + let segment_has_yield = segment_has_yield_value(&self.blocks, &segment); + let mut stored_locals = + collect_protected_stored_fast_locals_until(block, block.instructions.len()); + if stored_locals.is_empty() && segment_has_yield { + stored_locals = collect_protected_predecessor_stored_fast_locals( + &self.blocks, + &predecessors, + BlockIdx::new(block_idx as u32), + ); + } if stored_locals.is_empty() { continue; } + let borrowed_stored_locals = + collect_borrowed_stored_locals_in_segment(&self.blocks, &segment, &stored_locals); + if !handler_has_nested_exception_match + && handler_chain_resumes_after_assigning_locals( + &self.blocks, + block, + &in_segment, + &borrowed_stored_locals, + ) + { + continue; + } let handler_has_explicit_raise = handler_chain_has_explicit_raise(&self.blocks, block); + if !handler_has_explicit_raise + && segment_has_yield + && segment.iter().any(|(block_idx, start)| { + block_has_tail_deopt_trigger_from(&self.blocks[block_idx.idx()], *start) + }) + && segment.iter().any(|(block_idx, start)| { + borrows_any_local_from(&self.blocks[block_idx.idx()], &stored_locals, *start) + }) + { + let (yield_tail_region, _) = collect_unprotected_tail_region(&self.blocks, tail); + for (block_idx, start) in yield_tail_region { + to_deopt.push((block_idx, start)); + } + continue; + } if tail != BlockIdx::NULL && !block_is_exceptional(&self.blocks[tail.idx()]) + && !self.blocks[tail.idx()].try_else_orelse_entry && !block_has_protected_instructions(&self.blocks[tail.idx()]) + && !is_plain_protected_resume_successor(&self.blocks, &predecessors, tail) && starts_with_borrowed_local_bool_guard(&self.blocks[tail.idx()], &stored_locals) && !handler_has_explicit_raise { @@ -10164,6 +11206,9 @@ impl CodeInfo { && block_has_tail_deopt_trigger_from(work_block, 0) && block_is_simple_exit_branch(exit_block) { + if normal_path_reaches_for_loop_back(&self.blocks, work) { + continue; + } to_deopt.push((tail, 0)); if borrows_any_local_from(work_block, &stored_locals, 0) { to_deopt.push((work, 0)); @@ -10185,7 +11230,17 @@ impl CodeInfo { || !segment.iter().any(|(block_idx, start)| { block_has_tail_deopt_trigger_from(&self.blocks[block_idx.idx()], *start) }) - || handler_chain_can_resume_to_segment(&self.blocks, block, &in_segment) + || segment_has_named_except_cleanup_predecessor( + &self.blocks, + &predecessors, + &segment, + ) + || (linear_segment_has_for_loop_back + && !linear_segment_starts_with_builtin_any_all_fast_path) + || (handler_chain_can_resume_to_segment(&self.blocks, block, &in_segment) + && (linear_segment_reaches_external_backward_jump + || (!handler_has_nested_exception_match + && linear_segment_has_backward_jump))) || segment.iter().any(|(block_idx, _)| { predecessors[block_idx.idx()].iter().any(|pred| { let pred_block = &self.blocks[pred.idx()]; @@ -10202,10 +11257,33 @@ impl CodeInfo { { continue; } - for (block_idx, start) in segment { - if predecessors[block_idx.idx()] - .iter() - .any(|pred| is_handler_resume_predecessor(&self.blocks[pred.idx()], block_idx)) + let (deopt_segment, deopt_in_segment) = if handler_has_nested_exception_match + && !linear_segment_reaches_external_backward_jump + { + collect_unprotected_tail_region(&self.blocks, tail) + } else { + (segment, in_segment) + }; + if segment_has_for_loop_back(&self.blocks, &deopt_segment) + && !segment_starts_with_builtin_any_all_fast_path( + &self.blocks, + &self.metadata.names, + &deopt_segment, + ) + { + continue; + } + let deopt_segment_reaches_external_backward_jump = + segment_reaches_external_backward_jump( + &self.blocks, + &deopt_segment, + &deopt_in_segment, + ); + for (block_idx, start) in deopt_segment { + if deopt_segment_reaches_external_backward_jump + && predecessors[block_idx.idx()].iter().any(|pred| { + is_handler_resume_predecessor(&self.blocks[pred.idx()], block_idx) + }) { continue; } @@ -10287,10 +11365,41 @@ impl CodeInfo { ) }); if tail_jumps_back_to_target { + if normal_path_reaches_for_loop_back(&self.blocks, tail) + && !block_suffix_starts_with_builtin_any_all_fast_path( + &self.blocks[tail.idx()], + &self.metadata.names, + 0, + ) + { + continue; + } to_deopt.push((tail, 0)); } } + for block in &self.blocks { + if block.cold + || block_is_exceptional(block) + || !block_is_normal_cleanup_call(block, &self.metadata) + { + continue; + } + let tail = next_nonempty_block(&self.blocks, block.next); + let (region, _) = collect_unprotected_tail_region(&self.blocks, tail); + if region.is_empty() + || !segment_has_yield_value(&self.blocks, ®ion) + || !region.iter().any(|(block_idx, start)| { + block_has_tail_deopt_trigger_from(&self.blocks[block_idx.idx()], *start) + }) + { + continue; + } + for (block_idx, start) in region { + to_deopt.push((block_idx, start)); + } + } + to_deopt.sort_by_key(|(idx, start)| (idx.idx(), *start)); let mut merged: Vec<(BlockIdx, usize)> = Vec::new(); for (idx, start) in to_deopt { @@ -10448,16 +11557,37 @@ impl CodeInfo { } for (block_idx, block) in self.blocks.iter_mut().enumerate() { - if predecessor_count[block_idx] < 2 - || !block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::ImportName { .. }))) - { + if predecessor_count[block_idx] < 2 { continue; } let len = block.instructions.len(); + let first_import_from = block + .instructions + .iter() + .position(|info| matches!(info.instr.real(), Some(Instruction::ImportFrom { .. }))); + if let Some(first_import_from) = first_import_from { + for idx in 0..first_import_from { + if matches!( + block.instructions[idx].instr.real(), + Some(Instruction::LoadFastBorrow { .. }) + ) { + block.instructions[idx].instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + } + } + + if !block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ImportName { .. }))) + { + continue; + } + for idx in 0..len.saturating_sub(1) { if !matches!( block.instructions[idx].instr.real(), @@ -10926,6 +12056,11 @@ impl CodeInfo { continue; }; if let DeoptKind::ReturnIter { tail_start_idx } = deopt_kind { + if !(reachable_from_protected_predecessor[block_idx.idx()] + || is_protected_source[block_idx.idx()]) + { + continue; + } let tail_instr_idx = real_instrs .get(tail_start_idx) .map_or(block_instr_len, |(instr_idx, _)| *instr_idx); @@ -11833,6 +12968,13 @@ impl CodeInfo { self.debug_block_dump(), )); + inline_small_fast_return_blocks(&mut self.blocks); + inline_unprotected_tuple_genexpr_assignment_return_blocks(&mut self.blocks); + trace.push(( + "after_inline_small_fast_return_blocks".to_owned(), + self.debug_block_dump(), + )); + duplicate_end_returns(&mut self.blocks, &self.metadata); duplicate_fallthrough_jump_back_targets(&mut self.blocks); duplicate_shared_jump_back_targets(&mut self.blocks); @@ -11965,11 +13107,6 @@ impl CodeInfo { "after_deoptimize_borrow_after_terminal_except_before_with".to_owned(), self.debug_block_dump(), )); - self.deoptimize_borrow_in_async_finally_early_return_tail(); - trace.push(( - "after_deoptimize_borrow_in_async_finally_early_return_tail".to_owned(), - self.debug_block_dump(), - )); self.deoptimize_borrow_after_handler_resume_loop_tail(); trace.push(( "after_deoptimize_borrow_after_handler_resume_loop_tail".to_owned(), @@ -12459,6 +13596,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }); jump_block.next = blocks[cold_idx.idx()].next; blocks[cold_idx.idx()].next = jump_block_idx; @@ -12677,6 +13815,17 @@ fn threaded_jump_instr( }) } +fn can_thread_conditional_through_forward_nointerrupt( + source: AnyInstruction, + target_pos: u32, + final_target_pos: u32, +) -> bool { + matches!( + source.real(), + Some(Instruction::PopJumpIfNone { .. } | Instruction::PopJumpIfNotNone { .. }) + ) && final_target_pos > target_pos +} + fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { let mut changed = true; while changed { @@ -12716,6 +13865,23 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { .last() .is_some_and(|instr| instr.instr.is_scope_exit()); if next_is_scope_exit { + let target_pos = block_order.get(target.idx()).copied().unwrap_or(u32::MAX); + let target_first_jump = blocks[target.idx()].instructions.first().copied(); + let threads_match_success_jump_to_forward_nointerrupt = + matches!(ins.instr.real(), Some(Instruction::PopJumpIfNone { .. })) + && target_first_jump + .filter(|target_ins| target_ins.instr.is_unconditional_jump()) + .filter(|target_ins| target_ins.target != BlockIdx::NULL) + .is_some_and(|target_ins| { + let final_target_pos = block_order + .get(target_ins.target.idx()) + .copied() + .unwrap_or(u32::MAX); + jump_thread_kind(target_ins.instr) + == Some(JumpThreadKind::NoInterrupt) + && target_ins.match_success_jump + && final_target_pos > target_pos + }); let next_raises = blocks[next.idx()].instructions.iter().any(|instr| { matches!(instr.instr.real(), Some(Instruction::RaiseVarargs { .. })) }); @@ -12728,7 +13894,10 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { final_target == BlockIdx(bi as u32) || comes_before(blocks, final_target, BlockIdx(bi as u32)) }); - if !(block_is_protected(&blocks[bi]) && next_raises && target_is_loop_backedge) + if !(threads_match_success_jump_to_forward_nointerrupt + || block_is_protected(&blocks[bi]) + && next_raises + && target_is_loop_backedge) { continue; } @@ -12822,17 +13991,30 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { // the line-anchored continue/break jump that follows. continue; } - if conditional - && !matches!( - jump_thread_kind(target_ins.instr), - Some(JumpThreadKind::Plain) - ) - { - continue; - } - let Some(threaded_instr) = - threaded_jump_instr(ins.instr, target_ins.instr, conditional) - else { + let Some(threaded_instr) = (if conditional { + match jump_thread_kind(target_ins.instr) { + Some(JumpThreadKind::Plain) => Some(ins.instr), + Some(JumpThreadKind::NoInterrupt) + if target_ins.match_success_jump + && can_thread_conditional_through_forward_nointerrupt( + ins.instr, + target_pos, + final_target_pos, + ) => + { + // A forward JUMP_NO_INTERRUPT assembles to the same + // JUMP_FORWARD opcode as CPython's plain match + // success jump. Limit this to None-check match + // success tests; boolean finally-cleanup jumps need + // the stronger no-interrupt shape for later CFG + // cleanup decisions. + Some(ins.instr) + } + _ => None, + } + } else { + threaded_jump_instr(ins.instr, target_ins.instr, false) + }) else { continue; }; if ins.target == final_target { @@ -12929,6 +14111,7 @@ fn normalize_jumps(blocks: &mut Vec) { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }; blocks[idx].instructions.push(not_taken); } else { @@ -12966,6 +14149,7 @@ fn normalize_jumps(blocks: &mut Vec) { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }); new_block.instructions.push(InstructionInfo { instr: PseudoOpcode::Jump.into(), @@ -12982,6 +14166,7 @@ fn normalize_jumps(blocks: &mut Vec) { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }); new_block.next = old_next; @@ -13792,6 +14977,7 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }); } @@ -13824,6 +15010,7 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, }, ); } @@ -17227,6 +18414,67 @@ fn inline_small_fast_return_blocks(blocks: &mut [Block]) { } } +fn is_fast_store_load_return_block(block: &Block) -> bool { + let [store, load, ret] = block.instructions.as_slice() else { + return false; + }; + let stored = match store.instr.real() { + Some(Instruction::StoreFast { var_num }) => usize::from(var_num.get(store.arg)), + _ => return false, + }; + let loaded = match load.instr.real() { + Some(Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num }) => { + usize::from(var_num.get(load.arg)) + } + _ => return false, + }; + stored == loaded && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) +} + +fn inline_unprotected_tuple_genexpr_assignment_return_blocks(blocks: &mut [Block]) { + for block_idx in 0..blocks.len() { + if block_is_protected(&blocks[block_idx]) { + continue; + } + + let Some(jump_idx) = blocks[block_idx].instructions.len().checked_sub(1) else { + continue; + }; + let jump = blocks[block_idx].instructions[jump_idx]; + if !jump.instr.is_unconditional_jump() || jump.target == BlockIdx::NULL { + continue; + } + + let previous_is_list_to_tuple = blocks[block_idx].instructions[..jump_idx] + .iter() + .rev() + .find_map(|info| match info.instr.real() { + Some(Instruction::CallIntrinsic1 { func }) => { + Some(func.get(info.arg) == IntrinsicFunction1::ListToTuple) + } + Some(Instruction::Nop | Instruction::NotTaken) => None, + Some(_) => Some(false), + None => None, + }) + .unwrap_or(false); + if !previous_is_list_to_tuple { + continue; + } + + let target = next_nonempty_block(blocks, jump.target); + if target == BlockIdx::NULL || !is_fast_store_load_return_block(&blocks[target.idx()]) { + continue; + } + + let mut cloned = blocks[target.idx()].instructions.clone(); + if let Some(first) = cloned.first_mut() { + overwrite_location(first, jump.location, jump.end_location); + } + blocks[block_idx].instructions.pop(); + blocks[block_idx].instructions.extend(cloned); + } +} + fn inline_with_suppress_return_blocks(blocks: &mut [Block]) { fn has_with_suppress_prefix(block: &Block, jump_idx: usize) -> bool { let tail: Vec<_> = block.instructions[..jump_idx] @@ -17317,6 +18565,22 @@ fn duplicate_named_except_cleanup_returns(blocks: &mut Vec, metadata: &Co continue; } + let target_lineno = blocks[target.idx()] + .instructions + .first() + .map(instruction_lineno) + .unwrap_or(-1); + let layout_pred_lineno = blocks[layout_pred.idx()] + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()) + .map(instruction_lineno) + .unwrap_or(-1); + if target_lineno > 0 && layout_pred_lineno > 0 && target_lineno != layout_pred_lineno { + continue; + } + for block_idx in 0..blocks.len() { if block_idx == target.idx() { continue; @@ -17583,7 +18847,11 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) PseudoInstruction::SetupCleanup { .. } | PseudoInstruction::SetupFinally { .. } | PseudoInstruction::SetupWith { .. } => { + let preserve_block_start_no_location_nop = + info.preserve_block_start_no_location_nop; set_to_nop(info); + info.preserve_block_start_no_location_nop = + preserve_block_start_no_location_nop; } // PopBlock in reachable blocks is converted to NOP by // label_exception_targets. Dead blocks may still have them. @@ -17695,6 +18963,7 @@ mod tests { folded_operand_nop: false, no_location_exit: false, preserve_block_start_no_location_nop: false, + match_success_jump: false, } } diff --git a/crates/codegen/src/preprocess.rs b/crates/codegen/src/preprocess.rs index 63246ff840..ae2e65bf3f 100644 --- a/crates/codegen/src/preprocess.rs +++ b/crates/codegen/src/preprocess.rs @@ -184,7 +184,7 @@ fn simple_format_arg_parse( let width = parse_digits(chars, pos, &mut ch)?; let precision = if ch == '.' { ch = next_char(chars, pos)?; - parse_digits(chars, pos, &mut ch)? + Some(parse_digits(chars, pos, &mut ch)?.unwrap_or(0)) } else { None }; diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 92f53d3aac..4e762ff8b1 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -632,6 +632,29 @@ impl SymbolTableAnalyzer { drop_class_free(symbol_table, &mut newfree); } + // CPython update_symbols(..., classflag): after class implicit frees + // are dropped, a class block, or an annotation/type-params block that + // can see a class scope, records existing child-free names with + // DEF_FREE_CLASS. This preserves the current scope's own lookup kind + // (for example GLOBAL_IMPLICIT via __classdict__) while still making + // the name available as a closure cell for nested children such as + // generator expressions. + if symbol_table.typ == CompilerScope::Class { + for name in &newfree { + if let Some(symbol) = symbol_table.symbols.get_mut(name) { + symbol.flags.insert(SymbolFlags::FREE_CLASS); + } + } + } else if symbol_table.can_see_class_scope { + for name in &newfree { + if let Some(symbol) = symbol_table.symbols.get_mut(name) + && !symbol.is_local() + { + symbol.flags.insert(SymbolFlags::FREE_CLASS); + } + } + } + Ok(newfree) } @@ -872,7 +895,10 @@ impl SymbolTableAnalyzer { return self.found_in_inner_scope(&st.sub_tables, name, st_typ); } let sym = st.symbols.get(name)?; - if sym.scope == SymbolScope::Free || sym.flags.contains(SymbolFlags::FREE_CLASS) { + if sym.scope == SymbolScope::Free + || (sym.flags.contains(SymbolFlags::FREE_CLASS) + && !matches!(st_typ, CompilerScope::Module)) + { if st_typ == CompilerScope::Class && name != "__class__" { None } else { From b3b8329c4df2e04d513b9e5e33dcfb5827387668 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 12 May 2026 01:30:12 +0900 Subject: [PATCH 74/76] Skip test_stack_overflow under -u cpu CFG cleanup passes exceed the 60-second worker timeout when compiling 200k chained if statements. --- Lib/test/test_compile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index fd1743e670..93d2601b12 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1043,6 +1043,7 @@ def test_path_like_objects(self): # bpo-31113: Stack overflow when compile a long sequence of # complex statements. @support.requires_resource('cpu') + @unittest.skip("TODO: RUSTPYTHON; CFG cleanup is too slow for 200k statements") def test_stack_overflow(self): # Android test devices have less memory. size = 100_000 if sys.platform == "android" else 200_000 From 7790e548362b8cd5d19dcfb7245b0c5a21796f7d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 12 May 2026 10:12:49 +0900 Subject: [PATCH 75/76] Fix CI lint and clippy errors Add cfws/CFWS/lslpp/nointerrupt/preds to cspell dictionary. Replace .map(...).unwrap_or(-1) with .map_or(-1, ...) in ir.rs. Drop expectedFailure on test_with.NestedWith.testExceptionLocation. --- .cspell.dict/cpython.txt | 5 +++++ Lib/test/test_with.py | 1 - crates/codegen/src/ir.rs | 6 ++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 4530ac9546..fe217a4b47 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -31,6 +31,8 @@ cellvar cellvars ceval cfield +cfws +CFWS CLASSDEREF classdict cmpop @@ -118,6 +120,7 @@ loadfast localsplus localspluskinds Lshift +lslpp lsprof MAXBLOCKS maxdepth @@ -137,6 +140,7 @@ nfrees nkwargs nkwelts nlocalsplus +nointerrupt Nondescriptor noninteger nops @@ -159,6 +163,7 @@ platstdlib posonlyarg posonlyargs prec +preds preinitialized pybuilddir pycore diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index 68ee372508..f16611b29a 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -807,7 +807,6 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' def testExceptionLocation(self): # The location of an exception raised from # __init__, __enter__ or __exit__ of a context diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d01e9c7d3c..a4f2016908 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -18568,15 +18568,13 @@ fn duplicate_named_except_cleanup_returns(blocks: &mut Vec, metadata: &Co let target_lineno = blocks[target.idx()] .instructions .first() - .map(instruction_lineno) - .unwrap_or(-1); + .map_or(-1, instruction_lineno); let layout_pred_lineno = blocks[layout_pred.idx()] .instructions .iter() .rev() .find(|info| info.instr.real().is_some()) - .map(instruction_lineno) - .unwrap_or(-1); + .map_or(-1, instruction_lineno); if target_lineno > 0 && layout_pred_lineno > 0 && target_lineno != layout_pred_lineno { continue; } From 19bf583000be7e436e098463beeb0b5a43c51b89 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 12 May 2026 20:40:40 +0900 Subject: [PATCH 76/76] Speed up CFG cleanup for large compile inputs --- Lib/test/test_compile.py | 1 - crates/codegen/src/ir.rs | 75 +++++++++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 93d2601b12..fd1743e670 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1043,7 +1043,6 @@ def test_path_like_objects(self): # bpo-31113: Stack overflow when compile a long sequence of # complex statements. @support.requires_resource('cpu') - @unittest.skip("TODO: RUSTPYTHON; CFG cleanup is too slow for 200k statements") def test_stack_overflow(self): # Android test devices have less memory. size = 100_000 if sys.platform == "android" else 200_000 diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a4f2016908..fc24eb01de 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -15996,16 +15996,28 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { fn has_other_conditional_predecessor_to( blocks: &[Block], + conditional_target_counts: &[usize], target: BlockIdx, current: BlockIdx, ) -> bool { - blocks.iter().enumerate().any(|(idx, block)| { - BlockIdx::new(idx as u32) != current - && block - .instructions - .iter() - .any(|info| info.target == target && is_conditional_jump(&info.instr)) - }) + let current_targets = blocks[current.idx()] + .instructions + .iter() + .filter(|info| info.target == target && is_conditional_jump(&info.instr)) + .count(); + conditional_target_counts[target.idx()] > current_targets + } + + let has_exceptional_lineno_sources = blocks + .iter() + .any(|block| block.cold || block_is_exceptional(block) || block_is_protected(block)); + let mut conditional_target_counts = vec![0usize; blocks.len()]; + for block in blocks.iter() { + for info in &block.instructions { + if info.target != BlockIdx::NULL && is_conditional_jump(&info.instr) { + conditional_target_counts[info.target.idx()] += 1; + } + } } let mut current = BlockIdx(0); @@ -16083,14 +16095,20 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; } - if is_generic_false_path_reorder + if has_exceptional_lineno_sources + && is_generic_false_path_reorder && has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&last)) { current = next; continue; } if is_generic_false_path_reorder - && has_other_conditional_predecessor_to(blocks, chain_start, current) + && has_other_conditional_predecessor_to( + blocks, + &conditional_target_counts, + chain_start, + current, + ) { current = next; continue; @@ -16235,8 +16253,13 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { cloned_jump.start_depth = None; let cloned_idx = BlockIdx::new(blocks.len() as u32); blocks.push(cloned_jump); + conditional_target_counts.push(0); blocks[idx].next = cloned_idx; let cond_mut = &mut blocks[idx].instructions[cond_idx]; + if cond_mut.target != BlockIdx::NULL { + conditional_target_counts[cond_mut.target.idx()] -= 1; + } + conditional_target_counts[chain_start.idx()] += 1; cond_mut.instr = reversed; cond_mut.target = chain_start; @@ -17865,24 +17888,36 @@ fn has_unique_fallthrough_origin( return false; } - let mut allowed = vec![false; blocks.len()]; - allowed[source.idx()] = true; - - let mut current = blocks[source.idx()].next; - while current != BlockIdx::NULL && current != target { + let chain_start = blocks[source.idx()].next; + let mut current = chain_start; + while current != target { + if current == BlockIdx::NULL { + return false; + } if !blocks[current.idx()].instructions.is_empty() { return false; } - allowed[current.idx()] = true; current = blocks[current.idx()].next; } - if current != target { - return false; + + fn empty_chain_contains( + blocks: &[Block], + mut current: BlockIdx, + target: BlockIdx, + needle: BlockIdx, + ) -> bool { + while current != target { + if current == needle { + return true; + } + current = blocks[current.idx()].next; + } + false } - incoming_origins[target.idx()] - .iter() - .all(|origin| allowed[origin.idx()]) + incoming_origins[target.idx()].iter().all(|&origin| { + origin == source || empty_chain_contains(blocks, chain_start, target, origin) + }) } fn has_unique_jump_origin(