diff --git a/.cspell.json b/.cspell.json index b7efb2e2311..f05f2adcd65 100644 --- a/.cspell.json +++ b/.cspell.json @@ -60,9 +60,11 @@ "dedentations", "dedents", "deduped", + "deoptimized", "deoptimize", "emscripten", "excs", + "fnfe", "interps", "jitted", "jitting", @@ -72,6 +74,9 @@ "oparg", "opargs", "pyc", + "reborrow", + "reraises", + "reraising", "significand", "summands", "unraisable", diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 604153354c1..99a675d11d5 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2065,7 +2065,6 @@ async def f(): run_async(f()), ([], {1: 1, 2: 2, 3: 3})) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: __aiter__ def test_nested_comp(self): async def run_list_inside_list(): return [[i + j async for i in asynciter([1, 2])] for j in [10, 20]] diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 0c39914bc06..6654a8830bf 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -243,7 +243,6 @@ def g(a): self.assertTrue(g(4)) self.check_lnotab(g) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_constant_folding_small_int(self): tests = [ ('(0, )[0]', 0), @@ -278,7 +277,6 @@ def test_constant_folding_small_int(self): self.assertNotInBytecode(code, 'LOAD_SMALL_INT') self.check_lnotab(code) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UNARY_NEGATIVE' starts with 'UNARY_' def test_constant_folding_unaryop(self): intrinsic_positive = 5 tests = [ @@ -324,7 +322,6 @@ def negzero(): self.assertNotStartsWith(instr.opname, 'UNARY_') self.check_lnotab(negzero) - @unittest.expectedFailure # TODO: RUSTPYTHON; BINARY_OP 26 ([]) def test_constant_folding_binop(self): tests = [ ('1 + 2', 'NB_ADD', True, 'LOAD_SMALL_INT', 3), @@ -672,7 +669,6 @@ def f(x): return 6 self.check_lnotab(f) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 2 != 1 def test_assignment_idiom_in_comprehensions(self): def listcomp(): return [y for x in a for y in [f(x)]] diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 34c6e34c525..debec3acd25 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -236,6 +236,20 @@ enum ComprehensionType { Dict, } +#[derive(Debug, Clone, Copy)] +enum ComprehensionLoopControl { + Iteration { + loop_block: BlockIdx, + if_cleanup_block: BlockIdx, + after_block: BlockIdx, + is_async: bool, + end_async_for_target: BlockIdx, + }, + IfCleanupOnly { + if_cleanup_block: BlockIdx, + }, +} + fn validate_duplicate_params(params: &ast::Parameters) -> Result<(), CodegenErrorType> { let mut seen_params = IndexSet::default(); for param in params { @@ -727,13 +741,10 @@ impl Compiler { let big = n + pushed > STACK_USE_GUIDELINE; - let can_fold_const_collection = match collection_type { - CollectionType::Tuple => n > 0, - // Match CPython's constant ordering for list/set literals by - // letting the late IR folding passes introduce their tuple-backed - // constants instead of inserting them during AST lowering. - CollectionType::List | CollectionType::Set => false, - }; + // Match CPython's constant ordering by letting the late flowgraph-style + // folding passes introduce tuple-backed constants after their operands + // have first been emitted as constants. + let can_fold_const_collection = false; if !self.disable_const_collection_folding && !seen_star && pushed == 0 @@ -1804,7 +1815,11 @@ impl Compiler { /// Unwind the fblock stack, emitting cleanup code for each block /// preserve_tos: if true, preserve the top of stack (e.g., return value) /// stop_at_loop: if true, stop when encountering a loop (for break/continue) - fn unwind_fblock_stack(&mut self, preserve_tos: bool, stop_at_loop: bool) -> CompileResult<()> { + fn unwind_fblock_stack( + &mut self, + preserve_tos: bool, + stop_at_loop: bool, + ) -> CompileResult { // Collect the info we need, with indices for FinallyTry blocks #[derive(Clone)] enum UnwindInfo { @@ -1848,6 +1863,7 @@ impl Compiler { } // Process each fblock + let mut unwound_finally = false; for info in unwind_infos { match info { UnwindInfo::Normal(fblock_info) => { @@ -1872,6 +1888,7 @@ impl Compiler { } self.compile_statements(&body)?; + unwound_finally = true; if preserve_tos { self.pop_fblock(FBlockType::PopValue); @@ -1884,7 +1901,7 @@ impl Compiler { } } - Ok(()) + Ok(unwound_finally) } // could take impl Into>, but everything is borrowed from ast structs; we never @@ -2834,20 +2851,37 @@ impl Compiler { emit!(self, Instruction::Nop); } self.set_source_range(stmt_range); - self.unwind_fblock_stack(preserve_tos, false)?; - self.set_source_range(stmt_range); - if let Some(constant) = folded_constant { - self.emit_load_const(constant); + let unwound_finally = self.unwind_fblock_stack(preserve_tos, false)?; + if !unwound_finally { + self.set_source_range(stmt_range); + } + match folded_constant { + Some(constant) if unwound_finally => { + self.emit_return_const_no_location(constant); + } + Some(constant) => { + self.emit_load_const(constant); + self.emit_return_value(); + } + None => { + self.emit_return_value(); + if unwound_finally { + self.set_no_location(); + } + } } - self.emit_return_value(); } None => { self.set_source_range(stmt_range); emit!(self, Instruction::Nop); // Unwind fblock stack with preserve_tos=false (no value to preserve) - self.unwind_fblock_stack(false, false)?; - self.set_source_range(stmt_range); - self.emit_return_const(ConstantData::None); + let unwound_finally = self.unwind_fblock_stack(false, false)?; + if unwound_finally { + self.emit_return_const_no_location(ConstantData::None); + } else { + self.set_source_range(stmt_range); + self.emit_return_const(ConstantData::None); + } } } self.set_source_range(prev_source_range); @@ -3356,7 +3390,9 @@ impl Compiler { // This prevents exception table from covering the normal path if !finalbody.is_empty() { emit!(self, PseudoInstruction::PopBlock); - if !preserve_finally_entry_nop { + if preserve_finally_entry_nop { + self.preserve_last_redundant_nop(); + } else { self.set_no_location(); self.remove_last_no_location_nop(); } @@ -3411,6 +3447,7 @@ impl Compiler { // handler (COPY 3, POP_EXCEPT, RERAISE 1) runs POP_EXCEPT to // restore exc_info before the exception reaches the outer handler. emit!(self, Instruction::Reraise { depth: 0 }); + self.set_no_location(); // PopBlock after RERAISE (dead code, but marks the exception // table range end so the cleanup covers RERAISE). @@ -3441,12 +3478,15 @@ impl Compiler { self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; self.compile_statements(body)?; emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + self.remove_last_no_location_nop(); self.pop_fblock(FBlockType::TryExcept); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: else_block } ); self.set_no_location(); + self.remove_last_no_location_nop(); let cleanup_block = self.new_block(); // We successfully ran the try block: @@ -3595,8 +3635,22 @@ impl Compiler { // the CFG and exception-table ranges match that structure. self.switch_to_block(finally_block); if !finalbody.is_empty() { + let preserve_finally_normal_pop_block_nop = orelse.is_empty() + && !Self::statements_end_with_scope_exit(body) + && handlers.iter().all(|handler| match handler { + ast::ExceptHandler::ExceptHandler(handler) => { + Self::statements_end_with_scope_exit(&handler.body) + } + }); + if preserve_finally_normal_pop_block_nop && let Some(last_body_stmt) = body.last() { + self.set_source_range(last_body_stmt.range()); + } emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); + if preserve_finally_normal_pop_block_nop { + self.preserve_last_redundant_nop(); + } else { + self.set_no_location(); + } self.pop_fblock(FBlockType::FinallyTry); // Snapshot sub_tables before first finally compilation (for double compilation issue) @@ -3649,6 +3703,7 @@ impl Compiler { // runs POP_EXCEPT to restore exc_info before re-raising to // the outer handler. emit!(self, Instruction::Reraise { depth: 0 }); + self.set_no_location(); // PopBlock after RERAISE (dead code, but marks the exception // table range end so the cleanup covers RERAISE). @@ -3716,12 +3771,14 @@ impl Compiler { self.pop_fblock(FBlockType::TryExcept); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); + self.remove_last_no_location_nop(); self.compile_statements(orelse)?; emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); + self.remove_last_no_location_nop(); self.switch_to_block(handler_block); emit!( @@ -3907,11 +3964,15 @@ impl Compiler { self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; self.compile_statements(body)?; emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + self.remove_last_no_location_nop(); self.pop_fblock(FBlockType::TryExcept); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: else_block } ); + self.set_no_location(); + self.remove_last_no_location_nop(); // Exception handler entry self.switch_to_block(handler_block); @@ -4041,8 +4102,8 @@ impl Compiler { self.compile_statements(body)?; // Handler body completed normally - self.set_no_location(); emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); self.pop_fblock(FBlockType::HandlerCleanup); // Cleanup name binding @@ -4164,6 +4225,7 @@ impl Compiler { // Stack: [prev_exc] emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); // POP_EXCEPT - restore previous exception context emit!(self, Instruction::PopExcept); // Stack: [] @@ -4177,10 +4239,10 @@ impl Compiler { // Reraise the result self.switch_to_block(reraise_block); - // Stack: [prev_exc, result] - self.set_no_location(); + // Stack: [prev_exc, result] emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); emit!(self, Instruction::Swap { i: 2 }); // Stack: [result, prev_exc] @@ -4208,6 +4270,7 @@ impl Compiler { delta: continuation_block } ); + self.set_no_location(); } if !finalbody.is_empty() { @@ -4251,6 +4314,7 @@ impl Compiler { } emit!(self, Instruction::Reraise { depth: 0 }); + self.set_no_location(); if let Some(cleanup) = finally_cleanup_block { self.switch_to_block(cleanup); @@ -5277,6 +5341,7 @@ impl Compiler { .map(|s| ConstantData::Str { value: s.into() }) .collect(), }); + self.set_no_location(); let static_attrs_name = self.name("__static_attributes__"); emit!( self, @@ -5284,12 +5349,14 @@ impl Compiler { namei: static_attrs_name } ); + self.set_no_location(); } // Store __classdictcell__ if __classdict__ is a cell variable if self.current_symbol_table().needs_classdict { let classdict_idx = u32::from(self.get_cell_var_index("__classdict__")?); emit!(self, PseudoInstruction::LoadClosure { i: classdict_idx }); + self.set_no_location(); let classdictcell = self.name("__classdictcell__"); emit!( self, @@ -5297,6 +5364,7 @@ impl Compiler { namei: classdictcell } ); + self.set_no_location(); } if let Some(classcell_idx) = classcell_idx { @@ -5306,15 +5374,20 @@ impl Compiler { i: classcell_idx.to_u32() } ); + self.set_no_location(); emit!(self, Instruction::Copy { i: 1 }); + self.set_no_location(); let classcell = self.name("__classcell__"); emit!(self, Instruction::StoreName { namei: classcell }); + self.set_no_location(); } else { self.emit_load_const(ConstantData::None); + self.set_no_location(); } // Return the class namespace self.emit_return_value(); + self.set_no_location(); // Exit scope and return the code object Ok(self.exit_scope()) @@ -5751,11 +5824,25 @@ impl Compiler { self.compile_with(items, body, is_async)?; } - // Pop fblock before normal exit + // Pop fblock before normal exit. CPython emits this POP_BLOCK with + // no location for sync with, but with the with-item location for + // async with. + if is_async { + self.set_source_range(with_range); + } + let preserve_pop_block_nop = !is_async && self.current_block_follows_end_async_for(); + if preserve_pop_block_nop { + emit!(self, Instruction::Nop); + self.preserve_last_redundant_nop(); + } emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.remove_last_no_location_nop(); - self.preserve_last_block_start_no_location_nop(); + if !is_async { + self.set_no_location(); + if !preserve_pop_block_nop { + self.remove_last_no_location_nop(); + } + self.set_source_range(with_range); + } self.pop_fblock(if is_async { FBlockType::AsyncWith } else { @@ -5765,7 +5852,6 @@ impl Compiler { // ===== Normal exit path ===== // Stack: [..., exit_func, self_exit] // Call exit_func(self_exit, None, None, None) - self.set_source_range(with_range); self.emit_load_const(ConstantData::None); self.emit_load_const(ConstantData::None); self.emit_load_const(ConstantData::None); @@ -5777,6 +5863,7 @@ impl Compiler { } emit!(self, Instruction::PopTop); // Pop __exit__ result emit!(self, PseudoInstruction::Jump { delta: after_block }); + self.set_no_location(); // ===== Exception handler path ===== // Stack at entry: [..., exit_func, self_exit, lasti, exc] @@ -5979,6 +6066,18 @@ impl Compiler { self.compile_expression(iter) } + fn singleton_comprehension_assignment_iter(iter: &ast::Expr) -> Option<&ast::Expr> { + let elts = match iter { + ast::Expr::List(ast::ExprList { elts, .. }) => elts, + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => elts, + _ => return None, + }; + match elts.as_slice() { + [elt] if !matches!(elt, ast::Expr::Starred(_)) => Some(elt), + _ => None, + } + } + fn forbidden_name(&mut self, name: &str, ctx: NameUsage) -> CompileResult { if ctx == NameUsage::Store && name == "__debug__" { return Err(self.error(CodegenErrorType::Assign("__debug__"))); @@ -6973,7 +7072,7 @@ impl Compiler { } self.compile_statements(&m.body)?; - emit!(self, PseudoInstruction::Jump { delta: end }); + emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); self.set_source_range(m.pattern.range()); self.emit_and_reset_fail_pop(pattern_context)?; } @@ -7611,6 +7710,14 @@ impl Compiler { && matches!(&comparators[0], ast::Expr::NoneLiteral(_)) => { self.compile_expression(left)?; + let source = self.source_file.to_source_code(); + let comparator_line = source.line_index(comparators[0].range().start()); + let left_line = source.line_index(left.range().start()); + if comparator_line != left_line { + self.set_source_range(comparators[0].range()); + emit!(self, Instruction::Nop); + self.set_source_range(source_range.unwrap_or_else(|| expression.range())); + } let is_not = matches!(ops[0], ast::CmpOp::IsNot); // is None + jump_if_false → POP_JUMP_IF_NOT_NONE // is None + jump_if_true → POP_JUMP_IF_NONE @@ -7958,13 +8065,11 @@ impl Compiler { // fail: CLEANUP_THROW // Stack when exception: [receiver, yielded_value, exc] // CLEANUP_THROW: [sub_iter, last_sent_val, exc] -> [None, value] - // Jump back to the shared END_SEND block like CPython's fail path. + // CPython lets this block fall through to END_SEND during codegen; + // push_cold_blocks_to_end later inserts the no-interrupt jump after + // moving the cold fail block behind the warm exit path. self.switch_to_block(fail_block); emit!(self, Instruction::CleanupThrow); - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: exit_block } - ); // exit: END_SEND // Stack: [receiver, value] (from SEND) or [None, value] (from CLEANUP_THROW) @@ -8022,6 +8127,13 @@ impl Compiler { return Ok(()); } + if matches!(expression, ast::Expr::BinOp(_)) + && let Some(constant) = self.try_fold_constant_expr(expression)? + { + self.emit_load_const(constant); + return Ok(()); + } + if !self.disable_const_boolop_folding && let ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) = expression { @@ -8341,7 +8453,10 @@ impl Compiler { self.ctx = prev_ctx; } ast::Expr::ListComp(ast::ExprListComp { - elt, generators, .. + elt, + generators, + range, + .. }) => { self.compile_comprehension( "", @@ -8352,22 +8467,26 @@ impl Compiler { .into(), ), generators, - &|compiler| { + &|compiler, collection_add_i| { compiler.compile_comprehension_element(elt)?; emit!( compiler, Instruction::ListAppend { - i: (generators.len() + 1).to_u32(), + i: collection_add_i.to_u32(), } ); Ok(()) }, ComprehensionType::List, Self::contains_await(elt) || Self::generators_contain_await(generators), + *range, )?; } ast::Expr::SetComp(ast::ExprSetComp { - elt, generators, .. + elt, + generators, + range, + .. }) => { self.compile_comprehension( "", @@ -8378,24 +8497,26 @@ impl Compiler { .into(), ), generators, - &|compiler| { + &|compiler, collection_add_i| { compiler.compile_comprehension_element(elt)?; emit!( compiler, Instruction::SetAdd { - i: (generators.len() + 1).to_u32(), + i: collection_add_i.to_u32(), } ); Ok(()) }, ComprehensionType::Set, Self::contains_await(elt) || Self::generators_contain_await(generators), + *range, )?; } ast::Expr::DictComp(ast::ExprDictComp { key, value, generators, + range, .. }) => { self.compile_comprehension( @@ -8407,7 +8528,7 @@ impl Compiler { .into(), ), generators, - &|compiler| { + &|compiler, collection_add_i| { // changed evaluation order for Py38 named expression PEP 572 compiler.compile_expression(key)?; compiler.compile_expression(value)?; @@ -8415,7 +8536,7 @@ impl Compiler { emit!( compiler, Instruction::MapAdd { - i: (generators.len() + 1).to_u32(), + i: collection_add_i.to_u32(), } ); @@ -8425,10 +8546,14 @@ impl Compiler { Self::contains_await(key) || Self::contains_await(value) || Self::generators_contain_await(generators), + *range, )?; } ast::Expr::Generator(ast::ExprGenerator { - elt, generators, .. + elt, + generators, + range, + .. }) => { // Check if element or generators contain async content // This makes the generator expression into an async generator @@ -8438,7 +8563,7 @@ impl Compiler { "", None, generators, - &|compiler| { + &|compiler, _collection_add_i| { // Compile the element expression // Note: if element is an async comprehension, compile_expression // already handles awaiting it, so we don't need to await again here @@ -8469,6 +8594,7 @@ impl Compiler { }, ComprehensionType::Generator, element_contains_await, + *range, )?; } ast::Expr::Starred(ast::ExprStarred { value, .. }) => { @@ -8484,12 +8610,19 @@ impl Compiler { ast::Expr::If(ast::ExprIf { test, body, orelse, .. }) => { + let folded_test_truthiness = self + .try_fold_constant_expr(test)? + .as_ref() + .map(Self::constant_truthiness); let else_block = self.new_block(); let after_block = self.new_block(); self.compile_jump_if(test, false, else_block)?; // True case self.compile_expression(body)?; + if folded_test_truthiness == Some(true) { + self.mark_last_instruction_folded_from_nonliteral_expr(); + } emit!( self, PseudoInstruction::JumpNoInterrupt { delta: after_block } @@ -8499,6 +8632,9 @@ impl Compiler { // False case self.switch_to_block(else_block); self.compile_expression(orelse)?; + if folded_test_truthiness == Some(false) { + self.mark_last_instruction_folded_from_nonliteral_expr(); + } // End self.switch_to_block(after_block); @@ -8805,7 +8941,8 @@ impl Compiler { .iter() .any(|arg| matches!(arg, ast::Expr::Starred(_))); let has_double_star = arguments.keywords.iter().any(|k| k.arg.is_none()); - let too_big = arguments.args.len() + arguments.keywords.len() > 15; + let too_big = + arguments.args.len() + arguments.keywords.len() * 2 > STACK_USE_GUIDELINE as usize; has_starred || has_double_star || too_big } @@ -8819,8 +8956,8 @@ impl Compiler { let n = end - begin; assert!(n > 0); - // For large kwargs, use BUILD_MAP(0) + MAP_ADD to avoid stack overflow - let big = n * 2 > 8; // STACK_USE_GUIDELINE approximation + // For large kwargs, use BUILD_MAP(0) + MAP_ADD to avoid stack overflow. + let big = n * 2 > STACK_USE_GUIDELINE as usize; if big { emit!(self, Instruction::BuildMap { count: 0 }); @@ -8864,10 +9001,10 @@ impl Compiler { .any(|arg| matches!(arg, ast::Expr::Starred(_))); let has_double_star = arguments.keywords.iter().any(|k| k.arg.is_none()); - // Check if exceeds stack guideline (STACK_USE_GUIDELINE / 2 = 15) + // Check if exceeds CPython's stack-use guideline. // With CALL_KW, kwargs values go on stack but keys go in a const tuple, // so stack usage is: func + null + positional_args + kwarg_values + kwnames_tuple - let too_big = nelts + nkwelts > 15; + let too_big = nelts + nkwelts * 2 > STACK_USE_GUIDELINE as usize; if !has_starred && !has_double_star && !too_big { // Simple call path: no * or ** args @@ -9145,14 +9282,16 @@ impl Compiler { Ok(()) } + #[allow(clippy::too_many_arguments)] fn compile_comprehension( &mut self, name: &str, init_collection: Option, generators: &[ast::Comprehension], - compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, + compile_element: &dyn Fn(&mut Self, usize) -> CompileResult<()>, comprehension_type: ComprehensionType, element_contains_await: bool, + comprehension_range: TextRange, ) -> CompileResult<()> { let prev_ctx = self.ctx; let has_an_async_gen = generators.iter().any(|g| g.is_async); @@ -9201,6 +9340,7 @@ impl Compiler { generators, compile_element, has_an_async_gen, + comprehension_range, ); } @@ -9262,12 +9402,33 @@ impl Compiler { } let mut loop_labels = vec![]; - for generator in generators { + let mut real_loop_depth = 0; + for (gen_index, generator) in generators.iter().enumerate() { + if gen_index > 0 + && !generator.is_async + && let Some(singleton_iter) = + Self::singleton_comprehension_assignment_iter(&generator.iter) + { + self.compile_expression(singleton_iter)?; + self.compile_store(&generator.target)?; + + if !generator.ifs.is_empty() { + let if_cleanup_block = self.new_block(); + for if_condition in &generator.ifs { + self.compile_jump_if(if_condition, false, if_cleanup_block)?; + } + let body_block = self.new_block(); + self.switch_to_block(body_block); + loop_labels.push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); + } + continue; + } + let loop_block = self.new_block(); let if_cleanup_block = self.new_block(); let after_block = self.new_block(); - if loop_labels.is_empty() { + if gen_index == 0 { // Load iterator onto stack (passed as first argument): emit!(self, Instruction::LoadFast { var_num: arg0 }); } else { @@ -9303,13 +9464,14 @@ impl Compiler { emit!(self, Instruction::ForIter { delta: after_block }); self.compile_store(&generator.target)?; } - loop_labels.push(( + real_loop_depth += 1; + loop_labels.push(ComprehensionLoopControl::Iteration { loop_block, if_cleanup_block, after_block, - generator.is_async, + is_async: generator.is_async, end_async_for_target, - )); + }); // CPython always lowers comprehension guards through codegen_jump_if // and leaves constant-folding to later CFG optimization passes. @@ -9322,25 +9484,36 @@ impl Compiler { } } - compile_element(self)?; + compile_element(self, real_loop_depth + 1)?; - for (loop_block, if_cleanup_block, after_block, is_async, end_async_for_target) in - loop_labels.iter().rev().copied() - { - emit!(self, PseudoInstruction::Jump { delta: loop_block }); + for loop_control in loop_labels.iter().rev().copied() { + match loop_control { + ComprehensionLoopControl::Iteration { + loop_block, + if_cleanup_block, + after_block, + is_async, + end_async_for_target, + } => { + emit!(self, PseudoInstruction::Jump { delta: loop_block }); - self.switch_to_block(if_cleanup_block); - emit!(self, PseudoInstruction::Jump { delta: loop_block }); + self.switch_to_block(if_cleanup_block); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); - self.switch_to_block(after_block); - if is_async { - // EndAsyncFor pops both the exception and the aiter - // (handler depth is before GetANext, so aiter is at handler depth) - self.emit_end_async_for(end_async_for_target); - } else { - // END_FOR + POP_ITER pattern (CPython 3.14) - emit!(self, Instruction::EndFor); - emit!(self, Instruction::PopIter); + self.switch_to_block(after_block); + if is_async { + // EndAsyncFor pops both the exception and the aiter + // (handler depth is before GetANext, so aiter is at handler depth) + self.emit_end_async_for(end_async_for_target); + } else { + // END_FOR + POP_ITER pattern (CPython 3.14) + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); + } + } + ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block } => { + self.switch_to_block(if_cleanup_block); + } } } @@ -9407,8 +9580,9 @@ impl Compiler { comp_table: SymbolTable, init_collection: Option, generators: &[ast::Comprehension], - compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, + compile_element: &dyn Fn(&mut Self, usize) -> CompileResult<()>, has_async: bool, + comprehension_range: TextRange, ) -> CompileResult<()> { fn collect_bound_names(target: &ast::Expr, out: &mut Vec) { match target { @@ -9440,6 +9614,7 @@ impl Compiler { .next_sub_table += 1; let was_in_inlined_comp = self.current_code_info().in_inlined_comp; + let saved_source_range = self.current_source_range; let in_class_block = { let ct = self.current_symbol_table(); ct.typ == CompilerScope::Class && !was_in_inlined_comp @@ -9552,6 +9727,7 @@ impl Compiler { // For merged CELL variables, LoadFastAndClear saves the cell object from // the merged slot, and MAKE_CELL creates a new empty cell in-place. // MAKE_CELL has no stack effect (operates only on fastlocals). + self.set_source_range(comprehension_range); let mut total_stack_items: usize = 0; for name in &pushed_locals { let var_num = self.varname(name)?; @@ -9607,8 +9783,30 @@ impl Compiler { } // Step 5: Compile the comprehension loop(s) - let mut loop_labels: Vec<(BlockIdx, BlockIdx, BlockIdx, bool, BlockIdx)> = vec![]; + let mut loop_labels: Vec = vec![]; + let mut real_loop_depth = 0; for (i, generator) in generators.iter().enumerate() { + if i > 0 + && !generator.is_async + && let Some(singleton_iter) = + Self::singleton_comprehension_assignment_iter(&generator.iter) + { + self.compile_expression(singleton_iter)?; + self.compile_store(&generator.target)?; + + if !generator.ifs.is_empty() { + let if_cleanup_block = self.new_block(); + for if_condition in &generator.ifs { + self.compile_jump_if(if_condition, false, if_cleanup_block)?; + } + let body_block = self.new_block(); + self.switch_to_block(body_block); + loop_labels + .push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); + } + continue; + } + let loop_block = self.new_block(); let if_cleanup_block = self.new_block(); let after_block = self.new_block(); @@ -9639,17 +9837,21 @@ impl Compiler { self.pop_fblock(FBlockType::AsyncComprehensionGenerator); self.compile_store(&generator.target)?; } else { + let saved_range = self.current_source_range; + self.set_source_range(generator.iter.range()); emit!(self, Instruction::ForIter { delta: after_block }); + self.set_source_range(saved_range); self.compile_store(&generator.target)?; } - loop_labels.push(( + real_loop_depth += 1; + loop_labels.push(ComprehensionLoopControl::Iteration { loop_block, if_cleanup_block, after_block, - generator.is_async, + is_async: generator.is_async, end_async_for_target, - )); + }); // CPython always lowers comprehension guards through codegen_jump_if // and leaves constant-folding to later CFG optimization passes. @@ -9659,27 +9861,39 @@ impl Compiler { } // Step 6: Compile the element expression and append to collection - compile_element(self)?; + compile_element(self, real_loop_depth + 1)?; // Step 7: Close all loops - for &(loop_block, if_cleanup_block, after_block, is_async, end_async_for_target) in - loop_labels.iter().rev() - { - emit!(self, PseudoInstruction::Jump { delta: loop_block }); + for loop_control in loop_labels.iter().rev().copied() { + match loop_control { + ComprehensionLoopControl::Iteration { + loop_block, + if_cleanup_block, + after_block, + is_async, + end_async_for_target, + } => { + emit!(self, PseudoInstruction::Jump { delta: loop_block }); - self.switch_to_block(if_cleanup_block); - emit!(self, PseudoInstruction::Jump { delta: loop_block }); + self.switch_to_block(if_cleanup_block); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); - self.switch_to_block(after_block); - if is_async { - self.emit_end_async_for(end_async_for_target); - } else { - emit!(self, Instruction::EndFor); - emit!(self, Instruction::PopIter); + self.switch_to_block(after_block); + if is_async { + self.emit_end_async_for(end_async_for_target); + } else { + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); + } + } + ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block } => { + self.switch_to_block(if_cleanup_block); + } } } // Step 8: Clean up - restore saved locals (and cell values) + self.set_source_range(comprehension_range); if total_stack_items > 0 { emit!(self, PseudoInstruction::PopBlock); self.pop_fblock(FBlockType::TryExcept); @@ -9707,8 +9921,8 @@ impl Compiler { } ); for name in pushed_locals.iter().rev() { - let var_num = self.varname(name)?; - emit!(self, Instruction::StoreFast { var_num }); + let var_num = self.varname(name)?.as_u32(); + emit!(self, PseudoInstruction::StoreFastMaybeNull { var_num }); } // Re-raise the exception emit!(self, Instruction::Reraise { depth: 0 }); @@ -9729,9 +9943,10 @@ impl Compiler { // Restore saved locals (StoreFast restores the saved cell object for merged cells) for name in pushed_locals.iter().rev() { - let var_num = self.varname(name)?; - emit!(self, Instruction::StoreFast { var_num }); + let var_num = self.varname(name)?.as_u32(); + emit!(self, PseudoInstruction::StoreFastMaybeNull { var_num }); } + self.set_source_range(saved_source_range); Ok(()) })(); @@ -9810,15 +10025,15 @@ impl Compiler { } } - fn remove_last_no_location_nop(&mut self) { + fn preserve_last_redundant_nop(&mut self) { if let Some(info) = self.current_block().instructions.last_mut() { - info.remove_no_location_nop = true; + info.preserve_block_start_no_location_nop = true; } } - fn preserve_last_block_start_no_location_nop(&mut self) { + fn remove_last_no_location_nop(&mut self) { if let Some(info) = self.current_block().instructions.last_mut() { - info.preserve_block_start_no_location_nop = true; + info.remove_no_location_nop = true; } } @@ -9998,6 +10213,107 @@ impl Compiler { Ok(Some(constant)) } + fn constant_as_fold_int(constant: &ConstantData) -> Option<(BigInt, bool)> { + match constant { + ConstantData::Boolean { value } => Some((BigInt::from(u8::from(*value)), true)), + ConstantData::Integer { value } => Some((value.clone(), false)), + _ => None, + } + } + + fn try_fold_constant_binop( + op: ast::Operator, + left: &ConstantData, + right: &ConstantData, + ) -> Option { + let (left_int, left_is_bool) = Self::constant_as_fold_int(left)?; + let (right_int, right_is_bool) = Self::constant_as_fold_int(right)?; + let zero = BigInt::from(0); + + if !left_is_bool && !right_is_bool { + return None; + } + + match op { + ast::Operator::Add => Some(ConstantData::Integer { + value: left_int + right_int, + }), + ast::Operator::Sub => Some(ConstantData::Integer { + value: left_int - right_int, + }), + ast::Operator::Mult => Some(ConstantData::Integer { + value: left_int * right_int, + }), + ast::Operator::Div => { + if right_int.is_zero() { + return None; + } + Some(ConstantData::Float { + value: left_int.to_f64()? / right_int.to_f64()?, + }) + } + ast::Operator::FloorDiv => { + if right_int.is_zero() || left_int < zero || right_int < zero { + return None; + } + Some(ConstantData::Integer { + value: left_int / right_int, + }) + } + ast::Operator::Mod => { + if right_int.is_zero() || left_int < zero || right_int < zero { + return None; + } + Some(ConstantData::Integer { + value: left_int % right_int, + }) + } + ast::Operator::Pow => { + let exponent = right_int.to_u32()?; + if exponent > 128 { + return None; + } + Some(ConstantData::Integer { + value: left_int.pow(exponent), + }) + } + ast::Operator::BitAnd => { + if left_is_bool && right_is_bool { + Some(ConstantData::Boolean { + value: !left_int.is_zero() & !right_int.is_zero(), + }) + } else { + Some(ConstantData::Integer { + value: left_int & right_int, + }) + } + } + ast::Operator::BitOr => { + if left_is_bool && right_is_bool { + Some(ConstantData::Boolean { + value: !left_int.is_zero() | !right_int.is_zero(), + }) + } else { + Some(ConstantData::Integer { + value: left_int | right_int, + }) + } + } + ast::Operator::BitXor => { + if left_is_bool && right_is_bool { + Some(ConstantData::Boolean { + value: !left_int.is_zero() ^ !right_int.is_zero(), + }) + } else { + Some(ConstantData::Integer { + value: left_int ^ right_int, + }) + } + } + ast::Operator::MatMult | ast::Operator::LShift | ast::Operator::RShift => None, + } + } + fn try_fold_constant_expr(&mut self, expr: &ast::Expr) -> CompileResult> { Ok(Some(match expr { ast::Expr::NumberLiteral(num) => match &num.value { @@ -10101,6 +10417,20 @@ impl Compiler { _ => return Ok(None), } } + ast::Expr::BinOp(ast::ExprBinOp { + left, op, right, .. + }) => { + let Some(left) = self.try_fold_constant_expr(left)? else { + return Ok(None); + }; + let Some(right) = self.try_fold_constant_expr(right)? else { + return Ok(None); + }; + let Some(constant) = Self::try_fold_constant_binop(*op, &left, &right) else { + return Ok(None); + }; + constant + } ast::Expr::UnaryOp(ast::ExprUnaryOp { op, operand, .. }) => { let Some(constant) = self.try_fold_constant_expr(operand)? else { return Ok(None); @@ -10179,6 +10509,14 @@ impl Compiler { upper: Option<&ast::Expr>, step: Option<&ast::Expr>, ) -> CompileResult> { + if [lower, upper, step] + .into_iter() + .flatten() + .any(|expr| !expr.is_constant()) + { + return Ok(None); + } + let start = match lower { Some(expr) => { let Some(constant) = self.try_fold_constant_expr(expr)? else { @@ -10536,6 +10874,29 @@ impl Compiler { &mut info.blocks[info.current_block] } + fn current_block_follows_end_async_for(&mut self) -> bool { + let info = self.current_code_info(); + let current = info.current_block; + if info.blocks[current] + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()) + .is_some_and(|instr| matches!(instr, Instruction::EndAsyncFor)) + { + return true; + } + info.blocks.iter().any(|block| { + block.next == current + && block + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()) + .is_some_and(|instr| matches!(instr, Instruction::EndAsyncFor)) + }) + } + fn new_block(&mut self) -> BlockIdx { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); @@ -10607,11 +10968,13 @@ impl Compiler { match expr { ast::Expr::Await(_) => self.found = true, - // Note: We do NOT check for async comprehensions here. - // Async list/set/dict comprehensions are handled by compile_comprehension - // which already awaits the result. A generator expression containing - // an async comprehension as its element does NOT become an async generator, - // because the async comprehension is awaited when evaluating the element. + ast::Expr::ListComp(ast::ExprListComp { generators, .. }) + | ast::Expr::SetComp(ast::ExprSetComp { generators, .. }) + | ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) + if generators.iter().any(|generator| generator.is_async) => + { + self.found = true + } _ => ast::visitor::walk_expr(self, expr), } } @@ -10641,20 +11004,42 @@ impl Compiler { } fn compile_expr_fstring(&mut self, fstring: &ast::ExprFString) -> CompileResult<()> { - let fstring = &fstring.value; + let fstring = fstring.value.as_slice(); + if self.count_fstring_parts(fstring) > STACK_USE_GUIDELINE { + return self.compile_fstring_parts_joined(fstring); + } + let mut element_count = 0; let mut pending_literal = None; for part in fstring { - self.compile_fstring_part_into(part, &mut pending_literal, &mut element_count)?; + self.compile_fstring_part_into(part, &mut pending_literal, &mut element_count, false)?; } self.finish_fstring(pending_literal, element_count) } + fn compile_fstring_parts_joined(&mut self, fstring: &[ast::FStringPart]) -> CompileResult<()> { + self.emit_load_const(ConstantData::Str { + value: Wtf8Buf::new(), + }); + let join_idx = self.get_global_name_index("join"); + self.emit_load_attr_method(join_idx); + emit!(self, Instruction::BuildList { count: 0 }); + + let mut element_count = 0; + let mut pending_literal = None; + for part in fstring { + self.compile_fstring_part_into(part, &mut pending_literal, &mut element_count, true)?; + } + self.finish_fstring_join(pending_literal, element_count); + Ok(()) + } + fn compile_fstring_part_into( &mut self, part: &ast::FStringPart, pending_literal: &mut Option, element_count: &mut u32, + append_to_join_list: bool, ) -> CompileResult<()> { match part { ast::FStringPart::Literal(string) => { @@ -10671,6 +11056,7 @@ impl Compiler { &fstring.elements, pending_literal, element_count, + append_to_join_list, ), } } @@ -10681,7 +11067,12 @@ impl Compiler { mut element_count: u32, ) -> CompileResult<()> { let keep_empty = element_count == 0; - self.emit_pending_fstring_literal(&mut pending_literal, &mut element_count, keep_empty); + self.emit_pending_fstring_literal( + &mut pending_literal, + &mut element_count, + keep_empty, + false, + ); if element_count == 0 { self.emit_load_const(ConstantData::Str { @@ -10699,11 +11090,27 @@ impl Compiler { Ok(()) } + fn finish_fstring_join( + &mut self, + mut pending_literal: Option, + mut element_count: u32, + ) { + let keep_empty = element_count == 0; + self.emit_pending_fstring_literal( + &mut pending_literal, + &mut element_count, + keep_empty, + true, + ); + emit!(self, Instruction::Call { argc: 1 }); + } + fn emit_pending_fstring_literal( &mut self, pending_literal: &mut Option, element_count: &mut u32, keep_empty: bool, + append_to_join_list: bool, ) { let Some(value) = pending_literal.take() else { return; @@ -10718,6 +11125,60 @@ impl Compiler { self.emit_load_const(ConstantData::Str { value }); *element_count += 1; + if append_to_join_list { + emit!(self, Instruction::ListAppend { i: 1 }); + } + } + + fn count_fstring_parts(&self, fstring: &[ast::FStringPart]) -> u32 { + let mut element_count = 0; + let mut pending_literal = None; + for part in fstring { + self.count_fstring_part_into(part, &mut pending_literal, &mut element_count); + } + let keep_empty = element_count == 0; + Self::count_pending_fstring_literal(&mut pending_literal, &mut element_count, keep_empty); + element_count + } + + fn count_fstring_part_into( + &self, + part: &ast::FStringPart, + pending_literal: &mut Option, + element_count: &mut u32, + ) { + match part { + ast::FStringPart::Literal(string) => { + let value = self.compile_fstring_part_literal_value(string); + if let Some(pending) = pending_literal.as_mut() { + pending.push_wtf8(value.as_ref()); + } else { + *pending_literal = Some(value); + } + } + ast::FStringPart::FString(fstring) => self.count_fstring_elements_into( + fstring.flags, + &fstring.elements, + pending_literal, + element_count, + ), + } + } + + fn count_pending_fstring_literal( + pending_literal: &mut Option, + element_count: &mut u32, + keep_empty: bool, + ) { + let Some(value) = pending_literal.take() else { + return; + }; + + if value.is_empty() && (!keep_empty || *element_count > 0) { + return; + } + + *element_count += 1; } /// Optimize `'format_str' % (args,)` into f-string bytecode. @@ -10844,6 +11305,10 @@ impl Compiler { flags: ast::FStringFlags, fstring_elements: &ast::InterpolatedStringElements, ) -> CompileResult<()> { + if self.count_fstring_elements(flags, fstring_elements) > STACK_USE_GUIDELINE { + return self.compile_fstring_elements_joined(flags, fstring_elements); + } + let mut element_count = 0; let mut pending_literal: Option = None; self.compile_fstring_elements_into( @@ -10851,16 +11316,43 @@ impl Compiler { fstring_elements, &mut pending_literal, &mut element_count, + false, )?; self.finish_fstring(pending_literal, element_count) } + fn compile_fstring_elements_joined( + &mut self, + flags: ast::FStringFlags, + fstring_elements: &ast::InterpolatedStringElements, + ) -> CompileResult<()> { + self.emit_load_const(ConstantData::Str { + value: Wtf8Buf::new(), + }); + let join_idx = self.get_global_name_index("join"); + self.emit_load_attr_method(join_idx); + emit!(self, Instruction::BuildList { count: 0 }); + + let mut element_count = 0; + let mut pending_literal: Option = None; + self.compile_fstring_elements_into( + flags, + fstring_elements, + &mut pending_literal, + &mut element_count, + true, + )?; + self.finish_fstring_join(pending_literal, element_count); + Ok(()) + } + fn compile_fstring_elements_into( &mut self, flags: ast::FStringFlags, fstring_elements: &ast::InterpolatedStringElements, pending_literal: &mut Option, element_count: &mut u32, + append_to_join_list: bool, ) -> CompileResult<()> { for element in fstring_elements { match element { @@ -10905,7 +11397,12 @@ impl Compiler { } } - self.emit_pending_fstring_literal(pending_literal, element_count, false); + self.emit_pending_fstring_literal( + pending_literal, + element_count, + false, + append_to_join_list, + ); self.compile_expression(&fstring_expr.expression)?; @@ -10930,6 +11427,9 @@ impl Compiler { } *element_count += 1; + if append_to_join_list { + emit!(self, Instruction::ListAppend { i: 1 }); + } } } } @@ -10937,6 +11437,65 @@ impl Compiler { Ok(()) } + fn count_fstring_elements( + &self, + flags: ast::FStringFlags, + fstring_elements: &ast::InterpolatedStringElements, + ) -> u32 { + let mut element_count = 0; + let mut pending_literal = None; + self.count_fstring_elements_into( + flags, + fstring_elements, + &mut pending_literal, + &mut element_count, + ); + let keep_empty = element_count == 0; + Self::count_pending_fstring_literal(&mut pending_literal, &mut element_count, keep_empty); + element_count + } + + fn count_fstring_elements_into( + &self, + flags: ast::FStringFlags, + fstring_elements: &ast::InterpolatedStringElements, + pending_literal: &mut Option, + element_count: &mut u32, + ) { + for element in fstring_elements { + match element { + ast::InterpolatedStringElement::Literal(string) => { + let value = self.compile_fstring_literal_value(string, flags); + if let Some(pending) = pending_literal.as_mut() { + pending.push_wtf8(value.as_ref()); + } else { + *pending_literal = Some(value); + } + } + ast::InterpolatedStringElement::Interpolation(fstring_expr) => { + if let Some(ast::DebugText { leading, trailing }) = &fstring_expr.debug_text { + let range = fstring_expr.expression.range(); + let source = self.source_file.slice(range); + let text = [ + strip_fstring_debug_comments(leading).as_str(), + source, + strip_fstring_debug_comments(trailing).as_str(), + ] + .concat(); + + let text: Wtf8Buf = text.into(); + pending_literal + .get_or_insert_with(Wtf8Buf::new) + .push_wtf8(text.as_ref()); + } + + Self::count_pending_fstring_literal(pending_literal, element_count, false); + *element_count += 1; + } + } + } + } + fn compile_expr_tstring(&mut self, expr_tstring: &ast::ExprTString) -> CompileResult<()> { // ast::TStringValue can contain multiple ast::TString parts (implicit // concatenation). Match CPython's stack order by materializing the @@ -11615,6 +12174,68 @@ def f(f, oldcls, newcls): } } + #[test] + #[ignore = "debug helper"] + fn debug_trace_dtrace_tail() { + let trace = compile_single_function_late_cfg_trace( + r#" +def f(proc, unittest): + try: + with proc: + version, stderr = proc.communicate() + if proc.returncode: + raise Exception(version, stderr) + except OSError: + raise unittest.SkipTest("x") + match = re.search("pat", version) + if match is None: + raise unittest.SkipTest(f"Unable to parse readelf version: {version}") + return int(match.group(1)), int(match.group(2)) +"#, + "f", + ); + for (label, dump) in trace { + if label == "after_raw_optimize_load_fast_borrow" + || label.contains("deoptimize_borrow_in_protected_conditional_tail") + || label.contains("deoptimize_borrow_after_terminal_except_tail") + { + eprintln!("=== {label} ===\n{dump}"); + } + } + } + + #[test] + #[ignore = "debug helper"] + fn debug_trace_colorize_tail() { + let trace = compile_single_function_late_cfg_trace( + r#" +def f(sys, os, file): + if sys.platform == "win32": + try: + import nt + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + + try: + return os.isatty(file.fileno()) + except OSError: + return hasattr(file, "isatty") and file.isatty() +"#, + "f", + ); + for (label, dump) in trace { + if label == "after_raw_optimize_load_fast_borrow" + || label == "after_deoptimize_borrow_after_protected_import" + || label == "after_optimize_load_fast_borrow" + || label == "after_borrow_deopts" + { + eprintln!("=== {label} ===\n{dump}"); + } + } + } + fn find_code<'a>(code: &'a CodeObject, name: &str) -> Option<&'a CodeObject> { if code.obj_name == name { return Some(code); @@ -12170,6 +12791,47 @@ def or_false(x): } } + #[test] + fn test_folded_nonliteral_tuple_unpack_tail_keeps_plain_load_fast() { + let code = compile_exec( + "\ +def f(self, mod): + optimize, opt = (1, 1) if __debug__ else (0, '') + mod.call(self.path, optimize=optimize) + cached = mod.cache(self.source_path, optimization=opt) + self.assertTrue(cached) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let tail_start = ops + .iter() + .position(|op| matches!(op, Instruction::LoadFast { .. })) + .expect("missing folded assignment tail load"); + let tail = &ops[tail_start..]; + + assert!( + tail.iter() + .any(|op| matches!(op, Instruction::LoadFast { .. })), + "expected folded nonliteral tuple-unpack tail to use strong LOAD_FAST, got tail={tail:?}" + ); + assert!( + !tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "folded nonliteral tuple-unpack tail should not borrow local loads, got tail={tail:?}" + ); + } + #[test] fn test_nested_double_async_with() { assert_dis_snapshot!(compile_exec( @@ -12243,6 +12905,38 @@ def f(cls, args, kwargs): } } + #[test] + fn test_large_plain_call_uses_direct_call_until_stack_guideline() { + let code = compile_exec( + "\ +def f(g): + return g(a0, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let direct_call_18 = f.instructions.iter().any(|unit| match unit.op { + Instruction::Call { argc } => argc.get(OpArg::new(u32::from(u8::from(unit.arg)))) == 18, + _ => false, + }); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + direct_call_18, + "18 positional arguments should stay on CPython's direct CALL path, got ops={ops:?}" + ); + assert!( + !ops.iter() + .any(|op| matches!(op, Instruction::CallFunctionEx)), + "18 positional arguments should not use CALL_FUNCTION_EX, got ops={ops:?}" + ); + } + #[test] fn test_simple_attribute_call_keeps_method_load() { let code = compile_exec( @@ -12746,6 +13440,40 @@ def f(x, y): assert_eq!(u8::from(compare.arg), 88); } + #[test] + fn test_multiline_is_none_conditional_keeps_comparator_nop() { + let code = compile_exec( + "\ +def f(x): + if x.find( + 'a') is not None: + return 1 + return 0 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::Call { .. }, + Instruction::Nop, + Instruction::PopJumpIfNone { .. }, + ] + ) + }), + "expected CPython-style comparator NOP before folded POP_JUMP_IF_NONE, got ops={ops:?}" + ); + } + #[test] fn test_chained_conditional_compares_use_bool_compare_oparg() { let code = compile_exec( @@ -13157,30 +13885,144 @@ def f(x, y): } #[test] - fn test_bare_function_annotations_check_attribute_and_subscript_expressions() { - assert_dis_snapshot!(compile_exec( - "\ -def f(one: int): - int.new_attr: int - [list][0].new_attr: [int, str] - my_lst = [1] - my_lst[one]: int - return my_lst -" - )); - } - - #[test] - fn test_non_simple_bare_name_annotation_does_not_create_local_binding() { + fn test_conditional_assert_message_target_uses_strong_load_fast() { let code = compile_exec( "\ -def f2bad(): - (no_such_global): int - print(no_such_global) +def f(fname): + if fname == 'a': + return 1 + assert False, 'Unknown attrname %s' % fname ", ); - let f = find_code(&code, "f2bad").expect("missing f2bad code"); - assert!( + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let assertion_error = ops + .iter() + .position(|op| matches!(op, Instruction::LoadCommonConstant { .. })) + .expect("missing LOAD_COMMON_CONSTANT AssertionError"); + let window = &ops[assertion_error..(assertion_error + 5).min(ops.len())]; + assert!( + matches!( + window, + [ + Instruction::LoadCommonConstant { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadFast { .. }, + Instruction::BinaryOp { .. }, + Instruction::Call { .. }, + .. + ] + ), + "expected CPython-style strong LOAD_FAST in targeted assert message block, got {window:?}" + ); + } + + #[test] + fn test_assert_message_after_condition_in_same_block_keeps_borrowed_loads() { + let code = compile_exec( + "\ +def f(expected_ns, namespace): + try: + assert expected_ns == namespace, ('expected %s, got %s' % (expected_ns, namespace)) + except AssertionError as e: + raise RuntimeError(e) + setattr(namespace, 'spam', expected_ns) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let assertion_error = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::LoadCommonConstant { .. })) + .expect("missing LOAD_COMMON_CONSTANT AssertionError"); + let raise = ops[assertion_error..] + .iter() + .position(|unit| matches!(unit.op, Instruction::RaiseVarargs { .. })) + .map(|idx| assertion_error + idx) + .expect("missing assert raise"); + let message_path = &ops[assertion_error..raise]; + + let load_fast_name = |unit: &&bytecode::CodeUnit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + let borrow_name = |unit: &&bytecode::CodeUnit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + + assert!( + message_path + .iter() + .filter_map(load_fast_name) + .all(|name| name != "expected_ns" && name != "namespace"), + "assert message after same-block condition should keep borrowed loads, got {message_path:?}" + ); + for name in ["expected_ns", "namespace"] { + assert!( + message_path + .iter() + .filter_map(borrow_name) + .any(|var| var == name), + "expected borrowed {name} load in assert message path, got {message_path:?}" + ); + } + + let setattr = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::LoadGlobal { .. })) + .expect("missing final setattr load"); + let tail = &ops[setattr..]; + for name in ["expected_ns", "namespace"] { + assert!( + tail.iter() + .filter_map(load_fast_name) + .any(|var| var == name), + "expected strong {name} load in post-try tail, got {tail:?}" + ); + } + } + + #[test] + fn test_bare_function_annotations_check_attribute_and_subscript_expressions() { + assert_dis_snapshot!(compile_exec( + "\ +def f(one: int): + int.new_attr: int + [list][0].new_attr: [int, str] + my_lst = [1] + my_lst[one]: int + return my_lst +" + )); + } + + #[test] + fn test_non_simple_bare_name_annotation_does_not_create_local_binding() { + let code = compile_exec( + "\ +def f2bad(): + (no_such_global): int + print(no_such_global) +", + ); + let f = find_code(&code, "f2bad").expect("missing f2bad code"); + assert!( f.instructions .iter() .any(|unit| matches!(unit.op, Instruction::LoadGlobal { .. })), @@ -13761,6 +14603,90 @@ def f(a, b, d): ); } + #[test] + fn test_try_except_finally_normal_cleanup_keeps_body_exit_nop() { + let code = compile_exec( + "\ +def f(self, x): + if x and self.sock: + saved = self.sock.gettimeout() + self.sock.settimeout(x) + try: + resp = self._get_line() + except TimeoutError as err: + raise self._timeout from err + finally: + self.sock.settimeout(saved) + else: + resp = self._get_line() + return resp +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let resp_store = ops + .iter() + .position(|unit| match unit.op { + Instruction::StoreFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == "resp" + } + _ => false, + }) + .expect("missing resp store"); + + assert!( + matches!( + ops.get(resp_store + 1).map(|unit| unit.op), + Some(Instruction::Nop) + ), + "expected CPython-style NOP between try/except normal body exit and finally cleanup, got ops={ops:?}", + ); + } + + #[test] + fn test_try_except_finally_suppressing_handler_drops_body_exit_nop() { + let code = compile_exec( + "\ +def f(self): + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError as exc: + if exc.errno != errno.ENOTCONN: + raise + finally: + self.sock.close() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let shutdown_pop = ops + .iter() + .position(|op| matches!(op, Instruction::PopTop)) + .expect("missing shutdown POP_TOP"); + + assert!( + !matches!(ops.get(shutdown_pop + 1), Some(Instruction::Nop)), + "suppressing except handler should fall directly into finally cleanup without a CPython body-exit NOP, got ops={ops:?}", + ); + assert!( + matches!( + ops.get(shutdown_pop + 1), + Some(Instruction::LoadFastBorrow { .. }) + ), + "suppressing except handler should keep CPython-style borrowed finally cleanup receiver, got ops={ops:?}", + ); + } + #[test] fn test_conditional_break_finally_does_not_keep_break_cleanup_nop() { let code = compile_exec( @@ -13900,6 +14826,44 @@ def f(): ); } + #[test] + fn test_try_import_pass_else_keeps_borrow() { + let code = compile_exec( + "\ +def f(self): + try: + from _ctypes import set_conversion_mode + except ImportError: + pass + else: + self.prev_conv_mode = set_conversion_mode('ascii', 'strict') +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &ops[..handler_start]; + + assert!( + normal_tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "try-import pass/else normal path should keep CPython-style borrows, got tail={normal_tail:?}" + ); + } + #[test] fn test_protected_attr_direct_return_keeps_borrow() { let code = compile_exec( @@ -13981,6 +14945,91 @@ def f(tarfile, tarinfo, self): ); } + #[test] + fn test_generator_protected_store_subscr_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(names, modules): + for name in names: + try: + mod = __import__(name) + except ImportError: + continue + modules[name] = mod + yield mod +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let store_subscr = ops + .iter() + .position(|op| matches!(op, Instruction::StoreSubscr)) + .expect("missing STORE_SUBSCR"); + let window = &ops[store_subscr.saturating_sub(2)..(store_subscr + 3).min(ops.len())]; + + assert!( + matches!( + window, + [ + Instruction::LoadFastLoadFast { .. }, + Instruction::LoadFast { .. }, + Instruction::StoreSubscr, + Instruction::LoadFast { .. }, + Instruction::YieldValue { .. }, + .. + ] + ), + "expected CPython-style strong LOAD_FAST around protected STORE_SUBSCR generator tail, got {window:?}" + ); + } + + #[test] + fn test_protected_call_function_ex_store_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(func, *args): + try: + result = func(*args) + except Exception: + return None + return type(result) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let tail_call = ops + .iter() + .rposition(|op| matches!(op, Instruction::Call { .. })) + .expect("missing tail CALL"); + let result_store = ops[..tail_call] + .iter() + .rposition(|op| matches!(op, Instruction::StoreFast { .. })) + .expect("missing protected result STORE_FAST"); + let tail = &ops[result_store + 1..tail_call]; + + assert!( + tail.iter() + .any(|op| matches!(op, Instruction::LoadFast { .. })), + "expected CPython-style strong LOAD_FAST after protected CALL_FUNCTION_EX store, got ops={ops:?}", + ); + assert!( + !tail + .iter() + .any(|op| matches!(op, Instruction::LoadFastBorrow { .. })), + "protected CALL_FUNCTION_EX store tail should not borrow result, got ops={ops:?}", + ); + } + #[test] fn test_protected_attr_subscript_tail_uses_strong_load_fast() { let code = compile_exec( @@ -14019,15 +15068,15 @@ def f(obj, idx): } #[test] - fn test_protected_attr_iter_chain_uses_strong_load_fast() { + fn test_protected_direct_subscript_tail_uses_strong_load_fast() { let code = compile_exec( "\ -def f(fields): +def f(seq): try: - x = 1 + items = [int(item) for item in seq] except ValueError: - return False - return tuple(v for v in fields.values()) + return None + return items[0] + items[1] ", ); let f = find_code(&code, "f").expect("missing function code"); @@ -14041,37 +15090,37 @@ def f(fields): .iter() .position(|op| matches!(op, Instruction::PushExcInfo)) .expect("missing handler entry"); - let protected_tail = &ops[..handler_start]; + let protected_store = ops[..handler_start] + .iter() + .rposition(|op| matches!(op, Instruction::StoreFast { .. })) + .expect("missing protected local store"); + let tail = &ops[protected_store + 1..handler_start]; assert!( - !protected_tail.iter().any(|op| { + !tail.iter().any(|op| { matches!( op, Instruction::LoadFastBorrow { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected protected attr-iter chain to keep strong LOAD_FAST ops, got tail={protected_tail:?}" + "expected protected direct-subscript tail to keep strong LOAD_FAST ops, got tail={tail:?}" ); } #[test] - fn test_protected_attr_subscript_store_tail_uses_strong_load_fast() { + fn test_protected_attr_iter_chain_uses_strong_load_fast() { let code = compile_exec( "\ -def f(f, oldcls, newcls): +def f(fields): try: - idx = f.__code__.co_freevars.index('__class__') + x = 1 except ValueError: return False - closure = f.__closure__[idx] - if closure.cell_contents is oldcls: - closure.cell_contents = newcls - return True - return False + return tuple(v for v in fields.values()) ", ); - let f = find_code(&code, "f").expect("missing f code"); + let f = find_code(&code, "f").expect("missing function code"); let ops: Vec<_> = f .instructions .iter() @@ -14083,37 +15132,30 @@ def f(f, oldcls, newcls): .position(|op| matches!(op, Instruction::PushExcInfo)) .expect("missing handler entry"); let protected_tail = &ops[..handler_start]; - let store_closure_idx = protected_tail - .windows(2) - .position(|window| { - matches!( - window, - [Instruction::BinaryOp { .. }, Instruction::StoreFast { .. }] - ) - }) - .map(|idx| idx + 1) - .expect("missing STORE_FAST for closure"); - let post_store_tail = &protected_tail[store_closure_idx + 1..]; assert!( - !post_store_tail.iter().any(|op| { + !protected_tail.iter().any(|op| { matches!( op, Instruction::LoadFastBorrow { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected protected attr-subscript store tail to keep strong LOAD_FAST ops, got tail={post_store_tail:?}" + "expected protected attr-iter chain to keep strong LOAD_FAST ops, got tail={protected_tail:?}" ); } #[test] - fn test_plain_attr_subscript_tail_keeps_borrow() { + fn test_generator_except_return_handler_deopts_normal_tail_borrows() { let code = compile_exec( "\ -def f(self, name): - annotations = self.method_annotations[name] - return annotations +def f(fields): + try: + x = 1 + except ValueError: + return + for fielddesc in fields: + yield fielddesc ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -14123,29 +15165,42 @@ def f(self, name): .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &ops[..handler_start]; assert!( - ops.windows(4).any(|window| { + normal_tail + .iter() + .any(|op| matches!(op, Instruction::LoadFast { .. })), + "generator tail after non-yielding except return should keep CPython-style strong LOAD_FAST, got tail={normal_tail:?}" + ); + assert!( + !normal_tail.iter().any(|op| { matches!( - window, - [ - Instruction::LoadFastBorrow { .. }, - Instruction::LoadAttr { .. }, - Instruction::LoadFastBorrow { .. }, - Instruction::BinaryOp { .. }, - ] + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected plain attr-subscript tail to keep borrowed receiver/index loads, got ops={ops:?}" + "generator tail after non-yielding except return should not borrow, got tail={normal_tail:?}" ); } #[test] - fn test_plain_attr_iter_chain_keeps_borrow() { + fn test_generator_except_yielding_handler_keeps_normal_tail_borrows() { let code = compile_exec( "\ -def f(fields): - return tuple(v for v in fields.values()) +def f(tp, parent=None): + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + yield fielddesc ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -14155,358 +15210,392 @@ def f(fields): .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let normal_tail = &ops[..handler_start]; assert!( - ops.windows(4).any(|window| { + normal_tail.iter().any(|op| { matches!( - window, - [ - Instruction::LoadFastBorrow { .. }, - Instruction::LoadAttr { .. }, - Instruction::Call { .. }, - Instruction::GetIter, - ] + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected plain attr-iter chain to keep borrowed receiver, got ops={ops:?}" + "generator tail after yielding except handler should keep CPython-style borrows, got tail={normal_tail:?}" ); } #[test] - fn test_genexpr_true_filter_omits_bool_scaffolding() { + fn test_generator_except_pass_resume_tail_keeps_borrows() { let code = compile_exec( "\ -def f(it): - return (x for x in it if True) +def f(self, msg): + if self.log_queue is not None: + yield + output = [] + try: + while True: + output.append(self.log_queue.get_nowait().getMessage()) + except queue.Empty: + pass + else: + with self.assertLogs('concurrent.futures', 'CRITICAL') as cm: + yield + output = cm.output + self.assertTrue(any(msg in line for line in output), output) ", ); - let genexpr = find_code(&code, "").expect("missing code"); + let f = find_code(&code, "f").expect("missing f code"); + + let has_strong_load = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + let has_borrow_load = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + + for name in ["msg", "output"] { + assert!( + has_borrow_load(name), + "generator except-pass resume tail should borrow {name}, got instructions={:?}", + f.instructions + ); + assert!( + !has_strong_load(name), + "generator except-pass resume tail should not force strong LOAD_FAST for {name}, got instructions={:?}", + f.instructions + ); + } + } + + #[test] + fn test_async_for_cleanup_resume_tail_uses_strong_loads() { + let code = compile_exec( + "\ +async def f(g, self, x): + async for val in g: + break + self.x(x) + await g.aclose() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); + let aclose_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "aclose" + } + _ => false, + }) + .expect("missing aclose load"); + assert!( - !genexpr.instructions.iter().any(|unit| { - matches!(unit.op, Instruction::LoadConst { .. }) - && matches!( - genexpr.constants.get(usize::from(u8::from(unit.arg))), - Some(ConstantData::Boolean { value: true }) - ) + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFast { .. }, + Instruction::Call { .. }, + ] + ) }), - "constant-true filter should not load True, got ops={:?}", - genexpr - .instructions - .iter() - .map(|unit| unit.op) - .collect::>() + "async-for cleanup resume tail should use strong LOAD_FAST ops before the await, got ops={ops:?}" ); assert!( - !genexpr - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::PopJumpIfTrue { .. })), - "constant-true filter should not leave POP_JUMP_IF_TRUE scaffolding, got ops={:?}", - genexpr - .instructions - .iter() - .map(|unit| unit.op) - .collect::>() + matches!( + instructions + .get(aclose_idx.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFast { .. }) + ), + "async-for cleanup resume tail should keep g strong before aclose, got ops={ops:?}" ); } #[test] - fn test_classdictcell_uses_load_closure_path_and_borrows_after_optimize() { + fn test_async_generator_async_with_yield_keeps_borrow() { let code = compile_exec( "\ -class C: - def method(self): - return 1 +async def f(self, my_cm): + async with self.exit_stack() as stack: + await stack.enter_async_context(my_cm()) + yield stack ", ); - let class_code = find_code(&code, "C").expect("missing class code"); - let store_classdictcell = class_code + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f .instructions .iter() - .position(|unit| { - matches!( - unit.op, - Instruction::StoreName { namei } - if class_code.names - [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] - .as_str() - == "__classdictcell__" - ) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); + let wrap_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::CallIntrinsic1 { func } => { + func.get(OpArg::new(u32::from(u8::from(unit.arg)))) + == IntrinsicFunction1::AsyncGenWrap + } + _ => false, }) - .expect("missing STORE_NAME __classdictcell__"); + .expect("missing async generator wrap"); assert!( matches!( - class_code - .instructions - .get(store_classdictcell.saturating_sub(1)) - .map(|unit| unit.op), + ops.get(wrap_idx.saturating_sub(1)), Some(Instruction::LoadFastBorrow { .. }) ), - "expected LOAD_FAST_BORROW before __classdictcell__ store, got ops={:?}", - class_code - .instructions - .iter() - .map(|unit| unit.op) - .collect::>() + "async generator yield inside async-with should borrow stack like CPython, got ops={ops:?}" ); } #[test] - fn test_nested_function_static_attributes_are_collected() { + fn test_deoptimized_async_with_enter_continuation_uses_strong_loads() { let code = compile_exec( "\ -class C: - def f(self): - self.x = 1 - self.y = 2 - self.x = 3 - - def g(self, obj): - self.y = 4 - self.z = 5 - - def h(self, a): - self.u = 6 - self.v = 7 - - obj.self = 8 +async def f(): + async def cm(): + pass + try: + async with cm(): + 1 / 0 + except ZeroDivisionError as e: + frames = e + class E(RuntimeError): + pass + try: + async with cm(): + raise E(42) + except E as e: + frames = e ", ); - let class_code = find_code(&code, "C").expect("missing class code"); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); assert!( - class_code.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if elements - == &[ - ConstantData::Str { value: "u".into() }, - ConstantData::Str { value: "v".into() }, - ConstantData::Str { value: "x".into() }, - ConstantData::Str { value: "y".into() }, - ConstantData::Str { value: "z".into() }, - ] - )), - "expected nested function static attributes in class consts" + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::PushNull, + Instruction::LoadSmallInt { .. }, + Instruction::Call { .. }, + Instruction::RaiseVarargs { .. }, + ] + ) + }), + "async-with enter continuation after a deoptimized setup block should keep raised class strong, got ops={ops:?}" ); } #[test] - fn test_static_attributes_match_cpython_store_rule() { + fn test_async_with_bare_raise_continuation_keeps_borrow() { let code = compile_exec( "\ -class C: - @staticmethod - def f(): - self.x = 1 - - @classmethod - def g(cls): - self.y = 2 - - def h(obj): - obj.z = 3 - tarinfo.uid = 4 - - def i(self): - self.a: int - self.b: int = 1 - self.c += 1 - del self.d +async def f(tg): + class E(Exception): + pass + try: + async with tg: + raise E + except ExceptionGroup: + pass ", ); - let class_code = find_code(&code, "C").expect("missing class code"); - - assert!( - class_code.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if elements - == &[ - ConstantData::Str { value: "b".into() }, - ConstantData::Str { value: "x".into() }, - ConstantData::Str { value: "y".into() }, - ] - )), - "expected only CPython-collected static attributes in class consts" - ); - } - - #[test] - fn test_decorated_class_uses_first_decorator_for_firstlineno() { - let code = compile_exec( - "\ -@dec1 -@dec2 -class C: - pass -", - ); - let class_code = find_code(&code, "C").expect("missing class code"); - let store_firstlineno = class_code + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f .instructions .iter() - .position(|unit| { - matches!( - unit.op, - Instruction::StoreName { namei } - if class_code.names - [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] - .as_str() - == "__firstlineno__" - ) - }) - .expect("missing STORE_NAME __firstlineno__"); - let load_firstlineno = class_code - .instructions - .get(store_firstlineno.saturating_sub(1)) - .expect("missing LOAD_CONST for __firstlineno__"); + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let raise_idx = ops + .iter() + .position(|op| matches!(op, Instruction::RaiseVarargs { .. })) + .expect("missing raise"); - let expected = ConstantData::Integer { - value: BigInt::from(1), - }; assert!( matches!( - load_firstlineno.op, - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. } + ops.get(raise_idx.saturating_sub(1)), + Some(Instruction::LoadFastBorrow { .. }) ), - "expected LOAD_SMALL_INT/LOAD_CONST before __firstlineno__, got {:?}", - load_firstlineno.op + "bare async-with raise continuation should keep the raised class borrowed like CPython, got ops={ops:?}" ); - if let Instruction::LoadConst { consti } = load_firstlineno.op { - let value = &class_code.constants - [consti.get(OpArg::new(u32::from(u8::from(load_firstlineno.arg))))]; - assert_eq!(value, &expected); - } else { - assert_eq!(u32::from(u8::from(load_firstlineno.arg)), 1); - } } #[test] - fn test_future_annotations_class_keeps_conditional_annotations_cell() { + fn test_except_star_tail_uses_strong_loads() { let code = compile_exec( "\ -from __future__ import annotations -class C: - x: int +def f(self): + try: + pass + except* ValueError: + pass + self.fail('x') ", ); - let class_code = find_code(&code, "C").expect("missing class code"); - - assert!( - class_code - .cellvars - .iter() - .any(|name| name.as_str() == "__conditional_annotations__"), - "expected __conditional_annotations__ cellvar, got cellvars={:?}", - class_code.cellvars - ); - } - - #[test] - fn test_plain_super_call_keeps_class_freevar() { - let code = compile_exec( - "\ -class A: - pass + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); -class B(A): - def method(self): - return super() -", - ); - let method = find_code(&code, "method").expect("missing method code"); assert!( - method.freevars.iter().any(|name| name == "__class__"), - "plain super() must keep __class__ freevar, got freevars={:?}", - method.freevars + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "except* tail should use strong LOAD_FAST like CPython, got ops={ops:?}" ); assert!( - method - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::CopyFreeVars { .. })), - "plain super() must keep COPY_FREE_VARS prelude, got ops={:?}", - method - .instructions - .iter() - .map(|unit| unit.op) - .collect::>() + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "except* tail should not borrow the receiver after the handler region, got ops={ops:?}" ); } #[test] - fn test_chained_compare_jump_uses_single_cleanup_copy() { + fn test_protected_attr_subscript_store_tail_uses_strong_load_fast() { let code = compile_exec( "\ -def f(code): - if not 1 <= code <= 2147483647: - raise ValueError('x') +def f(f, oldcls, newcls): + try: + idx = f.__code__.co_freevars.index('__class__') + except ValueError: + return False + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False ", ); - let f = find_code(&code, "f").expect("missing function code"); - let copy_count = f + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f .instructions .iter() - .filter(|unit| matches!(unit.op, Instruction::Copy { .. })) - .count(); - let pop_top_count = f - .instructions + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let handler_start = ops .iter() - .filter(|unit| matches!(unit.op, Instruction::PopTop)) - .count(); + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let protected_tail = &ops[..handler_start]; + let store_closure_idx = protected_tail + .windows(2) + .position(|window| { + matches!( + window, + [Instruction::BinaryOp { .. }, Instruction::StoreFast { .. }] + ) + }) + .map(|idx| idx + 1) + .expect("missing STORE_FAST for closure"); + let post_store_tail = &protected_tail[store_closure_idx + 1..]; - assert_eq!(copy_count, 1); - assert_eq!(pop_top_count, 1); + assert!( + !post_store_tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected protected attr-subscript store tail to keep strong LOAD_FAST ops, got tail={post_store_tail:?}" + ); } #[test] - fn test_yield_from_cleanup_jumps_to_shared_end_send() { + fn test_plain_attr_subscript_tail_keeps_borrow() { let code = compile_exec( "\ -def outer(): - def inner(): - yield from outer_gen - return inner +def f(self, name): + annotations = self.method_annotations[name] + return annotations ", ); - let inner = find_code(&code, "inner").expect("missing inner code"); - let ops: Vec<_> = inner + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f .instructions .iter() .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let cleanup_idx = ops - .iter() - .position(|op| matches!(op, Instruction::CleanupThrow)) - .expect("missing CLEANUP_THROW"); - assert!( - matches!( - ops.get(cleanup_idx + 1), - Some(Instruction::JumpBackwardNoInterrupt { .. } | Instruction::JumpForward { .. }) - ), - "expected CLEANUP_THROW to jump to shared END_SEND block, got ops={ops:?}" - ); assert!( - !matches!(ops.get(cleanup_idx + 1), Some(Instruction::EndSend)), - "CLEANUP_THROW should not inline END_SEND directly, got ops={ops:?}" + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::BinaryOp { .. }, + ] + ) + }), + "expected plain attr-subscript tail to keep borrowed receiver/index loads, got ops={ops:?}" ); } #[test] - fn test_try_except_falls_through_to_post_handler_code() { + fn test_plain_attr_iter_chain_keeps_borrow() { let code = compile_exec( "\ -def f(): - try: - line = 2 - raise KeyError - except: - line = 5 - line = 6 +def f(fields): + return tuple(v for v in fields.values()) ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -14517,348 +15606,503 @@ def f(): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let first_pop_except = ops - .iter() - .position(|op| matches!(op, Instruction::PopExcept)) - .expect("missing POP_EXCEPT"); - assert!( - !matches!( - ops.get(first_pop_except + 1), - Some(Instruction::JumpForward { .. }) - ), - "expected except body to fall through to post-handler code, got ops={ops:?}" - ); assert!( - matches!( - ops.get(first_pop_except + 1), - Some(Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }) - ), - "expected line-after-except code immediately after POP_EXCEPT, got ops={ops:?}" + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. }, + Instruction::LoadAttr { .. }, + Instruction::Call { .. }, + Instruction::GetIter, + ] + ) + }), + "expected plain attr-iter chain to keep borrowed receiver, got ops={ops:?}" ); } #[test] - fn test_named_except_cleanup_keeps_jump_over_cleanup_and_next_try() { + fn test_genexpr_true_filter_omits_bool_scaffolding() { let code = compile_exec( - r#" -def f(self): - try: - assert 0, 'msg' - except AssertionError as e: - self.assertEqual(e.args[0], 'msg') - else: - self.fail("AssertionError not raised by assert 0") + "\ +def f(it): + return (x for x in it if True) +", + ); + let genexpr = find_code(&code, "").expect("missing code"); + assert!( + !genexpr.instructions.iter().any(|unit| { + matches!(unit.op, Instruction::LoadConst { .. }) + && matches!( + genexpr.constants.get(usize::from(u8::from(unit.arg))), + Some(ConstantData::Boolean { value: true }) + ) + }), + "constant-true filter should not load True, got ops={:?}", + genexpr + .instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert!( + !genexpr + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::PopJumpIfTrue { .. })), + "constant-true filter should not leave POP_JUMP_IF_TRUE scaffolding, got ops={:?}", + genexpr + .instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } - try: - assert False - except AssertionError as e: - self.assertEqual(len(e.args), 0) - else: - self.fail("AssertionError not raised by 'assert False'") -"#, + #[test] + fn test_classdictcell_uses_load_closure_path_and_borrows_after_optimize() { + let code = compile_exec( + "\ +class C: + def method(self): + return 1 +", ); - let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let class_code = find_code(&code, "C").expect("missing class code"); + let store_classdictcell = class_code .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); + .position(|unit| { + matches!( + unit.op, + Instruction::StoreName { namei } + if class_code.names + [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] + .as_str() + == "__classdictcell__" + ) + }) + .expect("missing STORE_NAME __classdictcell__"); - let first_pop_except = ops - .iter() - .position(|op| matches!(op, Instruction::PopExcept)) - .expect("missing POP_EXCEPT"); - let window = &ops[first_pop_except..(first_pop_except + 6).min(ops.len())]; assert!( matches!( - window, - [ - Instruction::PopExcept, - Instruction::LoadConst { .. }, - Instruction::StoreName { .. } | Instruction::StoreFast { .. }, - Instruction::DeleteName { .. } | Instruction::DeleteFast { .. }, - Instruction::JumpForward { .. }, - .. - ] + class_code + .instructions + .get(store_classdictcell.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) ), - "expected named except cleanup to jump over cleanup reraise block, got ops={window:?}" + "expected LOAD_FAST_BORROW before __classdictcell__ store, got ops={:?}", + class_code + .instructions + .iter() + .map(|unit| unit.op) + .collect::>() ); } #[test] - fn test_bare_except_deopts_post_handler_load_fast_borrow() { + fn test_conditional_class_body_duplicates_no_location_exit_tail() { let code = compile_exec( "\ -def f(self): - try: - 1 / 0 - except: - pass - with self.assertRaises(SyntaxError): - pass +flag = False +class C: + if flag: + value = 1 ", ); - let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let class_code = find_code(&code, "C").expect("missing class code"); + let ops: Vec<_> = class_code .instructions .iter() .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - - let attr_idx = ops + let return_count = ops .iter() - .position(|op| matches!(op, Instruction::LoadAttr { .. })) - .expect("missing LOAD_ATTR for assertRaises"); - assert!( - matches!(ops.get(attr_idx - 1), Some(Instruction::LoadFast { .. })), - "bare except tail should deopt self to LOAD_FAST, got ops={ops:?}" + .filter(|op| matches!(op, Instruction::ReturnValue)) + .count(); + let static_attrs_count = class_code + .instructions + .iter() + .filter(|unit| { + matches!( + unit.op, + Instruction::StoreName { namei } + if class_code.names + [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] + .as_str() + == "__static_attributes__" + ) + }) + .count(); + + assert_eq!( + return_count, 2, + "conditional class body should duplicate CPython no-location return tail, got ops={ops:?}" + ); + assert_eq!( + static_attrs_count, 2, + "conditional class body should duplicate __static_attributes__ tail, got ops={ops:?}" ); } #[test] - fn test_typed_except_keeps_post_handler_load_fast_borrow() { + fn test_class_lambda_assignment_does_not_create_classdictcell() { let code = compile_exec( "\ -def f(self): - try: - 1 / 0 - except ZeroDivisionError: - pass - with self.assertRaises(SyntaxError): - pass +class C: + data = start = end = lambda *a: None ", ); - let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f - .instructions - .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); + let class_code = find_code(&code, "C").expect("missing class code"); - let attr_idx = ops - .iter() - .position(|op| matches!(op, Instruction::LoadAttr { .. })) - .expect("missing LOAD_ATTR for assertRaises"); assert!( - matches!( - ops.get(attr_idx - 1), - Some(Instruction::LoadFastBorrow { .. }) - ), - "typed except tail should keep LOAD_FAST_BORROW, got ops={ops:?}" + !class_code.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::StoreName { namei } + if class_code.names + [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] + .as_str() + == "__classdictcell__" + ) + }), + "lambda-only class should not create __classdictcell__, got ops={:?}", + class_code + .instructions + .iter() + .map(|unit| unit.op) + .collect::>() ); } #[test] - fn test_dunder_debug_constant_false_if_deopts_tail_borrow() { + fn test_nested_function_static_attributes_are_collected() { let code = compile_exec( "\ -def f(self): - if not __debug__: - self.skipTest('need asserts, run without -O') - self.do_disassembly_test() +class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 ", ); - let f = find_code(&code, "f").expect("missing f code"); - let instructions: Vec<_> = f - .instructions - .iter() - .filter(|unit| !matches!(unit.op, Instruction::Cache)) - .collect(); - let attr_idx = instructions - .iter() - .position(|unit| match unit.op { - Instruction::LoadAttr { namei } => { - let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); - f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() - == "do_disassembly_test" - } - _ => false, - }) - .expect("missing LOAD_ATTR for do_disassembly_test"); - let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); + let class_code = find_code(&code, "C").expect("missing class code"); + assert!( - matches!(ops.get(attr_idx - 1), Some(Instruction::LoadFast { .. })), - "constant-false __debug__ tail should deopt self to LOAD_FAST, got ops={ops:?}" + class_code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if elements + == &[ + ConstantData::Str { value: "u".into() }, + ConstantData::Str { value: "v".into() }, + ConstantData::Str { value: "x".into() }, + ConstantData::Str { value: "y".into() }, + ConstantData::Str { value: "z".into() }, + ] + )), + "expected nested function static attributes in class consts" ); } #[test] - fn test_constant_slice_folds_constant_bounds() { + fn test_static_attributes_match_cpython_store_rule() { let code = compile_exec( "\ -def f(obj): - return obj['a':123456789012345678901234567890] -", +class C: + @staticmethod + def f(): + self.x = 1 + + @classmethod + def g(cls): + self.y = 2 + + def h(obj): + obj.z = 3 + tarinfo.uid = 4 + + def i(self): + self.a: int + self.b: int = 1 + self.c += 1 + del self.d +", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f + let class_code = find_code(&code, "C").expect("missing class code"); + + assert!( + class_code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if elements + == &[ + ConstantData::Str { value: "b".into() }, + ConstantData::Str { value: "x".into() }, + ConstantData::Str { value: "y".into() }, + ] + )), + "expected only CPython-collected static attributes in class consts" + ); + } + + #[test] + fn test_decorated_class_uses_first_decorator_for_firstlineno() { + let code = compile_exec( + "\ +@dec1 +@dec2 +class C: + pass +", + ); + let class_code = find_code(&code, "C").expect("missing class code"); + let store_firstlineno = class_code .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); - let folded_slice = f - .constants - .iter() - .find_map(|constant| match constant { - ConstantData::Slice { elements } => Some(elements), - _ => None, + .position(|unit| { + matches!( + unit.op, + Instruction::StoreName { namei } + if class_code.names + [namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize] + .as_str() + == "__firstlineno__" + ) }) - .expect("missing folded slice constant"); + .expect("missing STORE_NAME __firstlineno__"); + let load_firstlineno = class_code + .instructions + .get(store_firstlineno.saturating_sub(1)) + .expect("missing LOAD_CONST for __firstlineno__"); + + let expected = ConstantData::Integer { + value: BigInt::from(1), + }; assert!( matches!( - folded_slice.as_ref(), - [ - ConstantData::Str { .. }, - ConstantData::Integer { .. }, - ConstantData::None, - ] + load_firstlineno.op, + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. } ), - "expected folded slice('a', 123456789012345678901234567890, None), got {folded_slice:?}" + "expected LOAD_SMALL_INT/LOAD_CONST before __firstlineno__, got {:?}", + load_firstlineno.op + ); + if let Instruction::LoadConst { consti } = load_firstlineno.op { + let value = &class_code.constants + [consti.get(OpArg::new(u32::from(u8::from(load_firstlineno.arg))))]; + assert_eq!(value, &expected); + } else { + assert_eq!(u32::from(u8::from(load_firstlineno.arg)), 1); + } + } + + #[test] + fn test_future_annotations_class_keeps_conditional_annotations_cell() { + let code = compile_exec( + "\ +from __future__ import annotations +class C: + x: int +", ); + let class_code = find_code(&code, "C").expect("missing class code"); + assert!( - matches!( - ops.as_slice(), - [ - Instruction::Resume { .. }, - Instruction::LoadFastBorrow { .. }, - Instruction::LoadConst { .. }, - Instruction::BinaryOp { .. }, - Instruction::ReturnValue, - ] - ), - "expected CPython-style LOAD_CONST(slice(...)) path for constant bounds, got ops={ops:?}" + class_code + .cellvars + .iter() + .any(|name| name.as_str() == "__conditional_annotations__"), + "expected __conditional_annotations__ cellvar, got cellvars={:?}", + class_code.cellvars ); } #[test] - fn test_tuple_bound_slice_uses_two_part_slice_path() { + fn test_plain_super_call_keeps_class_freevar() { let code = compile_exec( "\ -def f(obj): - return obj[(1, 2):] +class A: + pass + +class B(A): + def method(self): + return super() ", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f - .instructions - .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); + let method = find_code(&code, "method").expect("missing method code"); + assert!( + method.freevars.iter().any(|name| name == "__class__"), + "plain super() must keep __class__ freevar, got freevars={:?}", + method.freevars + ); + assert!( + method + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::CopyFreeVars { .. })), + "plain super() must keep COPY_FREE_VARS prelude, got ops={:?}", + method + .instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } + + #[test] + fn test_nested_class_super_does_not_create_outer_class_closure() { + let code = compile_exec( + "\ +class C: + def outer(self): + class D: + def __init__(self): + super().__init__() +", + ); + let outer_class = find_code(&code, "C").expect("missing outer class code"); + let nested_class = find_code(&code, "D").expect("missing nested class code"); + let init = find_code(&code, "__init__").expect("missing nested __init__ code"); assert!( - matches!( - ops.as_slice(), - [ - Instruction::Resume { .. }, - Instruction::LoadFastBorrow { .. }, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::BinarySlice, - Instruction::ReturnValue, - ] - ), - "expected CPython-style BINARY_SLICE path for tuple lower bound, got ops={ops:?}" + !outer_class.cellvars.iter().any(|name| name == "__class__"), + "nested super() must not force __class__ on outer class, got cellvars={:?}", + outer_class.cellvars + ); + assert!( + nested_class.cellvars.iter().any(|name| name == "__class__"), + "nested class should own __class__ cell, got cellvars={:?}", + nested_class.cellvars + ); + assert!( + init.freevars.iter().any(|name| name == "__class__"), + "method using super() should close over nested class, got freevars={:?}", + init.freevars ); } #[test] - fn test_exception_cleanup_jump_to_return_is_inlined() { + fn test_nested_closure_parameter_class_does_not_create_outer_class_closure() { let code = compile_exec( "\ -def f(names, cls): - try: - cls.attr = names - except: - pass - return names +class C: + def m(self): + def create_closure(__class__): + return (lambda: __class__).__closure__ +", + ); + let outer_class = find_code(&code, "C").expect("missing class code"); + let create_closure = + find_code(&code, "create_closure").expect("missing create_closure code"); + let lambda = find_code(&code, "").expect("missing lambda code"); + + assert!( + !outer_class.cellvars.iter().any(|name| name == "__class__"), + "nested __class__ parameter must not force outer class cell, got cellvars={:?}", + outer_class.cellvars + ); + assert!( + create_closure + .cellvars + .iter() + .any(|name| name == "__class__"), + "create_closure should own __class__ parameter cell, got cellvars={:?}", + create_closure.cellvars + ); + assert!( + lambda.freevars.iter().any(|name| name == "__class__"), + "lambda should close over create_closure parameter, got freevars={:?}", + lambda.freevars + ); + } + + #[test] + fn test_chained_compare_jump_uses_single_cleanup_copy() { + let code = compile_exec( + "\ +def f(code): + if not 1 <= code <= 2147483647: + raise ValueError('x') ", ); let f = find_code(&code, "f").expect("missing function code"); - let return_count = f + let copy_count = f .instructions .iter() - .filter(|unit| matches!(unit.op, Instruction::ReturnValue)) + .filter(|unit| matches!(unit.op, Instruction::Copy { .. })) + .count(); + let pop_top_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::PopTop)) .count(); - assert_eq!( - return_count, 2, - "expected CPython-style distinct return sites for normal and except paths" - ); + assert_eq!(copy_count, 1); + assert_eq!(pop_top_count, 1); } #[test] - fn test_nested_with_bare_except_keeps_handler_cleanup_before_following_code() { + fn test_yield_from_cleanup_jumps_to_shared_end_send() { let code = compile_exec( "\ -def f(cm, self): - try: - with cm: - raise Exception - except: - pass - self.g() +def outer(): + def inner(): + yield from outer_gen + return inner ", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f + let inner = find_code(&code, "inner").expect("missing inner code"); + let ops: Vec<_> = inner .instructions .iter() .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let outer_handler = ops + let cleanup_idx = ops .iter() - .enumerate() - .filter_map(|(idx, op)| matches!(op, Instruction::PushExcInfo).then_some(idx)) - .next_back() - .expect("missing outer handler"); + .position(|op| matches!(op, Instruction::CleanupThrow)) + .expect("missing CLEANUP_THROW"); assert!( - ops[outer_handler..].windows(6).any(|window| { - matches!( - window, - [ - Instruction::PopExcept, - Instruction::JumpForward { .. }, - Instruction::Copy { .. }, - Instruction::PopExcept, - Instruction::Reraise { .. }, - Instruction::LoadFast { .. }, - ] - ) - }), - "expected CPython-style handler cleanup before following code, got ops={ops:?}" + matches!( + ops.get(cleanup_idx + 1), + Some(Instruction::JumpBackwardNoInterrupt { .. } | Instruction::JumpForward { .. }) + ), + "expected CLEANUP_THROW to jump to shared END_SEND block, got ops={ops:?}" + ); + assert!( + !matches!(ops.get(cleanup_idx + 1), Some(Instruction::EndSend)), + "CLEANUP_THROW should not inline END_SEND directly, got ops={ops:?}" ); } #[test] - fn test_try_else_for_cleanup_drops_redundant_jump_nop() { + fn test_try_except_falls_through_to_post_handler_code() { let code = compile_exec( "\ -def f(self, xs, ys, cm1, cm2): - for x in xs: - with self.subTest(x=x): - try: - with cm1: - self.a() - except Exception: - if x: - pass - else: - raise - else: - for y in ys: - with self.subTest(y=y): - with cm2: - self.b() +def f(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 ", ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() @@ -14866,101 +16110,47 @@ def f(self, xs, ys, cm1, cm2): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); + let first_pop_except = ops + .iter() + .position(|op| matches!(op, Instruction::PopExcept)) + .expect("missing POP_EXCEPT"); assert!( - ops.windows(7).any(|window| { - matches!( - window, - [ - Instruction::EndFor, - Instruction::PopIter, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::Call { .. }, - Instruction::PopTop, - ] - ) - }), - "expected inner for cleanup to fall directly into surrounding with cleanup, got ops={ops:?}", + !matches!( + ops.get(first_pop_except + 1), + Some(Instruction::JumpForward { .. }) + ), + "expected except body to fall through to post-handler code, got ops={ops:?}" ); assert!( - !ops.windows(8).any(|window| { - matches!( - window, - [ - Instruction::EndFor, - Instruction::PopIter, - Instruction::Nop, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::Call { .. }, - Instruction::PopTop, - ] - ) - }), - "expected CPython-style removal of the redundant jump NOP after for cleanup, got ops={ops:?}", + matches!( + ops.get(first_pop_except + 1), + Some(Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }) + ), + "expected line-after-except code immediately after POP_EXCEPT, got ops={ops:?}" ); } #[test] - fn test_non_none_final_return_is_not_duplicated() { + fn test_named_except_cleanup_keeps_jump_over_cleanup_and_next_try() { let code = compile_exec( - "\ -def f(p, s): - if p == '': - if s == '': - return 0 - return -1 -", - ); - let f = find_code(&code, "f").expect("missing function code"); - let minus_one_loads = f - .instructions - .iter() - .filter(|unit| { - matches!( - unit.op, - Instruction::LoadConst { consti } - if matches!( - f.constants.get( - consti - .get(OpArg::new(u32::from(u8::from(unit.arg)))) - .as_usize() - ), - Some(ConstantData::Integer { value }) if value == &BigInt::from(-1) - ) - ) - }) - .count(); - - assert_eq!( - minus_one_loads, - 1, - "expected a single final return -1 epilogue, got ops={:?}", - f.instructions - .iter() - .map(|unit| unit.op) - .collect::>() - ); - } + r#" +def f(self): + try: + assert 0, 'msg' + except AssertionError as e: + self.assertEqual(e.args[0], 'msg') + else: + self.fail("AssertionError not raised by assert 0") - #[test] - fn test_try_else_if_return_keeps_conditional_target_nop() { - let code = compile_exec( - "\ -def f(cond): try: - x = cond - except E: - pass + assert False + except AssertionError as e: + self.assertEqual(len(e.args), 0) else: - if x: - return 1 - return 2 -", + self.fail("AssertionError not raised by 'assert False'") +"#, ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() @@ -14968,48 +16158,41 @@ def f(cond): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let has_cpython_nop_target = ops.windows(5).any(|window| { - matches!( - window, - [ - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, - Instruction::Nop, - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, - ] - ) - }); - let has_direct_fallthrough = ops.windows(4).any(|window| { + let first_pop_except = ops + .iter() + .position(|op| matches!(op, Instruction::PopExcept)) + .expect("missing POP_EXCEPT"); + let window = &ops[first_pop_except..(first_pop_except + 6).min(ops.len())]; + assert!( matches!( window, [ - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, - Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, - Instruction::ReturnValue, + Instruction::PopExcept, + Instruction::LoadConst { .. }, + Instruction::StoreName { .. } | Instruction::StoreFast { .. }, + Instruction::DeleteName { .. } | Instruction::DeleteFast { .. }, + Instruction::JumpForward { .. }, + .. ] - ) - }); - assert!( - has_cpython_nop_target || has_direct_fallthrough, - "expected adjacent try-else return and final return targets, got ops={ops:?}" + ), + "expected named except cleanup to jump over cleanup reraise block, got ops={window:?}" ); } #[test] - fn test_named_except_conditional_branch_duplicates_cleanup_return() { + fn test_bare_except_deopts_post_handler_load_fast_borrow() { let code = compile_exec( "\ def f(self): try: - raise TypeError('x') - except TypeError as e: - if '+' not in str(e): - self.fail('join() ate exception message') + 1 / 0 + except: + pass + with self.assertRaises(SyntaxError): + pass ", ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() @@ -15017,39 +16200,30 @@ def f(self): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let cleanup_return_count = ops - .windows(6) - .filter(|window| { - matches!( - window, - [ - Instruction::PopExcept, - Instruction::LoadConst { .. }, - Instruction::StoreFast { .. } | Instruction::StoreName { .. }, - Instruction::DeleteFast { .. } | Instruction::DeleteName { .. }, - Instruction::LoadConst { .. }, - Instruction::ReturnValue, - ] - ) - }) - .count(); - - assert_eq!( - cleanup_return_count, 2, - "expected duplicated named-except cleanup return blocks, got ops={ops:?}" + let attr_idx = ops + .iter() + .position(|op| matches!(op, Instruction::LoadAttr { .. })) + .expect("missing LOAD_ATTR for assertRaises"); + assert!( + matches!(ops.get(attr_idx - 1), Some(Instruction::LoadFast { .. })), + "bare except tail should deopt self to LOAD_FAST, got ops={ops:?}" ); } #[test] - fn test_listcomp_cleanup_tail_keeps_split_store_fast_pair() { + fn test_typed_except_keeps_post_handler_load_fast_borrow() { let code = compile_exec( "\ -def f(escaped_string, quote_types): - possible_quotes = [q for q in quote_types if q not in escaped_string] - return possible_quotes +def f(self): + try: + 1 / 0 + except ZeroDivisionError: + pass + with self.assertRaises(SyntaxError): + pass ", ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() @@ -15057,36 +16231,34 @@ def f(escaped_string, quote_types): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let pop_iter_idx = ops + let attr_idx = ops .iter() - .position(|op| matches!(op, Instruction::PopIter)) - .expect("missing POP_ITER"); - let tail = &ops[pop_iter_idx + 1..]; - + .position(|op| matches!(op, Instruction::LoadAttr { .. })) + .expect("missing LOAD_ATTR for assertRaises"); assert!( matches!( - tail, - [ - Instruction::StoreFast { .. }, - Instruction::StoreFast { .. }, - Instruction::LoadFastBorrow { .. }, - Instruction::ReturnValue, - .. - ] + ops.get(attr_idx - 1), + Some(Instruction::LoadFastBorrow { .. }) ), - "expected split STORE_FAST pair after listcomp cleanup, got ops={ops:?}" + "typed except tail should keep LOAD_FAST_BORROW, got ops={ops:?}" ); } #[test] - fn test_dictcomp_cleanup_tail_keeps_split_store_fast_pair() { + fn test_reraising_typed_except_deopts_post_handler_loads() { let code = compile_exec( "\ -def f(obj, g): - return {g(k): g(v) for k, v in obj.items()} +def f(x, os, self, pid, exitcode): + try: + y = 1 + except RuntimeError: + raise + if x: + os._exit(exitcode) + self.wait_impl(pid, exitcode=exitcode) ", ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() @@ -15094,827 +16266,4720 @@ def f(obj, g): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - let pop_iter_idx = ops + let guard_idx = ops .iter() - .position(|op| matches!(op, Instruction::PopIter)) - .expect("missing POP_ITER"); - let tail = &ops[pop_iter_idx + 1..]; + .position(|op| matches!(op, Instruction::ToBool)) + .and_then(|idx| idx.checked_sub(1)) + .expect("missing post-handler bool guard"); + assert!( + matches!(ops.get(guard_idx), Some(Instruction::LoadFast { .. })), + "reraising typed except tail should deopt guard load, got ops={ops:?}" + ); + let wait_idx = ops + .iter() + .position(|op| matches!(op, Instruction::CallKw { .. })) + .expect("missing wait_impl CALL_KW"); + let call_args = &ops[wait_idx.saturating_sub(3)..wait_idx]; assert!( - matches!( - tail, - [ - Instruction::Swap { .. }, - Instruction::StoreFast { .. }, - Instruction::StoreFast { .. }, - Instruction::ReturnValue, - .. - ] - ), - "expected split STORE_FAST pair after dictcomp cleanup, got ops={ops:?}" + call_args.iter().any(|op| matches!( + op, + Instruction::LoadFastLoadFast { .. } | Instruction::LoadFast { .. } + )), + "reraising typed except tail should keep strong fast loads for call args, got ops={ops:?}" + ); + assert!( + !call_args.iter().any(|op| matches!( + op, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastBorrow { .. } + )), + "reraising typed except tail should not borrow call args, got ops={ops:?}" ); } #[test] - fn test_static_swap_triple_assign_keeps_store_fast_store_fast() { + fn test_reraising_except_loop_backedge_keeps_loop_header_borrow() { let code = compile_exec( "\ -def f(x, y, z): - a, b, a = x, y, z - return a +def f(self, tag, expect_bye): + while 1: + result = self.tagged_commands[tag] + if result is not None: + del self.tagged_commands[tag] + return result + if expect_bye: + typ = 'BYE' + bye = self.untagged_responses.pop(typ, None) + if bye is not None: + return (typ, bye) + self._check_bye() + try: + self._get_response() + except self.abort as val: + if __debug__: + if self.debug >= 1: + self.print_log() + raise ", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_ops: Vec<_> = instructions[..handler_start] .iter() .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) .collect(); assert!( - ops.windows(3).any(|window| { - matches!( - window, - [ - Instruction::Swap { .. }, - Instruction::StoreFastStoreFast { .. }, - Instruction::StoreFast { .. } - ] - ) - }), - "expected CPython-style SWAP/STORE_FAST_STORE_FAST/STORE_FAST sequence, got ops={ops:?}" + warm_ops.iter().any(|op| matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + )), + "expected loop body before reraising handler to keep borrowed loads, got ops={warm_ops:?}" + ); + assert!( + warm_ops + .iter() + .all(|op| !matches!(op, Instruction::LoadFast { .. })), + "loop backedge into reraising handler should not deopt warm loop loads, got ops={warm_ops:?}" ); } #[test] - fn test_constant_if_expression_stmt_in_loop_removes_empty_body() { + fn test_protected_store_break_handler_deopts_bool_guard_tail() { let code = compile_exec( "\ -def f(x): - while x: - 0 if 1 else 0 +def f(self, size): + parts = [] + while size > 0: + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break + if not buf: + break + self._readbuf.append(buf) + size -= len(buf) + return b''.join(parts) ", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); + let guard_bool = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::ToBool)) + .expect("missing bool guard"); + let store_buf = instructions[..guard_bool] + .iter() + .rposition(|unit| matches!(unit.op, Instruction::StoreFast { .. })) + .expect("missing protected STORE_FAST before bool guard"); + let guard_load = instructions[store_buf + 1].op; + let append_call = instructions[store_buf + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::Call { .. })) + .map(|idx| idx + store_buf + 1) + .expect("missing append call"); + let append_arg = instructions[append_call - 1].op; assert!( - !ops.iter() - .any(|op| matches!(op, Instruction::LoadSmallInt { .. })), - "expected constant if-expression statement to compile away inside loop, got ops={ops:?}" + matches!(guard_load, Instruction::LoadFast { .. }), + "CPython uses strong LOAD_FAST for protected-store break guard, got ops={:?}", + instructions.iter().map(|unit| unit.op).collect::>() + ); + assert!( + matches!(append_arg, Instruction::LoadFast { .. }), + "CPython uses strong LOAD_FAST for protected-store append arg, got ops={:?}", + instructions.iter().map(|unit| unit.op).collect::>() ); } #[test] - fn test_if_expression_in_jump_context_skips_constant_true_arm_load() { + fn test_assertion_success_join_deopts_following_debug_tail() { let code = compile_exec( "\ -def f(): - a if (1 if b else c) else d +def f(self, typ, dat): + if self._idle_capture: + if self._idle_responses: + response = self._idle_responses[-1] + assert response[0] == typ + response[1].append(dat) + else: + self._idle_responses.append((typ, [dat])) + if self.debug >= 5: + self._mesg(f'idle: queue untagged {typ} {dat!r}') + return ", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); + let debug_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "debug" + } + _ => false, + }) + .expect("missing debug LOAD_ATTR"); + let mesg_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "_mesg" + } + _ => false, + }) + .expect("missing _mesg LOAD_ATTR"); assert!( - !ops.iter() - .any(|op| matches!(op, Instruction::LoadSmallInt { .. })), - "expected jump-context if-expression to avoid materializing constant truthy arm, got ops={ops:?}" + matches!( + instructions[debug_attr - 1].op, + Instruction::LoadFast { .. } + ), + "CPython uses strong LOAD_FAST after assertion success join, got ops={:?}", + instructions.iter().map(|unit| unit.op).collect::>() + ); + assert!( + matches!(instructions[mesg_attr - 1].op, Instruction::LoadFast { .. }), + "CPython uses strong LOAD_FAST in assertion-success debug body, got ops={:?}", + instructions.iter().map(|unit| unit.op).collect::>() ); } #[test] - fn test_with_suppress_tail_duplicates_final_return_none() { + fn test_multi_protected_method_call_terminal_handler_deopts_block() { let code = compile_exec( "\ -def f(cm, cond): - if cond: - with cm(): - pass +def f(self, literal): + try: + self.send(literal) + self.send(CRLF) + except OSError as val: + raise self.abort('socket error: %s' % val) ", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - let return_count = ops + let first_send = instructions .iter() - .filter(|op| matches!(op, Instruction::ReturnValue)) - .count(); + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "send" + } + _ => false, + }) + .expect("missing send LOAD_ATTR"); + let first_literal = instructions[first_send + 1].op; + let second_send = instructions[first_send + 1..] + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "send" + } + _ => false, + }) + .map(|idx| idx + first_send + 1) + .expect("missing second send LOAD_ATTR"); - assert_eq!( - return_count, 3, - "expected duplicated return-none epilogues, got ops={ops:?}" + assert!( + matches!( + instructions[first_send - 1].op, + Instruction::LoadFast { .. } + ), + "CPython uses strong LOAD_FAST for first protected send receiver, got ops={:?}", + instructions.iter().map(|unit| unit.op).collect::>() ); assert!( - !ops.iter() - .any(|op| matches!(op, Instruction::JumpBackwardNoInterrupt { .. })), - "with suppress tail should not jump back to shared return block, got ops={ops:?}" + matches!(first_literal, Instruction::LoadFast { .. }), + "CPython uses strong LOAD_FAST for first protected send arg, got ops={:?}", + instructions.iter().map(|unit| unit.op).collect::>() + ); + assert!( + matches!( + instructions[second_send - 1].op, + Instruction::LoadFast { .. } + ), + "CPython uses strong LOAD_FAST for second protected send receiver, got ops={:?}", + instructions.iter().map(|unit| unit.op).collect::>() ); } #[test] - fn test_with_conditional_bare_return_keeps_return_line_nop_before_exit_cleanup() { + fn test_dunder_debug_constant_false_if_deopts_tail_borrow() { let code = compile_exec( "\ -def f(cm, registry, altkey): - with cm: - if registry.get(altkey): - return - registry[altkey] = 1 +def f(self): + if not __debug__: + self.skipTest('need asserts, run without -O') + self.do_disassembly_test() ", ); - let f = find_code(&code, "f").expect("missing function code"); - let ops: Vec<_> = f + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - + let attr_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "do_disassembly_test" + } + _ => false, + }) + .expect("missing LOAD_ATTR for do_disassembly_test"); + let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); assert!( - ops.windows(8).any(|window| { - matches!( - window, - [ - Instruction::Nop, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::Call { .. }, - Instruction::PopTop, - Instruction::LoadConst { .. }, - Instruction::ReturnValue, - ] - ) - }), - "expected CPython-style return-line NOP before with-exit cleanup return, got ops={ops:?}" + matches!(ops.get(attr_idx - 1), Some(Instruction::LoadFast { .. })), + "constant-false __debug__ tail should deopt self to LOAD_FAST, got ops={ops:?}" ); } #[test] - fn test_genexpr_compare_header_uses_store_fast_load_fast_like_cpython() { + fn test_constant_slice_folds_constant_bounds() { let code = compile_exec( "\ -def f(it): - return (offset == (4, 10) for offset in it) +def f(obj): + return obj['a':123456789012345678901234567890] ", ); - let genexpr = find_code(&code, "").expect("missing code"); - let ops: Vec<_> = genexpr + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f .instructions .iter() .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); - + let folded_slice = f + .constants + .iter() + .find_map(|constant| match constant { + ConstantData::Slice { elements } => Some(elements), + _ => None, + }) + .expect("missing folded slice constant"); assert!( - ops.windows(3).any(|window| { - matches!( - window, - [ - Instruction::StoreFastLoadFast { .. }, - Instruction::LoadConst { .. }, - Instruction::CompareOp { .. }, - ] - ) - }), - "expected CPython-style STORE_FAST_LOAD_FAST compare header, got ops={ops:?}" - ); + matches!( + folded_slice.as_ref(), + [ + ConstantData::Str { .. }, + ConstantData::Integer { .. }, + ConstantData::None, + ] + ), + "expected folded slice('a', 123456789012345678901234567890, None), got {folded_slice:?}" + ); + assert!( + matches!( + ops.as_slice(), + [ + Instruction::Resume { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::LoadConst { .. }, + Instruction::BinaryOp { .. }, + Instruction::ReturnValue, + ] + ), + "expected CPython-style LOAD_CONST(slice(...)) path for constant bounds, got ops={ops:?}" + ); } #[test] - fn test_fstring_adjacent_literals_are_merged() { + fn test_negative_step_slice_uses_build_slice() { let code = compile_exec( "\ -def f(cls, proto): - raise TypeError( - f\"cannot pickle {cls.__name__!r} object: \" - f\"a class that defines __slots__ without \" - f\"defining __getstate__ cannot be pickled \" - f\"with protocol {proto}\" - ) +def f(obj): + return obj[::-1] ", ); let f = find_code(&code, "f").expect("missing function code"); - let string_consts = f + let ops: Vec<_> = f .instructions .iter() - .filter_map(|unit| match unit.op { - Instruction::LoadConst { consti } => { - Some(&f.constants[consti.get(OpArg::new(u32::from(u8::from(unit.arg))))]) - } - _ => None, - }) - .filter_map(|constant| match constant { - ConstantData::Str { value } => Some(value.to_string()), - _ => None, - }) - .collect::>(); + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); assert!( - string_consts.iter().any(|value| { - value - == " object: a class that defines __slots__ without defining __getstate__ cannot be pickled with protocol " - }), - "expected merged trailing f-string literal, got {string_consts:?}" - ); - assert!( - !string_consts.iter().any(|value| value == " object: "), - "did not expect split trailing literal, got {string_consts:?}" + matches!( + ops.as_slice(), + [ + Instruction::Resume { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::BuildSlice { .. }, + Instruction::BinaryOp { .. }, + Instruction::ReturnValue, + ] + ), + "expected CPython-style BUILD_SLICE path for non-literal negative step, got ops={ops:?}" ); } #[test] - fn test_literal_only_fstring_statement_is_optimized_away() { + fn test_bool_int_binop_constants_fold() { let code = compile_exec( "\ def f(): - f'''Not a docstring''' + return False + 2, True + 2, False + False, True / 1, True & False + +def g(): + return False + 2 ", ); let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); assert!( - !f.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::PopTop)), - "literal-only f-string statement should be removed" + !ops.iter() + .any(|op| matches!(op, Instruction::BinaryOp { .. })), + "expected CPython-style folded bool/int binops, got ops={ops:?}" ); assert!( - !f.constants.iter().any(|constant| matches!( - constant, - ConstantData::Str { value } if value.to_string() == "Not a docstring" - )), - "literal-only f-string should not survive in constants" + matches!( + ops.as_slice(), + [ + Instruction::Resume { .. }, + Instruction::LoadConst { .. }, + Instruction::ReturnValue + ] + ), + "expected folded constants for bool/int binops, got ops={ops:?}" + ); + + let g = find_code(&code, "g").expect("missing function code"); + let g_ops: Vec<_> = g + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + assert!( + !g_ops + .iter() + .any(|op| matches!(op, Instruction::BinaryOp { .. })), + "expected top-level bool/int binop to fold, got ops={g_ops:?}" ); } #[test] - fn test_empty_fstring_literals_are_elided_around_interpolation() { + fn test_double_not_expression_folds_to_bool_conversion() { let code = compile_exec( "\ def f(x): - if '' f'{x}': - return 1 - return 2 + return not not x ", ); let f = find_code(&code, "f").expect("missing function code"); - - let empty_string_loads = f - .instructions - .iter() - .filter_map(|unit| match unit.op { - Instruction::LoadConst { consti } => { - Some(&f.constants[consti.get(OpArg::new(u32::from(u8::from(unit.arg))))]) - } - _ => None, - }) - .filter(|constant| { - matches!( - constant, - ConstantData::Str { value } if value.is_empty() - ) - }) - .count(); - let build_string_count = f + let ops: Vec<_> = f .instructions .iter() - .filter(|unit| matches!(unit.op, Instruction::BuildString { .. })) - .count(); - - assert_eq!(empty_string_loads, 0); - assert_eq!(build_string_count, 0); - } - - #[test] - fn test_large_power_is_not_constant_folded() { - let code = compile_exec("x = 2**100\n"); + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); - assert!(code.instructions.iter().any(|unit| match unit.op { - Instruction::BinaryOp { op } => { - op.get(OpArg::new(u32::from(u8::from(unit.arg)))) == oparg::BinaryOperator::Power - } - _ => false, - })); + assert!( + matches!( + ops.as_slice(), + [ + Instruction::Resume { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::ToBool, + Instruction::ReturnValue, + ] + ), + "expected CPython-style double-not bool conversion, got ops={ops:?}" + ); } #[test] - fn test_string_and_bytes_binops_constant_fold_like_cpython() { + fn test_tuple_bound_slice_uses_two_part_slice_path() { let code = compile_exec( "\ -x = b'\\\\' + b'u1881'\n\ -y = 103 * 'a' + 'x'\n", +def f(obj): + return obj[(1, 2):] +", ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); assert!( - !code - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), - "unexpected runtime BINARY_OP in folded string/bytes constants: {:?}", - code.instructions + matches!( + ops.as_slice(), + [ + Instruction::Resume { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::BinarySlice, + Instruction::ReturnValue, + ] + ), + "expected CPython-style BINARY_SLICE path for tuple lower bound, got ops={ops:?}" ); - assert!(code.constants.iter().any(|constant| matches!( - constant, - ConstantData::Bytes { value } if value == b"\\u1881" - ))); - let expected = format!("{}x", "a".repeat(103)); - assert!(code.constants.iter().any(|constant| matches!( - constant, - ConstantData::Str { value } - if value.to_string() == expected - ))); } #[test] - fn test_constant_string_subscript_folds_inside_collection() { + fn test_exception_cleanup_jump_to_return_is_inlined() { let code = compile_exec( "\ -values = [item for item in [r\"\\\\'a\\\\'\", r\"\\t3\", r\"\\\\\"[0]]]\n", +def f(names, cls): + try: + cls.attr = names + except: + pass + return names +", ); + let f = find_code(&code, "f").expect("missing function code"); + let return_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::ReturnValue)) + .count(); - assert!( - !code - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), - "unexpected runtime BINARY_OP after constant subscript folding: {:?}", - code.instructions + assert_eq!( + return_count, 2, + "expected CPython-style distinct return sites for normal and except paths" ); - assert!(code.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if elements.len() == 3 - && matches!(&elements[2], ConstantData::Str { value } if value.to_string() == "\\") - ))); } #[test] - fn test_constant_string_subscript_with_surrogate_skips_lossy_fold() { - let code = compile_exec("value = \"\\ud800\"[0]\n"); + fn test_nested_with_bare_except_keeps_handler_cleanup_before_following_code() { + let code = compile_exec( + "\ +def f(cm, self): + try: + with cm: + raise Exception + except: + pass + self.g() +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); - assert!( - code.instructions.iter().any(|unit| match unit.op { + let outer_handler = ops + .iter() + .enumerate() + .filter_map(|(idx, op)| matches!(op, Instruction::PushExcInfo).then_some(idx)) + .next_back() + .expect("missing outer handler"); + assert!( + ops[outer_handler..].windows(6).any(|window| { + matches!( + window, + [ + Instruction::PopExcept, + Instruction::JumpForward { .. }, + Instruction::Copy { .. }, + Instruction::PopExcept, + Instruction::Reraise { .. }, + Instruction::LoadFast { .. }, + ] + ) + }), + "expected CPython-style handler cleanup before following code, got ops={ops:?}" + ); + } + + #[test] + fn test_try_else_for_cleanup_drops_redundant_jump_nop() { + let code = compile_exec( + "\ +def f(self, xs, ys, cm1, cm2): + for x in xs: + with self.subTest(x=x): + try: + with cm1: + self.a() + except Exception: + if x: + pass + else: + raise + else: + for y in ys: + with self.subTest(y=y): + with cm2: + self.b() +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "expected inner for cleanup to fall directly into surrounding with cleanup, got ops={ops:?}", + ); + assert!( + !ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + ] + ) + }), + "expected CPython-style removal of the redundant jump NOP after for cleanup, got ops={ops:?}", + ); + } + + #[test] + fn test_non_none_final_return_is_not_duplicated() { + let code = compile_exec( + "\ +def f(p, s): + if p == '': + if s == '': + return 0 + return -1 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let minus_one_loads = f + .instructions + .iter() + .filter(|unit| { + matches!( + unit.op, + Instruction::LoadConst { consti } + if matches!( + f.constants.get( + consti + .get(OpArg::new(u32::from(u8::from(unit.arg)))) + .as_usize() + ), + Some(ConstantData::Integer { value }) if value == &BigInt::from(-1) + ) + ) + }) + .count(); + + assert_eq!( + minus_one_loads, + 1, + "expected a single final return -1 epilogue, got ops={:?}", + f.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } + + #[test] + fn test_try_else_if_return_keeps_conditional_target_nop() { + let code = compile_exec( + "\ +def f(cond): + try: + x = cond + except E: + pass + else: + if x: + return 1 + return 2 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let has_cpython_nop_target = ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, + Instruction::ReturnValue, + Instruction::Nop, + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }); + let has_direct_fallthrough = ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, + Instruction::ReturnValue, + Instruction::LoadSmallInt { .. } | Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }); + assert!( + has_cpython_nop_target || has_direct_fallthrough, + "expected adjacent try-else return and final return targets, got ops={ops:?}" + ); + } + + #[test] + fn test_named_except_conditional_branch_duplicates_cleanup_return() { + let code = compile_exec( + "\ +def f(self): + try: + raise TypeError('x') + except TypeError as e: + if '+' not in str(e): + self.fail('join() ate exception message') +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let cleanup_return_count = ops + .windows(6) + .filter(|window| { + matches!( + window, + [ + Instruction::PopExcept, + Instruction::LoadConst { .. }, + Instruction::StoreFast { .. } | Instruction::StoreName { .. }, + Instruction::DeleteFast { .. } | Instruction::DeleteName { .. }, + Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }) + .count(); + + assert_eq!( + cleanup_return_count, 2, + "expected duplicated named-except cleanup return blocks, got ops={ops:?}" + ); + } + + #[test] + fn test_listcomp_cleanup_tail_keeps_split_store_fast_pair() { + let code = compile_exec( + "\ +def f(escaped_string, quote_types): + possible_quotes = [q for q in quote_types if q not in escaped_string] + return possible_quotes +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let pop_iter_idx = ops + .iter() + .position(|op| matches!(op, Instruction::PopIter)) + .expect("missing POP_ITER"); + let tail = &ops[pop_iter_idx + 1..]; + + assert!( + matches!( + tail, + [ + Instruction::StoreFast { .. }, + Instruction::StoreFast { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::ReturnValue, + .. + ] + ), + "expected split STORE_FAST pair after listcomp cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_dictcomp_cleanup_tail_keeps_split_store_fast_pair() { + let code = compile_exec( + "\ +def f(obj, g): + return {g(k): g(v) for k, v in obj.items()} +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let pop_iter_idx = ops + .iter() + .position(|op| matches!(op, Instruction::PopIter)) + .expect("missing POP_ITER"); + let tail = &ops[pop_iter_idx + 1..]; + + assert!( + matches!( + tail, + [ + Instruction::Swap { .. }, + Instruction::StoreFast { .. }, + Instruction::StoreFast { .. }, + Instruction::ReturnValue, + .. + ] + ), + "expected split STORE_FAST pair after dictcomp cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_static_swap_triple_assign_keeps_store_fast_store_fast() { + let code = compile_exec( + "\ +def f(x, y, z): + a, b, a = x, y, z + return a +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::Swap { .. }, + Instruction::StoreFastStoreFast { .. }, + Instruction::StoreFast { .. } + ] + ) + }), + "expected CPython-style SWAP/STORE_FAST_STORE_FAST/STORE_FAST sequence, got ops={ops:?}" + ); + } + + #[test] + fn test_static_swap_duplicate_pair_eliminates_swap() { + let code = compile_exec( + "\ +def f(x, y): + a, a = x, y + return a +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.iter().any(|op| matches!(op, Instruction::Swap { .. })), + "duplicate pair assignment should statically eliminate SWAP, got ops={ops:?}" + ); + assert!( + ops.windows(2).any(|window| { + matches!(window, [Instruction::StoreFast { .. }, Instruction::PopTop]) + }), + "expected CPython-style STORE_FAST/POP_TOP duplicate assignment, got ops={ops:?}" + ); + } + + #[test] + fn test_static_swap_duplicate_prefix_eliminates_swap() { + let code = compile_exec( + "\ +def f(x, y, z): + a, a, b = x, y, z + return a +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.iter().any(|op| matches!(op, Instruction::Swap { .. })), + "duplicate-prefix assignment should statically eliminate SWAP, got ops={ops:?}" + ); + assert!( + ops.windows(2).any(|window| { + matches!( + window, + [Instruction::StoreFastStoreFast { .. }, Instruction::PopTop] + ) + }), + "expected CPython-style STORE_FAST_STORE_FAST/POP_TOP duplicate prefix, got ops={ops:?}" + ); + } + + #[test] + fn test_constant_if_expression_stmt_in_loop_removes_empty_body() { + let code = compile_exec( + "\ +def f(x): + while x: + 0 if 1 else 0 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.iter() + .any(|op| matches!(op, Instruction::LoadSmallInt { .. })), + "expected constant if-expression statement to compile away inside loop, got ops={ops:?}" + ); + } + + #[test] + fn test_if_expression_in_jump_context_skips_constant_true_arm_load() { + let code = compile_exec( + "\ +def f(): + a if (1 if b else c) else d +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.iter() + .any(|op| matches!(op, Instruction::LoadSmallInt { .. })), + "expected jump-context if-expression to avoid materializing constant truthy arm, got ops={ops:?}" + ); + } + + #[test] + fn test_with_suppress_tail_duplicates_final_return_none() { + let code = compile_exec( + "\ +def f(cm, cond): + if cond: + with cm(): + pass +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let return_count = ops + .iter() + .filter(|op| matches!(op, Instruction::ReturnValue)) + .count(); + + assert_eq!( + return_count, 3, + "expected duplicated return-none epilogues, got ops={ops:?}" + ); + assert!( + !ops.iter() + .any(|op| matches!(op, Instruction::JumpBackwardNoInterrupt { .. })), + "with suppress tail should not jump back to shared return block, got ops={ops:?}" + ); + } + + #[test] + fn test_with_conditional_bare_return_keeps_return_line_nop_before_exit_cleanup() { + let code = compile_exec( + "\ +def f(cm, registry, altkey): + with cm: + if registry.get(altkey): + return + registry[altkey] = 1 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::LoadConst { .. }, + Instruction::ReturnValue, + ] + ) + }), + "expected CPython-style return-line NOP before with-exit cleanup return, got ops={ops:?}" + ); + } + + #[test] + fn test_try_finally_conditional_return_duplicates_finally_exit_return() { + let code = compile_exec( + "\ +def f(flag, data, callback): + try: + if flag: + return + value = 1 + finally: + if data: + callback(data) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let return_count = ops + .iter() + .filter(|op| matches!(op, Instruction::ReturnValue)) + .count(); + assert_eq!( + return_count, 4, + "try-finally return unwind should keep CPython-style distinct true/false finalbody exits, got ops={ops:?}" + ); + } + + #[test] + fn test_named_except_conditional_cleanup_is_inlined_per_branch() { + let code = compile_exec( + "\ +def f(self, logger): + try: + work() + except A as exc: + if not self.closing: + self.fatal(exc, 'msg') + elif self.loop.get_debug(): + logger.debug('closing', exc_info=True) + finally: + if self.length > -1: + self.recv() +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let cleanup_after_branch_count = ops + .windows(6) + .filter(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::PopExcept, + Instruction::LoadConst { .. }, + Instruction::StoreFast { .. }, + Instruction::DeleteFast { .. }, + Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }) + .count(); + assert_eq!( + cleanup_after_branch_count, 2, + "named except branch exits should inline cleanup like CPython, got ops={ops:?}" + ); + } + + #[test] + fn test_try_finally_exception_path_duplicates_conditional_reraise() { + let code = compile_exec( + "\ +def f(flag, callback): + try: + work() + finally: + if flag: + callback() +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let reraise_count = ops + .iter() + .filter(|op| matches!(op, Instruction::Reraise { .. })) + .count(); + assert_eq!( + reraise_count, 3, + "try-finally exception finalbody should duplicate CPython no-location RERAISE exits, got ops={ops:?}" + ); + } + + #[test] + fn test_genexpr_compare_header_uses_store_fast_load_fast_like_cpython() { + let code = compile_exec( + "\ +def f(it): + return (offset == (4, 10) for offset in it) +", + ); + let genexpr = find_code(&code, "").expect("missing code"); + let ops: Vec<_> = genexpr + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::StoreFastLoadFast { .. }, + Instruction::LoadConst { .. }, + Instruction::CompareOp { .. }, + ] + ) + }), + "expected CPython-style STORE_FAST_LOAD_FAST compare header, got ops={ops:?}" + ); + } + + #[test] + fn test_fstring_adjacent_literals_are_merged() { + let code = compile_exec( + "\ +def f(cls, proto): + raise TypeError( + f\"cannot pickle {cls.__name__!r} object: \" + f\"a class that defines __slots__ without \" + f\"defining __getstate__ cannot be pickled \" + f\"with protocol {proto}\" + ) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let string_consts = f + .instructions + .iter() + .filter_map(|unit| match unit.op { + Instruction::LoadConst { consti } => { + Some(&f.constants[consti.get(OpArg::new(u32::from(u8::from(unit.arg))))]) + } + _ => None, + }) + .filter_map(|constant| match constant { + ConstantData::Str { value } => Some(value.to_string()), + _ => None, + }) + .collect::>(); + + assert!( + string_consts.iter().any(|value| { + value + == " object: a class that defines __slots__ without defining __getstate__ cannot be pickled with protocol " + }), + "expected merged trailing f-string literal, got {string_consts:?}" + ); + assert!( + !string_consts.iter().any(|value| value == " object: "), + "did not expect split trailing literal, got {string_consts:?}" + ); + } + + #[test] + fn test_literal_only_fstring_statement_is_optimized_away() { + let code = compile_exec( + "\ +def f(): + f'''Not a docstring''' +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::PopTop)), + "literal-only f-string statement should be removed" + ); + assert!( + !f.constants.iter().any(|constant| matches!( + constant, + ConstantData::Str { value } if value.to_string() == "Not a docstring" + )), + "literal-only f-string should not survive in constants" + ); + } + + #[test] + fn test_empty_fstring_literals_are_elided_around_interpolation() { + let code = compile_exec( + "\ +def f(x): + if '' f'{x}': + return 1 + return 2 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + let empty_string_loads = f + .instructions + .iter() + .filter_map(|unit| match unit.op { + Instruction::LoadConst { consti } => { + Some(&f.constants[consti.get(OpArg::new(u32::from(u8::from(unit.arg))))]) + } + _ => None, + }) + .filter(|constant| { + matches!( + constant, + ConstantData::Str { value } if value.is_empty() + ) + }) + .count(); + let build_string_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::BuildString { .. })) + .count(); + + assert_eq!(empty_string_loads, 0); + assert_eq!(build_string_count, 0); + } + + #[test] + fn test_large_fstring_uses_join_list_like_cpython() { + let mut source = String::from("def f(x):\n return f\""); + for _ in 0..=STACK_USE_GUIDELINE { + source.push_str("{x}"); + } + source.push_str("\"\n"); + + let code = compile_exec(&source); + let f = find_code(&code, "f").expect("missing function code"); + let build_string_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::BuildString { .. })) + .count(); + let list_append_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::ListAppend { .. })) + .count(); + let join_attr_count = f + .instructions + .iter() + .filter(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + load_attr.is_method() + && f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "join" + } + _ => false, + }) + .count(); + + assert_eq!(build_string_count, 0); + assert_eq!( + list_append_count, + usize::try_from(STACK_USE_GUIDELINE + 1).unwrap() + ); + assert_eq!(join_attr_count, 1); + } + + #[test] + fn test_large_power_is_not_constant_folded() { + let code = compile_exec("x = 2**100\n"); + + assert!(code.instructions.iter().any(|unit| match unit.op { + Instruction::BinaryOp { op } => { + op.get(OpArg::new(u32::from(u8::from(unit.arg)))) == oparg::BinaryOperator::Power + } + _ => false, + })); + } + + #[test] + fn test_string_and_bytes_binops_constant_fold_like_cpython() { + let code = compile_exec( + "\ +x = b'\\\\' + b'u1881'\n\ +y = 103 * 'a' + 'x'\n", + ); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "unexpected runtime BINARY_OP in folded string/bytes constants: {:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Bytes { value } if value == b"\\u1881" + ))); + let expected = format!("{}x", "a".repeat(103)); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Str { value } + if value.to_string() == expected + ))); + } + + #[test] + fn test_float_floor_division_constant_folds_like_cpython() { + let code = compile_exec( + "\ +x = 1.0 // 0.1\n\ +y = 1.0 % 0.1\n\ +z = 1e300 * 1e300 * 0\n", + ); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "float constant floor-div/mod should fold away, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Float { value } if value.to_bits() == 9.0f64.to_bits() + ))); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Float { value } + if value.to_bits() == 0.09999999999999995f64.to_bits() + ))); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Float { value } if value.is_nan() + ))); + } + + #[test] + fn test_float_power_overflow_constant_does_not_fold() { + let code = compile_exec("x = 1e300 ** 2\n"); + + assert!( + code.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::BinaryOp { op } + if op.get(OpArg::new(u32::from(u8::from(unit.arg)))) + == oparg::BinaryOperator::Power + )), + "overflowing float power should stay runtime like CPython, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_large_string_and_bytes_binops_constant_fold_like_cpython() { + let code = compile_exec( + r#" +encoded = b'\xff\xfe\x00\x00' + b'\x00\x00\x01\x00' * 1024 +text = '\U00010000' * 1024 +"#, + ); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "large safe string/bytes constants should fold away, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Bytes { value } if value.len() == 4100 + ))); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Str { value } if value.code_points().count() == 1024 + ))); + } + + #[test] + fn test_constant_string_subscript_folds_inside_collection() { + let code = compile_exec( + "\ +values = [item for item in [r\"\\\\'a\\\\'\", r\"\\t3\", r\"\\\\\"[0]]]\n", + ); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "unexpected runtime BINARY_OP after constant subscript folding: {:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if elements.len() == 3 + && matches!(&elements[2], ConstantData::Str { value } if value.to_string() == "\\") + ))); + } + + #[test] + fn test_constant_string_subscript_with_surrogate_skips_lossy_fold() { + let code = compile_exec("value = \"\\ud800\"[0]\n"); + + assert!( + code.instructions.iter().any(|unit| match unit.op { Instruction::BinaryOp { op } => { op.get(OpArg::new(u32::from(u8::from(unit.arg)))) == oparg::BinaryOperator::Subscr } _ => false, }), - "expected runtime subscript for surrogate literal, got instructions={:?}", + "expected runtime subscript for surrogate literal, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_constant_subscript_folds_in_load_context() { + let cases = [ + ("value = (1, 2, 3)[0]\n", Some(BigInt::from(1)), None), + ("value = b\"abc\"[0]\n", Some(BigInt::from(97)), None), + ("value = \"abc\"[0]\n", None, Some("a")), + ]; + + for (source, expected_int, expected_str) in cases { + let code = compile_exec(source); + assert!( + !code.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::BinaryOp { op } + if op.get(OpArg::new(u32::from(u8::from(unit.arg)))) + == oparg::BinaryOperator::Subscr + )), + "expected folded constant subscript for {source:?}, got instructions={:?}", + code.instructions + ); + + if let Some(expected_int) = expected_int.as_ref() { + let has_small_int = code.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadSmallInt { i } + if BigInt::from(i.get(OpArg::new(u32::from(u8::from(unit.arg))))) + == *expected_int + ) + }); + let has_const_int = code.constants.iter().any(|constant| { + matches!(constant, ConstantData::Integer { value } if value == expected_int) + }); + assert!( + has_small_int || has_const_int, + "missing folded integer constant {expected_int} for {source:?}, instructions={:?}", + code.instructions + ); + } + + if let Some(expected_str) = expected_str { + assert!( + code.constants.iter().any(|constant| { + matches!(constant, ConstantData::Str { value } if value.to_string() == expected_str) + }), + "missing folded string constant {expected_str:?} for {source:?}", + ); + } + } + } + + #[test] + fn test_constant_slice_subscript_folds_in_load_context() { + let code = compile_exec( + "\ +a = 'hello'[:4]\n\ +b = b'abcd'[1:3]\n\ +c = (1, 2, 3)[:2]\n", + ); + + assert!( + !code.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::BinaryOp { op } + if op.get(OpArg::new(u32::from(u8::from(unit.arg)))) + == oparg::BinaryOperator::Subscr + )), + "expected folded constant slice subscripts, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Str { value } if value.to_string() == "hell" + ))); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Bytes { value } if value == b"bc" + ))); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if matches!( + elements.as_slice(), + [ + ConstantData::Integer { value: a }, + ConstantData::Integer { value: b }, + ] if *a == BigInt::from(1) + && *b == BigInt::from(2) + ) + ))); + } + + #[test] + fn test_list_of_constant_tuples_uses_list_extend() { + let code = compile_exec( + "\ +deprecated_cases = [('a', 'b'), ('c', 'd'), ('e', 'f'), ('g', 'h'), ('i', 'j')] +", + ); + + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), + "expected constant tuple list folding" + ); + } + + #[test] + fn test_large_list_of_unary_constants_uses_list_extend() { + let code = compile_exec( + "\ +values = [-1, not True, ~0, +True, 5] +", + ); + + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), + "expected unary-folded constants to participate in list folding, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if elements.len() == 5 + && matches!(&elements[0], ConstantData::Integer { value } if *value == BigInt::from(-1)) + && matches!(&elements[1], ConstantData::Boolean { value } if !value) + && matches!(&elements[2], ConstantData::Integer { value } if *value == BigInt::from(-1)) + && matches!(&elements[3], ConstantData::Integer { value } if *value == BigInt::from(1)) + && matches!(&elements[4], ConstantData::Integer { value } if *value == BigInt::from(5)) + ))); + } + + #[test] + fn test_outer_unary_after_binop_folds_before_list_folding() { + let code = compile_exec( + "\ +values = [2.0**53, -0.5, -2.0**-54] +", + ); + + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), + "expected binop-folded constants to participate in list folding, got instructions={:?}", + code.instructions + ); + assert!( + !code.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::BinaryOp { .. } | Instruction::UnaryNegative + )), + "constant expression list should not leave runtime ops, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if elements.len() == 3 + && matches!(&elements[0], ConstantData::Float { value } if *value == 9007199254740992.0) + && matches!(&elements[1], ConstantData::Float { value } if *value == -0.5) + && matches!(&elements[2], ConstantData::Float { value } if value.is_sign_negative()) + ))); + } + + #[test] + fn test_negative_integer_power_folds_to_float_constant() { + let code = compile_exec("value = -3.0 * 2**(-333)\n"); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "negative integer power should fold through the enclosing multiply, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Float { value } + if value.is_sign_negative() && *value < 0.0 && value.abs() < 1.0e-90 + ))); + } + + #[test] + fn test_complex_power_constants_fold_like_cpython() { + let code = compile_exec( + "\ +one = 3j ** 0j +zero = 0j ** 2 +", + ); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "safe complex power constants should fold away, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Complex { value } if value.re == 1.0 && value.im == 0.0 + ))); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Complex { value } if value.re == 0.0 && value.im == 0.0 + ))); + } + + #[test] + fn test_zero_complex_power_exception_constants_do_not_fold() { + let code = compile_exec("value = 0j ** (3 - 2j)\n"); + + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "zero complex to complex power should stay runtime so ZeroDivisionError is preserved, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_large_constant_list_keeps_streaming_build() { + let source = format!( + "values = [{}]\n", + (0..31) + .map(|i| format!("'v{i}'")) + .collect::>() + .join(", ") + ); + let code = compile_exec(&source); + + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListAppend { .. })), + "large constant lists should keep LIST_APPEND streaming form, got instructions={:?}", + code.instructions + ); + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), + "large constant lists should not fold to LIST_EXTEND, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_large_constant_tuple_stream_folds_to_tuple_const() { + let source = format!( + "values = ({},)\n", + (0..31) + .map(|i| format!("'v{i}'")) + .collect::>() + .join(", ") + ); + let code = compile_exec(&source); + + assert!( + !code.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::BuildList { .. } + | Instruction::ListAppend { .. } + | Instruction::CallIntrinsic1 { .. } + )), + "large constant tuple should fold the LIST_TO_TUPLE stream, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } if elements.len() == 31 + ))); + } + + #[test] + fn test_annotation_closure_uses_format_varname() { + let code = compile_exec( + "\ +class C: + x: int +", + ); + let annotate = find_code(&code, "__annotate__").expect("missing __annotate__ code"); + let varnames = annotate + .varnames + .iter() + .map(|name| name.as_str()) + .collect::>(); + assert_eq!(varnames, vec!["format"]); + } + + #[test] + fn test_type_param_evaluator_uses_dot_format_varname() { + let code = compile_exec( + "\ +class C[T: int]: + pass +", + ); + let evaluator = find_code(&code, "T").expect("missing type parameter evaluator"); + let varnames = evaluator + .varnames + .iter() + .map(|name| name.as_str()) + .collect::>(); + assert_eq!(varnames, vec![".format"]); + } + + #[test] + fn test_class_annotation_global_resolution_matches_cpython() { + let class_global = compile_exec( + "\ +X = 'global' +class C: + locals()['X'] = 'class' + global X + y: X +", + ); + let annotate = + find_code(&class_global, "__annotate__").expect("missing class __annotate__ code"); + assert!( + annotate + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadGlobal { .. })), + "expected explicit class global to use LOAD_GLOBAL, got instructions={:?}", + annotate.instructions + ); + assert!( + !annotate + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFromDictOrGlobals { .. })), + "did not expect class explicit global to use LOAD_FROM_DICT_OR_GLOBALS, got instructions={:?}", + annotate.instructions + ); + + let outer_global = compile_exec( + "\ +def f(): + global X + class C: + locals()['X'] = 'class' + y: X +", + ); + let annotate = find_code(&outer_global, "__annotate__") + .expect("missing nested class __annotate__ code"); + assert!( + annotate + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFromDictOrGlobals { .. })), + "expected outer explicit global in class annotation to use LOAD_FROM_DICT_OR_GLOBALS, got instructions={:?}", + annotate.instructions + ); + } + + #[test] + fn test_constant_tuple_binops_fold_like_cpython() { + let code = compile_exec("value = (1,) * 17 + ('spam',)\n"); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), + "tuple constant binops should fold away, got instructions={:?}", + code.instructions + ); + assert!(code.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if elements.len() == 18 + && elements[..17] + .iter() + .all(|elt| matches!(elt, ConstantData::Integer { value } if *value == BigInt::from(1))) + && matches!(&elements[17], ConstantData::Str { value } if value.to_string() == "spam") + ))); + } + + #[test] + fn test_constant_list_iterable_uses_tuple() { + let code = compile_exec( + "\ +def f(): + return {x: y for x, y in [(1, 2), ]} +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildList { .. })), + "constant list iterable should avoid BUILD_LIST before GET_ITER" + ); + assert!(f.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if matches!( + elements.as_slice(), + [ConstantData::Tuple { elements: inner }] + if matches!( + inner.as_slice(), + [ + ConstantData::Integer { .. }, + ConstantData::Integer { .. } + ] + ) + ) + ))); + } + + #[test] + fn test_constant_set_iterable_uses_frozenset_const() { + let code = compile_exec( + "\ +def f(): + return [x for x in {1, 2, 3}] +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildSet { .. })), + "constant set iterable should avoid BUILD_SET before GET_ITER" + ); + assert!(f.constants.iter().any(|constant| matches!( + constant, + ConstantData::Frozenset { elements } + if matches!( + elements.as_slice(), + [ + ConstantData::Integer { .. }, + ConstantData::Integer { .. }, + ConstantData::Integer { .. } + ] + ) + ))); + } + + #[test] + fn test_constant_list_membership_uses_tuple_const() { + let code = compile_exec( + "\ +f = lambda x: x in [1, 2, 3] +", + ); + let lambda = find_code(&code, "").expect("missing lambda code"); + + assert!( + !lambda + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildList { .. })), + "constant list membership should avoid BUILD_LIST before CONTAINS_OP" + ); + assert!(lambda.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if matches!( + elements.as_slice(), + [ + ConstantData::Integer { .. }, + ConstantData::Integer { .. }, + ConstantData::Integer { .. } + ] + ) + ))); + } + + #[test] + fn test_small_constant_set_membership_uses_frozenset_const() { + let code = compile_exec( + "\ +f = lambda x: x in {0} +", + ); + let lambda = find_code(&code, "").expect("missing lambda code"); + + assert!( + !lambda + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildSet { .. })), + "constant set membership should avoid BUILD_SET before CONTAINS_OP" + ); + assert!(lambda.constants.iter().any(|constant| matches!( + constant, + ConstantData::Frozenset { elements } + if matches!(elements.as_slice(), [ConstantData::Integer { value }] if *value == BigInt::from(0)) + ))); + } + + #[test] + fn test_nonconstant_list_membership_uses_tuple() { + let code = compile_exec( + "\ +def f(a, b, c, x): + return x in [a, b, c] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(2).any(|window| { + matches!( + window, + [ + Instruction::BuildTuple { .. }, + Instruction::ContainsOp { .. } + ] + ) + }), + "expected BUILD_TUPLE before CONTAINS_OP for non-constant list membership, got ops={ops:?}" + ); + } + + #[test] + fn test_starred_tuple_iterable_drops_list_to_tuple_before_get_iter() { + let code = compile_exec( + "\ +def f(a, b, c): + for x in *a, *b, *c: + pass +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !has_intrinsic_1(f, IntrinsicFunction1::ListToTuple), + "LIST_TO_TUPLE should be removed before GET_ITER in for-iterable context" + ); + assert!( + f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::GetIter)), + "expected GET_ITER in for loop" + ); + } + + #[test] + fn test_comprehension_single_list_iterable_uses_tuple() { + let code = compile_exec( + "\ +def g(): + [x for x in [(yield 1)]] +", + ); + let g = find_code(&code, "g").expect("missing g code"); + let ops: Vec<_> = g + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(2).any(|window| { + matches!( + window, + [Instruction::BuildTuple { .. }, Instruction::GetIter] + ) + }), + "expected BUILD_TUPLE before GET_ITER for single-item list iterable in comprehension, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_comprehension_list_iterable_uses_tuple() { + let code = compile_exec( + "\ +def f(): + return [[y for y in [x, x + 1]] for x in [1, 3, 5]] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(2).any(|window| { + matches!( + window, + [Instruction::BuildTuple { .. }, Instruction::GetIter] + ) + }), + "expected BUILD_TUPLE before GET_ITER for nested list iterable in comprehension, got ops={ops:?}" + ); + } + + #[test] + fn test_comprehension_singleton_sub_iter_uses_assignment_idiom() { + let code = compile_exec( + "\ +def f(): + return {j: j * j for i in range(4) for j in [i + 1]} +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let for_iter_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::ForIter { .. })) + .count(); + let has_map_add_depth_2 = f.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::MapAdd { i } + if i.get(OpArg::new(u32::from(u8::from(unit.arg)))) == 2 + ) + }); + + assert_eq!( + for_iter_count, 1, + "singleton sub-iter should not emit its own FOR_ITER, got instructions={:?}", + f.instructions + ); + assert!( + has_map_add_depth_2, + "assignment-idiom dictcomp should use MAP_ADD depth 2, got instructions={:?}", + f.instructions + ); + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildTuple { .. })), + "singleton sub-iter should not materialize an iterator tuple, got instructions={:?}", + f.instructions + ); + } + + #[test] + fn test_constant_comprehension_iterable_with_unary_int_uses_tuple_const() { + let code = compile_exec( + "\ +l = lambda : [2 < x for x in [-1, 3, 0]] +", + ); + let lambda = find_code(&code, "").expect("missing lambda code"); + + assert!( + lambda.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if matches!( + elements.as_slice(), + [ + ConstantData::Integer { .. }, + ConstantData::Integer { .. }, + ConstantData::Integer { .. } + ] + ) + )), + "expected folded tuple constant for comprehension iterable" + ); + } + + #[test] + fn test_module_scope_listcomp_is_inlined() { + let code = compile_exec("values = [i for i in range(3)]\n"); + + assert!( + find_code(&code, "").is_none(), + "module-scope list comprehension should be inlined" + ); + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastAndClear { .. })), + "inlined module-scope list comprehension should use LOAD_FAST_AND_CLEAR, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_module_scope_dictcomp_is_inlined() { + let code = compile_exec("mapping = {i: i for i in range(3)}\n"); + + assert!( + find_code(&code, "").is_none(), + "module-scope dict comprehension should be inlined" + ); + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastAndClear { .. })), + "inlined module-scope dict comprehension should use LOAD_FAST_AND_CLEAR, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_async_dictcomp_in_async_function_is_inlined() { + let code = compile_exec( + "\ +async def f(items): + return {item: item async for item in items} +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + find_code(&code, "").is_none(), + "async dict comprehension should be inlined" + ); + assert!( + ops.iter().any(|op| matches!(op, Instruction::GetAIter)), + "inlined async dict comprehension should keep GET_AITER in outer code, got ops={ops:?}" + ); + assert!( + ops.iter() + .any(|op| matches!(op, Instruction::LoadFastAndClear { .. })), + "inlined async dict comprehension should use LOAD_FAST_AND_CLEAR, got ops={ops:?}" + ); + assert!( + !ops.iter().any(|op| matches!(op, Instruction::MakeFunction)), + "inlined async dict comprehension should not materialize MAKE_FUNCTION, got ops={ops:?}" + ); + } + + #[test] + fn test_async_inlined_comprehension_inlines_restore_return_into_end_async_for() { + let code = compile_exec( + "\ +async def f(): + return [i + 1 async for i in g([10, 20])] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::EndAsyncFor, + Instruction::Swap { .. }, + Instruction::StoreFast { .. }, + Instruction::ReturnValue, + Instruction::Swap { .. }, + Instruction::PopTop, + Instruction::Swap { .. }, + Instruction::StoreFast { .. }, + ] + ) + }), + "expected CPython-style restore+return inlined into END_ASYNC_FOR before cleanup, got ops={ops:?}" + ); + assert!( + !ops.windows(2).any(|window| { + matches!( + window, + [ + Instruction::EndAsyncFor, + Instruction::JumpForward { .. } | Instruction::JumpBackward { .. }, + ] + ) + }), + "unexpected jump from END_ASYNC_FOR to the normal restore tail, got ops={ops:?}" + ); + } + + #[test] + fn test_await_cleanup_throw_falls_through_until_cold_reorder() { + let code = compile_exec( + "\ +async def f(): + await 1 +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::CleanupThrow, + Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::CallIntrinsic1 { .. }, + ] + ) + }), + "expected CPython-style cold CLEANUP_THROW jump before StopIteration handler, got ops={ops:?}" + ); + assert!( + !ops.windows(2).any(|window| { + matches!(window, [Instruction::CleanupThrow, Instruction::EndSend]) + }), + "CLEANUP_THROW should not inline the normal END_SEND return tail, got ops={ops:?}" + ); + } + + #[test] + fn test_match_async_inlined_comprehension_success_jump_no_interrupt() { + let code = compile_exec( + "\ +async def f(name_3, name_5): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: f async for name_2 in name_5} + case []: + pass + [[]] +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::StoreFast { .. }, + Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style no-interrupt match success backedge after async comprehension cleanup, got ops={ops:?}" + ); + assert!( + !ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::StoreFast { .. }, + Instruction::JumpBackward { .. }, + ] + ) + }), + "match success cleanup backedge should not be a regular interrupting jump, got ops={ops:?}" + ); + } + + #[test] + fn test_for_loop_if_return_reorders_continue_backedge_before_exit_body() { + let code = compile_exec( + "\ +def f(items, occurrence): + for item in items: + if item: + occurrence -= 1 + if not occurrence: + return item + return None +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. }, + ] + ) + }), + "expected CPython-style inverted return guard followed by loop backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + ] + ) + }), + "return guard should not fall through into the return body before the loop backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_sync_with_after_async_for_keeps_end_async_for_line_marker() { + let code = compile_exec( + "\ +async def f(cm, source, tgt): + with cm: + async for tgt[0] in source(): + pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::EndAsyncFor, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) + }), + "expected CPython-style line-marker NOP between END_ASYNC_FOR and with cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_genexpr_with_async_comprehension_element_is_async_generator() { + let code = compile_exec( + "\ +async def f(): + gen = ([i async for i in asynciter([1, 2])] for j in [10, 20]) + return [x async for x in gen] +", + ); + let genexpr = find_code(&code, "").expect("missing genexpr code"); + let units: Vec<_> = genexpr + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + + assert!( + units.windows(2).any(|window| { + let [wrap, yield_value] = window else { + return false; + }; + matches!(yield_value.op, Instruction::YieldValue { .. }) + && match wrap.op { + Instruction::CallIntrinsic1 { func } => { + func.get(OpArg::new(u32::from(u8::from(wrap.arg)))) + == bytecode::IntrinsicFunction1::AsyncGenWrap + } + _ => false, + } + }), + "expected CPython-style ASYNC_GEN_WRAP before genexpr yield, got units={units:?}" + ); + } + + #[test] + fn test_nested_module_scope_dictcomp_symbols_are_local() { + let symbol_table = scan_program_symbol_table( + "\ +deoptmap = { + specialized: base + for base, family in _specializations.items() + for specialized in family +} +", + ); + + for name in ["base", "family", "specialized"] { + let symbol = symbol_table + .lookup(name) + .unwrap_or_else(|| panic!("missing module symbol {name}")); + assert_eq!( + symbol.scope, + SymbolScope::Local, + "expected module-scope inlined comprehension symbol {name} to be Local, got {symbol:?}" + ); + } + + let comp = symbol_table + .sub_tables + .first() + .expect("missing comprehension symbol table"); + assert!(comp.comp_inlined, "expected comprehension to be inlined"); + for name in ["base", "family", "specialized"] { + let symbol = comp + .lookup(name) + .unwrap_or_else(|| panic!("missing comprehension symbol {name}")); + assert_eq!( + symbol.scope, + SymbolScope::Local, + "expected comprehension symbol {name} to be Local, got {symbol:?}" + ); + } + } + + #[test] + fn test_nested_module_scope_dictcomp_uses_fast_locals() { + let code = compile_exec( + "\ +deoptmap = { + specialized: base + for base, family in _specializations.items() + for specialized in family +} +", + ); + let ops: Vec<_> = code + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.iter() + .any(|op| matches!(op, Instruction::StoreFastStoreFast { .. })), + "expected outer target unpack to use STORE_FAST_STORE_FAST, got ops={ops:?}" + ); + assert!( + ops.iter().any(|op| matches!( + op, + Instruction::StoreFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + )), + "expected inner target/store-use path to use fast locals, got ops={ops:?}" + ); + assert!( + ops.iter() + .filter(|op| matches!(op, Instruction::LoadName { .. })) + .count() + <= 1, + "unexpected extra LOAD_NAME ops in nested inlined comprehension, got ops={ops:?}" + ); + assert!( + ops.iter() + .filter(|op| matches!(op, Instruction::StoreName { .. })) + .count() + <= 1, + "unexpected extra STORE_NAME ops in nested inlined comprehension, got ops={ops:?}" + ); + } + + #[test] + fn test_module_scope_inlined_comprehension_keeps_outer_iter_as_name_lookup() { + let code = compile_exec( + "\ +path_separators = ['/'] +_pathseps_with_colon = {f':{s}' for s in path_separators} +", + ); + let ops: Vec<_> = code + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let load_name_path = ops + .windows(2) + .any(|window| matches!(window, [Instruction::LoadName { .. }, Instruction::GetIter])); + assert!( + load_name_path, + "expected outer iterable to stay a NAME lookup before GET_ITER, got ops={ops:?}" + ); + assert!( + !ops.windows(2).any(|window| matches!( + window, + [ + Instruction::LoadFast { .. } | Instruction::LoadFastCheck { .. }, + Instruction::GetIter + ] + )), + "module local outer iterable should not become a fast local, got ops={ops:?}" + ); + assert!( + ops.iter().any(|op| matches!( + op, + Instruction::StoreFastLoadFast { .. } | Instruction::StoreFast { .. } + )), + "comprehension target should still use fast locals, got ops={ops:?}" + ); + } + + #[test] + fn test_function_scope_inlined_comprehension_restore_keeps_swap_before_duplicate_store() { + let code = compile_exec( + "\ +def f(): + a = [1 for a in [0]] + return 1 +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| matches!( + window, + [ + Instruction::PopIter, + Instruction::Swap { .. }, + Instruction::StoreFast { .. }, + Instruction::StoreFast { .. } + ] + )), + "expected PopIter/SWAP 2/STORE_FAST/STORE_FAST restore tail, got ops={ops:?}" + ); + } + + #[test] + fn test_inlined_comprehension_restore_does_not_form_store_fast_load_fast() { + let code = compile_exec( + "\ +def f(e): + e[1:3] = [g(i) for i in range(2)] + +def g(datadir): + files = [filename[:-4] for filename in sorted(os.listdir(datadir)) if filename.endswith('.xml')] + input_files = [filename for filename in files if filename.startswith('in')] + return files, input_files +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::Swap { .. }, + Instruction::StoreFast { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::LoadConst { .. }, + Instruction::StoreSubscr, + ] + ) + }), + "expected CPython-style inlined comprehension restore before slice store, got ops={ops:?}" + ); + assert!( + !ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::StoreFastLoadFast { .. }, + Instruction::LoadConst { .. }, + Instruction::StoreSubscr, + ] + ) + }), + "inlined comprehension restore should not be folded into STORE_FAST_LOAD_FAST, got ops={ops:?}" + ); + + let g = find_code(&code, "g").expect("missing g code"); + let g_ops: Vec<_> = g + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + assert!( + g_ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::StoreFast { .. }, + Instruction::StoreFast { .. }, + ] + ) + }), + "expected CPython-style static swap over STORE_FAST_MAYBE_NULL restore, got ops={g_ops:?}" + ); + assert!( + !g_ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::Swap { .. }, + Instruction::StoreFast { .. }, + Instruction::StoreFast { .. }, + ] + ) + }), + "inlined comprehension restore should statically remove SWAP before adjacent stores, got ops={g_ops:?}" + ); + } + + #[test] + fn test_single_mode_folded_multiline_constant_does_not_leave_nops() { + let code = compile_single( + "\ +(- + - + - + 1) +", + ); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::Nop)), + "expected folded single-mode multiline constant to drop NOP anchors, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_folded_multiline_tuple_constant_does_not_leave_operand_nops() { + let code = compile_exec( + "\ +values = ( + (1 + 1j, 0 + 0j), + (1 + 1j, 0.0), + (1 + 1j, 0), +) +", + ); + + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::Nop)), + "expected CPython nop_out-style folded tuple operands to have no surviving NOPs, got instructions={:?}", + code.instructions + ); + } + + #[test] + fn test_folded_multiline_bytes_binop_does_not_leave_operand_nops() { + let code = compile_exec( + "\ +def f(self, out): + self.assertIn( + b'gnu' + (b'/123' * 125) + b'/longlink' + (b'/123' * 125) + b'/longname', + out) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::Nop)), + "expected CPython nop_out-style folded operands to have no surviving NOPs, got instructions={:?}", + f.instructions + ); + } + + #[test] + fn test_folded_binop_at_branch_body_start_does_not_leave_nop() { + let code = compile_exec( + "\ +def f(sys): + if sys.platform == 'win32': + component = 'd' * 25 + return component +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(3).any(|window| { + matches!( + window, + [ + Instruction::NotTaken, + Instruction::Nop, + Instruction::LoadConst { .. } + ] + ) + }), + "expected CPython nop_out-style folded branch body to drop operand NOP, got ops={ops:?}", + ); + } + + #[test] + fn test_folded_iterable_at_assert_target_does_not_leave_nop() { + let code = compile_exec( + r#" +def f(caches, non_caches): + assert 1 / 3 <= caches / non_caches, "this test needs more caches!" + for show_caches in (False, True): + pass +"#, + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.windows(2).any(|window| { + matches!(window, [Instruction::Nop, Instruction::LoadConst { .. }]) + }), + "expected folded for-iterable at assert target to drop operand NOP, got ops={ops:?}", + ); + } + + #[test] + fn test_multiline_unpack_target_uses_element_locations() { + let code = compile_exec( + "\ +def f(cm): + with cm as (_, + filename_2): + return filename_2 +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + !ops.iter() + .any(|op| matches!(op, Instruction::StoreFastStoreFast { .. })), + "expected multiline target elements to keep separate STORE_FAST instructions, got ops={ops:?}", + ); + } + + #[test] + fn test_or_condition_in_jump_context_uses_shared_true_fallthrough() { + let code = compile_exec( + "\ +def f(lines): + for line in lines: + if line.startswith('--') or not line.strip(): + continue + return line +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let first_pop_jump = ops + .iter() + .find(|op| { + matches!( + op, + Instruction::PopJumpIfTrue { .. } | Instruction::PopJumpIfFalse { .. } + ) + }) + .copied() + .expect("missing conditional jump"); + assert!( + matches!(first_pop_jump, Instruction::PopJumpIfTrue { .. }), + "expected first OR branch to jump on true into shared fallthrough, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_break_bool_chain_reorders_false_path_to_jump_back() { + let code = compile_exec( + "\ +def f(filters, text, category, module, lineno, defaultaction): + for item in filters: + action, msg, cat, mod, ln = item + if ((msg is None or msg.match(text)) and + issubclass(category, cat) and + (mod is None or mod.match(module)) and + (ln == 0 or lineno == ln)): + break + else: + action = defaultaction + return action +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected CPython-style false path to fall through into loop jump-back, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_conditional_body_keeps_duplicate_jump_back_paths() { + let code = compile_exec( + "\ +def f(new, old): + for replace in ['__module__', '__name__', '__qualname__', '__doc__']: + if hasattr(old, replace): + setattr(new, replace, getattr(old, replace)) + return new +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let jump_back_count = ops + .iter() + .filter(|op| { + matches!( + op, + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } + ) + }) + .count(); + assert!( + jump_back_count >= 2, + "expected separate false-path and body jump-back blocks, got ops={ops:?}" + ); + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected false path to jump back before body, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_multiblock_conditional_body_keeps_body_before_jump_back() { + let code = compile_exec( + "\ +def f(random, d, f): + for dummy in range(100): + k = random.choice('abc') + if random.random() < 0.2: + if k in d: + del d[k] + del f[k] + else: + v = random.choice((1, 2)) + d[k] = v + f[k] = v + check(f[k], v) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadFastBorrowLoadFastBorrow { .. }, + Instruction::DeleteSubscr, + ] + ) + }), + "expected CPython-style multi-block body before false jump-back, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_not_conditional_body_threads_true_path_to_jump_back() { + let code = compile_exec( + "\ +def f(xs): + for x in xs: + if not x: + g(x) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected CPython-style true path to jump back before not-body, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { + let code = compile_exec( + "\ +def f(x, y): + for i in x: + if y: + pass +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style synthetic false-path jump-back plus body jump-back, got ops={ops:?}" + ); + assert!( + !ops.iter().any(|op| matches!(op, Instruction::Nop)), + "expected pass body line to attach to loop backedge instead of leaving a NOP, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_if_shared_jump_back_target_is_duplicated() { + let code = compile_exec( + "\ +def f(s, size, encodeSetO, encodeWhiteSpace): + inShift = True + base64bits = 0 + out = [] + for i, ch in enumerate(s): + if base64bits == 0: + if i + 1 < size: + ch2 = s[i + 1] + if E(ch2, encodeSetO, encodeWhiteSpace): + if B(ch2) or ch2 == '-': + out.append(b'-') + inShift = False + else: + out.append(b'-') + inShift = False + return out +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::PopTop, + Instruction::LoadConst { .. }, + Instruction::StoreFast { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, + ] + ) + }), + "expected separate nested-if and outer-if jump-back tails, got ops={ops:?}" + ); + } + + #[test] + fn test_protected_loop_conditional_keeps_forward_body_entry() { + let code = compile_exec( + "\ +def outer(it, C1): + def f(): + for x in it: + try: + if C1: + yield 2 + except OSError: + pass + return f +", + ); + let outer = find_code(&code, "outer").expect("missing outer code"); + let f = find_code(outer, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadSmallInt { .. }, + Instruction::YieldValue { .. }, + Instruction::Resume { .. }, + Instruction::PopTop, + ] + ) + }), + "expected protected conditional to keep CPython-style forward body entry, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_except_false_path_duplicates_pop_except_jump_back_tail() { + let code = compile_exec( + "\ +def f(it, C3): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::LoadSmallInt { .. }, + Instruction::StoreFast { .. }, + Instruction::PopExcept, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::PopExcept, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style duplicated false-path exit tail, got ops={ops:?}" + ); + } + + #[test] + fn test_more_nested_except_false_paths_duplicate_all_jump_back_tails() { + let code = compile_exec( + "\ +def f(it, C3, C4): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::LoadSmallInt { .. }, + Instruction::StoreFast { .. }, + Instruction::PopExcept, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::PopExcept, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::PopExcept, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style duplicated nested false-path exit tails, got ops={ops:?}" + ); + } + + #[test] + fn test_no_wraparound_jump_keeps_forward_hop_before_loop_backedge() { + let code = compile_exec( + "\ +def while_not_chained(a, b, c): + while not (a < b < c): + pass +", + ); + let f = find_code(&code, "while_not_chained").expect("missing while_not_chained code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpForward { .. }, + Instruction::PopTop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style no-wraparound forward hop before the loop backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_while_break_else_keeps_true_edge_into_forward_break_body() { + let code = compile_exec( + "\ +def f(i): + while i: + i -= 1 + if i < 4: + break + else: + print('x') + print('y') +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfTrue { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::JumpForward { .. }, + ] + ) + }), + "expected CPython-style true edge into forward break body with false path falling into the loop backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_if_continue_reorders_false_path_to_loop_backedge() { + let code = compile_exec( + "\ +def f(items, changes): + for x in items: + if not x: + if x in changes: + raise TypeError + continue +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(7).any(|window| { + matches!( + window, + [ + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::ContainsOp { .. }, + Instruction::PopJumpIfFalse { .. }, + ] + ) + }), + "expected nested if/continue to keep CPython-style false-edge jump-back tails, got ops={ops:?}" + ); + } + + #[test] + fn test_loop_assert_keeps_false_edge_into_raise_body() { + let code = compile_exec( + "\ +def f(bytecode): + for instr, positions in zip(bytecode, bytecode.codeobj.co_positions()): + assert instr.positions == positions +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadCommonConstant { .. }, + Instruction::RaiseVarargs { .. }, + ] + ) + }), + "expected loop assert to keep CPython-style false-edge into the raise body, got ops={ops:?}" + ); + } + + #[test] + fn test_and_is_not_none_loop_guard_uses_direct_jump_back_false_path() { + let code = compile_exec( + "\ +def f(code): + last_line = -2 + for _, _, line in code.co_lines(): + if line is not None and line != last_line: + last_line = line +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::PopJumpIfNotNone { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. }, + Instruction::CompareOp { .. }, + ] + ) + }), + "expected CPython-style direct jump-back false path for 'is not None and ...', got ops={ops:?}" + ); + } + + #[test] + fn test_large_is_not_none_loop_guard_uses_direct_jump_back_false_path() { + let code = compile_exec( + "\ +def f(cls, _FIELDS, _PARAMS): + all_frozen_bases = None + any_frozen_base = False + has_dataclass_bases = False + for b in cls.__mro__[-1:0:-1]: + base_fields = getattr(b, _FIELDS, None) + if base_fields is not None: + has_dataclass_bases = True + for field in base_fields.values(): + name = field.name + if all_frozen_bases is None: + all_frozen_bases = True + current_frozen = getattr(b, _PARAMS).frozen + all_frozen_bases = all_frozen_bases and current_frozen + any_frozen_base = any_frozen_base or current_frozen +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::PopJumpIfNotNone { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadConst { .. }, + Instruction::StoreFast { .. }, + ] + ) + }), + "expected CPython-style direct jump-back false path for large 'is not None' loop body, got ops={ops:?}" + ); + } + + #[test] + fn test_continue_inside_with_keeps_line_marker_nop_before_exit_cleanup() { + let code = compile_exec( + "\ +def f(it): + for func in it: + with cm(): + if cond(): + continue +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(9).any(|window| { + matches!( + window, + [ + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::PopTop, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + ] + ) + }), + "expected CPython-style line-marker NOP before with-exit cleanup on continue, got ops={ops:?}" + ); + } + + #[test] + fn test_nested_async_with_normal_cleanup_drops_pop_block_nop() { + let code = compile_exec( + "\ +async def foo(): + async with CM(): + async with CM(): + raise RuntimeError +", + ); + let f = find_code(&code, "foo").expect("missing foo code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::GetAwaitable { .. }, + ] + ) + }), + "expected CPython-style async-with normal cleanup without a POP_BLOCK NOP, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { + matches!( + window, + [ + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + Instruction::GetAwaitable { .. }, + ] + ) + }), + "unexpected POP_BLOCK NOP before async-with normal cleanup, got ops={ops:?}" + ); + } + + #[test] + fn test_try_loop_elif_places_return_before_orelse_tail() { + let code = compile_exec( + "\ +def f(source, suggest, tb, s): + if source is not None: + try: + tb = tb + except Exception: + suggest = False + tb = None + if tb is not None: + for frame in tb: + s += frame + elif suggest: + s += 'x' + return s +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let has_direct_return = ops.windows(8).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ReturnValue, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + ] + ) + }); + let has_nop_anchored_return = ops.windows(9).any(|window| { + matches!( + window, + [ + Instruction::EndFor, + Instruction::PopIter, + Instruction::Nop, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ReturnValue, + Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, + Instruction::ToBool, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + ] + ) + }); + assert!( + has_direct_return || has_nop_anchored_return, + "expected CPython-style duplicated return between loop exit and elif tail, got ops={ops:?}" + ); + } + + #[test] + fn test_constant_false_while_else_deopts_post_else_borrows() { + let code = compile_exec( + "\ +def f(self): + x = 0 + while 0: + x = 1 + else: + x = 2 + self.assertEqual(x, 2) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + let assert_idx = ops + .iter() + .position(|op| matches!(op, Instruction::LoadAttr { .. })) + .expect("missing assertEqual call"); + let window = &ops[assert_idx.saturating_sub(1)..(assert_idx + 3).min(ops.len())]; + assert!( + matches!( + window, + [ + Instruction::LoadFast { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFast { .. }, + .. + ] + ), + "expected post-else assertEqual call to use plain LOAD_FAST, got ops={window:?}" + ); + } + + #[test] + fn test_single_unpack_assignment_disables_constant_collection_folding() { + let code = compile_exec("a, b, c = 1, 2, 3\n"); + + assert!( + !code.instructions.iter().any(|unit| { + matches!(unit.op, Instruction::UnpackSequence { .. }) + || matches!(unit.op, Instruction::LoadConst { .. }) + && matches!( + code.constants.get(usize::from(u8::from(unit.arg))), + Some(ConstantData::Tuple { .. }) + ) + }), + "single unpack assignment should keep builder form for later lowering, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert!( + code.instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::LoadSmallInt { .. })) + .count() + >= 3, + "expected individual constant loads before unpack-target stores, got ops={:?}", code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() ); } #[test] - fn test_constant_subscript_folds_in_load_context() { - let cases = [ - ("value = (1, 2, 3)[0]\n", Some(BigInt::from(1)), None), - ("value = b\"abc\"[0]\n", Some(BigInt::from(97)), None), - ("value = \"abc\"[0]\n", None, Some("a")), - ]; + fn test_chained_unpack_assignment_keeps_constant_collection_folding() { + let code = compile_exec("(a, b) = c = d = (1, 2)\n"); - for (source, expected_int, expected_str) in cases { - let code = compile_exec(source); - assert!( - !code.instructions.iter().any(|unit| matches!( - unit.op, - Instruction::BinaryOp { op } - if op.get(OpArg::new(u32::from(u8::from(unit.arg)))) - == oparg::BinaryOperator::Subscr - )), - "expected folded constant subscript for {source:?}, got instructions={:?}", - code.instructions - ); + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadConst { .. })), + "chained unpack assignment should keep tuple constant, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::UnpackSequence { .. })), + "chained unpack assignment should still unpack the copied tuple, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } - if let Some(expected_int) = expected_int.as_ref() { - let has_small_int = code.instructions.iter().any(|unit| { - matches!( - unit.op, - Instruction::LoadSmallInt { i } - if BigInt::from(i.get(OpArg::new(u32::from(u8::from(unit.arg))))) - == *expected_int - ) - }); - let has_const_int = code.constants.iter().any(|constant| { - matches!(constant, ConstantData::Integer { value } if value == expected_int) - }); - assert!( - has_small_int || has_const_int, - "missing folded integer constant {expected_int} for {source:?}, instructions={:?}", - code.instructions - ); - } + #[test] + fn test_constant_true_assert_skips_message_nested_scope() { + let code = compile_exec("assert 1, (lambda x: x + 1)\n"); - if let Some(expected_str) = expected_str { - assert!( - code.constants.iter().any(|constant| { - matches!(constant, ConstantData::Str { value } if value.to_string() == expected_str) - }), - "missing folded string constant {expected_str:?} for {source:?}", - ); - } - } + assert_eq!( + code.constants + .iter() + .filter(|constant| matches!(constant, ConstantData::Code { .. })) + .count(), + 0, + "constant-true assert should not compile the skipped message lambda" + ); + assert!( + !code + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::RaiseVarargs { .. })), + "constant-true assert should be elided, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + } + + #[test] + fn test_constant_false_assert_uses_direct_raise_shape() { + let code = compile_exec("assert 0, (lambda x: x + 1)\n"); + + assert!( + !code.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::ToBool + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfFalse { .. } + ) + }), + "constant-false assert should use direct raise shape, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::RaiseVarargs { .. })), + "constant-false assert should still raise, got ops={:?}", + code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); + assert_eq!( + code.constants + .iter() + .filter(|constant| matches!(constant, ConstantData::Code { .. })) + .count(), + 1, + "constant-false assert should still compile the message lambda" + ); } #[test] - fn test_list_of_constant_tuples_uses_list_extend() { - let code = compile_exec( - "\ -deprecated_cases = [('a', 'b'), ('c', 'd'), ('e', 'f'), ('g', 'h'), ('i', 'j')] -", - ); + fn test_constant_unary_positive_and_invert_fold() { + let code = compile_exec("x = +1\nx = ~1\n"); assert!( + !code.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::CallIntrinsic1 { .. } | Instruction::UnaryInvert + ) + }), + "constant unary ops should fold away, got ops={:?}", code.instructions .iter() - .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), - "expected constant tuple list folding" + .map(|unit| unit.op) + .collect::>() ); } #[test] - fn test_large_list_of_unary_constants_uses_list_extend() { - let code = compile_exec( - "\ -values = [-1, not True, ~0, +True, 5] -", - ); + fn test_bool_invert_is_not_const_folded() { + let code = compile_exec("x = ~True\n"); assert!( code.instructions .iter() - .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), - "expected unary-folded constants to participate in list folding, got instructions={:?}", + .any(|unit| matches!(unit.op, Instruction::UnaryInvert)), + "~bool should remain unfurled to match CPython, got ops={:?}", code.instructions + .iter() + .map(|unit| unit.op) + .collect::>() ); - assert!(code.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if elements.len() == 5 - && matches!(&elements[0], ConstantData::Integer { value } if *value == BigInt::from(-1)) - && matches!(&elements[1], ConstantData::Boolean { value } if !value) - && matches!(&elements[2], ConstantData::Integer { value } if *value == BigInt::from(-1)) - && matches!(&elements[3], ConstantData::Integer { value } if *value == BigInt::from(1)) - && matches!(&elements[4], ConstantData::Integer { value } if *value == BigInt::from(5)) - ))); } #[test] - fn test_large_constant_list_keeps_streaming_build() { - let source = format!( - "values = [{}]\n", - (0..31) - .map(|i| format!("'v{i}'")) - .collect::>() - .join(", ") + fn test_optimized_assert_preserves_nested_scope_order() { + compile_exec_optimized( + "\ +class S: + def f(self, sequence): + _formats = [self._types_mapping[type(item)] for item in sequence] + _list_len = len(_formats) + assert sum(len(fmt) <= 8 for fmt in _formats) == _list_len + _recreation_codes = [self._extract_recreation_code(item) for item in sequence] +", ); - let code = compile_exec(&source); + } - assert!( - code.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::ListAppend { .. })), - "large constant lists should keep LIST_APPEND streaming form, got instructions={:?}", - code.instructions + #[test] + fn test_optimized_assert_with_nested_scope_in_first_iter() { + // First iterator of a comprehension is evaluated in the enclosing + // scope, so nested scopes inside it (the generator here) must also + // be consumed when the assert is optimized away. + compile_exec_optimized( + "\ +def f(items): + assert [x for x in (y for y in items)] + return [x for x in items] +", ); - assert!( - !code - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), - "large constant lists should not fold to LIST_EXTEND, got instructions={:?}", - code.instructions + } + + #[test] + fn test_optimized_assert_with_lambda_defaults() { + // Lambda default values are evaluated in the enclosing scope, + // so nested scopes inside defaults must be consumed. + compile_exec_optimized( + "\ +def f(items): + assert (lambda x=[i for i in items]: x)() + return [x for x in items] +", ); } #[test] - fn test_annotation_closure_uses_format_varname() { + fn test_try_else_nested_scopes_keep_subtable_cursor_aligned() { let code = compile_exec( "\ -class C: - x: int +try: + import missing_mod +except ImportError: + def fallback(): + return 0 +else: + def impl(): + return reversed('abc') ", ); - let annotate = find_code(&code, "__annotate__").expect("missing __annotate__ code"); - let varnames = annotate - .varnames - .iter() - .map(|name| name.as_str()) - .collect::>(); - assert_eq!(varnames, vec!["format"]); + + assert!( + find_code(&code, "fallback").is_some(), + "missing fallback code" + ); + let impl_code = find_code(&code, "impl").expect("missing impl code"); + assert!( + impl_code.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadGlobal { .. } | Instruction::LoadName { .. } + ) + }), + "expected impl to compile global name access, got ops={:?}", + impl_code + .instructions + .iter() + .map(|unit| unit.op) + .collect::>() + ); } #[test] - fn test_type_param_evaluator_uses_dot_format_varname() { + fn test_nested_try_else_multi_resume_join_keeps_strong_load_fast_tail() { let code = compile_exec( "\ -class C[T: int]: - pass +def f(msg): + s = '' + try: + import a + except Exception: + suggest = False + tb = None + else: + try: + suggest = not t() + tb = g(msg) + except Exception: + suggest = False + tb = None + if tb is not None: + for frame in tb: + s += frame + elif suggest: + s += 'y' + return s ", ); - let evaluator = find_code(&code, "T").expect("missing type parameter evaluator"); - let varnames = evaluator - .varnames + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions .iter() - .map(|name| name.as_str()) - .collect::>(); - assert_eq!(varnames, vec![".format"]); + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let tail_start = ops + .iter() + .position(|op| matches!(op, Instruction::PopJumpIfNone { .. })) + .expect("missing tail POP_JUMP_IF_NONE") + .saturating_sub(1); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &ops[tail_start..handler_start]; + + assert!( + !tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected nested try/except else-resume tail to keep strong LOAD_FAST ops, got tail={tail:?}" + ); + + assert!( + tail.iter() + .any(|op| matches!(op, Instruction::LoadFastLoadFast { .. })), + "expected loop body to keep LOAD_FAST_LOAD_FAST in the resume tail, got tail={tail:?}" + ); } #[test] - fn test_class_annotation_global_resolution_matches_cpython() { - let class_global = compile_exec( + fn test_protected_conditional_tail_keeps_strong_load_fast() { + let code = compile_exec( "\ -X = 'global' -class C: - locals()['X'] = 'class' - global X - y: X +def f(m, class_name, category, warning_base): + try: + cat = getattr(m, class_name) + except AttributeError: + raise ValueError(category) + if not issubclass(cat, warning_base): + raise TypeError(category) + return cat ", ); - let annotate = - find_code(&class_global, "__annotate__").expect("missing class __annotate__ code"); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let tail_start = ops + .iter() + .position(|op| matches!(op, Instruction::StoreFast { .. })) + .expect("missing STORE_FAST cat"); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &ops[tail_start + 1..handler_start]; + assert!( - annotate - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::LoadGlobal { .. })), - "expected explicit class global to use LOAD_GLOBAL, got instructions={:?}", - annotate.instructions + !tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected protected conditional tail to keep strong LOAD_FAST ops, got tail={tail:?}" ); + assert!( - !annotate - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::LoadFromDictOrGlobals { .. })), - "did not expect class explicit global to use LOAD_FROM_DICT_OR_GLOBALS, got instructions={:?}", - annotate.instructions + tail.iter() + .any(|op| matches!(op, Instruction::LoadFastLoadFast { .. })), + "expected protected tail to keep LOAD_FAST_LOAD_FAST for issubclass args, got tail={tail:?}" ); + } - let outer_global = compile_exec( + #[test] + fn test_nonresuming_protected_conditional_tail_keeps_strong_load_fast() { + let code = compile_exec( "\ -def f(): - global X - class C: - locals()['X'] = 'class' - y: X +def f(href, parse='xml'): + try: + data = XINCLUDE[href] + except KeyError: + raise OSError('resource not found') + if parse == 'xml': + data = ET.XML(data) + return data ", ); - let annotate = find_code(&outer_global, "__annotate__") - .expect("missing nested class __annotate__ code"); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + let tail_start = ops + .iter() + .position(|op| matches!(op, Instruction::StoreFast { .. })) + .expect("missing protected STORE_FAST data"); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &ops[tail_start + 1..handler_start]; + assert!( - annotate - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::LoadFromDictOrGlobals { .. })), - "expected outer explicit global in class annotation to use LOAD_FROM_DICT_OR_GLOBALS, got instructions={:?}", - annotate.instructions + !tail.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected non-resuming protected conditional tail to keep strong LOAD_FAST ops, got tail={tail:?}" ); } #[test] - fn test_constant_tuple_binops_fold_like_cpython() { - let code = compile_exec("value = (1,) * 17 + ('spam',)\n"); + fn test_optional_nonresuming_protected_tail_keeps_borrow() { + let code = compile_exec( + "\ +def f(b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError(f'bad {type(b).__name__}') from None + if b: + sink(b) + return len(b) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let b_index = f + .varnames + .iter() + .position(|name| name.as_str() == "b") + .expect("missing b varname"); + let store_b = instructions + .iter() + .position(|unit| match unit.op { + Instruction::StoreFast { var_num } => { + usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg))))) == b_index + } + _ => false, + }) + .expect("missing protected STORE_FAST b"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[store_b + 1..handler_start]; assert!( - !code - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::BinaryOp { .. })), - "tuple constant binops should fold away, got instructions={:?}", - code.instructions + tail.iter() + .filter(|unit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg))))) + == b_index + } + _ => false, + }) + .all(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. })), + "optional protected tail should keep CPython-style borrowed b loads, got tail={tail:?}" ); - assert!(code.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if elements.len() == 18 - && elements[..17] - .iter() - .all(|elt| matches!(elt, ConstantData::Integer { value } if *value == BigInt::from(1))) - && matches!(&elements[17], ConstantData::Str { value } if value.to_string() == "spam") - ))); } #[test] - fn test_constant_list_iterable_uses_tuple() { + fn test_handled_except_conditional_tail_keeps_borrow() { let code = compile_exec( "\ -def f(): - return {x: y for x, y in [(1, 2), ]} +def f(self): + try: + if self.active: + self.step() + if self.waiter is not None and self.pending is None: + self.waiter.set_result(None) + except ConnectionResetError as exc: + self.close(exc) + except OSError as exc: + self.fail(exc, 'x') ", ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let waiter_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "waiter" + } + _ => false, + }) + .expect("missing waiter LOAD_ATTR"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[waiter_idx.saturating_sub(1)..handler_start]; assert!( - !f.instructions + tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "handled-except conditional tail should keep borrowed loads, got tail={tail:?}" + ); + assert!( + !tail .iter() - .any(|unit| matches!(unit.op, Instruction::BuildList { .. })), - "constant list iterable should avoid BUILD_LIST before GET_ITER" + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "handled-except conditional tail should not force strong LOAD_FAST, got tail={tail:?}" ); - assert!(f.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if matches!( - elements.as_slice(), - [ConstantData::Tuple { elements: inner }] - if matches!( - inner.as_slice(), - [ - ConstantData::Integer { .. }, - ConstantData::Integer { .. } - ] - ) - ) - ))); } #[test] - fn test_constant_set_iterable_uses_frozenset_const() { + fn test_handled_except_else_tail_keeps_borrow() { let code = compile_exec( "\ -def f(): - return [x for x in {1, 2, 3}] +def f(self, fut=None): + try: + if self.closed: + return + item = self.queue.popleft() + self.size -= len(item) + if self.addr is not None: + self.future = self.loop.send(self.sock, item) + else: + self.future = self.loop.sendto(self.sock, item, addr=item) + except OSError as exc: + self.protocol.error_received(exc) + except Exception as exc: + self.fatal(exc, 'x') + else: + self.future.add_done_callback(self.loop_writing) + self.resume() ", ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let done_callback_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "add_done_callback" + } + _ => false, + }) + .expect("missing add_done_callback LOAD_ATTR"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[done_callback_idx.saturating_sub(3)..handler_start]; assert!( - !f.instructions + tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "handled-except else tail should keep borrowed loads, got tail={tail:?}" + ); + assert!( + !tail .iter() - .any(|unit| matches!(unit.op, Instruction::BuildSet { .. })), - "constant set iterable should avoid BUILD_SET before GET_ITER" + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "handled-except else tail should not force strong LOAD_FAST, got tail={tail:?}" ); - assert!(f.constants.iter().any(|constant| matches!( - constant, - ConstantData::Frozenset { elements } - if matches!( - elements.as_slice(), - [ - ConstantData::Integer { .. }, - ConstantData::Integer { .. }, - ConstantData::Integer { .. } - ] - ) - ))); } #[test] - fn test_constant_list_membership_uses_tuple_const() { + fn test_reraising_handler_with_handled_returns_keeps_borrow() { let code = compile_exec( "\ -f = lambda x: x in [1, 2, 3] +def f(self, fut=None): + try: + if fut is not None: + fut.result() + if self.future is not fut: + return + fut = self.reader.recv(self.sock, 4096) + except CancelledError: + return + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self.handle({'exception': exc, 'loop': self}) + else: + self.future = fut + fut.add_done_callback(self.loop_reading) ", ); - let lambda = find_code(&code, "").expect("missing lambda code"); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let recv_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "recv" + } + _ => false, + }) + .expect("missing recv LOAD_ATTR"); + let done_callback_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "add_done_callback" + } + _ => false, + }) + .expect("missing add_done_callback LOAD_ATTR"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = + &instructions[recv_idx.saturating_sub(3)..handler_start.min(done_callback_idx + 3)]; assert!( - !lambda - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::BuildList { .. })), - "constant list membership should avoid BUILD_LIST before CONTAINS_OP" - ); - assert!(lambda.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if matches!( - elements.as_slice(), - [ - ConstantData::Integer { .. }, - ConstantData::Integer { .. }, - ConstantData::Integer { .. } - ] + tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) - ))); + }), + "handler chain with handled returns should keep borrowed warm/else loads, got tail={tail:?}" + ); + assert!( + !tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. } + ) + }), + "handler chain with handled returns should not force strong warm/else loads, got tail={tail:?}" + ); } #[test] - fn test_small_constant_set_membership_uses_frozenset_const() { + fn test_with_protected_conditional_tail_without_exception_match_keeps_borrow() { let code = compile_exec( "\ -f = lambda x: x in {0} +def f(self, cm, p, platform): + with cm: + if p.returncode != 0: + if platform.machine() == 'x86_64': + p.check_returncode() + else: + self.skipTest(f'could not compile indirect function: {p}') + done() ", ); - let lambda = find_code(&code, "").expect("missing lambda code"); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); - assert!( - !lambda - .instructions + let attr_load_uses_borrow = |name: &str| { + let attr_idx = instructions .iter() - .any(|unit| matches!(unit.op, Instruction::BuildSet { .. })), - "constant set membership should avoid BUILD_SET before CONTAINS_OP" + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == name + } + _ => false, + }) + .unwrap_or_else(|| panic!("missing {name} attr load")); + matches!( + instructions + .get(attr_idx.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ) + }; + + assert!( + attr_load_uses_borrow("check_returncode"), + "plain with-protected conditional tail should keep borrowed p load, got instructions={instructions:?}" + ); + assert!( + attr_load_uses_borrow("skipTest"), + "plain with-protected conditional tail should keep borrowed self load, got instructions={instructions:?}" ); - assert!(lambda.constants.iter().any(|constant| matches!( - constant, - ConstantData::Frozenset { elements } - if matches!(elements.as_slice(), [ConstantData::Integer { value }] if *value == BigInt::from(0)) - ))); } #[test] - fn test_nonconstant_list_membership_uses_tuple() { + fn test_listcomp_cleanup_predecessor_does_not_deopt_following_conditional_tail() { let code = compile_exec( "\ -def f(a, b, c, x): - return x in [a, b, c] +def f(self, compile_snippet): + sizes = [compile_snippet(i).co_stacksize for i in range(2, 5)] + if len(set(sizes)) != 1: + import dis, io + out = io.StringIO() + dis.dis(compile_snippet(1), file=out) + self.fail('%s\\n%s' % (sizes, out.getvalue())) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f - .instructions - .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); - assert!( - ops.windows(2).any(|window| { - matches!( - window, - [ - Instruction::BuildTuple { .. }, - Instruction::ContainsOp { .. } - ] - ) - }), - "expected BUILD_TUPLE before CONTAINS_OP for non-constant list membership, got ops={ops:?}" - ); + let has_strong_load = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + let has_borrow_load = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + + for name in ["sizes", "io", "dis", "compile_snippet", "out", "self"] { + assert!( + has_borrow_load(name), + "expected listcomp-following conditional tail to borrow {name}, got instructions={:?}", + f.instructions + ); + } + for name in ["sizes", "io", "dis", "compile_snippet", "out", "self"] { + assert!( + !has_strong_load(name), + "listcomp cleanup predecessor should not force strong LOAD_FAST for {name}, got instructions={:?}", + f.instructions + ); + } } #[test] - fn test_starred_tuple_iterable_drops_list_to_tuple_before_get_iter() { + fn test_handler_resume_loop_conditional_tail_keeps_strong_load_fast() { let code = compile_exec( "\ -def f(a, b, c): - for x in *a, *b, *c: - pass +def f(self): + is_utf8 = (self.ENCODING == 'utf-8') + encode_errors = 'surrogateescape' if is_utf8 else 'strict' + strings = list(self.BYTES_STRINGS) + for text in self.STRINGS: + try: + encoded = text.encode(self.ENCODING, encode_errors) + if encoded not in strings: + strings.append(encoded) + except UnicodeEncodeError: + encoded = None + if is_utf8: + encoded2 = text.encode(self.ENCODING, 'surrogatepass') + if encoded2 != encoded: + strings.append(encoded2) + for encoded in strings: + self.consume(encoded) ", ); - let f = find_code(&code, "f").expect("missing function code"); + let f = find_code(&code, "f").expect("missing f code"); + + let has_strong_load = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + let has_borrow_load = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + for name in ["is_utf8", "text", "self", "strings", "encoded2"] { + assert!( + has_strong_load(name), + "expected handler-resume loop tail to use strong LOAD_FAST for {name}, got instructions={:?}", + f.instructions + ); + } assert!( - !has_intrinsic_1(f, IntrinsicFunction1::ListToTuple), - "LIST_TO_TUPLE should be removed before GET_ITER in for-iterable context" + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFastLoadFast { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == "encoded2" + && f.varnames[usize::from(right)] == "encoded" + } + _ => false, + }), + "expected encoded2/encoded comparison to use strong LOAD_FAST_LOAD_FAST, got instructions={:?}", + f.instructions ); assert!( + !f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == "encoded2" + && f.varnames[usize::from(right)] == "encoded" + } + _ => false, + }), + "handler-resume loop tail should not borrow encoded2/encoded comparison, got instructions={:?}", f.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::GetIter)), - "expected GET_ITER in for loop" + ); + assert!( + has_borrow_load("strings"), + "expected later loop/list uses outside the deopt tail to keep borrowing strings" ); } #[test] - fn test_comprehension_single_list_iterable_uses_tuple() { + fn test_async_early_return_send_tail_uses_strong_load_fast_after_entry() { let code = compile_exec( "\ -def g(): - [x for x in [(yield 1)]] +class C: + async def _sock_sendfile_native(self, sock, file, offset, count): + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise exceptions.SendfileNotAvailableError('not a regular file') + try: + fsize = os.fstat(fileno).st_size + except OSError: + raise exceptions.SendfileNotAvailableError('not a regular file') + blocksize = count if count else fsize + if not blocksize: + return 0 + blocksize = min(blocksize, 0xffff_ffff) + end_pos = min(offset + count, fsize) if count else fsize + offset = min(offset, fsize) + total_sent = 0 + try: + while True: + blocksize = min(end_pos - offset, blocksize) + if blocksize <= 0: + return total_sent + await self._proactor.sendfile(sock, file, offset, blocksize) + offset += blocksize + total_sent += blocksize + finally: + if total_sent > 0: + file.seek(offset) ", ); - let g = find_code(&code, "g").expect("missing g code"); - let ops: Vec<_> = g + let f = find_code(&code, "_sock_sendfile_native").expect("missing method code"); + + let names_for_unit = |unit: &bytecode::CodeUnit| -> Vec { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + vec![f.varnames[usize::from(var_num.get(arg))].to_string()] + } + Instruction::LoadFastLoadFast { var_nums } + | Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let (left, right) = var_nums.get(arg).indexes(); + vec![ + f.varnames[usize::from(left)].to_string(), + f.varnames[usize::from(right)].to_string(), + ] + } + _ => Vec::new(), + } + }; + + let borrowed_names: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter_map(|unit| match unit.op { + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } => Some(names_for_unit(unit)), + _ => None, + }) .collect(); + assert_eq!( + borrowed_names, + vec![vec!["file".to_owned()]], + "only the initial file.fileno() receiver should borrow, got instructions={:?}", + f.instructions + ); - assert!( - ops.windows(2).any(|window| { + let has_strong_name = |name: &str| { + f.instructions.iter().any(|unit| { matches!( - window, - [Instruction::BuildTuple { .. }, Instruction::GetIter] - ) - }), - "expected BUILD_TUPLE before GET_ITER for single-item list iterable in comprehension, got ops={ops:?}" - ); + unit.op, + Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. } + ) && names_for_unit(unit).iter().any(|loaded| loaded == name) + }) + }; + + for name in [ + "fileno", + "count", + "blocksize", + "offset", + "fsize", + "end_pos", + "total_sent", + "file", + "self", + "sock", + ] { + assert!( + has_strong_name(name), + "async early-return send tail should use strong LOAD_FAST for {name}, got instructions={:?}", + f.instructions + ); + } } #[test] - fn test_nested_comprehension_list_iterable_uses_tuple() { + fn test_protected_import_tail_keeps_strong_load_fast() { let code = compile_exec( "\ -def f(): - return [[y for y in [x, x + 1]] for x in [1, 3, 5]] +def f(s, size, pos, errors): + message = 'x' + look = pos + try: + import unicodedata + except ImportError: + return None + if look < size and chr(s[look]) == '{': + while look < size and chr(s[look]) != '}': + look += 1 + if look > pos + 1 and look < size and chr(s[look]) == '}': + message = 'y' + return message ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -15925,321 +20990,584 @@ def f(): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); + let import_idx = ops + .iter() + .position(|op| matches!(op, Instruction::ImportName { .. })) + .expect("missing IMPORT_NAME"); + let protected_tail = &ops[import_idx + 1..]; + assert!( - ops.windows(2).any(|window| { + !protected_tail.iter().any(|op| { matches!( - window, - [Instruction::BuildTuple { .. }, Instruction::GetIter] + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected BUILD_TUPLE before GET_ITER for nested list iterable in comprehension, got ops={ops:?}" - ); - } - - #[test] - fn test_constant_comprehension_iterable_with_unary_int_uses_tuple_const() { - let code = compile_exec( - "\ -l = lambda : [2 < x for x in [-1, 3, 0]] -", + "expected protected import tail to keep strong LOAD_FAST ops, got tail={protected_tail:?}" ); - let lambda = find_code(&code, "").expect("missing lambda code"); assert!( - lambda.constants.iter().any(|constant| matches!( - constant, - ConstantData::Tuple { elements } - if matches!( - elements.as_slice(), - [ - ConstantData::Integer { .. }, - ConstantData::Integer { .. }, - ConstantData::Integer { .. } - ] - ) - )), - "expected folded tuple constant for comprehension iterable" + protected_tail + .iter() + .any(|op| matches!(op, Instruction::LoadFastLoadFast { .. })), + "expected protected import tail to keep LOAD_FAST_LOAD_FAST ops, got tail={protected_tail:?}" ); } #[test] - fn test_module_scope_listcomp_is_inlined() { - let code = compile_exec("values = [i for i in range(3)]\n"); - - assert!( - find_code(&code, "").is_none(), - "module-scope list comprehension should be inlined" - ); - assert!( - code.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::LoadFastAndClear { .. })), - "inlined module-scope list comprehension should use LOAD_FAST_AND_CLEAR, got instructions={:?}", - code.instructions + fn test_unprotected_import_before_with_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, document): + from xml.etree import ElementInclude + document = self.xinclude_loader('C1.xml') + with self.assertRaises(OSError) as cm: + ElementInclude.include(document, self.xinclude_loader) + self.assertEqual(str(cm.exception), 'resource not found') +", ); - } + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); - #[test] - fn test_module_scope_dictcomp_is_inlined() { - let code = compile_exec("mapping = {i: i for i in range(3)}\n"); + let import_idx = ops + .iter() + .position(|op| matches!(op, Instruction::ImportName { .. })) + .expect("missing IMPORT_NAME"); + let handler_start = ops + .iter() + .position(|op| matches!(op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_path = &ops[import_idx + 1..handler_start]; - assert!( - find_code(&code, "").is_none(), - "module-scope dict comprehension should be inlined" + assert!( + warm_path.iter().any(|op| { + matches!( + op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected unprotected import before with-block to keep LOAD_FAST_BORROW ops, got warm_path={warm_path:?}" ); assert!( - code.instructions + warm_path .iter() - .any(|unit| matches!(unit.op, Instruction::LoadFastAndClear { .. })), - "inlined module-scope dict comprehension should use LOAD_FAST_AND_CLEAR, got instructions={:?}", - code.instructions + .any(|op| matches!(op, Instruction::LoadFastBorrowLoadFastBorrow { .. })), + "expected with body arguments to keep LOAD_FAST_BORROW_LOAD_FAST_BORROW, got warm_path={warm_path:?}" ); } #[test] - fn test_async_dictcomp_in_async_function_is_inlined() { + fn test_unprotected_prefix_before_try_keeps_attr_subscript_borrow() { let code = compile_exec( "\ -async def f(items): - return {item: item async for item in items} +def f(): + import sys, getopt + usage = f'usage: {sys.argv[0]}' + try: + opts, args = getopt.getopt(sys.argv[1:], 'h') + except getopt.error as msg: + sys.stdout = sys.stderr + print(msg) + print(usage) + sys.exit(2) + return usage ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let first_argv_idx = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "argv" + } + _ => false, + }) + .expect("missing argv attr load"); + let receiver = f.instructions[..first_argv_idx] + .iter() + .rev() + .find(|unit| !matches!(unit.op, Instruction::Cache)) + .expect("missing argv receiver") + .op; assert!( - find_code(&code, "").is_none(), - "async dict comprehension should be inlined" - ); - assert!( - ops.iter().any(|op| matches!(op, Instruction::GetAIter)), - "inlined async dict comprehension should keep GET_AITER in outer code, got ops={ops:?}" + matches!(receiver, Instruction::LoadFastBorrow { .. }), + "unprotected prefix before try should keep CPython-style LOAD_FAST_BORROW receiver, got {receiver:?}" ); - assert!( - ops.iter() - .any(|op| matches!(op, Instruction::LoadFastAndClear { .. })), - "inlined async dict comprehension should use LOAD_FAST_AND_CLEAR, got ops={ops:?}" + } + + #[test] + fn test_terminal_except_inlined_comprehension_keeps_borrowed_warm_loads() { + let code = compile_exec( + r##" +def f(output): + output = re.sub(r"\[[0-9]+ refs\]", "", output) + try: + result = [ + row.split("\t") + for row in output.splitlines() + if row and not row.startswith('#') + ] + result.sort(key=lambda row: int(row[0])) + result = [row[1] for row in result] + return "\n".join(result) + except (IndexError, ValueError): + raise AssertionError( + "tracer produced unparsable output:\n{}".format(output) + ) +"##, ); + let f = find_code(&code, "f").expect("missing f code"); + let handler_start = f + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let warm_path = &f.instructions[..handler_start]; + let load_fast_name = |unit: &bytecode::CodeUnit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + let borrow_name = |unit: &bytecode::CodeUnit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + Some(f.varnames[usize::from(var_num.get(arg))].as_str()) + } + _ => None, + }; + assert!( - !ops.iter().any(|op| matches!(op, Instruction::MakeFunction)), - "inlined async dict comprehension should not materialize MAKE_FUNCTION, got ops={ops:?}" + warm_path + .iter() + .filter_map(load_fast_name) + .all(|name| name != "row" && name != "result"), + "terminal-except inlined comprehension warm path should keep CPython-style borrowed row/result loads, got warm_path={warm_path:?}" ); + for name in ["row", "result"] { + assert!( + warm_path + .iter() + .filter_map(borrow_name) + .any(|var| var == name), + "expected borrowed {name} load in terminal-except inlined comprehension warm path, got warm_path={warm_path:?}" + ); + } } #[test] - fn test_nested_module_scope_dictcomp_symbols_are_local() { - let symbol_table = scan_program_symbol_table( + fn test_outer_guarded_protected_import_keeps_borrow_tail() { + let code = compile_exec( "\ -deoptmap = { - specialized: base - for base, family in _specializations.items() - for specialized in family -} +def f(sys, os, file): + if sys.platform == 'win32': + try: + import nt + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + try: + return os.isatty(file.fileno()) + except OSError: + return hasattr(file, 'isatty') and file.isatty() ", ); + let f = find_code(&code, "f").expect("missing f code"); + let borrows_name = |name: &str| { + f.instructions.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == name || f.varnames[usize::from(right)] == name + } + _ => false, + }) + }; - for name in ["base", "family", "specialized"] { - let symbol = symbol_table - .lookup(name) - .unwrap_or_else(|| panic!("missing module symbol {name}")); - assert_eq!( - symbol.scope, - SymbolScope::Local, - "expected module-scope inlined comprehension symbol {name} to be Local, got {symbol:?}" - ); - } - - let comp = symbol_table - .sub_tables - .first() - .expect("missing comprehension symbol table"); - assert!(comp.comp_inlined, "expected comprehension to be inlined"); - for name in ["base", "family", "specialized"] { - let symbol = comp - .lookup(name) - .unwrap_or_else(|| panic!("missing comprehension symbol {name}")); - assert_eq!( - symbol.scope, - SymbolScope::Local, - "expected comprehension symbol {name} to be Local, got {symbol:?}" + for name in ["nt", "os", "file"] { + assert!( + borrows_name(name), + "outer-guarded protected import should keep CPython-style borrow for {name}, got instructions={:?}", + f.instructions ); } } #[test] - fn test_nested_module_scope_dictcomp_uses_fast_locals() { + fn test_loop_or_break_continue_orders_break_before_backedge() { let code = compile_exec( "\ -deoptmap = { - specialized: base - for base, family in _specializations.items() - for specialized in family -} +def f(self, quoted): + while True: + if self.state == 'x': + if self.token or (self.posix and quoted): + break + else: + continue + elif self.state == 'y': + self.consume() + x = self.a + self.b + self.c + return x ", ); - let ops: Vec<_> = code + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - + let quoted_load = ops + .iter() + .position(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == "quoted" + } + _ => false, + }) + .expect("missing quoted LOAD_FAST_BORROW"); + let final_cond = ops[quoted_load + 1..] + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + }) + .map(|idx| quoted_load + 1 + idx) + .expect("missing final conditional jump"); assert!( - ops.iter() - .any(|op| matches!(op, Instruction::StoreFastStoreFast { .. })), - "expected outer target unpack to use STORE_FAST_STORE_FAST, got ops={ops:?}" + matches!(ops[final_cond].op, Instruction::PopJumpIfFalse { .. }), + "expected CPython-style inverted final condition, got ops={ops:?}" ); + let break_jump_idx = ops[final_cond + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpForward { .. })) + .map(|idx| final_cond + 1 + idx) + .expect("missing break jump after condition"); + let jump_back_idx = ops[final_cond + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| final_cond + 1 + idx) + .expect("missing continue backedge"); assert!( - ops.iter().any(|op| matches!( - op, - Instruction::StoreFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - )), - "expected inner target/store-use path to use fast locals, got ops={ops:?}" + break_jump_idx < jump_back_idx, + "expected break jump before continue backedge, got ops={ops:?}" + ); + } + + #[test] + fn test_for_continue_before_return_orders_backedge_before_return_body() { + let code = compile_exec( + "\ +def f(self): + for version in AllowedVersions: + if not version in self.capabilities: + continue + self.PROTOCOL_VERSION = version + return + raise self.error('x') +", ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let contains_idx = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::ContainsOp { .. })) + .expect("missing containment test"); + let cond_idx = ops[contains_idx + 1..] + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + }) + .map(|idx| contains_idx + 1 + idx) + .expect("missing conditional jump"); assert!( - ops.iter() - .filter(|op| matches!(op, Instruction::LoadName { .. })) - .count() - <= 1, - "unexpected extra LOAD_NAME ops in nested inlined comprehension, got ops={ops:?}" + matches!(ops[cond_idx].op, Instruction::PopJumpIfTrue { .. }), + "expected CPython-style condition targeting the return body, got ops={ops:?}" ); + + let backedge_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing continue backedge"); + let store_attr_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| match unit.op { + Instruction::StoreAttr { namei } => { + let namei = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(namei).unwrap()].as_str() == "PROTOCOL_VERSION" + } + _ => false, + }) + .map(|idx| cond_idx + 1 + idx) + .expect("missing PROTOCOL_VERSION store"); assert!( - ops.iter() - .filter(|op| matches!(op, Instruction::StoreName { .. })) - .count() - <= 1, - "unexpected extra STORE_NAME ops in nested inlined comprehension, got ops={ops:?}" + backedge_idx < store_attr_idx, + "expected continue backedge before return body, got ops={ops:?}" ); } #[test] - fn test_module_scope_inlined_comprehension_keeps_outer_iter_as_name_lookup() { + fn test_while_conditional_return_orders_backedge_before_return_body() { let code = compile_exec( "\ -path_separators = ['/'] -_pathseps_with_colon = {f':{s}' for s in path_separators} +def f(self, tag): + while self._get_response(): + if self.tagged_commands[tag]: + return tag ", ); - let ops: Vec<_> = code + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - let load_name_path = ops - .windows(2) - .any(|window| matches!(window, [Instruction::LoadName { .. }, Instruction::GetIter])); - assert!( - load_name_path, - "expected outer iterable to stay a NAME lookup before GET_ITER, got ops={ops:?}" - ); + let subscript_idx = ops + .iter() + .position(|unit| matches!(unit.op, Instruction::BinaryOp { .. })) + .expect("missing tagged_commands subscript"); + let cond_idx = ops[subscript_idx + 1..] + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + }) + .map(|idx| subscript_idx + 1 + idx) + .expect("missing conditional jump"); assert!( - !ops.windows(2).any(|window| matches!( - window, - [ - Instruction::LoadFast { .. } | Instruction::LoadFastCheck { .. }, - Instruction::GetIter - ] - )), - "module local outer iterable should not become a fast local, got ops={ops:?}" + matches!(ops[cond_idx].op, Instruction::PopJumpIfTrue { .. }), + "expected CPython-style condition targeting return body, got ops={ops:?}" ); + let backedge_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing loop backedge"); + let return_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::ReturnValue)) + .map(|idx| cond_idx + 1 + idx) + .expect("missing return"); assert!( - ops.iter().any(|op| matches!( - op, - Instruction::StoreFastLoadFast { .. } | Instruction::StoreFast { .. } - )), - "comprehension target should still use fast locals, got ops={ops:?}" + backedge_idx < return_idx, + "expected loop backedge before return body, got ops={ops:?}" ); } #[test] - fn test_function_scope_inlined_comprehension_restore_keeps_swap_before_duplicate_store() { + fn test_for_break_to_return_orders_backedge_before_return() { let code = compile_exec( "\ -def f(): - a = [1 for a in [0]] - return 1 +def f(it): + best = 10 + body = None + for prio, part in it: + if prio < best: + best = prio + body = part + if prio == 0: + break + return body +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let compare_idx = ops + .iter() + .enumerate() + .filter(|(_, unit)| matches!(unit.op, Instruction::CompareOp { .. })) + .nth(1) + .map(|(idx, _)| idx) + .expect("missing break comparison"); + let cond_idx = ops[compare_idx + 1..] + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + }) + .map(|idx| compare_idx + 1 + idx) + .expect("missing break conditional jump"); + assert!( + matches!(ops[cond_idx].op, Instruction::PopJumpIfTrue { .. }), + "expected CPython-style true jump to break return path, got ops={ops:?}" + ); + let jump_back_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing loop backedge before break return"); + let return_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::ReturnValue)) + .map(|idx| cond_idx + 1 + idx) + .expect("missing break return path"); + assert!( + jump_back_idx < return_idx, + "expected loop backedge before break return block, got ops={ops:?}" + ); + } + + #[test] + fn test_for_conditional_raise_orders_backedge_before_raise() { + let code = compile_exec( + "\ +def f(items, limit): + found = 0 + for item in items: + if item: + found += 1 + if found >= limit: + raise ValueError(found) + return found ", ); let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - + let compare_idx = ops + .iter() + .enumerate() + .find(|(_, unit)| matches!(unit.op, Instruction::CompareOp { .. })) + .map(|(idx, _)| idx) + .expect("missing raise comparison"); + let cond_idx = ops[compare_idx + 1..] + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + }) + .map(|idx| compare_idx + 1 + idx) + .expect("missing raise conditional jump"); assert!( - ops.windows(4).any(|window| matches!( - window, - [ - Instruction::PopIter, - Instruction::Swap { .. }, - Instruction::StoreFast { .. }, - Instruction::StoreFast { .. } - ] - )), - "expected PopIter/SWAP 2/STORE_FAST/STORE_FAST restore tail, got ops={ops:?}" - ); - } - - #[test] - fn test_single_mode_folded_multiline_constant_does_not_leave_nops() { - let code = compile_single( - "\ -(- - - - - - 1) -", + matches!(ops[cond_idx].op, Instruction::PopJumpIfTrue { .. }), + "expected CPython-style true jump to raise path, got ops={ops:?}" ); - + let jump_back_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing loop backedge before raise"); + let raise_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::RaiseVarargs { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing raise path"); assert!( - !code - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::Nop)), - "expected folded single-mode multiline constant to drop NOP anchors, got instructions={:?}", - code.instructions + jump_back_idx < raise_idx, + "expected loop backedge before conditional raise block, got ops={ops:?}" ); } #[test] - fn test_folded_multiline_bytes_binop_does_not_leave_operand_nops() { + fn test_exception_handler_loop_conditional_raise_orders_backedge_before_raise() { let code = compile_exec( "\ -def f(self, out): - self.assertIn( - b'gnu' + (b'/123' * 125) + b'/longlink' + (b'/123' * 125) + b'/longname', - out) +def f(chunk, dec, i): + try: + for c in chunk: + acc = dec[c] + except TypeError: + for j, c in enumerate(chunk): + if dec[c] is None: + raise ValueError('%d' % (i + j)) from None + raise ", ); let f = find_code(&code, "f").expect("missing f code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); assert!( - !f.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::Nop)), - "expected CPython nop_out-style folded operands to have no surviving NOPs, got instructions={:?}", - f.instructions + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::BinaryOp { .. }, + Instruction::PopJumpIfNone { .. }, + Instruction::NotTaken, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "expected exception-handler loop false path to jump back before raise body, got ops={ops:?}" + ); + assert!( + !ops.windows(4).any(|window| { + matches!( + window, + [ + Instruction::BinaryOp { .. }, + Instruction::PopJumpIfNotNone { .. }, + Instruction::NotTaken, + Instruction::LoadGlobal { .. }, + ] + ) + }), + "unexpected exception-handler loop raise body before backedge, got ops={ops:?}" ); } #[test] - fn test_folded_binop_at_branch_body_start_does_not_leave_nop() { + fn test_loop_if_body_keeps_fallthrough_before_implicit_continue_backedge() { let code = compile_exec( "\ -def f(sys): - if sys.platform == 'win32': - component = 'd' * 25 - return component +def f(b, curr, curr_append, decoded_append, packI, curr_clear): + for x in b: + if 33 <= x <= 117: + curr_append(x) + if len(curr) == 5: + acc = 0 + for x in curr: + acc = 85 * acc + (x - 33) + decoded_append(packI(acc)) + curr_clear() + elif x == 122: + decoded_append(0) ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -16251,181 +21579,277 @@ def f(sys): .collect(); assert!( - !ops.windows(3).any(|window| { + ops.windows(5).any(|window| { + matches!( + window, + [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfFalse { .. }, + Instruction::NotTaken, + Instruction::LoadSmallInt { .. }, + Instruction::StoreFast { .. }, + ] + ) + }), + "expected CPython-style conditional body fallthrough before implicit continue backedge, got ops={ops:?}" + ); + assert!( + !ops.windows(6).any(|window| { matches!( window, [ + Instruction::CompareOp { .. }, + Instruction::PopJumpIfTrue { .. }, Instruction::NotTaken, - Instruction::Nop, - Instruction::LoadConst { .. } + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadSmallInt { .. }, + Instruction::StoreFast { .. }, ] ) }), - "expected CPython nop_out-style folded branch body to drop operand NOP, got ops={ops:?}", + "unexpected inverted conditional with implicit continue backedge before body, got ops={ops:?}" ); } #[test] - fn test_multiline_unpack_target_uses_element_locations() { + fn test_explicit_continue_after_return_orders_return_before_backedge() { let code = compile_exec( "\ -def f(cm): - with cm as (_, - filename_2): - return filename_2 +def f(j, n): + while j < n: + if j < 0: + return j + continue + return -1 ", ); let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - + let compare_idx = ops + .iter() + .enumerate() + .filter(|(_, unit)| matches!(unit.op, Instruction::CompareOp { .. })) + .nth(1) + .map(|(idx, _)| idx) + .expect("missing inner comparison"); + let cond_idx = ops[compare_idx + 1..] + .iter() + .position(|unit| { + matches!( + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } + ) + }) + .map(|idx| compare_idx + 1 + idx) + .expect("missing conditional jump"); assert!( - !ops.iter() - .any(|op| matches!(op, Instruction::StoreFastStoreFast { .. })), - "expected multiline target elements to keep separate STORE_FAST instructions, got ops={ops:?}", + matches!(ops[cond_idx].op, Instruction::PopJumpIfFalse { .. }), + "expected CPython-style false jump to explicit continue, got ops={ops:?}" + ); + let return_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::ReturnValue)) + .map(|idx| cond_idx + 1 + idx) + .expect("missing return path"); + let jump_back_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing explicit continue backedge"); + assert!( + return_idx < jump_back_idx, + "expected return block before explicit continue backedge, got ops={ops:?}" ); } #[test] - fn test_or_condition_in_jump_context_uses_shared_true_fallthrough() { + fn test_implicit_while_tail_return_orders_backedge_before_return() { let code = compile_exec( "\ -def f(lines): - for line in lines: - if line.startswith('--') or not line.strip(): - continue - return line +def f(self, j, n): + while j < n: + name, j = self.scan(j) + if j < 0: + return j + return -1 ", ); let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - let first_pop_jump = ops + let compare_idx = ops .iter() - .find(|op| { + .enumerate() + .filter(|(_, unit)| matches!(unit.op, Instruction::CompareOp { .. })) + .nth(1) + .map(|(idx, _)| idx) + .expect("missing inner comparison"); + let cond_idx = ops[compare_idx + 1..] + .iter() + .position(|unit| { matches!( - op, - Instruction::PopJumpIfTrue { .. } | Instruction::PopJumpIfFalse { .. } + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } ) }) - .copied() + .map(|idx| compare_idx + 1 + idx) .expect("missing conditional jump"); assert!( - matches!(first_pop_jump, Instruction::PopJumpIfTrue { .. }), - "expected first OR branch to jump on true into shared fallthrough, got ops={ops:?}" + matches!(ops[cond_idx].op, Instruction::PopJumpIfTrue { .. }), + "expected CPython-style true jump to return, got ops={ops:?}" + ); + let jump_back_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing implicit backedge"); + let return_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::ReturnValue)) + .map(|idx| cond_idx + 1 + idx) + .expect("missing return path"); + assert!( + jump_back_idx < return_idx, + "expected implicit loop backedge before return block, got ops={ops:?}" ); } #[test] - fn test_loop_break_bool_chain_reorders_false_path_to_jump_back() { + fn test_branch_arm_implicit_continue_keeps_return_before_backedge() { let code = compile_exec( "\ -def f(filters, text, category, module, lineno, defaultaction): - for item in filters: - action, msg, cat, mod, ln = item - if ((msg is None or msg.match(text)) and - issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break - else: - action = defaultaction - return action +def f(self, j, n, c): + while j < n: + if c == 'x': + j = self.step(j) + if j < 0: + return j + elif c == 'y': + j = j + 1 + return -1 ", ); let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - assert!( - ops.windows(5).any(|window| { + let compare_idx = ops + .iter() + .enumerate() + .filter(|(_, unit)| matches!(unit.op, Instruction::CompareOp { .. })) + .nth(2) + .map(|(idx, _)| idx) + .expect("missing branch-arm return comparison"); + let cond_idx = ops[compare_idx + 1..] + .iter() + .position(|unit| { matches!( - window, - [ - Instruction::ToBool, - Instruction::PopJumpIfTrue { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadGlobal { .. }, - ] + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } ) - }), - "expected CPython-style false path to fall through into loop jump-back, got ops={ops:?}" + }) + .map(|idx| compare_idx + 1 + idx) + .expect("missing branch-arm conditional jump"); + assert!( + matches!(ops[cond_idx].op, Instruction::PopJumpIfFalse { .. }), + "expected CPython-style false jump to branch-arm continuation, got ops={ops:?}" + ); + let return_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::ReturnValue)) + .map(|idx| cond_idx + 1 + idx) + .expect("missing branch-arm return path"); + let jump_back_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing branch-arm loop backedge"); + assert!( + return_idx < jump_back_idx, + "expected branch-arm return before loop backedge, got ops={ops:?}" ); } #[test] - fn test_loop_conditional_body_keeps_duplicate_jump_back_paths() { + fn test_nested_implicit_while_tail_return_orders_backedge_before_return() { let code = compile_exec( "\ -def f(new, old): - for replace in ['__module__', '__name__', '__qualname__', '__doc__']: - if hasattr(old, replace): - setattr(new, replace, getattr(old, replace)) - return new +def f(self, rawdata, j, match): + while 1: + c = rawdata[j:j + 1] + if c in \"'\\\"\": + m = match(rawdata, j) + if not m: + return -1 + j = m.end() + else: + name, j = self.scan(j) + if j < 0: + return j ", ); let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - let jump_back_count = ops + let compare_idx = ops .iter() - .filter(|op| { + .enumerate() + .rfind(|(_, unit)| matches!(unit.op, Instruction::CompareOp { .. })) + .map(|(idx, _)| idx) + .expect("missing nested tail comparison"); + let cond_idx = ops[compare_idx + 1..] + .iter() + .position(|unit| { matches!( - op, - Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } + unit.op, + Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } ) }) - .count(); + .map(|idx| compare_idx + 1 + idx) + .expect("missing nested tail conditional jump"); assert!( - jump_back_count >= 2, - "expected separate false-path and body jump-back blocks, got ops={ops:?}" + matches!(ops[cond_idx].op, Instruction::PopJumpIfTrue { .. }), + "expected CPython-style true jump to nested return path, got ops={ops:?}" ); + let jump_back_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::JumpBackward { .. })) + .map(|idx| cond_idx + 1 + idx) + .expect("missing nested tail loop backedge"); + let return_idx = ops[cond_idx + 1..] + .iter() + .position(|unit| matches!(unit.op, Instruction::ReturnValue)) + .map(|idx| cond_idx + 1 + idx) + .expect("missing nested tail return path"); assert!( - ops.windows(5).any(|window| { - matches!( - window, - [ - Instruction::ToBool, - Instruction::PopJumpIfTrue { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadGlobal { .. }, - ] - ) - }), - "expected false path to jump back before body, got ops={ops:?}" + jump_back_idx < return_idx, + "expected nested implicit loop backedge before return block, got ops={ops:?}" ); } - #[test] - fn test_loop_if_pass_uses_line_bearing_jump_back_instead_of_nop() { - let code = compile_exec( - "\ -def f(x, y): - for i in x: - if y: - pass + #[test] + fn test_join_store_global_before_import_keeps_strong_load_fast() { + let code = compile_exec( + "\ +def f(module=None): + global ET + if module is None: + module = pyET + ET = module + from xml.etree import ElementPath ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -16437,95 +21861,101 @@ def f(x, y): .collect(); assert!( - ops.windows(5).any(|window| { + ops.windows(2).any(|window| { matches!( window, [ - Instruction::ToBool, - Instruction::PopJumpIfTrue { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFast { .. }, + Instruction::StoreGlobal { .. }, ] ) }), - "expected CPython-style synthetic false-path jump-back plus body jump-back, got ops={ops:?}" - ); - assert!( - !ops.iter().any(|op| matches!(op, Instruction::Nop)), - "expected pass body line to attach to loop backedge instead of leaving a NOP, got ops={ops:?}" + "expected CPython-style strong LOAD_FAST before join STORE_GLOBAL followed by import, got ops={ops:?}" ); } #[test] - fn test_nested_if_shared_jump_back_target_is_duplicated() { + fn test_handler_resume_join_keeps_borrow_in_common_tail() { let code = compile_exec( "\ -def f(s, size, encodeSetO, encodeWhiteSpace): - inShift = True - base64bits = 0 - out = [] - for i, ch in enumerate(s): - if base64bits == 0: - if i + 1 < size: - ch2 = s[i + 1] - if E(ch2, encodeSetO, encodeWhiteSpace): - if B(ch2) or ch2 == '-': - out.append(b'-') - inShift = False - else: - out.append(b'-') - inShift = False - return out +def f(p, errors, s, pos, look, final, escape_start, st): + try: + chr_codec = unicodedata.lookup('%s' % st) + except LookupError as e: + x = unicode_call_errorhandler( + errors, 'unicodeescape', 'unknown Unicode character name', s, pos - 1, look + 1 + ) + else: + x = chr_codec, look + 1 + p.append(x[0]) + pos = x[1] + if not final: + pos = escape_start + return p, pos + return unicode_call_errorhandler( + errors, 'unicodeescape', 'unknown Unicode character name', s, pos - 1, look + 1 + ) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let append_idx = f .instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "append" + } + _ => false, + }) + .expect("missing append tail"); + let tail: Vec<_> = f.instructions[append_idx.saturating_sub(1)..] .iter() .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) .collect(); assert!( - ops.windows(6).any(|window| { + matches!( + tail.as_slice(), + [ + Instruction::LoadFastBorrow { .. }, + Instruction::LoadAttr { .. }, + Instruction::LoadFastBorrow { .. }, + .., + ] + ), + "expected handler resume common tail to start with borrowed append receiver/arg loads, got tail={tail:?}" + ); + assert!( + tail.iter().any(|op| { matches!( - window, - [ - Instruction::PopTop, - Instruction::LoadConst { .. }, - Instruction::StoreFast { .. }, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }, - ] + op, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + | Instruction::LoadFastBorrow { .. } ) }), - "expected separate nested-if and outer-if jump-back tails, got ops={ops:?}" + "expected handler resume common tail to keep borrowed LOAD_FAST ops, got tail={tail:?}" ); } #[test] - fn test_protected_loop_conditional_keeps_forward_body_entry() { + fn test_multi_handler_guarded_resume_tail_keeps_borrow() { let code = compile_exec( "\ -def outer(it, C1): - def f(): - for x in it: - try: - if C1: - yield 2 - except OSError: - pass - return f +def f(a): + try: + g() + except ValueError: + pass + except TypeError: + pass + if a: + return a.x + return 0 ", ); - let outer = find_code(&code, "outer").expect("missing outer code"); - let f = find_code(outer, "f").expect("missing f code"); + let f = find_code(&code, "f").expect("missing f code"); let ops: Vec<_> = f .instructions .iter() @@ -16534,90 +21964,47 @@ def outer(it, C1): .collect(); assert!( - ops.windows(7).any(|window| { + ops.windows(5).any(|window| { matches!( window, [ + Instruction::LoadFastBorrow { .. }, Instruction::ToBool, Instruction::PopJumpIfFalse { .. }, Instruction::NotTaken, - Instruction::LoadSmallInt { .. }, - Instruction::YieldValue { .. }, - Instruction::Resume { .. }, - Instruction::PopTop, + Instruction::LoadFastBorrow { .. }, ] ) }), - "expected protected conditional to keep CPython-style forward body entry, got ops={ops:?}" - ); - } - - #[test] - fn test_nested_except_false_path_duplicates_pop_except_jump_back_tail() { - let code = compile_exec( - "\ -def f(it, C3): - for x in it: - try: - X = 3 - except OSError: - try: - if C3: - X = 4 - except OSError: - pass - return 42 -", + "expected guarded resume tail to keep borrowed guard/body loads, got ops={ops:?}" ); - let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f - .instructions - .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); - assert!( - ops.windows(6).any(|window| { + ops.windows(2).any(|window| { matches!( window, [ - Instruction::LoadSmallInt { .. }, - Instruction::StoreFast { .. }, - Instruction::PopExcept, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::PopExcept, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::LoadAttr { .. } ] ) }), - "expected CPython-style duplicated false-path exit tail, got ops={ops:?}" + "expected guarded resume tail attr access to keep borrowed receiver, got ops={ops:?}" ); } #[test] - fn test_more_nested_except_false_paths_duplicate_all_jump_back_tails() { + fn test_multi_handler_method_tail_keeps_borrow() { let code = compile_exec( "\ -def f(it, C3, C4): - for x in it: +def f(self, xs): + for vals, expected in xs: try: - X = 3 - except OSError: - try: - if C3: - if C4: - X = 4 - except OSError: - try: - if C3: - if C4: - X = 5 - except OSError: - pass - return 42 + actual = g(vals) + except OverflowError: + self.fail(expected) + except ValueError: + self.fail(expected) + self.assertEqual(actual, expected) ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -16628,76 +22015,82 @@ def f(it, C3, C4): .filter(|op| !matches!(op, Instruction::Cache)) .collect(); + let assert_equal_idx = ops + .iter() + .position(|op| matches!(op, Instruction::LoadAttr { .. })) + .expect("missing assertEqual LOAD_ATTR"); + let tail = &ops[assert_equal_idx.saturating_sub(1)..]; + assert!( - ops.windows(8).any(|window| { - matches!( - window, - [ - Instruction::LoadSmallInt { .. }, - Instruction::StoreFast { .. }, - Instruction::PopExcept, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::PopExcept, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::PopExcept, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - ] - ) - }), - "expected CPython-style duplicated nested false-path exit tails, got ops={ops:?}" + matches!(tail.first(), Some(Instruction::LoadFastBorrow { .. })), + "expected multi-handler method-call tail receiver to keep LOAD_FAST_BORROW, got tail={tail:?}" + ); + assert!( + tail.iter() + .any(|op| matches!(op, Instruction::LoadFastBorrow { .. })), + "expected multi-handler method-call tail args to keep borrowed loads, got tail={tail:?}" ); } #[test] - fn test_no_wraparound_jump_keeps_forward_hop_before_loop_backedge() { + fn test_named_except_cleanup_loop_header_keeps_borrow_in_for_loop() { let code = compile_exec( "\ -def while_not_chained(a, b, c): - while not (a < b < c): - pass +def f(args): + for arg in args: + try: + _wm._setoption(arg) + except _wm._OptionError as msg: + print('Invalid -W option ignored:', msg, file=sys.stderr) ", ); - let f = find_code(&code, "while_not_chained").expect("missing while_not_chained code"); - let ops: Vec<_> = f + let f = find_code(&code, "f").expect("missing f code"); + let attr_idx = f .instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "_setoption" + } + _ => false, + }) + .expect("missing _setoption attr load"); + let window: Vec<_> = f.instructions[attr_idx + 1..] .iter() .map(|unit| unit.op) .filter(|op| !matches!(op, Instruction::Cache)) + .take(3) .collect(); - assert!( - ops.windows(5).any(|window| { - matches!( - window, - [ - Instruction::PopJumpIfTrue { .. }, - Instruction::NotTaken, - Instruction::JumpForward { .. }, - Instruction::PopTop, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - ] - ) - }), - "expected CPython-style no-wraparound forward hop before the loop backedge, got ops={ops:?}" + matches!( + window.as_slice(), + [ + Instruction::LoadFastBorrow { .. }, + Instruction::Call { .. }, + Instruction::PopTop + ] + ), + "expected loop body call to keep borrowed arg load after named-except cleanup, got window={window:?}" ); } #[test] - fn test_while_break_else_keeps_true_edge_into_forward_break_body() { + fn test_multi_named_except_loop_header_keeps_borrow_for_normal_path() { let code = compile_exec( "\ -def f(i): - while i: - i -= 1 - if i < 4: - break - else: - print('x') - print('y') +def f(self): + for badval in ['illegal', -1, 1 << 32]: + class A: + def __len__(self): + return badval + try: + bool(A()) + except (Exception) as e_bool: + try: + len(A()) + except (Exception) as e_len: + self.assertEqual(str(e_bool), str(e_len)) ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -16713,805 +22106,1027 @@ def f(i): matches!( window, [ - Instruction::PopJumpIfTrue { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::JumpForward { .. }, + Instruction::LoadBuildClass, + Instruction::PushNull, + Instruction::LoadFastBorrow { .. }, + Instruction::BuildTuple { .. }, ] ) }), - "expected CPython-style true edge into forward break body with false path falling into the loop backedge, got ops={ops:?}" - ); - } - - #[test] - fn test_nested_if_continue_reorders_false_path_to_loop_backedge() { - let code = compile_exec( - "\ -def f(items, changes): - for x in items: - if not x: - if x in changes: - raise TypeError - continue -", + "expected class closure setup in loop header to borrow badval, got ops={ops:?}" ); - let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f - .instructions - .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); - assert!( - ops.windows(7).any(|window| { + ops.windows(5).any(|window| { matches!( window, [ - Instruction::ToBool, - Instruction::PopJumpIfFalse { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. }, - Instruction::ContainsOp { .. }, - Instruction::PopJumpIfFalse { .. }, + Instruction::LoadFastBorrow { .. }, + Instruction::PushNull, + Instruction::Call { .. }, + Instruction::Call { .. }, + Instruction::PopTop, ] ) }), - "expected nested if/continue to keep CPython-style false-edge jump-back tails, got ops={ops:?}" + "expected normal bool(A()) path in loop header to borrow A, got ops={ops:?}" ); } #[test] - fn test_loop_assert_keeps_false_edge_into_raise_body() { + fn test_named_except_cleanup_simple_resume_tail_keeps_borrow() { let code = compile_exec( "\ -def f(bytecode): - for instr, positions in zip(bytecode, bytecode.codeobj.co_positions()): - assert instr.positions == positions +def f(self): + try: + 1 / 0 + except Exception as e: + tb = e.__traceback__ + self.get_disassemble_as_string(tb.tb_frame.f_code, tb.tb_lasti) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - + let attr_idx = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "get_disassemble_as_string" + } + _ => false, + }) + .expect("missing LOAD_ATTR for get_disassemble_as_string"); + let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); assert!( - ops.windows(6).any(|window| { - matches!( - window, - [ - Instruction::CompareOp { .. }, - Instruction::PopJumpIfFalse { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadCommonConstant { .. }, - Instruction::RaiseVarargs { .. }, - ] - ) - }), - "expected loop assert to keep CPython-style false-edge into the raise body, got ops={ops:?}" + matches!( + ops.get(attr_idx - 1), + Some(Instruction::LoadFastBorrow { .. }) + ), + "expected named-except resume tail to keep borrowed self load, got ops={ops:?}" + ); + assert!( + matches!( + ops.get(attr_idx + 4), + Some(Instruction::LoadFastBorrow { .. }) + ), + "expected named-except resume tail to keep borrowed tb load, got ops={ops:?}" ); } #[test] - fn test_and_is_not_none_loop_guard_uses_direct_jump_back_false_path() { + fn test_named_except_cleanup_conditional_raise_tail_keeps_borrow() { let code = compile_exec( "\ -def f(code): - last_line = -2 - for _, _, line in code.co_lines(): - if line is not None and line != last_line: - last_line = line +def f(self): + try: + output = self.trace() + output = output.strip() + except (A, B, C) as fnfe: + output = str(fnfe) + if output != 'probe: success': + raise E('{} {}'.format(self.command[0], output)) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); + let raise_idx = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::RaiseVarargs { .. })) + .expect("missing conditional raise"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[..handler_start.min(raise_idx)]; assert!( - ops.windows(6).any(|window| { + tail.iter().any(|unit| { matches!( - window, - [ - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::PopJumpIfNotNone { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. }, - Instruction::CompareOp { .. }, - ] + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected CPython-style direct jump-back false path for 'is not None and ...', got ops={ops:?}" + "named-except cleanup conditional raise tail should keep borrowed loads, got tail={tail:?}" + ); + assert!( + !tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "named-except cleanup conditional raise tail should not force strong LOAD_FAST, got tail={tail:?}" ); } #[test] - fn test_large_is_not_none_loop_guard_uses_direct_jump_back_false_path() { + fn test_with_suppress_named_except_resume_tail_uses_strong_loads() { let code = compile_exec( "\ -def f(cls, _FIELDS, _PARAMS): - all_frozen_bases = None - any_frozen_base = False - has_dataclass_bases = False - for b in cls.__mro__[-1:0:-1]: - base_fields = getattr(b, _FIELDS, None) - if base_fields is not None: - has_dataclass_bases = True - for field in base_fields.values(): - name = field.name - if all_frozen_bases is None: - all_frozen_bases = True - current_frozen = getattr(b, _PARAMS).frozen - all_frozen_bases = all_frozen_bases and current_frozen - any_frozen_base = any_frozen_base or current_frozen +def f(self, cm, E): + try: + with cm: + pass + except E as e: + frames = e + self.x(frames) + self.y(frames) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); + let first_tail_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "x" + } + _ => false, + }) + .expect("missing x attr load"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[first_tail_attr.saturating_sub(1)..handler_start]; assert!( - ops.windows(6).any(|window| { - matches!( - window, - [ - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::PopJumpIfNotNone { .. }, - Instruction::NotTaken, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - Instruction::LoadConst { .. }, - Instruction::StoreFast { .. }, - ] + tail.iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "expected with-suppress/named-except resume tail to use strong LOAD_FAST, got tail={tail:?}" + ); + assert!( + tail.iter().all(|unit| { + !matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected CPython-style direct jump-back false path for large 'is not None' loop body, got ops={ops:?}" + "expected with-suppress/named-except resume tail not to borrow, got tail={tail:?}" ); } #[test] - fn test_continue_inside_with_keeps_line_marker_nop_before_exit_cleanup() { + fn test_with_except_else_with_resume_loop_tail_uses_strong_loads() { let code = compile_exec( "\ -def f(it): - for func in it: - with cm(): - if cond(): - continue +def f(self, cm, E): + with cm: + try: + g() + except E: + pass + else: + with self.z(E): + h() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT, 'not ready'): + if self.x: + break + self.y() ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); + let get_iter = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::GetIter)) + .expect("missing loop iterator"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[get_iter.saturating_sub(1)..handler_start]; assert!( - ops.windows(9).any(|window| { - matches!( - window, - [ - Instruction::PopJumpIfFalse { .. }, - Instruction::NotTaken, - Instruction::Nop, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::Call { .. }, - Instruction::PopTop, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. }, - ] + tail.iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "expected with except/else-with resume loop tail to use strong LOAD_FAST ops, got tail={tail:?}" + ); + assert!( + tail.iter().all(|unit| { + !matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected CPython-style line-marker NOP before with-exit cleanup on continue, got ops={ops:?}" + "with except/else-with resume loop tail should not borrow LOAD_FAST ops, got tail={tail:?}" ); } #[test] - fn test_try_loop_elif_places_return_before_orelse_tail() { + fn test_plain_with_then_global_loop_tail_keeps_borrow() { let code = compile_exec( "\ -def f(source, suggest, tb, s): - if source is not None: - try: - tb = tb - except Exception: - suggest = False - tb = None - if tb is not None: - for frame in tb: - s += frame - elif suggest: - s += 'x' - return s +def f(self, cm): + with cm: + self.x() + for value in ITEMS: + self.y(value) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); + let y_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "y" + } + _ => false, + }) + .expect("missing y attr load"); - let has_direct_return = ops.windows(8).any(|window| { - matches!( - window, - [ - Instruction::EndFor, - Instruction::PopIter, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::ReturnValue, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::ToBool, - Instruction::PopJumpIfFalse { .. }, - Instruction::NotTaken, - ] - ) - }); - let has_nop_anchored_return = ops.windows(9).any(|window| { - matches!( - window, - [ - Instruction::EndFor, - Instruction::PopIter, - Instruction::Nop, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::ReturnValue, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::ToBool, - Instruction::PopJumpIfFalse { .. }, - Instruction::NotTaken, - ] - ) - }); assert!( - has_direct_return || has_nop_anchored_return, - "expected CPython-style duplicated return between loop exit and elif tail, got ops={ops:?}" + matches!( + instructions + .get(y_attr.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "plain with/global-loop tail should keep CPython-style borrowed self load, got instructions={instructions:?}" ); } #[test] - fn test_constant_false_while_else_deopts_post_else_borrows() { + fn test_context_manager_for_join_tail_keeps_borrow() { let code = compile_exec( "\ -def f(self): - x = 0 - while 0: - x = 1 - else: - x = 2 - self.assertEqual(x, 2) +def f(self, factory): + with factory() as e: + executor = e + self.x(e) + for t in executor._threads: + t.join() ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - let assert_idx = ops + let join_attr = instructions .iter() - .position(|op| matches!(op, Instruction::LoadAttr { .. })) - .expect("missing assertEqual call"); - let window = &ops[assert_idx.saturating_sub(1)..(assert_idx + 3).min(ops.len())]; + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "join" + } + _ => false, + }) + .expect("missing join attr load"); + assert!( matches!( - window, - [ - Instruction::LoadFast { .. }, - Instruction::LoadAttr { .. }, - Instruction::LoadFast { .. }, - .. - ] + instructions + .get(join_attr.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) ), - "expected post-else assertEqual call to use plain LOAD_FAST, got ops={window:?}" + "context-manager for-join tail should keep CPython-style borrowed t load, got instructions={instructions:?}" ); } #[test] - fn test_single_unpack_assignment_disables_constant_collection_folding() { - let code = compile_exec("a, b, c = 1, 2, 3\n"); - - assert!( - !code.instructions.iter().any(|unit| { - matches!(unit.op, Instruction::UnpackSequence { .. }) - || matches!(unit.op, Instruction::LoadConst { .. }) - && matches!( - code.constants.get(usize::from(u8::from(unit.arg))), - Some(ConstantData::Tuple { .. }) - ) - }), - "single unpack assignment should keep builder form for later lowering, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() - ); - assert!( - code.instructions - .iter() - .filter(|unit| matches!(unit.op, Instruction::LoadSmallInt { .. })) - .count() - >= 3, - "expected individual constant loads before unpack-target stores, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() + fn test_with_except_resume_normal_tail_uses_strong_loads() { + let code = compile_exec( + "\ +def f(self, cm, E): + try: + with self.assertRaises(E): + with cm: + h() + except TimeoutError: + self._fail_on_deadlock(cm) + cm.shutdown(wait=True) +", ); - } - - #[test] - fn test_chained_unpack_assignment_keeps_constant_collection_folding() { - let code = compile_exec("(a, b) = c = d = (1, 2)\n"); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let shutdown_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "shutdown" + } + _ => false, + }) + .expect("missing shutdown attr load"); assert!( - code.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::LoadConst { .. })), - "chained unpack assignment should keep tuple constant, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() - ); - assert!( - code.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::UnpackSequence { .. })), - "chained unpack assignment should still unpack the copied tuple, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() + matches!( + instructions + .get(shutdown_attr.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFast { .. }) + ), + "with/except resume normal tail should keep CPython-style strong cm load, got instructions={instructions:?}" ); } #[test] - fn test_constant_true_assert_skips_message_nested_scope() { - let code = compile_exec("assert 1, (lambda x: x + 1)\n"); - - assert_eq!( - code.constants - .iter() - .filter(|constant| matches!(constant, ConstantData::Code { .. })) - .count(), - 0, - "constant-true assert should not compile the skipped message lambda" - ); - assert!( - !code - .instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::RaiseVarargs { .. })), - "constant-true assert should be elided, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() + fn test_with_except_else_attr_subscript_tail_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, cm, E, obj): + try: + with cm: + pass + except E as exc: + self.x(exc) + else: + self.fail('Expected') + inner = obj.saved_details[1] + self.x(inner) +", ); - } - - #[test] - fn test_constant_false_assert_uses_direct_raise_shape() { - let code = compile_exec("assert 0, (lambda x: x + 1)\n"); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let saved_details = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "saved_details" + } + _ => false, + }) + .expect("missing saved_details attr load"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[saved_details.saturating_sub(1)..handler_start]; assert!( - !code.instructions.iter().any(|unit| { - matches!( - unit.op, - Instruction::ToBool - | Instruction::PopJumpIfTrue { .. } - | Instruction::PopJumpIfFalse { .. } - ) - }), - "constant-false assert should use direct raise shape, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() - ); - assert!( - code.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::RaiseVarargs { .. })), - "constant-false assert should still raise, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() - ); - assert_eq!( - code.constants - .iter() - .filter(|constant| matches!(constant, ConstantData::Code { .. })) - .count(), - 1, - "constant-false assert should still compile the message lambda" + tail.iter().any(|unit| matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + )), + "expected except-else attr-subscript tail to keep borrowed LOAD_FAST ops, got tail={tail:?}" + ); + assert!( + tail.iter() + .all(|unit| !matches!(unit.op, Instruction::LoadFast { .. })), + "except-else attr-subscript tail should not be deoptimized to strong LOAD_FAST, got tail={tail:?}" ); } #[test] - fn test_constant_unary_positive_and_invert_fold() { - let code = compile_exec("x = +1\nx = ~1\n"); + fn test_with_suppress_attr_subscript_tail_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, cm): + stack = self.exit_stack() + with self.assertRaisesRegex(TypeError, 'the context manager'): + stack.enter_context(cm) + stack.push(cm) + self.assertIs(stack._exit_callbacks[-1][1], cm) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let exit_callbacks = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "_exit_callbacks" + } + _ => false, + }) + .expect("missing _exit_callbacks attr load"); assert!( - !code.instructions.iter().any(|unit| { - matches!( - unit.op, - Instruction::CallIntrinsic1 { .. } | Instruction::UnaryInvert - ) - }), - "constant unary ops should fold away, got ops={:?}", - code.instructions - .iter() - .map(|unit| unit.op) - .collect::>() + matches!( + instructions + .get(exit_callbacks.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "with-suppress attr-subscript tail should keep CPython-style borrowed stack load, got instructions={instructions:?}" ); } #[test] - fn test_bool_invert_is_not_const_folded() { - let code = compile_exec("x = ~True\n"); + fn test_named_except_conditional_reraise_deopts_with_chain_tail() { + let code = compile_exec( + "\ +def f(self, arc, tmp_filename, new_mode): + try: + os.chmod(tmp_filename, new_mode) + except OSError as exc: + if exc.errno == ERR: + self.skipTest() + else: + raise + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_file('a') + with self.check_context(arc.open(), 'tar'): + self.expect_file('b') +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let first_check_context = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "check_context" + } + _ => false, + }) + .expect("missing check_context load"); + let first_handler = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .unwrap_or(instructions.len()); + let warm_tail = &instructions[first_check_context.saturating_sub(1)..first_handler]; assert!( - code.instructions - .iter() - .any(|unit| matches!(unit.op, Instruction::UnaryInvert)), - "~bool should remain unfurled to match CPython, got ops={:?}", - code.instructions + warm_tail .iter() - .map(|unit| unit.op) - .collect::>() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "expected conditional named-except reraise tail to use strong LOAD_FAST ops, got tail={warm_tail:?}" + ); + assert!( + warm_tail.iter().all(|unit| { + !matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "expected all warm with-chain tail loads to stay strong after named-except reraise, got tail={warm_tail:?}" ); } #[test] - fn test_optimized_assert_preserves_nested_scope_order() { - compile_exec_optimized( + fn test_terminal_except_before_with_deopts_with_body_borrows() { + let code = compile_exec( "\ -class S: - def f(self, sequence): - _formats = [self._types_mapping[type(item)] for item in sequence] - _list_len = len(_formats) - assert sum(len(fmt) <= 8 for fmt in _formats) == _list_len - _recreation_codes = [self._extract_recreation_code(item) for item in sequence] +def f(self, cm): + try: + g() + except OSError: + raise Exception('skip') + with cm: + self.x() + self.y() ", ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let first_tail_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "x" + } + _ => false, + }) + .expect("missing x attr load"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let with_tail = &instructions[first_tail_attr.saturating_sub(1)..handler_start]; + + assert!( + with_tail + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "expected terminal-except before with to use strong LOAD_FAST ops, got tail={with_tail:?}" + ); + assert!( + with_tail.iter().all(|unit| { + !matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "terminal-except before with should not borrow protected with body loads, got tail={with_tail:?}" + ); } #[test] - fn test_optimized_assert_with_nested_scope_in_first_iter() { - // First iterator of a comprehension is evaluated in the enclosing - // scope, so nested scopes inside it (the generator here) must also - // be consumed when the assert is optimized away. - compile_exec_optimized( + fn test_terminal_except_resume_tail_uses_strong_loads() { + let code = compile_exec( "\ -def f(items): - assert [x for x in (y for y in items)] - return [x for x in items] +def f(re, proc, unittest): + try: + version = proc.communicate() + except OSError: + raise unittest.SkipTest('x') + match = re.search('pat', version) + if match is None: + raise unittest.SkipTest(f'Unable to parse readelf version: {version}') + return int(match.group(1)), int(match.group(2)) ", ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let search_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "search" + } + _ => false, + }) + .expect("missing re.search attr load"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let tail = &instructions[search_attr.saturating_sub(1)..handler_start]; + + let strong_loads_name = |name: &str| { + tail.iter().any(|unit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + Instruction::LoadFastLoadFast { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == name || f.varnames[usize::from(right)] == name + } + _ => false, + }) + }; + let borrows_name = |name: &str| { + tail.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + let (left, right) = var_nums.get(arg).indexes(); + f.varnames[usize::from(left)] == name || f.varnames[usize::from(right)] == name + } + _ => false, + }) + }; + + for name in ["re", "version", "match"] { + assert!( + strong_loads_name(name), + "terminal-except resume tail should use strong LOAD_FAST for {name}, got tail={tail:?}" + ); + assert!( + !borrows_name(name), + "terminal-except resume tail should not borrow {name}, got tail={tail:?}" + ); + } } #[test] - fn test_optimized_assert_with_lambda_defaults() { - // Lambda default values are evaluated in the enclosing scope, - // so nested scopes inside defaults must be consumed. - compile_exec_optimized( + fn test_terminal_except_conditional_return_tail_uses_strong_loads() { + let code = compile_exec( "\ -def f(items): - assert (lambda x=[i for i in items]: x)() - return [x for x in items] +def f(param, value, quote): + try: + value.encode('ascii') + except UnicodeEncodeError: + return param + if quote: + return param + return value ", ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let quote_idx = instructions[..handler_start] + .iter() + .position(|unit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == "quote" + } + _ => false, + }) + .expect("missing quote guard load"); + let tail = &instructions[quote_idx..handler_start]; + + let strong_loads_name = |name: &str| { + tail.iter().any(|unit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + let borrows_name = |name: &str| { + tail.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + + for name in ["quote", "param", "value"] { + assert!( + strong_loads_name(name), + "terminal-except conditional tail should use strong LOAD_FAST for {name}, got tail={tail:?}" + ); + assert!( + !borrows_name(name), + "terminal-except conditional tail should not borrow {name}, got tail={tail:?}" + ); + } } #[test] - fn test_try_else_nested_scopes_keep_subtable_cursor_aligned() { + fn test_terminal_except_successor_call_tail_uses_strong_load() { let code = compile_exec( "\ -try: - import missing_mod -except ImportError: - def fallback(): - return 0 -else: - def impl(): - return reversed('abc') +def f(curr, decoded_append, packI, curr_clear, Error): + if len(curr) == 5: + acc = 0 + for x in curr: + acc = 85 * acc + (x - 33) + try: + decoded_append(packI(acc)) + except Error: + raise ValueError('overflow') from None + curr_clear() ", ); - - assert!( - find_code(&code, "fallback").is_some(), - "missing fallback code" - ); - let impl_code = find_code(&code, "impl").expect("missing impl code"); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let curr_clear_load = instructions[..handler_start] + .iter() + .rev() + .find(|unit| match unit.op { + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == "curr_clear" + } + _ => false, + }) + .expect("missing curr_clear load"); + assert!( - impl_code.instructions.iter().any(|unit| { - matches!( - unit.op, - Instruction::LoadGlobal { .. } | Instruction::LoadName { .. } - ) - }), - "expected impl to compile global name access, got ops={:?}", - impl_code - .instructions - .iter() - .map(|unit| unit.op) - .collect::>() + matches!(curr_clear_load.op, Instruction::LoadFast { .. }), + "terminal except successor call tail should use strong LOAD_FAST for curr_clear, got instructions={instructions:?}" ); } #[test] - fn test_nested_try_else_multi_resume_join_keeps_strong_load_fast_tail() { + fn test_protected_method_call_after_terminal_except_tail_uses_strong_loads() { let code = compile_exec( "\ -def f(msg): - s = '' - try: - import a - except Exception: - suggest = False - tb = None - else: +def f(items, chunk, out, packI, Error): + for i in items: + acc = 0 try: - suggest = not t() - tb = g(msg) - except Exception: - suggest = False - tb = None - if tb is not None: - for frame in tb: - s += frame - elif suggest: - s += 'y' - return s + for c in chunk: + acc = acc * 85 + c + except TypeError: + raise + try: + out.append(packI(acc)) + except Error: + raise ValueError from None ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - let tail_start = ops - .iter() - .position(|op| matches!(op, Instruction::PopJumpIfNone { .. })) - .expect("missing tail POP_JUMP_IF_NONE") - .saturating_sub(1); - let handler_start = ops + let handler_start = instructions .iter() - .position(|op| matches!(op, Instruction::PushExcInfo)) + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) .expect("missing handler entry"); - let tail = &ops[tail_start..handler_start]; + let append_attr = instructions[..handler_start] + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "append" + } + _ => false, + }) + .expect("missing append LOAD_ATTR"); + let tail = &instructions[append_attr.saturating_sub(1)..handler_start]; - assert!( - !tail.iter().any(|op| { - matches!( - op, - Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - }), - "expected nested try/except else-resume tail to keep strong LOAD_FAST ops, got tail={tail:?}" + let strong_loads_name = |name: &str| { + tail.iter().any(|unit| match unit.op { + Instruction::LoadFast { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + let borrows_name = |name: &str| { + tail.iter().any(|unit| match unit.op { + Instruction::LoadFastBorrow { var_num } => { + let arg = OpArg::new(u32::from(u8::from(unit.arg))); + f.varnames[usize::from(var_num.get(arg))] == name + } + _ => false, + }) + }; + + for name in ["out", "packI", "acc"] { + assert!( + strong_loads_name(name), + "protected method-call after terminal except tail should use strong LOAD_FAST for {name}, got tail={tail:?}" + ); + assert!( + !borrows_name(name), + "protected method-call after terminal except tail should not borrow {name}, got tail={tail:?}" + ); + } + } + + #[test] + fn test_terminal_except_loop_successor_augassign_uses_strong_load_pair() { + let code = compile_exec( + "\ +def f(items, decoded, b32rev): + for i in range(0, len(items), 8): + quanta = items[i:i + 8] + acc = 0 + try: + for c in quanta: + acc = (acc << 5) + b32rev[c] + except KeyError: + raise ValueError from None + decoded += acc.to_bytes(5) + return decoded +", ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let to_bytes_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "to_bytes" + } + _ => false, + }) + .expect("missing to_bytes LOAD_ATTR"); + let pair = instructions[to_bytes_attr.saturating_sub(1)].op; assert!( - tail.iter() - .any(|op| matches!(op, Instruction::LoadFastLoadFast { .. })), - "expected loop body to keep LOAD_FAST_LOAD_FAST in the resume tail, got tail={tail:?}" + matches!(pair, Instruction::LoadFastLoadFast { .. }), + "terminal-except loop successor augassign should use strong LOAD_FAST_LOAD_FAST, got instructions={instructions:?}" ); } #[test] - fn test_protected_conditional_tail_keeps_strong_load_fast() { + fn test_except_handler_with_conditional_raise_and_resume_keeps_borrow() { let code = compile_exec( "\ -def f(m, class_name, category, warning_base): +def f(formatstr, args, output, overflowok): try: - cat = getattr(m, class_name) - except AttributeError: - raise ValueError(category) - if not issubclass(cat, warning_base): - raise TypeError(category) - return cat + result = formatstr % args + except OverflowError: + if not overflowok: + raise + print('overflow') + else: + if output and result != output: + raise AssertionError(result, output) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - let tail_start = ops + let assertion_error = instructions .iter() - .position(|op| matches!(op, Instruction::StoreFast { .. })) - .expect("missing STORE_FAST cat"); - let handler_start = ops + .position(|unit| match unit.op { + Instruction::LoadGlobal { namei } => { + let load_global = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_global >> 1).unwrap()].as_str() == "AssertionError" + } + _ => false, + }) + .expect("missing AssertionError raise"); + let handler_start = instructions .iter() - .position(|op| matches!(op, Instruction::PushExcInfo)) + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) .expect("missing handler entry"); - let tail = &ops[tail_start + 1..handler_start]; + let tail = &instructions[..handler_start.min(assertion_error)]; assert!( - !tail.iter().any(|op| { + tail.iter().any(|unit| { matches!( - op, + unit.op, Instruction::LoadFastBorrow { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected protected conditional tail to keep strong LOAD_FAST ops, got tail={tail:?}" + "conditional-raise handler with a resume path should keep borrowed loads, got tail={tail:?}" ); - assert!( tail.iter() - .any(|op| matches!(op, Instruction::LoadFastLoadFast { .. })), - "expected protected tail to keep LOAD_FAST_LOAD_FAST for issubclass args, got tail={tail:?}" + .all(|unit| !matches!(unit.op, Instruction::LoadFast { .. })), + "conditional-raise handler with a resume path should not force strong LOAD_FAST, got tail={tail:?}" ); } #[test] - fn test_protected_import_tail_keeps_strong_load_fast() { + fn test_reraising_except_else_tail_keeps_borrow() { let code = compile_exec( "\ -def f(s, size, pos, errors): - message = 'x' - look = pos - try: - import unicodedata - except ImportError: - return None - if look < size and chr(s[look]) == '{': - while look < size and chr(s[look]) != '}': - look += 1 - if look > pos + 1 and look < size and chr(s[look]) == '}': - message = 'y' - return message +def f(self, data, length): + if self._paused: + self._pending_data_length = length + return + if length == 0: + self._eof_received() + return + if isinstance(self._protocol, protocols.BufferedProtocol): + try: + protocols._feed_data_to_buffered_proto(self._protocol, data) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error(exc, 'x') + return + else: + self._protocol.data_received(data) ", ); let f = find_code(&code, "f").expect("missing f code"); - let ops: Vec<_> = f + let instructions: Vec<_> = f .instructions .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) + .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - - let import_idx = ops + let data_received_attr = instructions .iter() - .position(|op| matches!(op, Instruction::ImportName { .. })) - .expect("missing IMPORT_NAME"); - let protected_tail = &ops[import_idx + 1..]; + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == "data_received" + } + _ => false, + }) + .expect("missing data_received LOAD_ATTR"); + let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); assert!( - !protected_tail.iter().any(|op| { - matches!( - op, - Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - }), - "expected protected import tail to keep strong LOAD_FAST ops, got tail={protected_tail:?}" + matches!( + ops.get(data_received_attr - 2), + Some(Instruction::LoadFastBorrow { .. }) + ), + "normal else tail after reraising handler should keep borrowed self load, got ops={ops:?}" ); - assert!( - protected_tail - .iter() - .any(|op| matches!(op, Instruction::LoadFastLoadFast { .. })), - "expected protected import tail to keep LOAD_FAST_LOAD_FAST ops, got tail={protected_tail:?}" + matches!( + ops.get(data_received_attr + 1), + Some(Instruction::LoadFastBorrow { .. }) + ), + "normal else tail after reraising handler should keep borrowed data load, got ops={ops:?}" ); } #[test] - fn test_handler_resume_join_keeps_borrow_in_common_tail() { + fn test_try_else_finally_with_keeps_context_manager_borrow() { let code = compile_exec( "\ -def f(p, errors, s, pos, look, final, escape_start, st): +def f(i): try: - chr_codec = unicodedata.lookup('%s' % st) - except LookupError as e: - x = unicode_call_errorhandler( - errors, 'unicodeescape', 'unknown Unicode character name', s, pos - 1, look + 1 - ) + 1 / 0 + except ZeroDivisionError: + pass else: - x = chr_codec, look + 1 - p.append(x[0]) - pos = x[1] - if not final: - pos = escape_start - return p, pos - return unicode_call_errorhandler( - errors, 'unicodeescape', 'unknown Unicode character name', s, pos - 1, look + 1 - ) + with i as dodgy: + pass + finally: + pass ", ); let f = find_code(&code, "f").expect("missing f code"); - let append_idx = f + let instructions: Vec<_> = f .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let first_with_exit = instructions .iter() .position(|unit| match unit.op { - Instruction::LoadAttr { namei } => { - let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); - f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "append" + Instruction::LoadSpecial { method } => { + method.get(OpArg::new(u32::from(u8::from(unit.arg)))) == SpecialMethod::Exit } _ => false, }) - .expect("missing append tail"); - let tail: Vec<_> = f.instructions[append_idx.saturating_sub(1)..] - .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .collect(); + .expect("missing __exit__ load"); - assert!( - matches!( - tail.as_slice(), - [ - Instruction::LoadFastBorrow { .. }, - Instruction::LoadAttr { .. }, - Instruction::LoadFastBorrow { .. }, - .., - ] - ), - "expected handler resume common tail to start with borrowed append receiver/arg loads, got tail={tail:?}" - ); - assert!( - tail.iter().any(|op| { - matches!( - op, - Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::LoadFastBorrow { .. } - ) - }), - "expected handler resume common tail to keep borrowed LOAD_FAST ops, got tail={tail:?}" + assert!( + matches!( + instructions + .get(first_with_exit.saturating_sub(2)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "try/except/else/finally with setup should keep CPython-style borrowed context manager load, got instructions={instructions:?}" ); } #[test] - fn test_multi_handler_guarded_resume_tail_keeps_borrow() { + fn test_except_star_handler_pop_block_does_not_leave_nop_before_with_exit() { let code = compile_exec( "\ -def f(a): - try: - g() - except ValueError: - pass - except TypeError: - pass - if a: - return a.x - return 0 +def f(self): + with self.assertRaises(TypeError): + try: + raise OSError('blah') + except* ExceptionGroup as e: + pass ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -17523,87 +23138,75 @@ def f(a): .collect(); assert!( - ops.windows(5).any(|window| { - matches!( - window, - [ - Instruction::LoadFastBorrow { .. }, - Instruction::ToBool, - Instruction::PopJumpIfFalse { .. }, - Instruction::NotTaken, - Instruction::LoadFastBorrow { .. }, - ] - ) - }), - "expected guarded resume tail to keep borrowed guard/body loads, got ops={ops:?}" - ); - assert!( - ops.windows(2).any(|window| { + !ops.windows(5).any(|window| { matches!( window, [ - Instruction::LoadFastBorrow { .. }, - Instruction::LoadAttr { .. } + Instruction::Reraise { .. }, + Instruction::Nop, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, ] ) }), - "expected guarded resume tail attr access to keep borrowed receiver, got ops={ops:?}" + "except* handler cleanup should not leave an extra NOP before with-exit None loads, got ops={ops:?}" ); } #[test] - fn test_named_except_cleanup_loop_header_keeps_borrow_in_for_loop() { + fn test_except_star_body_to_else_jump_drops_without_line_nop() { let code = compile_exec( "\ -def f(args): - for arg in args: - try: - _wm._setoption(arg) - except _wm._OptionError as msg: - print('Invalid -W option ignored:', msg, file=sys.stderr) +async def f(self, cm): + try: + async with cm: + pass + except* Exception: + pass + else: + self.fail() ", ); let f = find_code(&code, "f").expect("missing f code"); - let attr_idx = f + let instructions: Vec<_> = f .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let fail_attr = instructions .iter() .position(|unit| match unit.op { Instruction::LoadAttr { namei } => { let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); - f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "_setoption" + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "fail" } _ => false, }) - .expect("missing _setoption attr load"); - let window: Vec<_> = f.instructions[attr_idx + 1..] - .iter() - .map(|unit| unit.op) - .filter(|op| !matches!(op, Instruction::Cache)) - .take(3) - .collect(); + .expect("missing self.fail load"); + assert!( - matches!( - window.as_slice(), - [ - Instruction::LoadFastBorrow { .. }, - Instruction::Call { .. }, - Instruction::PopTop - ] + !matches!( + instructions + .get(fail_attr.saturating_sub(2)) + .map(|unit| unit.op), + Some(Instruction::Nop) ), - "expected loop body call to keep borrowed arg load after named-except cleanup, got window={window:?}" + "body-to-else jump should be no-location and disappear, got instructions={instructions:?}" ); } #[test] - fn test_named_except_cleanup_simple_resume_tail_keeps_borrow() { + fn test_resuming_except_before_with_keeps_with_body_borrows() { let code = compile_exec( "\ -def f(self): +def f(self, cm): try: - 1 / 0 - except Exception as e: - tb = e.__traceback__ - self.get_disassemble_as_string(tb.tb_frame.f_code, tb.tb_lasti) + g() + except OSError: + pass + with cm: + self.x() ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -17612,50 +23215,48 @@ def f(self): .iter() .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - let attr_idx = instructions + let first_tail_attr = instructions .iter() .position(|unit| match unit.op { Instruction::LoadAttr { namei } => { let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); - f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() - == "get_disassemble_as_string" + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "x" } _ => false, }) - .expect("missing LOAD_ATTR for get_disassemble_as_string"); - let ops: Vec<_> = instructions.iter().map(|unit| unit.op).collect(); - assert!( - matches!( - ops.get(attr_idx - 1), - Some(Instruction::LoadFastBorrow { .. }) - ), - "expected named-except resume tail to keep borrowed self load, got ops={ops:?}" - ); + .expect("missing x attr load"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let with_tail = &instructions[first_tail_attr.saturating_sub(1)..handler_start]; + assert!( - matches!( - ops.get(attr_idx + 4), - Some(Instruction::LoadFastBorrow { .. }) - ), - "expected named-except resume tail to keep borrowed tb load, got ops={ops:?}" + with_tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "resuming except before with should keep borrowed LOAD_FAST ops, got tail={with_tail:?}" ); } #[test] - fn test_named_except_conditional_reraise_deopts_with_chain_tail() { + fn test_nested_finally_except_resume_loop_uses_strong_loads() { let code = compile_exec( "\ -def f(self, arc, tmp_filename, new_mode): +def f(self, xs): try: - os.chmod(tmp_filename, new_mode) - except OSError as exc: - if exc.errno == ERR: - self.skipTest() - else: - raise - with self.check_context(arc.open(), 'fully_trusted'): - self.expect_file('a') - with self.check_context(arc.open(), 'tar'): - self.expect_file('b') + try: + g() + finally: + h() + except OSError: + self.skipTest('x') + for x in xs: + self.x(x) ", ); let f = find_code(&code, "f").expect("missing f code"); @@ -17664,38 +23265,141 @@ def f(self, arc, tmp_filename, new_mode): .iter() .filter(|unit| !matches!(unit.op, Instruction::Cache)) .collect(); - let first_check_context = instructions + let for_iter = instructions .iter() - .position(|unit| match unit.op { - Instruction::LoadAttr { namei } => { - let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); - f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() - == "check_context" - } - _ => false, - }) - .expect("missing check_context load"); - let first_handler = instructions + .position(|unit| matches!(unit.op, Instruction::ForIter { .. })) + .expect("missing FOR_ITER"); + let handler_start = instructions .iter() .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) - .unwrap_or(instructions.len()); - let warm_tail = &instructions[first_check_context.saturating_sub(1)..first_handler]; + .expect("missing handler entry"); + let loop_tail = &instructions[for_iter.saturating_sub(2)..handler_start]; assert!( - warm_tail + loop_tail .iter() .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), - "expected conditional named-except reraise tail to use strong LOAD_FAST ops, got tail={warm_tail:?}" + "expected nested finally/except resume loop to use strong LOAD_FAST ops, got tail={loop_tail:?}" ); assert!( - warm_tail.iter().all(|unit| { + loop_tail.iter().all(|unit| { !matches!( unit.op, Instruction::LoadFastBorrow { .. } | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) }), - "expected all warm with-chain tail loads to stay strong after named-except reraise, got tail={warm_tail:?}" + "nested finally/except resume loop should not borrow LOAD_FAST ops, got tail={loop_tail:?}" + ); + } + + #[test] + fn test_finally_protected_loop_without_except_resume_keeps_borrows() { + let code = compile_exec( + "\ +def f(self, obj, expected, buf): + try: + lines = obj.readlines() + except ValueError: + self.fail('x') + if lines != expected: + self.fail('bad') + obj.close() + obj = self.open() + try: + for line in obj: + pass + try: + obj.readline() + obj.readinto(buf) + except ValueError: + self.fail('inner') + finally: + obj.close() +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let get_iter = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::GetIter)) + .expect("missing GET_ITER"); + assert!( + matches!( + instructions + .get(get_iter.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "finally-protected loop without except resume should keep borrowed iterable load, got instructions={instructions:?}" + ); + + for attr_name in ["close", "open"] { + let attr_idx = instructions[..get_iter] + .iter() + .rposition(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() + == attr_name + } + _ => false, + }) + .unwrap_or_else(|| panic!("missing {attr_name} attr load before loop")); + assert!( + matches!( + instructions + .get(attr_idx.saturating_sub(1)) + .map(|unit| unit.op), + Some(Instruction::LoadFastBorrow { .. }) + ), + "pre-loop {attr_name} call should keep borrowed receiver load, got instructions={instructions:?}" + ); + } + } + + #[test] + fn test_plain_except_resume_loop_keeps_borrows() { + let code = compile_exec( + "\ +def f(self, xs): + try: + g() + except OSError: + self.skipTest('x') + for x in xs: + self.x(x) +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let for_iter = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::ForIter { .. })) + .expect("missing FOR_ITER"); + let handler_start = instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::PushExcInfo)) + .expect("missing handler entry"); + let loop_tail = &instructions[for_iter.saturating_sub(2)..handler_start]; + + assert!( + loop_tail.iter().any(|unit| { + matches!( + unit.op, + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + }), + "plain except resume loop should keep borrowed LOAD_FAST ops, got tail={loop_tail:?}" ); } @@ -17812,4 +23516,80 @@ def f(s, size, errors, final): "expected outer invalid-escape tail to keep borrowed p/ch loads" ); } + + #[test] + fn test_imap_idle_status_debug_tail_keeps_borrow() { + let code = compile_exec( + "\ +def f(self, exc_type, CRLF, OSError): + imap = self._imap + try: + imap.send(b'DONE' + CRLF) + status, [msg] = imap._command_complete('IDLE', self._tag) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle status: {status} {msg!r}') + except OSError: + if not exc_type: + raise + return False +", + ); + let f = find_code(&code, "f").expect("missing f code"); + let instructions: Vec<_> = f + .instructions + .iter() + .filter(|unit| !matches!(unit.op, Instruction::Cache)) + .collect(); + let mesg_attr = instructions + .iter() + .position(|unit| match unit.op { + Instruction::LoadAttr { namei } => { + let load_attr = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + f.names[usize::try_from(load_attr.name_idx()).unwrap()].as_str() == "_mesg" + } + _ => false, + }) + .expect("missing _mesg attr load"); + let tail = &instructions[mesg_attr.saturating_sub(1)..]; + + assert!( + tail.iter() + .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. })), + "expected idle status debug tail to keep borrowed loads, got tail={tail:?}" + ); + } + + #[test] + fn test_match_async_comprehension_iter_keeps_capture_borrow() { + let code = compile_exec( + r#" +async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] +"#, + ); + let name_4 = find_code(&code, "name_4").expect("missing name_4 code"); + let Some(get_aiter_pos) = name_4 + .instructions + .iter() + .position(|unit| matches!(unit.op, Instruction::GetAIter)) + else { + panic!("missing GET_AITER in name_4"); + }; + let prev = &name_4.instructions[get_aiter_pos - 1]; + assert!( + matches!( + prev.op, + Instruction::LoadFastBorrow { var_num } + if name_4.varnames[usize::from(var_num.get(OpArg::new(u32::from(u8::from(prev.arg)))))] == "name_5" + ), + "expected async comprehension iterator capture to borrow name_5 before GET_AITER, got {prev:?}" + ); + } } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 87d002c98b2..21052d43cbb 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5,11 +5,12 @@ use crate::{IndexMap, IndexSet, error::InternalError}; use malachite_bigint::BigInt; use num_complex::Complex; use num_traits::{ToPrimitive, Zero}; +use rustpython_wtf8::Wtf8Buf; use rustpython_compiler_core::{ OneIndexed, SourceLocation, bytecode::{ - AnyInstruction, AnyOpcode, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, + AnyInstruction, AnyOpcode, Arg, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, ExceptionTableEntry, InstrDisplayContext, Instruction, InstructionMetadata, IntrinsicFunction1, Label, OpArg, Opcode, PseudoInstruction, PseudoOpcode, PyCodeLocationInfoKind, encode_exception_table, @@ -30,6 +31,7 @@ struct LineTableLocation { const MAX_INT_SIZE_BITS: u64 = 128; const MAX_COLLECTION_SIZE: usize = 256; const MAX_TOTAL_ITEMS: isize = 1024; +const MAX_STR_SIZE: usize = 4096; const MIN_CONST_SEQUENCE_SIZE: usize = 3; const STACK_USE_GUIDELINE: usize = 30; @@ -163,14 +165,14 @@ fn is_named_except_cleanup_normal_exit_block(block: &Block) -> bool { } let tail = &block.instructions[len - 5..]; matches!(tail[0].instr.real(), Some(Instruction::PopExcept)) - && matches!(tail[1].instr.real_opcode(), Some(Opcode::LoadConst)) + && matches!(tail[1].instr.real(), Some(Instruction::LoadConst { .. })) && matches!( - tail[2].instr.real_opcode(), - Some(Opcode::StoreName | Opcode::StoreFast) + tail[2].instr.real(), + Some(Instruction::StoreName { .. } | Instruction::StoreFast { .. }) ) && matches!( - tail[3].instr.real_opcode(), - Some(Opcode::DeleteName | Opcode::DeleteFast) + tail[3].instr.real(), + Some(Instruction::DeleteName { .. } | Instruction::DeleteFast { .. }) ) && tail[4].instr.is_unconditional_jump() } @@ -251,7 +253,9 @@ impl CodeInfo { self.fold_binop_constants(); self.fold_unary_constants(); self.fold_binop_constants(); // re-run after unary folding: -1 + 2 → 1 + self.fold_unary_constants(); // re-run after binop folding: -(2.0 ** -54) → const self.fold_tuple_constants(); + self.fold_binop_constants(); // re-run after tuple folding: (1,) * 2 → const self.fold_list_constants(); self.fold_set_constants(); self.optimize_lists_and_sets(); @@ -270,6 +274,7 @@ impl CodeInfo { // Peephole optimizer handles constant and compare folding. self.peephole_optimize(); self.fold_tuple_constants(); + self.fold_binop_constants(); self.fold_list_constants(); self.fold_set_constants(); self.optimize_lists_and_sets(); @@ -305,16 +310,29 @@ impl CodeInfo { inline_single_predecessor_artificial_expr_exit_blocks(&mut self.blocks); push_cold_blocks_to_end(&mut self.blocks); reorder_conditional_chain_and_jump_back_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) normalize_jumps(&mut self.blocks); reorder_conditional_exit_and_jump_blocks(&mut self.blocks); reorder_conditional_jump_and_exit_blocks(&mut self.blocks); + reorder_conditional_break_continue_blocks(&mut self.blocks); + reorder_conditional_explicit_continue_scope_exit_blocks(&mut self.blocks); + reorder_conditional_implicit_continue_scope_exit_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); + reorder_exception_handler_conditional_continue_raise_blocks(&mut self.blocks); + deduplicate_adjacent_jump_back_blocks(&mut self.blocks); + reorder_conditional_body_and_implicit_continue_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); reorder_jump_over_exception_cleanup_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, false); self.dce(); // re-run within-block DCE after normalize_jumps creates new instructions self.eliminate_unreachable_blocks(); resolve_line_numbers(&mut self.blocks); materialize_empty_conditional_exit_targets(&mut self.blocks); + materialize_exception_cleanup_jump_targets(&mut self.blocks); redirect_empty_block_targets(&mut self.blocks); duplicate_end_returns(&mut self.blocks, &self.metadata); duplicate_shared_jump_back_targets(&mut self.blocks); @@ -331,6 +349,7 @@ impl CodeInfo { remove_redundant_nops_and_jumps(&mut self.blocks); inline_with_suppress_return_blocks(&mut self.blocks); inline_pop_except_return_blocks(&mut self.blocks); + inline_named_except_cleanup_normal_exit_jumps(&mut self.blocks); duplicate_named_except_cleanup_returns(&mut self.blocks, &self.metadata); self.eliminate_unreachable_blocks(); resolve_line_numbers(&mut self.blocks); @@ -349,21 +368,38 @@ impl CodeInfo { // CFG entry depth for each block. let max_stackdepth = self.max_stackdepth()?; // Match CPython order: pseudo ops are lowered after stackdepth - // calculation but before optimize_load_fast. + // calculation, then redundant NOPs from pseudo lowering are removed + // before optimize_load_fast. convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); + remove_redundant_nops_and_jumps(&mut self.blocks); + self.mark_assertion_success_tail_borrow_disabled(); + self.mark_unprotected_debug_four_tails_borrow_disabled(); self.compute_load_fast_start_depths(); // optimize_load_fast: after normalize_jumps self.optimize_load_fast_borrow(); + self.deoptimize_borrow_in_targeted_assert_message_blocks(); self.deoptimize_borrow_for_folded_nonliteral_exprs(); + self.deoptimize_borrow_after_generator_exception_return(); + self.deoptimize_borrow_after_async_for_cleanup_resume(); self.deoptimize_borrow_after_multi_handler_resume_join(); self.deoptimize_borrow_after_named_except_cleanup_join(); + self.deoptimize_borrow_after_reraising_except_handler(); self.deoptimize_borrow_in_protected_conditional_tail(); + self.deoptimize_borrow_after_terminal_except_tail(); + self.deoptimize_borrow_after_except_star_try_tail(); + self.deoptimize_borrow_in_protected_method_call_after_terminal_except_tail(); + self.deoptimize_borrow_after_terminal_except_before_with(); + self.deoptimize_borrow_in_async_finally_early_return_tail(); + self.deoptimize_borrow_after_handler_resume_loop_tail(); self.deoptimize_borrow_after_protected_import(); + self.deoptimize_borrow_before_import_after_join_store(); self.deoptimize_borrow_after_protected_store_tail(); + self.deoptimize_borrow_after_deoptimized_async_with_enter(); self.deoptimize_borrow_after_push_exc_info(); self.deoptimize_borrow_for_handler_return_paths(); self.deoptimize_borrow_for_match_keys_attr(); self.deoptimize_borrow_in_protected_attr_chain_tail(); + self.reborrow_after_suppressing_handler_resume_cleanup(); self.deoptimize_store_fast_store_fast_after_cleanup(); self.apply_static_swaps(); self.deoptimize_store_fast_store_fast_after_cleanup(); @@ -762,8 +798,8 @@ impl CodeInfo { .iter() .take_while(|info| { matches!( - info.instr.real_opcode(), - Some(Opcode::MakeCell | Opcode::CopyFreeVars) + info.instr.real(), + Some(Instruction::MakeCell { .. } | Instruction::CopyFreeVars { .. }) ) }) .count(); @@ -778,7 +814,7 @@ impl CodeInfo { if nfrees > 0 { entry.instructions.push(InstructionInfo { - instr: Opcode::CopyFreeVars.into(), + instr: Instruction::CopyFreeVars { n: Arg::marker() }.into(), arg: OpArg::new(nfrees as u32), ..anchor }); @@ -795,7 +831,7 @@ impl CodeInfo { } for oldindex in sorted.into_iter().flatten() { entry.instructions.push(InstructionInfo { - instr: Opcode::MakeCell.into(), + instr: Instruction::MakeCell { i: Arg::marker() }.into(), arg: OpArg::new(oldindex as u32), ..anchor }); @@ -857,47 +893,47 @@ impl CodeInfo { op: Instruction, intrinsic: Option, ) -> Option { - match (operand, op.into(), intrinsic) { - (ConstantData::Integer { value }, Opcode::UnaryNegative, None) => { + match (operand, op, intrinsic) { + (ConstantData::Integer { value }, Instruction::UnaryNegative, None) => { Some(ConstantData::Integer { value: -value }) } - (ConstantData::Float { value }, Opcode::UnaryNegative, None) => { + (ConstantData::Float { value }, Instruction::UnaryNegative, None) => { Some(ConstantData::Float { value: -value }) } - (ConstantData::Complex { value }, Opcode::UnaryNegative, None) => { + (ConstantData::Complex { value }, Instruction::UnaryNegative, None) => { Some(ConstantData::Complex { value: -value }) } - (ConstantData::Boolean { value }, Opcode::UnaryNegative, None) => { + (ConstantData::Boolean { value }, Instruction::UnaryNegative, None) => { Some(ConstantData::Integer { value: BigInt::from(-i32::from(*value)), }) } - (ConstantData::Integer { value }, Opcode::UnaryInvert, None) => { + (ConstantData::Integer { value }, Instruction::UnaryInvert, None) => { Some(ConstantData::Integer { value: !value }) } - (ConstantData::Boolean { .. }, Opcode::UnaryInvert, None) => None, + (ConstantData::Boolean { .. }, Instruction::UnaryInvert, None) => None, ( ConstantData::Integer { value }, - Opcode::CallIntrinsic1, + Instruction::CallIntrinsic1 { .. }, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Integer { value: value.clone(), }), ( ConstantData::Float { value }, - Opcode::CallIntrinsic1, + Instruction::CallIntrinsic1 { .. }, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Float { value: *value }), ( ConstantData::Boolean { value }, - Opcode::CallIntrinsic1, + Instruction::CallIntrinsic1 { .. }, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Integer { value: BigInt::from(i32::from(*value)), }), ( ConstantData::Complex { value }, - Opcode::CallIntrinsic1, + Instruction::CallIntrinsic1 { .. }, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Complex { value: *value }), _ => None, @@ -919,7 +955,12 @@ impl CodeInfo { oparg::IntrinsicFunction1::UnaryPositive ) => { - (Opcode::CallIntrinsic1.into(), Some(func.get(instr.arg))) + ( + Instruction::CallIntrinsic1 { + func: Arg::marker(), + }, + Some(func.get(instr.arg)), + ) } _ => { i += 1; @@ -950,7 +991,10 @@ impl CodeInfo { block.instructions[idx].end_location = block.instructions[i].end_location; prev = idx; } - block.instructions[i].instr = Opcode::LoadConst.into(); + block.instructions[i].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); block.instructions[i].arg = OpArg::new(const_idx as u32); block.instructions[i].folded_from_nonliteral_expr = false; i = i.saturating_sub(1); @@ -1030,7 +1074,7 @@ impl CodeInfo { for block in &mut self.blocks { let mut i = 0; while i < block.instructions.len() { - let Some(Opcode::BinaryOp) = block.instructions[i].instr.real_opcode() else { + let Some(Instruction::BinaryOp { .. }) = block.instructions[i].instr.real() else { i += 1; continue; }; @@ -1066,11 +1110,6 @@ impl CodeInfo { let result = Self::eval_binop(&left_val, &right_val, op); if let Some(result_const) = result { - // Check result size limit (CPython limits to 4096 bytes) - if Self::const_too_big(&result_const) { - i += 1; - continue; - } let (const_idx, _) = self.metadata.consts.insert_full(result_const); let folded_from_nonliteral_expr = operand_indices .iter() @@ -1078,7 +1117,10 @@ impl CodeInfo { for &idx in &operand_indices { nop_out_no_location(&mut block.instructions[idx]); } - block.instructions[i].instr = Opcode::LoadConst.into(); + block.instructions[i].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); block.instructions[i].arg = OpArg::new(const_idx as u32); block.instructions[i].folded_from_nonliteral_expr = folded_from_nonliteral_expr; i = i.saturating_sub(1); // re-check with previous instruction @@ -1090,8 +1132,8 @@ impl CodeInfo { } fn get_const_value_from_dummy(info: &InstructionInfo) -> Option<()> { - match info.instr.real_opcode() { - Some(Opcode::LoadConst | Opcode::LoadSmallInt) => Some(()), + match info.instr.real() { + Some(Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }) => Some(()), _ => None, } } @@ -1100,12 +1142,12 @@ impl CodeInfo { metadata: &CodeUnitMetadata, info: &InstructionInfo, ) -> Option { - match info.instr.real_opcode() { - Some(Opcode::LoadConst) => { + match info.instr.real() { + Some(Instruction::LoadConst { .. }) => { let idx = u32::from(info.arg) as usize; metadata.consts.get_index(idx).cloned() } - Some(Opcode::LoadSmallInt) => { + Some(Instruction::LoadSmallInt { .. }) => { let v = u32::from(info.arg) as i32; Some(ConstantData::Integer { value: BigInt::from(v), @@ -1134,11 +1176,30 @@ impl CodeInfo { op: oparg::BinaryOperator, ) -> Option { use oparg::BinaryOperator as BinOp; + fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Wtf8Buf { + let mut result = Wtf8Buf::with_capacity(value.len().saturating_mul(n)); + for _ in 0..n { + result.push_wtf8(value); + } + result + } + fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { + let n = n.to_isize()?; + if item_size != 0 && (n < 0 || n as usize > MAX_STR_SIZE / item_size) { + return None; + } + Some(n.max(0) as usize) + } fn eval_complex_binop( left: Complex, right: Complex, op: BinOp, ) -> Option { + fn complex_const(value: Complex) -> Option { + (value.re.is_finite() && value.im.is_finite()) + .then_some(ConstantData::Complex { value }) + } + let value = match op { BinOp::Add => left + right, BinOp::Subtract => { @@ -1163,12 +1224,201 @@ impl CodeInfo { } left / right } + BinOp::Power => { + if left == Complex::new(0.0, 0.0) { + if right.im != 0.0 || right.re < 0.0 { + return None; + } + return complex_const(if right.re == 0.0 { + Complex::new(1.0, 0.0) + } else { + Complex::new(0.0, 0.0) + }); + } + if right.im == 0.0 + && right.re.fract() == 0.0 + && right.re >= f64::from(i32::MIN) + && right.re <= f64::from(i32::MAX) + { + left.powi(right.re as i32) + } else { + left.powc(right) + } + } _ => return None, }; - if !value.re.is_finite() || !value.im.is_finite() { + complex_const(value) + } + fn float_div_mod(left: f64, right: f64) -> Option<(f64, f64)> { + if right == 0.0 { + return None; + } + + let mut modulo = left % right; + let div = (left - modulo) / right; + let floordiv = if modulo != 0.0 { + let div = if (right < 0.0) != (modulo < 0.0) { + modulo += right; + div - 1.0 + } else { + div + }; + let mut floordiv = div.floor(); + if div - floordiv > 0.5 { + floordiv += 1.0; + } + floordiv + } else { + modulo = 0.0f64.copysign(right); + 0.0f64.copysign(left / right) + }; + + Some((floordiv, modulo)) + } + fn constant_as_index(value: &ConstantData) -> Option { + match value { + ConstantData::Integer { value } => value.to_i64().or_else(|| { + if value < &BigInt::from(0) { + Some(i64::MIN) + } else { + Some(i64::MAX) + } + }), + ConstantData::Boolean { value } => Some(i64::from(*value)), + _ => None, + } + } + fn slice_bound(value: &ConstantData) -> Option> { + match value { + ConstantData::None => Some(None), + _ => constant_as_index(value).map(Some), + } + } + fn adjusted_slice_indices(len: usize, slice: &[ConstantData; 3]) -> Option> { + let len = i64::try_from(len).ok()?; + let start = slice_bound(&slice[0])?; + let stop = slice_bound(&slice[1])?; + let step = slice_bound(&slice[2])?.unwrap_or(1); + if step == 0 || step == i64::MIN { + return None; + } + + let step_is_negative = step < 0; + let lower = if step_is_negative { -1 } else { 0 }; + let upper = if step_is_negative { len - 1 } else { len }; + let adjust = |value: Option, default: i64| { + let mut value = value.unwrap_or(default); + if value < 0 { + value = value.saturating_add(len); + if value < 0 { + value = lower; + } + } else if value >= len { + value = upper; + } + value + }; + let start = adjust(start, if step_is_negative { upper } else { lower }); + let stop = adjust(stop, if step_is_negative { lower } else { upper }); + + let mut indices = Vec::new(); + let mut index = i128::from(start); + let stop = i128::from(stop); + let step = i128::from(step); + if step > 0 { + while index < stop { + indices.push(usize::try_from(index).ok()?); + index += step; + } + } else { + while index > stop { + indices.push(usize::try_from(index).ok()?); + index += step; + } + } + Some(indices) + } + fn adjusted_const_index(len: usize, index: &ConstantData) -> Option { + let len = i64::try_from(len).ok()?; + let index = constant_as_index(index)?; + let index = if index < 0 { + index.saturating_add(len) + } else { + index + }; + if index < 0 || index >= len { return None; } - Some(ConstantData::Complex { value }) + usize::try_from(index).ok() + } + fn eval_const_subscript( + container: &ConstantData, + index: &ConstantData, + ) -> Option { + match (container, index) { + ( + ConstantData::Str { value }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let string = value.to_string(); + if string.contains(char::REPLACEMENT_CHARACTER) { + return None; + } + let chars = string.chars().collect::>(); + let index = adjusted_const_index(chars.len(), index)?; + Some(ConstantData::Str { + value: chars[index].to_string().into(), + }) + } + (ConstantData::Str { value }, ConstantData::Slice { elements }) => { + let string = value.to_string(); + if string.contains(char::REPLACEMENT_CHARACTER) { + return None; + } + let chars = string.chars().collect::>(); + let mut result = String::new(); + for index in adjusted_slice_indices(chars.len(), elements)? { + result.push(chars[index]); + } + Some(ConstantData::Str { + value: result.into(), + }) + } + ( + ConstantData::Bytes { value }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let index = adjusted_const_index(value.len(), index)?; + Some(ConstantData::Integer { + value: BigInt::from(value[index]), + }) + } + (ConstantData::Bytes { value }, ConstantData::Slice { elements }) => { + let mut result = Vec::new(); + for index in adjusted_slice_indices(value.len(), elements)? { + result.push(value[index]); + } + Some(ConstantData::Bytes { value: result }) + } + ( + ConstantData::Tuple { elements }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let index = adjusted_const_index(elements.len(), index)?; + Some(elements[index].clone()) + } + (ConstantData::Tuple { elements }, ConstantData::Slice { elements: slice }) => { + let elements = adjusted_slice_indices(elements.len(), slice)? + .into_iter() + .map(|index| elements[index].clone()) + .collect(); + Some(ConstantData::Tuple { elements }) + } + _ => None, + } + } + if matches!(op, BinOp::Subscr) { + return eval_const_subscript(left, right); } match (left, right) { (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { @@ -1218,6 +1468,24 @@ impl CodeInfo { } } BinOp::Power => { + if r < &BigInt::from(0) { + if l.is_zero() { + return None; + } + let base = l.to_f64()?; + if !base.is_finite() { + return None; + } + let result = if let Some(exp) = r.to_i32() { + base.powi(exp) + } else { + base.powf(r.to_f64()?) + }; + if !result.is_finite() { + return None; + } + return Some(ConstantData::Float { value: result }); + } let exp: u64 = r.try_into().ok()?; let exp_usize = usize::try_from(exp).ok()?; if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE_BITS / exp { @@ -1258,25 +1526,17 @@ impl CodeInfo { l / r } BinOp::FloorDivide => { - // Float floor division uses runtime semantics; skip folding - return None; + let (floordiv, _) = float_div_mod(*l, *r)?; + floordiv } BinOp::Remainder => { - if *r == 0.0 { - return None; - } - let mut result = l % r; - if result != 0.0 && (*r < 0.0) != (result < 0.0) { - result += r; - } else if result == 0.0 { - result = 0.0f64.copysign(*r); - } - result + let (_, modulo) = float_div_mod(*l, *r)?; + modulo } BinOp::Power => l.powf(*r), _ => return None, }; - if !result.is_finite() { + if matches!(op, BinOp::Power) && !result.is_finite() { return None; } Some(ConstantData::Float { value: result }) @@ -1317,23 +1577,16 @@ impl CodeInfo { (ConstantData::Str { value: l }, ConstantData::Str { value: r }) if matches!(op, BinOp::Add) => { - let mut result = l.to_string(); - result.push_str(&r.to_string()); - Some(ConstantData::Str { - value: result.into(), - }) + let mut result = l.clone(); + result.push_wtf8(r); + Some(ConstantData::Str { value: result }) } (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) if matches!(op, BinOp::Multiply) => { - let n: usize = n.try_into().ok()?; - if n > 4096 { - return None; - } - let result = s.to_string().repeat(n); - Some(ConstantData::Str { - value: result.into(), - }) + let n = checked_repeat_count(n, s.code_points().count())?; + let result = repeat_wtf8(s, n); + Some(ConstantData::Str { value: result }) } (ConstantData::Tuple { elements: l }, ConstantData::Tuple { elements: r }) if matches!(op, BinOp::Add) => @@ -1387,14 +1640,9 @@ impl CodeInfo { (ConstantData::Integer { value: n }, ConstantData::Str { value: s }) if matches!(op, BinOp::Multiply) => { - let n: usize = n.try_into().ok()?; - if n > 4096 { - return None; - } - let result = s.to_string().repeat(n); - Some(ConstantData::Str { - value: result.into(), - }) + let n = checked_repeat_count(n, s.code_points().count())?; + let result = repeat_wtf8(s, n); + Some(ConstantData::Str { value: result }) } (ConstantData::Bytes { value: l }, ConstantData::Bytes { value: r }) if matches!(op, BinOp::Add) => @@ -1406,142 +1654,233 @@ impl CodeInfo { (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) if matches!(op, BinOp::Multiply) => { - let n: usize = n.try_into().ok()?; - if n > 4096 { - return None; - } + let n = checked_repeat_count(n, b.len())?; Some(ConstantData::Bytes { value: b.repeat(n) }) } (ConstantData::Integer { value: n }, ConstantData::Bytes { value: b }) if matches!(op, BinOp::Multiply) => { - let n: usize = n.try_into().ok()?; - if n > 4096 { - return None; - } + let n = checked_repeat_count(n, b.len())?; Some(ConstantData::Bytes { value: b.repeat(n) }) } _ => None, } } - fn const_too_big(c: &ConstantData) -> bool { - match c { - ConstantData::Integer { value } => value.bits() > 4096 * 8, - ConstantData::Str { value } => value.len() > 4096, - ConstantData::Bytes { value } => value.len() > 4096, - _ => false, + fn fold_tuple_constant_at( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, + ) -> bool { + let Some(Instruction::BuildTuple { .. }) = block.instructions[i].instr.real() else { + return false; + }; + + let tuple_size = u32::from(block.instructions[i].arg) as usize; + if block + .instructions + .get(i + 1) + .and_then(|next| next.instr.real()) + .is_some_and(|next| { + matches!( + next, + Instruction::UnpackSequence { .. } + if usize::try_from(u32::from(block.instructions[i + 1].arg)).ok() + == Some(tuple_size) + ) + }) + { + return false; + } + if tuple_size == 0 { + let (const_idx, _) = metadata.consts.insert_full(ConstantData::Tuple { + elements: Vec::new(), + }); + block.instructions[i].instr = Opcode::LoadConst.into(); + block.instructions[i].arg = OpArg::new(const_idx as u32); + return true; + } + + let folded_from_nonliteral_expr = block.instructions[i].folded_from_nonliteral_expr; + let Some((operand_indices, elements)) = + Self::get_const_sequence(metadata, block, i, tuple_size) + else { + return false; + }; + + let (const_idx, _) = metadata + .consts + .insert_full(ConstantData::Tuple { elements }); + + for &j in &operand_indices { + nop_out_no_location(&mut block.instructions[j]); } + + block.instructions[i].instr = Opcode::LoadConst.into(); + block.instructions[i].arg = OpArg::new(const_idx as u32); + block.instructions[i].folded_from_nonliteral_expr = folded_from_nonliteral_expr; + true } - /// Constant folding: fold LOAD_CONST/LOAD_SMALL_INT + BUILD_TUPLE into LOAD_CONST tuple - /// fold_tuple_of_constants - fn fold_tuple_constants(&mut self) { - for block in &mut self.blocks { - let mut i = 0; - while i < block.instructions.len() { - let instr = &block.instructions[i]; - // Look for BUILD_TUPLE - let Some(Opcode::BuildTuple) = instr.instr.real_opcode() else { - i += 1; - continue; - }; + fn fold_constant_intrinsic_list_to_tuple_at( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, + ) -> bool { + let Some(Instruction::CallIntrinsic1 { func }) = block.instructions[i].instr.real() else { + return false; + }; + if func.get(block.instructions[i].arg) != IntrinsicFunction1::ListToTuple { + return false; + } - let tuple_size = u32::from(instr.arg) as usize; - if block - .instructions - .get(i + 1) - .and_then(|next| next.instr.real_opcode()) - .is_some_and(|next| { - matches!( - next, - Opcode::UnpackSequence if - usize::try_from(u32::from(block.instructions[i + 1].arg)) - .ok() == Some(tuple_size) - ) - }) - { - i += 1; - continue; - } - if tuple_size == 0 { - // BUILD_TUPLE 0 → LOAD_CONST () - let (const_idx, _) = self.metadata.consts.insert_full(ConstantData::Tuple { - elements: Vec::new(), - }); - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - i += 1; - continue; - } - let Some(operand_indices) = i.checked_sub(1).and_then(|start| { - Self::get_const_loading_instr_indices(block, start, tuple_size) - }) else { - i += 1; - continue; - }; + let mut consts_found = 0usize; + let mut expect_append = true; + let mut pos = i; + while let Some(prev) = pos.checked_sub(1) { + pos = prev; + let instr = &block.instructions[pos]; + if matches!(instr.instr.real(), Some(Instruction::Nop)) { + continue; + } - let mut elements = Vec::with_capacity(tuple_size); - let mut all_const = true; + if matches!(instr.instr.real(), Some(Instruction::BuildList { .. })) + && u32::from(instr.arg) == 0 + { + if !expect_append { + return false; + } - for &j in &operand_indices { - let load_instr = &block.instructions[j]; - if load_instr.folded_from_nonliteral_expr { - all_const = false; - break; + let mut elements = Vec::with_capacity(consts_found); + let mut folded_from_nonliteral_expr = false; + let mut expect_load = true; + for idx in pos + 1..i { + let instr = &block.instructions[idx]; + if matches!(instr.instr.real(), Some(Instruction::Nop)) { + continue; } - - match load_instr.instr.real_opcode() { - Some(Opcode::LoadConst) => { - let const_idx = u32::from(load_instr.arg) as usize; - if let Some(constant) = - self.metadata.consts.get_index(const_idx).cloned() - { - elements.push(constant); - } else { - all_const = false; - break; - } - } - Some(Opcode::LoadSmallInt) => { - // arg is the i32 value stored as u32 (two's complement) - let value = u32::from(load_instr.arg) as i32; - elements.push(ConstantData::Integer { - value: BigInt::from(value), - }); - } - _ => { - all_const = false; - break; - } + if expect_load { + let Some(value) = Self::get_const_value_from(metadata, instr) else { + return false; + }; + folded_from_nonliteral_expr |= instr.folded_from_nonliteral_expr; + elements.push(value); + } else if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) + || u32::from(instr.arg) != 1 + { + return false; } + expect_load = !expect_load; + } + if !expect_load || elements.len() != consts_found { + return false; } - if !all_const { - i += 1; - continue; + let (const_idx, _) = metadata + .consts + .insert_full(ConstantData::Tuple { elements }); + for idx in pos..i { + nop_out_no_location(&mut block.instructions[idx]); } + block.instructions[i].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[i].arg = OpArg::new(const_idx as u32); + block.instructions[i].folded_from_nonliteral_expr = folded_from_nonliteral_expr; + return true; + } - // Note: The first small int is added to co_consts during compilation - // (in compile_default_arguments). - // We don't need to add it here again. + if expect_append { + if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) + || u32::from(instr.arg) != 1 + { + return false; + } + } else { + if Self::get_const_value_from_dummy(instr).is_none() { + return false; + } + consts_found += 1; + } + expect_append = !expect_append; + } - // Create tuple constant and add to consts - let tuple_const = ConstantData::Tuple { elements }; - let (const_idx, _) = self.metadata.consts.insert_full(tuple_const); + false + } - // Replace preceding LOAD instructions with NOP at the - // BUILD_TUPLE location so remove_nops() can eliminate them. - let folded_loc = block.instructions[i].location; - for &j in &operand_indices { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - } + fn fold_list_constant_at(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { + let Some(Instruction::BuildList { .. }) = block.instructions[i].instr.real() else { + return false; + }; - // Replace BUILD_TUPLE with LOAD_CONST - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); + let list_size = u32::from(block.instructions[i].arg) as usize; + if list_size == 0 || list_size > STACK_USE_GUIDELINE { + return false; + } + + let Some((operand_indices, elements)) = + Self::get_const_sequence(metadata, block, i, list_size) + else { + return false; + }; + if list_size < MIN_CONST_SEQUENCE_SIZE { + return false; + } + + let (const_idx, _) = metadata + .consts + .insert_full(ConstantData::Tuple { elements }); + + let folded_loc = block.instructions[i].location; + let end_loc = block.instructions[i].end_location; + let eh = block.instructions[i].except_handler; + let build_idx = operand_indices[0]; + let const_idx_slot = operand_indices[1]; + + block.instructions[build_idx].instr = Instruction::BuildList { + count: Arg::marker(), + } + .into(); + block.instructions[build_idx].arg = OpArg::new(0); + block.instructions[build_idx].location = folded_loc; + block.instructions[build_idx].end_location = end_loc; + block.instructions[build_idx].except_handler = eh; + + block.instructions[const_idx_slot].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); + block.instructions[const_idx_slot].location = folded_loc; + block.instructions[const_idx_slot].end_location = end_loc; + block.instructions[const_idx_slot].except_handler = eh; + + for &j in &operand_indices[2..] { + set_to_nop(&mut block.instructions[j]); + block.instructions[j].location = folded_loc; + } + + block.instructions[i].instr = Opcode::ListExtend.into(); + block.instructions[i].arg = OpArg::new(1); + true + } + + /// Constant folding: fold LOAD_CONST/LOAD_SMALL_INT + BUILD_TUPLE into LOAD_CONST tuple + /// fold_tuple_of_constants. This also folds constant list/set literals + /// in block order to match CPython's optimize_basic_block() const-table order. + fn fold_tuple_constants(&mut self) { + for block in &mut self.blocks { + let mut i = 0; + while i < block.instructions.len() { + if Self::fold_tuple_constant_at(&mut self.metadata, block, i) + || Self::fold_list_constant_at(&mut self.metadata, block, i) + || Self::fold_set_constant_at(&mut self.metadata, block, i) + { + i += 1; + continue; + } i += 1; } } @@ -1554,7 +1893,7 @@ impl CodeInfo { let mut i = 0; while i < block.instructions.len() { let instr = &block.instructions[i]; - let Some(Opcode::BuildList) = instr.instr.real_opcode() else { + let Some(Instruction::BuildList { .. }) = instr.instr.real() else { i += 1; continue; }; @@ -1571,7 +1910,6 @@ impl CodeInfo { i += 1; continue; }; - if list_size < MIN_CONST_SEQUENCE_SIZE { i += 1; continue; @@ -1587,13 +1925,19 @@ impl CodeInfo { let build_idx = operand_indices[0]; let const_idx_slot = operand_indices[1]; - block.instructions[build_idx].instr = Opcode::BuildList.into(); + block.instructions[build_idx].instr = Instruction::BuildList { + count: Arg::marker(), + } + .into(); block.instructions[build_idx].arg = OpArg::new(0); block.instructions[build_idx].location = folded_loc; block.instructions[build_idx].end_location = end_loc; block.instructions[build_idx].except_handler = eh; - block.instructions[const_idx_slot].instr = Opcode::LoadConst.into(); + block.instructions[const_idx_slot].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); block.instructions[const_idx_slot].location = folded_loc; block.instructions[const_idx_slot].end_location = end_loc; @@ -1629,36 +1973,43 @@ impl CodeInfo { block.instructions[i].instr.real(), Some(Instruction::CallIntrinsic1 { func }) if func.get(block.instructions[i].arg) == IntrinsicFunction1::ListToTuple - ) && matches!( - block - .instructions - .get(i + 1) - .and_then(|instr| instr.instr.real()), - Some(Instruction::GetIter) ) { - set_to_nop(&mut block.instructions[i]); - i += 2; - continue; + if matches!( + block + .instructions + .get(i + 1) + .and_then(|instr| instr.instr.real()), + Some(Instruction::GetIter) + ) { + set_to_nop(&mut block.instructions[i]); + i += 2; + continue; + } + if Self::fold_constant_intrinsic_list_to_tuple_at(&mut self.metadata, block, i) + { + i += 1; + continue; + } } if let Some(non_nop4) = Self::get_non_nop_instr_indices(block, i, 4) { let is_build_list = non_nop4[0] == i && matches!( - block.instructions[non_nop4[0]].instr.real_opcode(), - Some(Opcode::BuildList) + block.instructions[non_nop4[0]].instr.real(), + Some(Instruction::BuildList { .. }) ) && u32::from(block.instructions[non_nop4[0]].arg) == 0; let is_const = matches!( - block.instructions[non_nop4[1]].instr.real_opcode(), - Some(Opcode::LoadConst) + block.instructions[non_nop4[1]].instr.real(), + Some(Instruction::LoadConst { .. }) ); let is_list_extend = matches!( - block.instructions[non_nop4[2]].instr.real_opcode(), - Some(Opcode::ListExtend) + block.instructions[non_nop4[2]].instr.real(), + Some(Instruction::ListExtend { .. }) ) && u32::from(block.instructions[non_nop4[2]].arg) == 1; let uses_iter_or_contains = matches!( - block.instructions[non_nop4[3]].instr.real_opcode(), - Some(Opcode::GetIter | Opcode::ContainsOp) + block.instructions[non_nop4[3]].instr.real(), + Some(Instruction::GetIter | Instruction::ContainsOp { .. }) ); if is_build_list && is_const && is_list_extend && uses_iter_or_contains { @@ -1673,13 +2024,13 @@ impl CodeInfo { let is_build_set = non_nop4[0] == i && matches!( - block.instructions[non_nop4[0]].instr.real_opcode(), - Some(Opcode::BuildSet) + block.instructions[non_nop4[0]].instr.real(), + Some(Instruction::BuildSet { .. }) ) && u32::from(block.instructions[non_nop4[0]].arg) == 0; let is_set_update = matches!( - block.instructions[non_nop4[2]].instr.real_opcode(), - Some(Opcode::SetUpdate) + block.instructions[non_nop4[2]].instr.real(), + Some(Instruction::SetUpdate { .. }) ) && u32::from(block.instructions[non_nop4[2]].arg) == 1; if is_build_set && is_const && is_set_update && uses_iter_or_contains { @@ -1699,18 +2050,17 @@ impl CodeInfo { }; let uses_iter_or_contains = non_nop2[0] == i && matches!( - block.instructions[non_nop2[1]].instr.real_opcode(), - Some(Opcode::GetIter | Opcode::ContainsOp) + block.instructions[non_nop2[1]].instr.real(), + Some(Instruction::GetIter | Instruction::ContainsOp { .. }) ); - if !uses_iter_or_contains { i += 1; continue; } if matches!( - block.instructions[i].instr.real_opcode(), - Some(Opcode::BuildList) + block.instructions[i].instr.real(), + Some(Instruction::BuildList { .. }) ) { let seq_size = u32::from(block.instructions[i].arg) as usize; if seq_size > STACK_USE_GUIDELINE { @@ -1744,8 +2094,8 @@ impl CodeInfo { block.instructions[i].instr = Opcode::BuildTuple.into(); i += 2; } else if matches!( - block.instructions[i].instr.real_opcode(), - Some(Opcode::BuildSet) + block.instructions[i].instr.real(), + Some(Instruction::BuildSet { .. }) ) { let seq_size = u32::from(block.instructions[i].arg) as usize; if seq_size > STACK_USE_GUIDELINE { @@ -1783,14 +2133,68 @@ impl CodeInfo { } } - /// Fold constant set literals: LOAD_CONST* + BUILD_SET N → - /// BUILD_SET 0 + LOAD_CONST (frozenset-as-tuple) + SET_UPDATE 1 - fn fold_set_constants(&mut self) { - for block in &mut self.blocks { - let mut i = 0; - while i < block.instructions.len() { - let instr = &block.instructions[i]; - let Some(Opcode::BuildSet) = instr.instr.real_opcode() else { + fn fold_set_constant_at(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { + let Some(Instruction::BuildSet { .. }) = block.instructions[i].instr.real() else { + return false; + }; + + let set_size = u32::from(block.instructions[i].arg) as usize; + if !(3..=STACK_USE_GUIDELINE).contains(&set_size) { + return false; + } + + let Some((operand_indices, elements)) = + Self::get_const_sequence(metadata, block, i, set_size) + else { + return false; + }; + let (const_idx, _) = metadata + .consts + .insert_full(ConstantData::Frozenset { elements }); + + let folded_loc = block.instructions[i].location; + let end_loc = block.instructions[i].end_location; + let eh = block.instructions[i].except_handler; + + let build_idx = operand_indices[0]; + let const_idx_slot = operand_indices[1]; + + block.instructions[build_idx].instr = Instruction::BuildSet { + count: Arg::marker(), + } + .into(); + block.instructions[build_idx].arg = OpArg::new(0); + block.instructions[build_idx].location = folded_loc; + block.instructions[build_idx].end_location = end_loc; + block.instructions[build_idx].except_handler = eh; + + block.instructions[const_idx_slot].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); + block.instructions[const_idx_slot].location = folded_loc; + block.instructions[const_idx_slot].end_location = end_loc; + block.instructions[const_idx_slot].except_handler = eh; + + for &j in &operand_indices[2..] { + set_to_nop(&mut block.instructions[j]); + block.instructions[j].location = folded_loc; + } + + block.instructions[i].instr = Opcode::SetUpdate.into(); + block.instructions[i].arg = OpArg::new(1); + true + } + + /// Fold constant set literals: LOAD_CONST* + BUILD_SET N → + /// BUILD_SET 0 + LOAD_CONST (frozenset-as-tuple) + SET_UPDATE 1 + fn fold_set_constants(&mut self) { + for block in &mut self.blocks { + let mut i = 0; + while i < block.instructions.len() { + let instr = &block.instructions[i]; + let Some(Instruction::BuildSet { .. }) = instr.instr.real() else { i += 1; continue; }; @@ -1817,13 +2221,19 @@ impl CodeInfo { let build_idx = operand_indices[0]; let const_idx_slot = operand_indices[1]; - block.instructions[build_idx].instr = Opcode::BuildSet.into(); + block.instructions[build_idx].instr = Instruction::BuildSet { + count: Arg::marker(), + } + .into(); block.instructions[build_idx].arg = OpArg::new(0); block.instructions[build_idx].location = folded_loc; block.instructions[build_idx].end_location = end_loc; block.instructions[build_idx].except_handler = eh; - block.instructions[const_idx_slot].instr = Opcode::LoadConst.into(); + block.instructions[const_idx_slot].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); block.instructions[const_idx_slot].location = folded_loc; block.instructions[const_idx_slot].end_location = end_loc; @@ -1852,11 +2262,12 @@ impl CodeInfo { let instructions = &mut block.instructions; let len = instructions.len(); for i in 0..len.saturating_sub(1) { - let Some(Opcode::BuildTuple) = instructions[i].instr.real_opcode() else { + let Some(Instruction::BuildTuple { .. }) = instructions[i].instr.real() else { continue; }; let n = u32::from(instructions[i].arg); - let Some(Opcode::UnpackSequence) = instructions[i + 1].instr.real_opcode() else { + let Some(Instruction::UnpackSequence { .. }) = instructions[i + 1].instr.real() + else { continue; }; if u32::from(instructions[i + 1].arg) != n { @@ -1895,17 +2306,19 @@ impl CodeInfo { const VISITED: i32 = -1; /// Instruction classes that are safe to reorder around SWAP. - const fn is_swappable(instr: AnyInstruction) -> bool { + fn is_swappable(instr: &AnyInstruction) -> bool { matches!( - instr.real_opcode(), - Some(Opcode::StoreFast | Opcode::PopTop) + (*instr).into(), + AnyOpcode::Real(Opcode::StoreFast | Opcode::PopTop) + | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) ) } /// Variable index that a STORE_FAST writes to, or None. - fn stores_to(info: InstructionInfo) -> Option { - match info.instr.real_opcode() { - Some(Opcode::StoreFast) => Some(u32::from(info.arg)), + fn stores_to(info: &InstructionInfo) -> Option { + match info.instr.into() { + AnyOpcode::Real(Opcode::StoreFast) => Some(u32::from(info.arg)), + AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) => Some(u32::from(info.arg)), _ => None, } } @@ -1920,26 +2333,20 @@ impl CodeInfo { ) -> Option { loop { i += 1; - if i >= instructions.len() { return None; } - let info = &instructions[i]; let info_lineno = info.location.line.get() as i32; - if lineno >= 0 && info_lineno > 0 && info_lineno != lineno { return None; } - - if matches!(info.instr.real_opcode(), Some(Opcode::Nop)) { + if matches!(info.instr, AnyInstruction::Real(Instruction::Nop)) { continue; } - - if is_swappable(info.instr) { + if is_swappable(&info.instr) { return Some(i); } - return None; } } @@ -1947,7 +2354,7 @@ impl CodeInfo { fn optimize_swap_block(instructions: &mut [InstructionInfo]) { let mut i = 0usize; while i < instructions.len() { - let Some(Opcode::Swap) = instructions[i].instr.real_opcode() else { + let AnyInstruction::Real(Instruction::Swap { .. }) = instructions[i].instr else { i += 1; continue; }; @@ -1957,14 +2364,14 @@ impl CodeInfo { let mut more = false; while i + len < instructions.len() { let info = &instructions[i + len]; - match info.instr.real_opcode() { - Some(Opcode::Swap) => { + match info.instr.real() { + Some(Instruction::Swap { .. }) => { let oparg = u32::from(info.arg) as usize; depth = depth.max(oparg); more |= len > 0; len += 1; } - Some(Opcode::Nop) => { + Some(Instruction::Nop) => { len += 1; } _ => break, @@ -1978,7 +2385,7 @@ impl CodeInfo { let mut stack: Vec = (0..depth as i32).collect(); for info in &instructions[i..i + len] { - if matches!(info.instr.real_opcode(), Some(Opcode::Swap)) { + if matches!(info.instr.real(), Some(Instruction::Swap { .. })) { let oparg = u32::from(info.arg) as usize; stack.swap(0, oparg - 1); } @@ -2018,9 +2425,19 @@ impl CodeInfo { fn apply_from(instructions: &mut [InstructionInfo], mut i: isize) { while i >= 0 { let idx = i as usize; - let swap_arg = match instructions[idx].instr.real_opcode() { - Some(Opcode::Swap) => u32::from(instructions[idx].arg), - Some(Opcode::Nop | Opcode::PopTop | Opcode::StoreFast) => { + let swap_arg = match instructions[idx].instr.real() { + Some(Instruction::Swap { .. }) => u32::from(instructions[idx].arg), + Some( + Instruction::Nop | Instruction::PopTop | Instruction::StoreFast { .. }, + ) => { + i -= 1; + continue; + } + _ if matches!( + instructions[idx].instr.pseudo(), + Some(PseudoInstruction::StoreFastMaybeNull { .. }) + ) => + { i -= 1; continue; } @@ -2043,20 +2460,19 @@ impl CodeInfo { k = next; } - let store_j = stores_to(instructions[j]); - let store_k = stores_to(instructions[k]); + let store_j = stores_to(&instructions[j]); + let store_k = stores_to(&instructions[k]); if store_j.is_some() || store_k.is_some() { if store_j == store_k { return; } - let conflict = instructions[(j + 1)..k].iter().any(|&info| { + let conflict = instructions[(j + 1)..k].iter().any(|info| { if let Some(store_idx) = stores_to(info) { Some(store_idx) == store_j || Some(store_idx) == store_k } else { false } }); - if conflict { return; } @@ -2074,8 +2490,8 @@ impl CodeInfo { let len = block.instructions.len(); for i in 0..len { if matches!( - block.instructions[i].instr.real_opcode(), - Some(Opcode::Swap) + block.instructions[i].instr.real(), + Some(Instruction::Swap { .. }) ) { apply_from(&mut block.instructions, i as isize); } @@ -2100,8 +2516,8 @@ impl CodeInfo { while i < len { // Look for UNPACK_SEQUENCE or UNPACK_EX let is_unpack = matches!( - instructions[i].instr.real_opcode(), - Some(Opcode::UnpackSequence | Opcode::UnpackEx) + instructions[i].instr.into(), + AnyOpcode::Real(Opcode::UnpackSequence | Opcode::UnpackEx) ); if !is_unpack { i += 1; @@ -2112,8 +2528,8 @@ impl CodeInfo { let mut run_end = run_start; while run_end < len && matches!( - instructions[run_end].instr.real_opcode(), - Some(Opcode::StoreFast) + instructions[run_end].instr.into(), + AnyOpcode::Real(Opcode::StoreFast) ) { run_end += 1; @@ -2142,13 +2558,10 @@ impl CodeInfo { for i in 0..instructions.len().saturating_sub(1) { let lhs = &instructions[i]; let rhs = &instructions[i + 1]; - let preceded_by_swap = - i > 0 && matches!(instructions[i - 1].instr.real_opcode(), Some(Opcode::Swap)); - if !matches!(lhs.instr.real_opcode(), Some(Opcode::StoreFast)) - || !matches!(rhs.instr.real_opcode(), Some(Opcode::StoreFast)) + if !matches!(lhs.instr.real(), Some(Instruction::StoreFast { .. })) + || !matches!(rhs.instr.real(), Some(Instruction::StoreFast { .. })) || u32::from(lhs.arg) != u32::from(rhs.arg) || instruction_lineno(lhs) != instruction_lineno(rhs) - || preceded_by_swap { continue; } @@ -2198,10 +2611,32 @@ impl CodeInfo { continue; }; + if matches!(curr_instr, Instruction::ToBool) + && matches!(next_instr, Instruction::UnaryNot) + && let Some(Instruction::ToBool) = block + .instructions + .get(i + 2) + .and_then(|info| info.instr.real()) + && let Some(Instruction::UnaryNot) = block + .instructions + .get(i + 3) + .and_then(|info| info.instr.real()) + { + block.instructions.drain(i + 1..=i + 3); + continue; + } + + if matches!(curr_instr, Instruction::UnaryNot | Instruction::ToBool) + && matches!(next_instr, Instruction::ToBool) + { + block.instructions.remove(i + 1); + continue; + } + if let Some(is_true) = const_truthiness(curr_instr, curr.arg, &self.metadata) { - let jump_if_true = match next_instr.into() { - Opcode::PopJumpIfTrue => Some(true), - Opcode::PopJumpIfFalse => Some(false), + let jump_if_true = match next_instr { + Instruction::PopJumpIfTrue { .. } => Some(true), + Instruction::PopJumpIfFalse { .. } => Some(false), _ => None, }; if let Some(jump_if_true) = jump_if_true { @@ -2212,7 +2647,10 @@ impl CodeInfo { }; set_to_nop(&mut block.instructions[i]); if is_true == jump_if_true { - block.instructions[i + 1].instr = PseudoOpcode::Jump.into(); + block.instructions[i + 1].instr = PseudoInstruction::Jump { + delta: Arg::marker(), + } + .into(); block.instructions[i + 1].arg = OpArg::new(u32::from(target)); } else { set_to_nop(&mut block.instructions[i + 1]); @@ -2234,8 +2672,8 @@ impl CodeInfo { } if matches!( - block.instructions[jump_idx].instr.real_opcode(), - Some(Opcode::ToBool) + block.instructions[jump_idx].instr.real(), + Some(Instruction::ToBool) ) { set_to_nop(&mut block.instructions[jump_idx]); jump_idx += 1; @@ -2271,9 +2709,13 @@ impl CodeInfo { set_to_nop(&mut block.instructions[i]); set_to_nop(&mut block.instructions[i + 1]); block.instructions[jump_idx].instr = if invert { - Opcode::PopJumpIfNotNone + Instruction::PopJumpIfNotNone { + delta: Arg::marker(), + } } else { - Opcode::PopJumpIfNone + Instruction::PopJumpIfNone { + delta: Arg::marker(), + } } .into(); block.instructions[jump_idx].arg = OpArg::new(u32::from(delta)); @@ -2282,8 +2724,10 @@ impl CodeInfo { } } - if matches!(curr_instr.into(), Opcode::LoadConst | Opcode::LoadSmallInt) - && matches!(next_instr, Instruction::PopTop) + if matches!( + curr_instr, + Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } + ) && matches!(next_instr, Instruction::PopTop) { set_to_nop(&mut block.instructions[i]); set_to_nop(&mut block.instructions[i + 1]); @@ -2315,7 +2759,12 @@ impl CodeInfo { .metadata .consts .insert_full(ConstantData::Boolean { value }); - Some((Opcode::LoadConst.into(), OpArg::new(const_idx as u32))) + Some(( + Instruction::LoadConst { + consti: Arg::marker(), + }, + OpArg::new(const_idx as u32), + )) } else { None } @@ -2367,8 +2816,8 @@ impl CodeInfo { let curr = &block.instructions[i]; let next = &block.instructions[i + 1]; - let (Some(Opcode::LoadGlobal), Some(Opcode::PushNull)) = - (curr.instr.real_opcode(), next.instr.real_opcode()) + let (Some(Instruction::LoadGlobal { .. }), Some(Instruction::PushNull)) = + (curr.instr.real(), next.instr.real()) else { i += 1; continue; @@ -2402,8 +2851,11 @@ impl CodeInfo { }; let redundant = matches!( - (curr_instr.into(), next_instr.into()), - (Opcode::LoadConst | Opcode::LoadSmallInt, Opcode::PopTop) + (curr_instr, next_instr), + ( + Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }, + Instruction::PopTop + ) ) || matches!(curr_instr, Instruction::Copy { i } if i.get(curr.arg) == 1) && matches!(next_instr, Instruction::PopTop); @@ -2424,7 +2876,7 @@ impl CodeInfo { for block in &mut self.blocks { for instr in &mut block.instructions { // Check if it's a LOAD_CONST instruction - let Some(Opcode::LoadConst) = instr.instr.real_opcode() else { + let Some(Instruction::LoadConst { .. }) = instr.instr.real() else { continue; }; @@ -2465,7 +2917,7 @@ impl CodeInfo { for block in &self.blocks { for instr in &block.instructions { - if let Some(Opcode::LoadConst) = instr.instr.real_opcode() { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { let idx = u32::from(instr.arg) as usize; if idx < nconsts { used[idx] = true; @@ -2502,7 +2954,7 @@ impl CodeInfo { // Update LOAD_CONST instruction arguments for block in &mut self.blocks { for instr in &mut block.instructions { - if let Some(Opcode::LoadConst) = instr.instr.real_opcode() { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { let old_idx = u32::from(instr.arg) as usize; if old_idx < nconsts { instr.arg = OpArg::new(old_to_new[old_idx] as u32); @@ -2515,43 +2967,13 @@ impl CodeInfo { /// Remove NOP instructions from all blocks, but keep NOPs that introduce /// a new source line (they serve as line markers for monitoring LINE events). fn remove_nops(&mut self) { - fn ends_with_for_cleanup(block: &Block) -> bool { - let mut reals = block - .instructions - .iter() - .rev() - .filter_map(|info| info.instr.real_opcode()); - matches!( - (reals.next(), reals.next()), - (Some(Opcode::PopIter), Some(Opcode::EndFor)) - ) - } - - let jump_targets = compute_target_predecessor_flags(&self.blocks).jump; - let mut fallthrough_predecessors = vec![None; self.blocks.len()]; - for (pred_idx, block) in self.blocks.iter().enumerate() { - let next = next_nonempty_block(&self.blocks, block.next); - if next != BlockIdx::NULL { - fallthrough_predecessors[next.idx()] = Some(pred_idx); - } - } - let starts_after_for_cleanup: Vec<_> = fallthrough_predecessors - .iter() - .map(|pred_idx| pred_idx.is_some_and(|idx| ends_with_for_cleanup(&self.blocks[idx]))) - .collect(); - for (block_idx, block) in self.blocks.iter_mut().enumerate() { + for block in &mut self.blocks { let mut prev_line = None; let mut src = 0usize; block.instructions.retain(|ins| { let keep = 'keep: { if matches!(ins.instr.real(), Some(Instruction::Nop)) { - if ins.remove_no_location_nop - && instruction_lineno(ins) < 0 - && !(src == 0 - && (jump_targets[block_idx] - || (ins.preserve_block_start_no_location_nop - && !starts_after_for_cleanup[block_idx]))) - { + if ins.remove_no_location_nop && instruction_lineno(ins) < 0 { break 'keep false; } let line = ins.location.line.get() as i32; @@ -2582,8 +3004,8 @@ impl CodeInfo { continue; } - match (curr.instr.real_opcode(), next.instr.real_opcode()) { - (Some(Opcode::LoadFast), Some(Opcode::LoadFast)) => { + match (curr.instr.real(), next.instr.real()) { + (Some(Instruction::LoadFast { .. }), Some(Instruction::LoadFast { .. })) => { let idx1 = u32::from(curr.arg); let idx2 = u32::from(next.arg); if idx1 >= 16 || idx2 >= 16 { @@ -2591,11 +3013,14 @@ impl CodeInfo { continue; } let packed = (idx1 << 4) | idx2; - block.instructions[i].instr = Opcode::LoadFastLoadFast.into(); + block.instructions[i].instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); block.instructions[i].arg = OpArg::new(packed); block.instructions.remove(i + 1); } - (Some(Opcode::StoreFast), Some(Opcode::LoadFast)) => { + (Some(Instruction::StoreFast { .. }), Some(Instruction::LoadFast { .. })) => { let store_idx = u32::from(curr.arg); let load_idx = u32::from(next.arg); if store_idx >= 16 || load_idx >= 16 { @@ -2603,11 +3028,14 @@ impl CodeInfo { continue; } let packed = (store_idx << 4) | load_idx; - block.instructions[i].instr = Opcode::StoreFastLoadFast.into(); + block.instructions[i].instr = Instruction::StoreFastLoadFast { + var_nums: Arg::marker(), + } + .into(); block.instructions[i].arg = OpArg::new(packed); block.instructions.remove(i + 1); } - (Some(Opcode::StoreFast), Some(Opcode::StoreFast)) => { + (Some(Instruction::StoreFast { .. }), Some(Instruction::StoreFast { .. })) => { let idx1 = u32::from(curr.arg); let idx2 = u32::from(next.arg); if idx1 >= 16 || idx2 >= 16 { @@ -2615,7 +3043,10 @@ impl CodeInfo { continue; } let packed = (idx1 << 4) | idx2; - block.instructions[i].instr = Opcode::StoreFastStoreFast.into(); + block.instructions[i].instr = Instruction::StoreFastStoreFast { + var_nums: Arg::marker(), + } + .into(); block.instructions[i].arg = OpArg::new(packed); block.instructions.remove(i + 1); } @@ -2961,17 +3392,21 @@ impl CodeInfo { if instr_flags[i] != 0 { continue; } - - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; - - info.instr = match opcode { - Opcode::LoadFast => Opcode::LoadFastBorrow, - Opcode::LoadFastLoadFast => Opcode::LoadFastBorrowLoadFastBorrow, - _ => continue, + match info.instr.real() { + Some(Instruction::LoadFast { .. }) => { + info.instr = Instruction::LoadFastBorrow { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastLoadFast { .. }) => { + info.instr = Instruction::LoadFastBorrowLoadFastBorrow { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} } - .into(); } } } @@ -3029,118 +3464,180 @@ impl CodeInfo { } } - fn deoptimize_borrow_for_handler_return_paths(&mut self) { - for block in &mut self.blocks { - let len = block.instructions.len(); - for i in 0..len { - let Some(Opcode::LoadFastBorrow) = block.instructions[i].instr.real_opcode() else { - continue; - }; + fn deoptimize_borrow_in_targeted_assert_message_blocks(&mut self) { + fn is_assertion_error_load(info: &InstructionInfo) -> bool { + matches!( + info.instr.real(), + Some(Instruction::LoadCommonConstant { idx }) + if idx.get(info.arg) == oparg::CommonConstant::AssertionError + ) + } - let tail = &block.instructions[i + 1..]; - if tail.len() < 3 { - continue; + fn is_direct_call_zero(info: &InstructionInfo) -> bool { + matches!( + info.instr.real(), + Some(Instruction::Call { argc }) if argc.get(info.arg) == 0 + ) + } + + fn has_prior_real_work(block: &Block, start: usize) -> bool { + block.instructions[..start].iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + }) + } + + fn deoptimize_borrow(info: &mut InstructionInfo) { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); } + _ => {} + } + } - if !matches!(tail[0].instr.real(), Some(Instruction::Swap { .. })) { + let target_flags = compute_target_predecessor_flags(&self.blocks); + for (block_idx, block) in self.blocks.iter_mut().enumerate() { + if block_idx == 0 || !target_flags.targeted[block_idx] { + continue; + } + + let mut assert_start = None; + for i in 0..block.instructions.len() { + if is_assertion_error_load(&block.instructions[i]) { + assert_start = Some(i); continue; } - if !matches!(tail[1].instr.real(), Some(Instruction::PopExcept)) { + let Some(start) = assert_start else { + continue; + }; + if !is_direct_call_zero(&block.instructions[i]) { continue; } - - if !matches!(tail[2].instr.real(), Some(Instruction::ReturnValue)) { + if !block.instructions[i + 1..] + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))) + { + assert_start = None; + continue; + } + if has_prior_real_work(block, start) { + assert_start = None; continue; } - block.instructions[i].instr = Opcode::LoadFast.into(); + for info in &mut block.instructions[start + 1..i] { + deoptimize_borrow(info); + } + assert_start = None; } } } - fn deoptimize_borrow_after_multi_handler_resume_join(&mut self) { - fn is_handler_resume_jump_block(block: &Block) -> bool { - let Some(last_info) = block.instructions.last() else { - return false; - }; - if last_info.target == BlockIdx::NULL || !last_info.instr.is_unconditional_jump() { - return false; - } + fn mark_assertion_success_tail_borrow_disabled(&mut self) { + fn starts_with_fast_load(block: &Block) -> bool { block .instructions .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + .find(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .is_some_and(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) + }) } - fn deoptimize_block_borrows(block: &mut Block) { - for info in &mut block.instructions { - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; + fn starts_with_assertion_error(block: &Block) -> bool { + block + .instructions + .iter() + .find(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .is_some_and(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadCommonConstant { idx }) + if idx.get(info.arg) == oparg::CommonConstant::AssertionError + ) + }) + } - info.instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast, - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, - _ => continue, - } - .into(); - } + fn block_contains_assertion_error(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadCommonConstant { idx }) + if idx.get(info.arg) == oparg::CommonConstant::AssertionError + ) + }) } - fn starts_with_conditional_guard(block: &Block) -> bool { - let infos: Vec<_> = block - .instructions - .iter() - .filter(|info| info.instr.real().is_some()) - .take(3) - .collect(); - if infos.len() < 2 { - return false; + fn assert_check_success_target(blocks: &[Block], block: &Block) -> Option { + if block.next != BlockIdx::NULL + && starts_with_assertion_error(&blocks[block.next.idx()]) + { + return block + .instructions + .iter() + .find(|info| is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL) + .map(|info| info.target); } - let starts_with_load_fast = matches!( - infos[0].instr.real_opcode(), - Some(Opcode::LoadFast | Opcode::LoadFastBorrow) - ); - if !starts_with_load_fast { - return false; - } - matches!( - infos.get(1).and_then(|info| info.instr.real_opcode()), - Some( - Opcode::PopJumpIfFalse - | Opcode::PopJumpIfTrue - | Opcode::PopJumpIfNone - | Opcode::PopJumpIfNotNone - ) - ) || (matches!(infos[1].instr.real(), Some(Instruction::ToBool)) - && matches!( - infos.get(2).and_then(|info| info.instr.real_opcode()), + let conditional = block.instructions.iter().enumerate().find(|(_, info)| { + is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL + }); + let (cond_idx, conditional) = conditional?; + let has_inline_assertion_failure = + block.instructions[cond_idx + 1..].iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadCommonConstant { idx }) + if idx.get(info.arg) == oparg::CommonConstant::AssertionError + ) + }) && block.instructions[cond_idx + 1..].iter().any(|info| { + matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. })) + }); + has_inline_assertion_failure.then_some(conditional.target) + } + + fn block_stores_fast(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), Some( - Opcode::PopJumpIfFalse - | Opcode::PopJumpIfTrue - | Opcode::PopJumpIfNone - | Opcode::PopJumpIfNotNone + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } ) - )) + ) + }) } - let mut handler_resume_predecessors = vec![0usize; self.blocks.len()]; - let mut is_handler_resume_block = vec![false; self.blocks.len()]; let mut predecessors = vec![Vec::new(); self.blocks.len()]; - for (block_idx, block) in self.blocks.iter().enumerate() { - if !is_handler_resume_jump_block(block) { - continue; - } - is_handler_resume_block[block_idx] = true; - let target = block - .instructions - .last() - .expect("resume jump block has a last instruction") - .target; - handler_resume_predecessors[target.idx()] += 1; - } for (pred_idx, block) in self.blocks.iter().enumerate() { if block.next != BlockIdx::NULL { predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); @@ -3152,401 +3649,418 @@ impl CodeInfo { } } + let mut assertion_success_tail = vec![false; self.blocks.len()]; + for (tail_idx, tail) in self.blocks.iter().enumerate() { + if tail.cold + || block_is_exceptional(tail) + || !starts_with_fast_load(tail) + || block_contains_assertion_error(tail) + || assert_check_success_target(&self.blocks, tail).is_some() + { + continue; + } + let tail_idx = BlockIdx::new(tail_idx as u32); + for pred in &predecessors[tail_idx.idx()] { + if assert_check_success_target(&self.blocks, &self.blocks[pred.idx()]) + == Some(tail_idx) + && trailing_conditional_jump_index(tail).is_some() + && !block_contains_assertion_error(tail) + { + assertion_success_tail[tail_idx.idx()] = true; + break; + } + for assert_check in &predecessors[pred.idx()] { + if assert_check_success_target(&self.blocks, &self.blocks[assert_check.idx()]) + == Some(*pred) + { + assertion_success_tail[tail_idx.idx()] = true; + break; + } + } + if assertion_success_tail[tail_idx.idx()] { + break; + } + } + } + let mut visited = vec![false; self.blocks.len()]; - for (idx, &count) in handler_resume_predecessors.iter().enumerate() { - if count < 2 { + for (idx, is_assertion_success_tail) in assertion_success_tail.iter().enumerate() { + if !*is_assertion_success_tail { continue; } - let seed = BlockIdx::new(idx as u32); let mut segment = Vec::new(); - let mut cursor = seed; - while cursor != BlockIdx::NULL { - if block_is_exceptional(&self.blocks[cursor.idx()]) { + let mut cursor = BlockIdx::new(idx as u32); + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + let block = &self.blocks[cursor.idx()]; + if block.cold || block_is_exceptional(block) { break; } segment.push(cursor); - cursor = self.blocks[cursor.idx()].next; - } - let has_complex_tail = segment.iter().any(|block_idx| { - self.blocks[block_idx.idx()] - .instructions - .iter() - .any(|info| { - matches!( - info.instr.real(), - Some( - Instruction::ForIter { .. } - | Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - | Instruction::EndFor - | Instruction::PopIter - | Instruction::LoadFastAndClear { .. } - | Instruction::LoadFastCheck { .. } - | Instruction::ListAppend { .. } - | Instruction::MapAdd { .. } - | Instruction::SetAdd { .. } - ) - ) - }) - }); - if starts_with_conditional_guard(&self.blocks[seed.idx()]) && !has_complex_tail { - continue; - } - - let mut in_segment = vec![false; self.blocks.len()]; - for block_idx in &segment { - in_segment[block_idx.idx()] = true; + if block_stores_fast(block) { + segment.clear(); + break; + } + if block + .instructions + .last() + .is_some_and(|info| info.instr.is_scope_exit()) + { + break; + } + cursor = self.blocks[cursor.idx()].next; } - for block_idx in segment { if visited[block_idx.idx()] { continue; } - if block_idx != seed - && predecessors[block_idx.idx()] - .iter() - .any(|pred| !in_segment[pred.idx()] && !is_handler_resume_block[pred.idx()]) - { - continue; - } visited[block_idx.idx()] = true; - deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + self.blocks[block_idx.idx()].disable_load_fast_borrow = true; } } } - fn deoptimize_borrow_after_named_except_cleanup_join(&mut self) { - fn first_real_instr(block: &Block) -> Option { - block.instructions.iter().find_map(|info| info.instr.real()) + fn mark_unprotected_debug_four_tails_borrow_disabled(&mut self) { + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) } - fn leading_bool_guard_local(block: &Block) -> Option { - let infos: Vec<_> = block + fn debug_four_guard_name_load( + block: &Block, + names: &IndexSet, + varnames: &IndexSet, + ) -> bool { + let reals: Vec<_> = block .instructions .iter() - .filter(|info| info.instr.real().is_some()) - .take(3) + .filter(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .take(6) .collect(); - if infos.len() < 3 { - return None; - } - let load_local = match infos[0].instr.real() { - Some(Instruction::LoadFast { var_num }) => usize::from(var_num.get(infos[0].arg)), - Some(Instruction::LoadFastBorrow { var_num }) => { - usize::from(var_num.get(infos[0].arg)) - } - _ => return None, - }; - if !matches!(infos[1].instr.real(), Some(Instruction::ToBool)) { - return None; + if reals.len() < 5 { + return false; } - if !matches!( - infos[2].instr.real_opcode(), + let loads_imap_fast = match reals[0].instr.real() { Some( - Opcode::PopJumpIfFalse - | Opcode::PopJumpIfTrue - | Opcode::PopJumpIfNone - | Opcode::PopJumpIfNotNone + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num }, + ) => varnames + .get_index(usize::from(var_num.get(reals[0].arg))) + .is_some_and(|name| name.as_str() == "imap"), + _ => false, + }; + let loads_debug_attr = matches!( + reals[1].instr.real(), + Some(Instruction::LoadAttr { namei }) + if names[usize::try_from(namei.get(reals[1].arg).name_idx()).unwrap()].as_str() + == "debug" + ); + let compares_with_four = reals.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadSmallInt { i }) if i.get(info.arg) == 4 ) - ) { - return None; - } - Some(load_local) + }) && reals + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::CompareOp { .. }))); + let has_conditional = reals.iter().any(|info| is_conditional_jump(&info.instr)); + loads_imap_fast && loads_debug_attr && compares_with_four && has_conditional } - fn deoptimize_block_borrows(block: &mut Block) { - for info in &mut block.instructions { - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; + fn block_has_jump_back_predecessor_to( + blocks: &[Block], + predecessors: &[Vec], + target: BlockIdx, + ) -> bool { + predecessors[target.idx()].iter().any(|pred| { + blocks[pred.idx()].instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) + }) + } - info.instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast, - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, - _ => continue, - } - .into(); - } + fn block_has_mesg_call(block: &Block, names: &IndexSet) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) + if names[usize::try_from(namei.get(info.arg).name_idx()).unwrap()].as_str() + == "_mesg" + ) + }) && block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) } - fn normal_successors(block: &Block) -> Vec { - let Some(last_info) = block.instructions.last() else { - return (block.next != BlockIdx::NULL) - .then_some(block.next) - .into_iter() - .collect(); - }; - if let Some(cond_idx) = trailing_conditional_jump_index(block) { - let mut successors = Vec::with_capacity(2); - let target = block.instructions[cond_idx].target; - if target != BlockIdx::NULL { - successors.push(target); - } - if block.next != BlockIdx::NULL { - successors.push(block.next); + fn normal_successors( + blocks: &[Block], + predecessors: &[Vec], + block_idx: BlockIdx, + ) -> Vec { + let block = &blocks[block_idx.idx()]; + let mut successors = Vec::new(); + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + successors.push(block.next); + } + let is_loop_header = + block_has_jump_back_predecessor_to(blocks, predecessors, block_idx); + for info in &block.instructions { + if info.target != BlockIdx::NULL && !is_loop_header { + successors.push(info.target); } - return successors; - } - if last_info.instr.is_scope_exit() { - return Vec::new(); - } - if last_info.instr.is_unconditional_jump() { - return (last_info.target != BlockIdx::NULL) - .then_some(last_info.target) - .into_iter() - .collect(); } - (block.next != BlockIdx::NULL) - .then_some(block.next) - .into_iter() - .collect() + successors } - fn path_reaches_named_cleanup( - blocks: &[Block], - start: BlockIdx, - cleanup: BlockIdx, - resume_target: BlockIdx, - ) -> bool { - if start == BlockIdx::NULL || start == resume_target { - return false; + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); } - let mut visited = vec![false; blocks.len()]; - let mut stack = vec![start]; - while let Some(block_idx) = stack.pop() { - if block_idx == BlockIdx::NULL - || block_idx == resume_target - || visited[block_idx.idx()] - { - continue; - } - if block_idx == cleanup { - return true; - } - visited[block_idx.idx()] = true; - for successor in normal_successors(&blocks[block_idx.idx()]) { - stack.push(successor); + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); } } - false } - fn path_reaches_explicit_raise( - blocks: &[Block], - start: BlockIdx, - cleanup: BlockIdx, - resume_target: BlockIdx, - ) -> bool { - if start == BlockIdx::NULL || start == cleanup || start == resume_target { - return false; + let mut to_disable = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + let has_protected_predecessor = predecessors[idx] + .iter() + .any(|pred| block_has_protected_instructions(&self.blocks[pred.idx()])); + if block.cold + || block_is_exceptional(block) + || block_has_protected_instructions(block) + || has_protected_predecessor + || !debug_four_guard_name_load(block, &self.metadata.names, &self.metadata.varnames) + { + continue; } - let mut visited = vec![false; blocks.len()]; - let mut stack = vec![start]; - while let Some(block_idx) = stack.pop() { - if block_idx == BlockIdx::NULL - || block_idx == cleanup - || block_idx == resume_target - || visited[block_idx.idx()] - { + let block_idx = BlockIdx::new(idx as u32); + to_disable.push(block_idx); + let mut seen = vec![false; self.blocks.len()]; + let mut stack = normal_successors(&self.blocks, &predecessors, block_idx); + while let Some(successor) = stack.pop() { + if successor == BlockIdx::NULL || seen[successor.idx()] { continue; } - let block = &blocks[block_idx.idx()]; - if block - .instructions - .iter() - .any(|info| matches!(info.instr.real_opcode(), Some(Opcode::RaiseVarargs))) + seen[successor.idx()] = true; + let successor_block = &self.blocks[successor.idx()]; + if successor_block.cold || block_is_exceptional(successor_block) { + continue; + } + to_disable.push(successor); + if block_has_protected_instructions(successor_block) + || successor_block + .instructions + .last() + .is_some_and(|info| info.instr.is_scope_exit()) + || successor_block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) { - return true; + continue; } - visited[block_idx.idx()] = true; - for successor in normal_successors(block) { - stack.push(successor); + if block_has_mesg_call(successor_block, &self.metadata.names) { + let has_loop_successor = + normal_successors(&self.blocks, &predecessors, successor) + .into_iter() + .any(|next| { + next != BlockIdx::NULL + && block_has_jump_back_predecessor_to( + &self.blocks, + &predecessors, + next, + ) + }); + if !has_loop_successor { + continue; + } + } + for next in normal_successors(&self.blocks, &predecessors, successor) { + stack.push(next); } } - false } + to_disable.sort_by_key(|idx| idx.idx()); + to_disable.dedup(); + for block_idx in to_disable { + self.blocks[block_idx.idx()].disable_load_fast_borrow = true; + } + } - fn named_cleanup_has_conditional_raise_sibling( - blocks: &[Block], - cleanup: BlockIdx, - resume_target: BlockIdx, - ) -> bool { - for block in blocks { - let Some(cond_idx) = trailing_conditional_jump_index(block) else { + fn deoptimize_borrow_for_handler_return_paths(&mut self) { + for block in &mut self.blocks { + let len = block.instructions.len(); + for i in 0..len { + let Some(Instruction::LoadFastBorrow { .. }) = block.instructions[i].instr.real() + else { continue; }; - let jump_target = block.instructions[cond_idx].target; - let fallthrough = block.next; - if jump_target == BlockIdx::NULL || fallthrough == BlockIdx::NULL { + let tail = &block.instructions[i + 1..]; + if tail.len() < 3 { continue; } - - let jump_reaches_cleanup = - path_reaches_named_cleanup(blocks, jump_target, cleanup, resume_target); - let fallthrough_reaches_cleanup = - path_reaches_named_cleanup(blocks, fallthrough, cleanup, resume_target); - if jump_reaches_cleanup == fallthrough_reaches_cleanup { + if !matches!(tail[0].instr.real(), Some(Instruction::Swap { .. })) { continue; } - - let sibling = if jump_reaches_cleanup { - fallthrough - } else { - jump_target - }; - if path_reaches_explicit_raise(blocks, sibling, cleanup, resume_target) { - return true; + if !matches!(tail[1].instr.real(), Some(Instruction::PopExcept)) { + continue; } - } - false - } - - let mut named_cleanup_predecessors = vec![0usize; self.blocks.len()]; - let mut named_cleanup_requires_deopt = vec![false; self.blocks.len()]; - let mut is_allowed_cleanup_resume_block = vec![false; self.blocks.len()]; - let mut predecessors = vec![Vec::new(); self.blocks.len()]; - - for (block_idx, block) in self.blocks.iter().enumerate() { - let Some(last_info) = block.instructions.last() else { - continue; - }; - if last_info.target == BlockIdx::NULL || !last_info.instr.is_unconditional_jump() { - continue; - } - if block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) - { - is_allowed_cleanup_resume_block[block_idx] = true; - } - if !is_named_except_cleanup_normal_exit_block(block) { - continue; - } - if matches!( - first_real_instr(&self.blocks[last_info.target.idx()]), - Some(Instruction::ForIter { .. }) - ) { - continue; - } - named_cleanup_predecessors[last_info.target.idx()] += 1; - if named_cleanup_has_conditional_raise_sibling( - &self.blocks, - BlockIdx::new(block_idx as u32), - last_info.target, - ) { - named_cleanup_requires_deopt[last_info.target.idx()] = true; - } - } - for (pred_idx, block) in self.blocks.iter().enumerate() { - if block.next != BlockIdx::NULL { - predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); - } - for info in &block.instructions { - if info.target != BlockIdx::NULL { - predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + if !matches!(tail[2].instr.real(), Some(Instruction::ReturnValue)) { + continue; } + block.instructions[i].instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); } } + } - let mut visited = vec![false; self.blocks.len()]; - for (idx, &count) in named_cleanup_predecessors.iter().enumerate() { - if count == 0 { - continue; - } - let seed = BlockIdx::new(idx as u32); - let mut segment = Vec::new(); - let mut cursor = seed; - let mut fallback_guard_local = None; - while cursor != BlockIdx::NULL { - let block = &self.blocks[cursor.idx()]; - if block_is_exceptional(block) { - break; - } - if cursor != seed - && let Some(local) = leading_bool_guard_local(block) - { - match fallback_guard_local { - None => fallback_guard_local = Some(local), - Some(expected) if expected != local => break, - Some(_) => {} + fn deoptimize_borrow_after_generator_exception_return(&mut self) { + if !self.flags.contains(CodeFlags::GENERATOR) { + return; + } + + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); } + _ => {} } - segment.push(cursor); - cursor = block.next; - } - if fallback_guard_local.is_none() && !named_cleanup_requires_deopt[idx] { - continue; - } - - let mut in_segment = vec![false; self.blocks.len()]; - for block_idx in &segment { - in_segment[block_idx.idx()] = true; } + } - for block_idx in segment { - if visited[block_idx.idx()] { + fn handler_checks_exception(blocks: &[Block], handler: BlockIdx) -> bool { + let mut stack = vec![handler]; + let mut visited = vec![false; blocks.len()]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL { continue; } - let is_same_guard_fallback = fallback_guard_local.is_some_and(|local| { - leading_bool_guard_local(&self.blocks[block_idx.idx()]) == Some(local) - }); - if block_idx != seed - && !is_same_guard_fallback - && predecessors[block_idx.idx()].iter().any(|pred| { - !in_segment[pred.idx()] && !is_allowed_cleanup_resume_block[pred.idx()] - }) - { + let idx = block_idx.idx(); + if visited[idx] { continue; } - visited[block_idx.idx()] = true; - deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); - } - } - } + visited[idx] = true; - fn deoptimize_borrow_in_protected_conditional_tail(&mut self) { - fn second_last_real_instr(block: &Block) -> Option { - let mut reals = block - .instructions - .iter() - .rev() - .filter_map(|info| info.instr.real()); - let _last = reals.next()?; - reals.next() + let block = &blocks[idx]; + let mut can_fallthrough = true; + for info in &block.instructions { + if matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) { + return true; + } + if info.target != BlockIdx::NULL { + stack.push(info.target); + } + if info.instr.is_scope_exit() || info.instr.is_unconditional_jump() { + can_fallthrough = false; + break; + } + } + if can_fallthrough && block.next != BlockIdx::NULL { + stack.push(block.next); + } + } + false } - fn deoptimize_block_borrows(block: &mut Block) { - for info in &mut block.instructions { - let Some(opcode) = info.instr.real_opcode() else { + fn handler_returns_without_yield(blocks: &[Block], handler: BlockIdx) -> bool { + let mut stack = vec![(handler, false)]; + let mut visited = vec![[false; 2]; blocks.len()]; + while let Some((block_idx, mut saw_yield)) = stack.pop() { + if block_idx == BlockIdx::NULL { continue; - }; + } + let idx = block_idx.idx(); + let yield_idx = usize::from(saw_yield); + if visited[idx][yield_idx] { + continue; + } + visited[idx][yield_idx] = true; - info.instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast, - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, - _ => continue, + let block = &blocks[idx]; + let handler_resume_jump = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + && block.instructions.last().is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }); + let mut can_fallthrough = true; + for info in &block.instructions { + if matches!(info.instr.real(), Some(Instruction::YieldValue { .. })) { + saw_yield = true; + } + if matches!(info.instr.real(), Some(Instruction::ReturnValue)) { + if !saw_yield { + return true; + } + can_fallthrough = false; + break; + } + if info.target != BlockIdx::NULL + && !(handler_resume_jump && info.instr.is_unconditional_jump()) + { + stack.push((info.target, saw_yield)); + } + if info.instr.is_scope_exit() || info.instr.is_unconditional_jump() { + can_fallthrough = false; + break; + } + } + if can_fallthrough && block.next != BlockIdx::NULL { + stack.push((block.next, saw_yield)); } - .into(); } + false } - let mut predecessors = vec![Vec::new(); self.blocks.len()]; - let mut is_handler_resume_block = vec![false; self.blocks.len()]; - for (pred_idx, block) in self.blocks.iter().enumerate() { - if matches!(second_last_real_instr(block), Some(Instruction::PopExcept)) - && block.instructions.last().is_some_and(|info| { - info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() - }) + let mut returning_handler = vec![false; self.blocks.len()]; + for block in &self.blocks { + for handler in block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) { - is_handler_resume_block[pred_idx] = true; - } - if block.next != BlockIdx::NULL { - predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); - } - for info in &block.instructions { - if info.target != BlockIdx::NULL { - predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + if !returning_handler[handler.idx()] { + returning_handler[handler.idx()] = + handler_checks_exception(&self.blocks, handler) + && handler_returns_without_yield(&self.blocks, handler); } } } @@ -3556,428 +4070,355 @@ impl CodeInfo { .iter() .enumerate() .filter_map(|(idx, block)| { - let prev_protected = predecessors[idx].iter().any(|pred| { - self.blocks[pred.idx()] - .instructions - .iter() - .any(|info| info.except_handler.is_some()) + if block_is_exceptional(block) || block.cold { + return None; + } + let seed = BlockIdx::new(idx as u32); + let prev_protected_return = self.blocks.iter().any(|pred| { + pred.next == seed + && pred.instructions.iter().any(|info| { + info.except_handler.is_some_and(|handler| { + returning_handler[handler.handler_block.idx()] + }) + }) }); - (!block_is_exceptional(block) - && trailing_conditional_jump_index(block).is_some() - && prev_protected - && block - .instructions - .iter() - .any(|info| matches!(info.instr.real_opcode(), Some(Opcode::Call)))) - .then_some(BlockIdx::new(idx as u32)) + prev_protected_return.then_some(seed) }) .collect(); let mut visited = vec![false; self.blocks.len()]; for seed in seeds { - let mut segment = Vec::new(); let mut cursor = seed; while cursor != BlockIdx::NULL { - if block_is_exceptional(&self.blocks[cursor.idx()]) { + let idx = cursor.idx(); + if visited[idx] || block_is_exceptional(&self.blocks[idx]) || self.blocks[idx].cold + { break; } - segment.push(cursor); - cursor = self.blocks[cursor.idx()].next; + visited[idx] = true; + deoptimize_block_borrows(&mut self.blocks[idx]); + cursor = self.blocks[idx].next; } + } + } - let segment_ops: Vec<_> = segment - .iter() - .flat_map(|block_idx| { - self.blocks[block_idx.idx()] - .instructions - .iter() - .filter_map(|info| info.instr.real()) - }) - .collect(); - let call_count = segment_ops - .iter() - .filter(|instr| matches!(instr, Instruction::Call { .. })) - .count(); - let raise_count = segment_ops - .iter() - .filter(|instr| matches!(instr, Instruction::RaiseVarargs { .. })) - .count(); - let return_count = segment_ops - .iter() - .filter(|instr| matches!(instr, Instruction::ReturnValue)) - .count(); - let conditional_count = segment_ops - .iter() - .filter(|instr| { - matches!( - instr, - Instruction::PopJumpIfFalse { .. } - | Instruction::PopJumpIfTrue { .. } - | Instruction::PopJumpIfNone { .. } - | Instruction::PopJumpIfNotNone { .. } - ) - }) - .count(); - let has_complex_tail = segment_ops.iter().any(|instr| { - matches!( - instr, - Instruction::StoreFast { .. } - | Instruction::StoreFastLoadFast { .. } - | Instruction::StoreFastStoreFast { .. } - | Instruction::ForIter { .. } - | Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - | Instruction::EndFor - | Instruction::PopIter - | Instruction::LoadFastAndClear { .. } - | Instruction::LoadFastCheck { .. } - | Instruction::ListAppend { .. } - | Instruction::MapAdd { .. } - | Instruction::SetAdd { .. } - ) - }); - if has_complex_tail - || call_count != 2 - || raise_count != 1 - || return_count != 1 - || conditional_count != 1 - { - continue; + fn deoptimize_borrow_after_async_for_cleanup_resume(&mut self) { + fn deoptimize_block_borrows_from(block: &mut Block, start: usize) { + for info in block.instructions.iter_mut().skip(start) { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } } + } - let mut in_segment = vec![false; self.blocks.len()]; - for block_idx in &segment { - in_segment[block_idx.idx()] = true; - } - - for block_idx in segment { - if visited[block_idx.idx()] { + let mut same_block_starts = Vec::new(); + let mut seeds = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + for (instr_idx, info) in block.instructions.iter().enumerate() { + if !matches!(info.instr.real(), Some(Instruction::EndAsyncFor)) { continue; } - if block_idx != seed - && predecessors[block_idx.idx()] - .iter() - .any(|pred| !in_segment[pred.idx()] && !is_handler_resume_block[pred.idx()]) + if block.instructions[instr_idx + 1..] + .iter() + .any(|info| info.instr.real().is_some()) { - continue; + same_block_starts.push((BlockIdx::new(idx as u32), instr_idx + 1)); + } else if block.next != BlockIdx::NULL { + let next = &self.blocks[block.next.idx()]; + let seed = next + .instructions + .last() + .filter(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) + .map_or(block.next, |info| info.target); + seeds.push(seed); } - visited[block_idx.idx()] = true; - deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); } } - } - - fn deoptimize_borrow_for_folded_nonliteral_exprs(&mut self) { - for block in &mut self.blocks { - for info in &mut block.instructions { - if !info.folded_from_nonliteral_expr { - continue; - } - - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; - info.instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast, - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, - _ => continue, - } - .into(); + for (block_idx, start) in same_block_starts { + if !block_is_exceptional(&self.blocks[block_idx.idx()]) { + deoptimize_block_borrows_from(&mut self.blocks[block_idx.idx()], start); + } + } + for seed in seeds { + if seed != BlockIdx::NULL && !block_is_exceptional(&self.blocks[seed.idx()]) { + deoptimize_block_borrows_from(&mut self.blocks[seed.idx()], 0); } } } - fn deoptimize_borrow_after_push_exc_info(&mut self) { - for block in &mut self.blocks { - let mut in_exception_state = false; + fn deoptimize_borrow_after_deoptimized_async_with_enter(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { for info in &mut block.instructions { - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; - - match opcode { - Opcode::PushExcInfo => { - in_exception_state = true; - } - Opcode::PopExcept | Opcode::Reraise => { - in_exception_state = false; - } - Opcode::LoadFastBorrow if in_exception_state => { - info.instr = Opcode::LoadFast.into(); + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); } - Opcode::LoadFastBorrowLoadFastBorrow if in_exception_state => { - info.instr = Opcode::LoadFastLoadFast.into(); + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); } _ => {} } } } - } - - fn deoptimize_borrow_after_protected_import(&mut self) { - fn deoptimize_block_borrows(block: &mut Block, after_import_only: bool) { - let mut after_import = !after_import_only; - for info in &mut block.instructions { - if matches!(info.instr.real_opcode(), Some(Opcode::ImportName)) { - after_import = true; - continue; - } - - if !after_import { - continue; - } - - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; - - info.instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast, - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, - _ => continue, - } - .into(); - } - } - fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { - let has_pop_except = block + fn block_has_deoptimized_async_with_enter(block: &Block) -> bool { + let has_async_enter = block .instructions .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); - let jumps_to_target = block.instructions.iter().any(|info| { - info.target == target - && matches!( - info.instr.real_opcode(), - Some( - Opcode::JumpForward - | Opcode::JumpBackward - | Opcode::JumpBackwardNoInterrupt - ) - ) + .any(|info| match info.instr.real() { + Some(Instruction::LoadSpecial { method }) => { + method.get(info.arg) == oparg::SpecialMethod::AEnter + } + _ => false, + }); + let has_strong_fast = block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. }) + ) }); - has_pop_except && jumps_to_target + has_async_enter && has_strong_fast } - let mut predecessors = vec![Vec::new(); self.blocks.len()]; - for (pred_idx, block) in self.blocks.iter().enumerate() { - if block.next != BlockIdx::NULL { - predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); - } + fn send_targets(block: &Block) -> impl Iterator + '_ { + block.instructions.iter().filter_map(|info| { + matches!(info.instr.real(), Some(Instruction::Send { .. })) + .then_some(info.target) + .filter(|target| *target != BlockIdx::NULL) + }) + } + + fn block_calls_before_raise(block: &Block) -> bool { + let mut saw_call = false; for info in &block.instructions { - if info.target != BlockIdx::NULL { - predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + match info.instr.real() { + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx, + ) => saw_call = true, + Some(Instruction::RaiseVarargs { .. }) => return saw_call, + _ => {} } } + false } - let seeds: Vec<_> = self - .blocks - .iter() - .enumerate() - .filter_map(|(idx, block)| { - (!block_is_exceptional(block) - && block - .instructions - .iter() - .any(|info| info.except_handler.is_some()) - && block.instructions.iter().any(|info| { - matches!(info.instr.real(), Some(Instruction::ImportName { .. })) - })) - .then_some(BlockIdx::new(idx as u32)) - }) - .collect(); + let mut seeds = Vec::new(); + for block in &self.blocks { + if !block_has_deoptimized_async_with_enter(block) { + continue; + } + seeds.extend(send_targets(block)); + if block.next != BlockIdx::NULL { + seeds.extend(send_targets(&self.blocks[block.next.idx()])); + } + } - let mut visited = vec![false; self.blocks.len()]; for seed in seeds { - let mut seed_handler_chain = vec![false; self.blocks.len()]; - let seed_handler_blocks: Vec<_> = self.blocks[seed.idx()] - .instructions - .iter() - .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) - .collect(); - for handler_block in seed_handler_blocks { - let mut cursor = handler_block; - while cursor != BlockIdx::NULL && !seed_handler_chain[cursor.idx()] { - seed_handler_chain[cursor.idx()] = true; - cursor = self.blocks[cursor.idx()].next; - } + if !block_is_exceptional(&self.blocks[seed.idx()]) + && block_calls_before_raise(&self.blocks[seed.idx()]) + { + deoptimize_block_borrows(&mut self.blocks[seed.idx()]); } + } + } - let mut in_segment = vec![false; self.blocks.len()]; - in_segment[seed.idx()] = true; - let mut segment = vec![seed]; - let mut cursor = self.blocks[seed.idx()].next; - while cursor != BlockIdx::NULL && !block_is_exceptional(&self.blocks[cursor.idx()]) { - if predecessors[cursor.idx()].iter().any(|pred| { - !in_segment[pred.idx()] - && seed_handler_chain[pred.idx()] - && is_handler_resume_predecessor(&self.blocks[pred.idx()], cursor) - }) { - break; - } - in_segment[cursor.idx()] = true; - segment.push(cursor); - cursor = self.blocks[cursor.idx()].next; - } + fn deoptimize_borrow_after_multi_handler_resume_join(&mut self) { + fn first_real_instr(block: &Block) -> Option { + block.instructions.iter().find_map(|info| info.instr.real()) + } - for (i, block_idx) in segment.into_iter().enumerate() { - if visited[block_idx.idx()] { - continue; - } - visited[block_idx.idx()] = true; - deoptimize_block_borrows(&mut self.blocks[block_idx.idx()], i == 0); + fn is_handler_resume_jump_block(block: &Block) -> bool { + let Some(last_info) = block.instructions.last() else { + return false; + }; + if last_info.target == BlockIdx::NULL || !last_info.instr.is_unconditional_jump() { + return false; } + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) } - } - fn deoptimize_borrow_after_protected_store_tail(&mut self) { - fn deoptimize_block_borrows_from(block: &mut Block, start: usize) { - for info in block.instructions.iter_mut().skip(start) { - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; + fn is_with_suppress_resume_jump_block(block: &Block) -> bool { + if !is_handler_resume_jump_block(block) { + return false; + } - info.instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast, - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, - _ => continue, + let mut saw_pop_except = false; + let mut pop_top_after_pop_except = 0usize; + for info in &block.instructions { + match info.instr.real() { + Some(Instruction::PopExcept) => saw_pop_except = true, + Some(Instruction::PopTop) if saw_pop_except => { + pop_top_after_pop_except += 1; + } + _ => {} } - .into(); } + saw_pop_except && pop_top_after_pop_except >= 3 } - fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { - let has_pop_except = block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); - let jumps_to_target = block.instructions.iter().any(|info| { - info.target == target - && matches!( - info.instr.real_opcode(), - Some( - Opcode::JumpForward - | Opcode::JumpBackward - | Opcode::JumpBackwardNoInterrupt - ) - ) - }); - has_pop_except && jumps_to_target + fn block_has_check_exc_match(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) } - fn handler_chain_can_resume_to_segment( + fn predecessor_chain_has_check_exc_match( blocks: &[Block], - block: &Block, - in_segment: &[bool], + predecessors: &[Vec], + start: BlockIdx, + stop: BlockIdx, ) -> bool { let mut visited = vec![false; blocks.len()]; - let handler_blocks: Vec<_> = block - .instructions - .iter() - .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) - .collect(); - for handler_block in handler_blocks { - let mut cursor = handler_block; - while cursor != BlockIdx::NULL && !visited[cursor.idx()] { - visited[cursor.idx()] = true; - let handler = &blocks[cursor.idx()]; - let mut after_pop_except = false; - for info in &handler.instructions { - if matches!(info.instr.real(), Some(Instruction::PopExcept)) { - after_pop_except = true; - continue; - } - if after_pop_except - && info.target != BlockIdx::NULL - && in_segment[info.target.idx()] - && matches!( - info.instr.real_opcode(), - Some( - Opcode::JumpForward - | Opcode::JumpBackward - | Opcode::JumpBackwardNoInterrupt - ) - ) - { - return true; - } - } - cursor = handler.next; + let mut stack = vec![start]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || cursor == stop || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + if block_has_check_exc_match(&blocks[cursor.idx()]) { + return true; + } + for pred in &predecessors[cursor.idx()] { + stack.push(*pred); } } false } - fn block_has_tail_deopt_trigger_from(block: &Block, start: usize) -> bool { - block.instructions.iter().skip(start).any(|info| { - matches!( - info.instr.real_opcode(), - Some(Opcode::Call | Opcode::CallKw | Opcode::StoreAttr) - ) + fn has_exception_match_resume_predecessor( + blocks: &[Block], + predecessors: &[Vec], + target: BlockIdx, + ) -> bool { + predecessors[target.idx()].iter().any(|pred| { + let block = &blocks[pred.idx()]; + is_handler_resume_jump_block(block) + && !is_with_suppress_resume_jump_block(block) + && predecessor_chain_has_check_exc_match(blocks, predecessors, *pred, target) }) } - fn block_has_protected_instructions(block: &Block) -> bool { - block + fn starts_with_fast_attr_call(block: &Block) -> bool { + let infos: Vec<_> = block .instructions .iter() - .any(|info| info.except_handler.is_some()) + .filter(|info| info.instr.real().is_some()) + .take(2) + .collect(); + matches!( + infos.as_slice(), + [ + first, + second, + .. + ] if matches!( + first.instr.real(), + Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) + ) && matches!(second.instr.real(), Some(Instruction::LoadAttr { .. })) + ) } - fn first_unprotected_suffix(block: &Block) -> Option { - let mut saw_protected = false; - for (idx, info) in block.instructions.iter().enumerate() { - if info.except_handler.is_some() { - saw_protected = true; - } else if saw_protected { - return Some(idx); + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} } } - None } - fn collect_stored_fast_locals_until(block: &Block, end: usize) -> Vec { - let mut locals = Vec::new(); - for info in block.instructions.iter().take(end) { - match info.instr.real() { - Some(Instruction::StoreFast { var_num }) => { - locals.push(usize::from(var_num.get(info.arg))); - } - Some(Instruction::StoreFastLoadFast { var_nums }) => { - let (store_idx, _) = var_nums.get(info.arg).indexes(); - locals.push(usize::from(store_idx)); - } - Some(Instruction::StoreFastStoreFast { var_nums }) => { - let (idx1, idx2) = var_nums.get(info.arg).indexes(); - locals.push(usize::from(idx1)); - locals.push(usize::from(idx2)); - } - _ => {} - } - } - locals - } - - fn borrows_any_local_from(block: &Block, locals: &[usize], start: usize) -> bool { - block + fn starts_with_conditional_guard(block: &Block) -> bool { + let infos: Vec<_> = block .instructions .iter() - .skip(start) - .any(|info| match info.instr.real() { - Some(Instruction::LoadFastBorrow { var_num }) => { - locals.contains(&usize::from(var_num.get(info.arg))) - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { var_nums }) => { - let (idx1, idx2) = var_nums.get(info.arg).indexes(); - locals.contains(&usize::from(idx1)) || locals.contains(&usize::from(idx2)) - } - _ => false, - }) + .filter(|info| info.instr.real().is_some()) + .take(3) + .collect(); + if infos.len() < 2 { + return false; + } + let starts_with_load_fast = matches!( + infos[0].instr.real(), + Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) + ); + if !starts_with_load_fast { + return false; + } + matches!( + infos.get(1).and_then(|info| info.instr.real()), + Some( + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + ) + ) || (matches!(infos[1].instr.real(), Some(Instruction::ToBool)) + && matches!( + infos.get(2).and_then(|info| info.instr.real()), + Some( + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + ) + )) } + let mut handler_resume_predecessors = vec![0usize; self.blocks.len()]; + let mut is_handler_resume_block = vec![false; self.blocks.len()]; let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (block_idx, block) in self.blocks.iter().enumerate() { + if !is_handler_resume_jump_block(block) { + continue; + } + is_handler_resume_block[block_idx] = true; + let target = block + .instructions + .last() + .expect("resume jump block has a last instruction") + .target; + handler_resume_predecessors[target.idx()] += 1; + } for (pred_idx, block) in self.blocks.iter().enumerate() { if block.next != BlockIdx::NULL { predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); @@ -3989,127 +4430,3828 @@ impl CodeInfo { } } - let mut to_deopt = Vec::new(); - for block in &self.blocks { - if block_is_exceptional(block) - || !block - .instructions - .iter() - .any(|info| info.except_handler.is_some()) - || !block.instructions.iter().any(|info| { - matches!( - info.instr.real_opcode(), - Some(Opcode::Call | Opcode::CallKw) - ) - }) - || !block_has_exception_match_handler(&self.blocks, block) - { - continue; - } - let same_block_tail_start = first_unprotected_suffix(block); - if same_block_tail_start.is_some() { + let mut visited = vec![false; self.blocks.len()]; + for (idx, &count) in handler_resume_predecessors.iter().enumerate() { + if count < 2 { continue; } - let stored_locals = collect_stored_fast_locals_until(block, block.instructions.len()); - if stored_locals.is_empty() { + let seed = BlockIdx::new(idx as u32); + if matches!( + first_real_instr(&self.blocks[seed.idx()]), + Some(Instruction::ForIter { .. }) + ) { continue; } - let mut in_segment = vec![false; self.blocks.len()]; let mut segment = Vec::new(); - let mut cursor = { - let tail = next_nonempty_block(&self.blocks, block.next); - if tail == BlockIdx::NULL - || block_is_exceptional(&self.blocks[tail.idx()]) - || block_has_protected_instructions(&self.blocks[tail.idx()]) - { - continue; - } - tail - }; + let mut cursor = seed; while cursor != BlockIdx::NULL { - let segment_block = &self.blocks[cursor.idx()]; - if block_is_exceptional(segment_block) - || block_has_protected_instructions(segment_block) - { + if block_is_exceptional(&self.blocks[cursor.idx()]) { break; } - segment.push((cursor, 0)); - in_segment[cursor.idx()] = true; - let last_real = segment_block + segment.push(cursor); + cursor = self.blocks[cursor.idx()].next; + } + let has_complex_tail = segment.iter().any(|block_idx| { + self.blocks[block_idx.idx()] .instructions .iter() - .rev() - .find_map(|info| info.instr.real()); - if last_real.is_some_and(|instr| { - instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() - }) { - break; - } - cursor = next_nonempty_block(&self.blocks, segment_block.next); - } - if segment.is_empty() - || !segment.iter().any(|(block_idx, start)| { - block_has_tail_deopt_trigger_from(&self.blocks[block_idx.idx()], *start) - }) - || handler_chain_can_resume_to_segment(&self.blocks, block, &in_segment) - || !segment.iter().any(|(block_idx, start)| { - borrows_any_local_from(&self.blocks[block_idx.idx()], &stored_locals, *start) - }) + .any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::ForIter { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::EndFor + | Instruction::PopIter + | Instruction::LoadFastAndClear { .. } + | Instruction::LoadFastCheck { .. } + | Instruction::ListAppend { .. } + | Instruction::MapAdd { .. } + | Instruction::SetAdd { .. } + ) + ) + }) + }); + let has_simple_with_except_resume_tail = + starts_with_fast_attr_call(&self.blocks[seed.idx()]) + && has_exception_match_resume_predecessor(&self.blocks, &predecessors, seed) + && predecessors[seed.idx()] + .iter() + .any(|pred| is_with_suppress_resume_jump_block(&self.blocks[pred.idx()])); + if !(starts_with_conditional_guard(&self.blocks[seed.idx()]) && has_complex_tail + || has_simple_with_except_resume_tail) { continue; } - for (block_idx, start) in segment { - if predecessors[block_idx.idx()] - .iter() - .any(|pred| is_handler_resume_predecessor(&self.blocks[pred.idx()], block_idx)) + + let mut in_segment = vec![false; self.blocks.len()]; + for block_idx in &segment { + in_segment[block_idx.idx()] = true; + } + + for block_idx in segment { + if visited[block_idx.idx()] { + continue; + } + if block_idx != seed + && predecessors[block_idx.idx()].iter().any(|pred| { + !in_segment[pred.idx()] + && !is_handler_resume_block[pred.idx()] + && self.blocks[pred.idx()] + .instructions + .iter() + .any(|info| info.instr.real().is_some()) + }) { continue; } - to_deopt.push((block_idx, start)); + visited[block_idx.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); } } + } - let mut continue_targets = Vec::new(); - for (handler_idx, block) in self.blocks.iter().enumerate() { - if !block.cold - || !block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::CheckExcMatch))) - { - continue; + fn deoptimize_borrow_after_named_except_cleanup_join(&mut self) { + fn first_real_instr(block: &Block) -> Option { + block.instructions.iter().find_map(|info| info.instr.real()) + } + + fn leading_bool_guard_local(block: &Block) -> Option { + let infos: Vec<_> = block + .instructions + .iter() + .filter(|info| info.instr.real().is_some()) + .take(3) + .collect(); + if infos.len() < 3 { + return None; } - let mut visited = vec![false; self.blocks.len()]; - let mut cursor = BlockIdx::new(handler_idx as u32); - while cursor != BlockIdx::NULL && !visited[cursor.idx()] { - visited[cursor.idx()] = true; - let handler = &self.blocks[cursor.idx()]; - let has_pop_except = handler - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); - if has_pop_except { - for info in &handler.instructions { - if info.target != BlockIdx::NULL - && matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) - { - continue_targets.push(info.target); + let load_local = match infos[0].instr.real() { + Some(Instruction::LoadFast { var_num }) => usize::from(var_num.get(infos[0].arg)), + Some(Instruction::LoadFastBorrow { var_num }) => { + usize::from(var_num.get(infos[0].arg)) + } + _ => return None, + }; + if !matches!(infos[1].instr.real(), Some(Instruction::ToBool)) { + return None; + } + if !matches!( + infos[2].instr.real(), + Some( + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + ) + ) { + return None; + } + Some(load_local) + } + + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), } + .into(); } + _ => {} } - cursor = handler.next; } } - continue_targets.sort_by_key(|idx| idx.idx()); - continue_targets.dedup(); - for target in continue_targets { + fn normal_successors(block: &Block) -> Vec { + let Some(last_info) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL && !successors.contains(&block.next) { + successors.push(block.next); + } + return successors; + } + if last_info.instr.is_scope_exit() { + return Vec::new(); + } + if last_info.instr.is_unconditional_jump() { + return (last_info.target != BlockIdx::NULL) + .then_some(last_info.target) + .into_iter() + .collect(); + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + fn path_reaches_named_cleanup( + blocks: &[Block], + start: BlockIdx, + cleanup: BlockIdx, + resume_target: BlockIdx, + ) -> bool { + if start == BlockIdx::NULL || start == resume_target { + return false; + } + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL + || block_idx == resume_target + || visited[block_idx.idx()] + { + continue; + } + if block_idx == cleanup { + return true; + } + visited[block_idx.idx()] = true; + for successor in normal_successors(&blocks[block_idx.idx()]) { + stack.push(successor); + } + } + false + } + + fn path_reaches_explicit_raise( + blocks: &[Block], + start: BlockIdx, + cleanup: BlockIdx, + resume_target: BlockIdx, + ) -> bool { + if start == BlockIdx::NULL || start == cleanup || start == resume_target { + return false; + } + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL + || block_idx == cleanup + || block_idx == resume_target + || visited[block_idx.idx()] + { + continue; + } + let block = &blocks[block_idx.idx()]; + if block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))) + { + return true; + } + visited[block_idx.idx()] = true; + for successor in normal_successors(block) { + stack.push(successor); + } + } + false + } + + fn named_cleanup_has_conditional_raise_sibling( + blocks: &[Block], + cleanup: BlockIdx, + resume_target: BlockIdx, + ) -> bool { + for block in blocks { + let Some(cond_idx) = trailing_conditional_jump_index(block) else { + continue; + }; + let jump_target = block.instructions[cond_idx].target; + let fallthrough = block.next; + if jump_target == BlockIdx::NULL || fallthrough == BlockIdx::NULL { + continue; + } + + let jump_reaches_cleanup = + path_reaches_named_cleanup(blocks, jump_target, cleanup, resume_target); + let fallthrough_reaches_cleanup = + path_reaches_named_cleanup(blocks, fallthrough, cleanup, resume_target); + if jump_reaches_cleanup == fallthrough_reaches_cleanup { + continue; + } + + let sibling = if jump_reaches_cleanup { + fallthrough + } else { + jump_target + }; + if path_reaches_explicit_raise(blocks, sibling, cleanup, resume_target) { + return true; + } + } + false + } + + fn is_with_suppress_resume_block(block: &Block) -> bool { + let Some(last_info) = block.instructions.last() else { + return false; + }; + if last_info.target == BlockIdx::NULL || !last_info.instr.is_unconditional_jump() { + return false; + } + + let mut saw_pop_except = false; + let mut pop_top_after_pop_except = 0usize; + for info in &block.instructions { + match info.instr.real() { + Some(Instruction::PopExcept) => saw_pop_except = true, + Some(Instruction::PopTop) if saw_pop_except => { + pop_top_after_pop_except += 1; + } + _ => {} + } + } + saw_pop_except && pop_top_after_pop_except >= 3 + } + + let mut named_cleanup_predecessors = vec![0usize; self.blocks.len()]; + let mut named_cleanup_requires_deopt = vec![false; self.blocks.len()]; + let mut has_with_suppress_resume_predecessor = vec![false; self.blocks.len()]; + let mut is_allowed_cleanup_resume_block = vec![false; self.blocks.len()]; + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + + for (block_idx, block) in self.blocks.iter().enumerate() { + let Some(last_info) = block.instructions.last() else { + continue; + }; + if last_info.target == BlockIdx::NULL || !last_info.instr.is_unconditional_jump() { + continue; + } + if is_with_suppress_resume_block(block) { + has_with_suppress_resume_predecessor[last_info.target.idx()] = true; + } + if block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + { + is_allowed_cleanup_resume_block[block_idx] = true; + } + if !is_named_except_cleanup_normal_exit_block(block) { + continue; + } + if matches!( + first_real_instr(&self.blocks[last_info.target.idx()]), + Some(Instruction::ForIter { .. }) + ) { + continue; + } + named_cleanup_predecessors[last_info.target.idx()] += 1; + if named_cleanup_has_conditional_raise_sibling( + &self.blocks, + BlockIdx::new(block_idx as u32), + last_info.target, + ) { + named_cleanup_requires_deopt[last_info.target.idx()] = true; + } + } + for (idx, has_with_suppress_resume) in has_with_suppress_resume_predecessor + .iter() + .copied() + .enumerate() + { + if has_with_suppress_resume && named_cleanup_predecessors[idx] > 0 { + named_cleanup_requires_deopt[idx] = true; + } + } + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let mut visited = vec![false; self.blocks.len()]; + for (idx, &count) in named_cleanup_predecessors.iter().enumerate() { + if count == 0 { + continue; + } + let seed = BlockIdx::new(idx as u32); + let mut segment = Vec::new(); + let mut cursor = seed; + let mut fallback_guard_local = None; + while cursor != BlockIdx::NULL { + let block = &self.blocks[cursor.idx()]; + if block_is_exceptional(block) { + break; + } + if cursor != seed + && let Some(local) = leading_bool_guard_local(block) + { + match fallback_guard_local { + None => fallback_guard_local = Some(local), + Some(expected) if expected != local => break, + Some(_) => {} + } + } + segment.push(cursor); + cursor = block.next; + } + if fallback_guard_local.is_none() && !named_cleanup_requires_deopt[idx] { + continue; + } + + let mut in_segment = vec![false; self.blocks.len()]; + for block_idx in &segment { + in_segment[block_idx.idx()] = true; + } + + for block_idx in segment { + if visited[block_idx.idx()] { + continue; + } + let is_same_guard_fallback = fallback_guard_local.is_some_and(|local| { + leading_bool_guard_local(&self.blocks[block_idx.idx()]) == Some(local) + }); + if block_idx != seed + && !is_same_guard_fallback + && predecessors[block_idx.idx()].iter().any(|pred| { + !in_segment[pred.idx()] && !is_allowed_cleanup_resume_block[pred.idx()] + }) + { + continue; + } + visited[block_idx.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + } + + fn deoptimize_borrow_after_reraising_except_handler(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn starts_with_fast_load(block: &Block) -> bool { + block + .instructions + .iter() + .find(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .is_some_and(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) + }) + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_non_nop_real_instructions(block: &Block) -> bool { + block.instructions.iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop | Instruction::Cache)) + }) + } + + fn block_jumps_backward_to(block: &Block, target: BlockIdx) -> bool { + block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) + } + + fn handler_chain_has_explicit_reraise(blocks: &[Block], handler: BlockIdx) -> bool { + let mut cursor = handler; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + if block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::RaiseVarargs { argc }) + if argc.get(info.arg) == oparg::RaiseKind::BareRaise + ) + }) { + return true; + } + if block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::PopExcept | Instruction::Reraise { .. }) + ) + }) { + return false; + } + cursor = block.next; + } + false + } + + fn handler_chain_resumes_normally(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let last_real_info = block + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()); + let last_real = last_real_info.and_then(|info| info.instr.real()); + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL + && info.instr.is_unconditional_jump() + && has_pop_except + }) { + return true; + } + if has_pop_except + && !matches!( + last_real, + Some(Instruction::RaiseVarargs { .. } | Instruction::Reraise { .. }) + ) + { + return true; + } + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) { + let target = last_real_info.unwrap().target; + if !has_pop_except && target != BlockIdx::NULL { + stack.push(target); + } + } else if !matches!( + last_real, + Some( + Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue + ) + ) && block.next != BlockIdx::NULL + { + stack.push(block.next); + } + } + false + } + + fn block_has_nonresuming_reraise_handler(blocks: &[Block], block: &Block) -> bool { + let mut seen_handlers = Vec::new(); + for handler in block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + { + if seen_handlers.contains(&handler) { + continue; + } + seen_handlers.push(handler); + if handler_chain_has_explicit_reraise(blocks, handler) + && !handler_chain_resumes_normally(blocks, handler) + { + return true; + } + } + false + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let has_reraising_except_handler = self.blocks.iter().any(|block| { + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| handler_chain_has_explicit_reraise(&self.blocks, handler)) + }); + if !has_reraising_except_handler { + return; + } + + let mut follows_protected_body = vec![false; self.blocks.len()]; + for (idx, block) in self.blocks.iter().enumerate() { + if block_is_exceptional(block) || block.cold || !starts_with_fast_load(block) { + continue; + } + let mut seen = vec![false; self.blocks.len()]; + let mut stack = predecessors[idx].clone(); + while let Some(pred) = stack.pop() { + if pred == BlockIdx::NULL || seen[pred.idx()] { + continue; + } + seen[pred.idx()] = true; + let pred_block = &self.blocks[pred.idx()]; + if block_has_protected_instructions(pred_block) { + if block_jumps_backward_to(pred_block, BlockIdx::new(idx as u32)) { + continue; + } + follows_protected_body[idx] = + block_has_nonresuming_reraise_handler(&self.blocks, pred_block); + break; + } + if !block_is_exceptional(pred_block) + && !pred_block.cold + && !block_has_non_nop_real_instructions(pred_block) + { + stack.extend(predecessors[pred.idx()].iter().copied()); + } + } + } + + let mut visited = vec![false; self.blocks.len()]; + for (idx, follows_protected_body) in follows_protected_body.iter().enumerate() { + if !*follows_protected_body { + continue; + } + let mut cursor = BlockIdx::new(idx as u32); + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + let block = &self.blocks[cursor.idx()]; + if block_is_exceptional(block) || block.cold { + break; + } + visited[cursor.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[cursor.idx()]); + if self.blocks[cursor.idx()] + .instructions + .last() + .is_some_and(|info| info.instr.is_scope_exit()) + { + break; + } + cursor = self.blocks[cursor.idx()].next; + } + } + } + + fn deoptimize_borrow_in_protected_conditional_tail(&mut self) { + fn second_last_real_instr(block: &Block) -> Option { + let mut reals = block + .instructions + .iter() + .rev() + .filter_map(|info| info.instr.real()); + let _last = reals.next()?; + reals.next() + } + + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_non_nop_real_instructions(block: &Block) -> bool { + block.instructions.iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop)) + }) + } + + fn starts_with_assertion_error(block: &Block) -> bool { + block + .instructions + .iter() + .find(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .is_some_and(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadCommonConstant { idx }) + if idx.get(info.arg) == oparg::CommonConstant::AssertionError + ) + }) + } + + fn success_path_stays_protected(blocks: &[Block], start: BlockIdx) -> bool { + let mut cursor = start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + let has_non_marker_real = block.instructions.iter().any(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }); + if has_non_marker_real { + return block_has_protected_instructions(block); + } + cursor = block.next; + } + false + } + + fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let pop_top_count = block + .instructions + .iter() + .filter(|info| matches!(info.instr.real(), Some(Instruction::PopTop))) + .count(); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + has_pop_except && pop_top_count == 0 && jumps_to_target + } + + fn handler_chain_resumes_normally(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let last_real_info = block + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()); + let last_real = last_real_info.and_then(|info| info.instr.real()); + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL + && info.instr.is_unconditional_jump() + && has_pop_except + }) { + return true; + } + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) { + let target = last_real_info.unwrap().target; + if !has_pop_except && target != BlockIdx::NULL { + stack.push(target); + } + } else if !matches!( + last_real, + Some( + Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue + ) + ) && block.next != BlockIdx::NULL + { + stack.push(block.next); + } + } + false + } + + fn handler_chain_has_explicit_raise(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let last_real_info = block + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()); + let last_real = last_real_info.and_then(|info| info.instr.real()); + if block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))) + { + return true; + } + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) { + let target = last_real_info.unwrap().target; + if target != BlockIdx::NULL { + stack.push(target); + } + } else if !matches!(last_real, Some(Instruction::ReturnValue)) + && block.next != BlockIdx::NULL + { + stack.push(block.next); + } + } + false + } + + fn handler_chain_has_multiple_handled_returns(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + let mut handled_returns = 0usize; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let has_return = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))); + let has_raise = block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::RaiseVarargs { .. } | Instruction::Reraise { .. }) + ) + }); + if has_pop_except && has_return && !has_raise { + handled_returns += 1; + if handled_returns >= 2 { + return true; + } + } + let last_real_info = block + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()); + let last_real = last_real_info.and_then(|info| info.instr.real()); + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) { + let target = last_real_info.unwrap().target; + if target != BlockIdx::NULL { + stack.push(target); + } + } else if !matches!(last_real, Some(Instruction::ReturnValue)) + && block.next != BlockIdx::NULL + { + stack.push(block.next); + } + } + false + } + + fn block_has_nonresuming_exception_match_handler(blocks: &[Block], block: &Block) -> bool { + let mut seen_handlers = Vec::new(); + for handler in block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + { + if seen_handlers.contains(&handler) { + continue; + } + seen_handlers.push(handler); + let mut cursor = handler; + let mut visited = vec![false; blocks.len()]; + let mut has_exception_match = false; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + if blocks[cursor.idx()].instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) { + has_exception_match = true; + break; + } + cursor = blocks[cursor.idx()].next; + } + if has_exception_match + && handler_chain_has_explicit_raise(blocks, handler) + && !handler_chain_has_multiple_handled_returns(blocks, handler) + && !handler_chain_resumes_normally(blocks, handler) + { + return true; + } + } + false + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + let mut is_handler_resume_block = vec![false; self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if matches!(second_last_real_instr(block), Some(Instruction::PopExcept)) + && block.instructions.last().is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) + { + is_handler_resume_block[pred_idx] = true; + } + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let seeds: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + let cond_idx = trailing_conditional_jump_index(block)?; + let prev_protected = predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + block_has_protected_instructions(pred_block) + && block_has_exception_match_handler(&self.blocks, pred_block) + && block_has_nonresuming_exception_match_handler(&self.blocks, pred_block) + }); + let has_unprotected_normal_predecessor = predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + !block_is_exceptional(pred_block) + && !pred_block.cold + && !is_handler_resume_block[pred.idx()] + && !is_handler_resume_predecessor(pred_block, BlockIdx::new(idx as u32)) + && !block_has_protected_instructions(pred_block) + && block_has_non_nop_real_instructions(pred_block) + }); + let assertion_message_fallthrough = block.next != BlockIdx::NULL + && starts_with_assertion_error(&self.blocks[block.next.idx()]); + let protected_assert = assertion_message_fallthrough + && block_has_protected_instructions(block) + && block_has_exception_match_handler(&self.blocks, block) + && block_has_nonresuming_exception_match_handler(&self.blocks, block); + let seed = if assertion_message_fallthrough { + block.instructions[cond_idx].target + } else { + BlockIdx::new(idx as u32) + }; + let force_deopt = assertion_message_fallthrough + && !success_path_stays_protected(&self.blocks, seed); + let seed_enabled = if assertion_message_fallthrough { + force_deopt && (prev_protected || protected_assert) + } else { + prev_protected + }; + (!block_is_exceptional(block) + && seed != BlockIdx::NULL + && seed_enabled + && !has_unprotected_normal_predecessor) + .then_some((seed, force_deopt)) + }) + .collect(); + + let mut visited = vec![false; self.blocks.len()]; + for (seed, force_deopt) in seeds { + let mut segment = Vec::new(); + let mut cursor = seed; + while cursor != BlockIdx::NULL { + if block_is_exceptional(&self.blocks[cursor.idx()]) { + break; + } + segment.push(cursor); + cursor = self.blocks[cursor.idx()].next; + } + + let segment_ops: Vec<_> = segment + .iter() + .flat_map(|block_idx| { + self.blocks[block_idx.idx()] + .instructions + .iter() + .filter_map(|info| info.instr.real()) + }) + .collect(); + let call_count = segment_ops + .iter() + .filter(|instr| matches!(instr, Instruction::Call { .. })) + .count(); + let raise_count = segment_ops + .iter() + .filter(|instr| matches!(instr, Instruction::RaiseVarargs { .. })) + .count(); + let return_count = segment_ops + .iter() + .filter(|instr| matches!(instr, Instruction::ReturnValue)) + .count(); + let conditional_count = segment_ops + .iter() + .filter(|instr| { + matches!( + instr, + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + ) + }) + .count(); + let has_handler_resume_predecessor = predecessors[seed.idx()].iter().any(|pred| { + is_handler_resume_block[pred.idx()] + || is_handler_resume_predecessor(&self.blocks[pred.idx()], seed) + }); + if has_handler_resume_predecessor { + continue; + } + let has_loop_cleanup_predecessor = predecessors[seed.idx()].iter().any(|pred| { + self.blocks[pred.idx()].instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::EndFor | Instruction::EndAsyncFor | Instruction::PopIter) + ) + }) + }); + let has_named_except_cleanup_predecessor = predecessors[seed.idx()] + .iter() + .any(|pred| is_named_except_cleanup_normal_exit_block(&self.blocks[pred.idx()])); + let has_complex_tail = segment_ops.iter().any(|instr| { + matches!( + instr, + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + | Instruction::ForIter { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::EndFor + | Instruction::PopIter + | Instruction::LoadFastAndClear { .. } + | Instruction::LoadFastCheck { .. } + | Instruction::ListAppend { .. } + | Instruction::MapAdd { .. } + | Instruction::SetAdd { .. } + ) + }); + let has_loop_or_comprehension_tail = segment_ops.iter().any(|instr| { + matches!( + instr, + Instruction::ForIter { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::EndFor + | Instruction::PopIter + | Instruction::LoadFastAndClear { .. } + | Instruction::LoadFastCheck { .. } + | Instruction::ListAppend { .. } + | Instruction::MapAdd { .. } + | Instruction::SetAdd { .. } + ) + }); + let has_store_fast_tail = segment_ops.iter().any(|instr| { + matches!( + instr, + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + ) + }); + let has_nonresuming_protected_conditional_tail = !has_handler_resume_predecessor + && !has_loop_cleanup_predecessor + && !has_loop_or_comprehension_tail + && has_store_fast_tail + && call_count >= 1 + && return_count >= 1 + && conditional_count == 1; + let has_existing_protected_conditional_tail = !has_loop_cleanup_predecessor + && !has_handler_resume_predecessor + && !has_named_except_cleanup_predecessor + && !has_complex_tail + && call_count == 2 + && raise_count == 1 + && return_count == 1 + && conditional_count == 1; + if !(force_deopt + || has_nonresuming_protected_conditional_tail + || has_existing_protected_conditional_tail) + { + continue; + } + + let mut in_segment = vec![false; self.blocks.len()]; + for block_idx in &segment { + in_segment[block_idx.idx()] = true; + } + + for block_idx in segment { + if visited[block_idx.idx()] { + continue; + } + if predecessors[block_idx.idx()].iter().any(|pred| { + is_handler_resume_block[pred.idx()] + || is_handler_resume_predecessor(&self.blocks[pred.idx()], block_idx) + }) { + continue; + } + if !force_deopt + && block_idx != seed + && predecessors[block_idx.idx()] + .iter() + .any(|pred| !in_segment[pred.idx()] && !is_handler_resume_block[pred.idx()]) + { + continue; + } + visited[block_idx.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + } + + fn deoptimize_borrow_after_terminal_except_tail(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_real_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.instr.real().is_some()) + } + + fn block_has_non_nop_real_instructions(block: &Block) -> bool { + block.instructions.iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop)) + }) + } + + fn handler_chain_resumes_normally(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut has_terminal_exit = false; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let last_real_info = block + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()); + let last_real = last_real_info.and_then(|info| info.instr.real()); + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL + && info.instr.is_unconditional_jump() + && has_pop_except + }) { + return true; + } + if block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue + ) + ) + }) { + has_terminal_exit = true; + } + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) { + let target = last_real_info.unwrap().target; + if !has_pop_except && target != BlockIdx::NULL { + stack.push(target); + } + } else if !matches!( + last_real, + Some( + Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue + ) + ) && block.next != BlockIdx::NULL + { + stack.push(block.next); + } + } + !has_terminal_exit + } + + fn handler_reaches_match_before_terminal(blocks: &[Block], handler: BlockIdx) -> bool { + let mut cursor = handler; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + for info in &block.instructions { + match info.instr.real() { + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) => { + return true; + } + Some( + Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue, + ) => return false, + _ => {} + } + } + cursor = block.next; + } + false + } + + fn handler_chain_has_multiple_handled_returns(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![handler]; + let mut handled_returns = 0usize; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let has_return = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))); + let has_raise = block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::RaiseVarargs { .. } | Instruction::Reraise { .. }) + ) + }); + if has_pop_except && has_return && !has_raise { + handled_returns += 1; + if handled_returns >= 2 { + return true; + } + } + let last_real_info = block + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()); + let last_real = last_real_info.and_then(|info| info.instr.real()); + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) { + let target = last_real_info.unwrap().target; + if target != BlockIdx::NULL { + stack.push(target); + } + } else if !matches!(last_real, Some(Instruction::ReturnValue)) + && block.next != BlockIdx::NULL + { + stack.push(block.next); + } + } + false + } + + fn protected_block_has_terminal_exception_handler(blocks: &[Block], block: &Block) -> bool { + let mut seen_handlers = Vec::new(); + for handler in block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + { + if seen_handlers.contains(&handler) { + continue; + } + seen_handlers.push(handler); + let has_exception_match = handler_reaches_match_before_terminal(blocks, handler); + if has_exception_match + && !handler_chain_has_multiple_handled_returns(blocks, handler) + && !handler_chain_resumes_normally(blocks, handler) + { + return true; + } + } + false + } + + fn normal_successors(block: &Block) -> Vec { + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + fn has_call_store_before_trailing_conditional(block: &Block) -> bool { + let Some(cond_idx) = trailing_conditional_jump_index(block) else { + return false; + }; + block.instructions[..cond_idx] + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) + && block.instructions[..cond_idx].iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + ) + ) + }) + } + + fn has_call_and_store(block: &Block) -> bool { + let mut has_call = false; + let mut has_store_fast = false; + for info in &block.instructions { + match info.instr.real() { + Some(Instruction::Call { .. }) => has_call = true, + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. }, + ) => has_store_fast = true, + _ => {} + } + } + has_call && has_store_fast + } + + fn has_load_fast_pair(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) + }) + } + + fn has_call(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) + } + + fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + has_pop_except && jumps_to_target + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let mut seeds = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + let has_protected_call_predecessor = predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + block_has_protected_instructions(pred_block) && has_call(pred_block) + }); + let has_call_store_tail = has_call_and_store(block) && has_load_fast_pair(block); + let has_conditional_tail = trailing_conditional_jump_index(block).is_some() + || has_call_store_before_trailing_conditional(block) + || predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + !block_is_exceptional(pred_block) + && !pred_block.cold + && has_call_and_store(pred_block) + && has_load_fast_pair(pred_block) + }); + let has_structured_terminal_tail_shape = has_conditional_tail; + if block_is_exceptional(block) + || block.cold + || block_has_protected_instructions(block) + || !block_has_real_instructions(block) + || (has_conditional_tail + && block.start_depth.is_some_and(|depth| depth > 0) + && !has_protected_call_predecessor + && !has_call_store_tail) + || predecessors[idx].iter().any(|pred| { + is_handler_resume_predecessor( + &self.blocks[pred.idx()], + BlockIdx::new(idx as u32), + ) + }) + || predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + !block_is_exceptional(pred_block) + && !pred_block.cold + && !block_has_protected_instructions(pred_block) + && block_has_non_nop_real_instructions(pred_block) + }) + || !(has_structured_terminal_tail_shape + || has_protected_call_predecessor + || has_call_store_tail) + { + continue; + } + + let mut seen = vec![false; self.blocks.len()]; + let mut stack = predecessors[idx].clone(); + while let Some(pred) = stack.pop() { + if pred == BlockIdx::NULL || seen[pred.idx()] { + continue; + } + seen[pred.idx()] = true; + let pred_block = &self.blocks[pred.idx()]; + if block_has_protected_instructions(pred_block) { + if protected_block_has_terminal_exception_handler(&self.blocks, pred_block) { + seeds.push(( + BlockIdx::new(idx as u32), + (has_protected_call_predecessor || has_call_store_tail) + && !has_structured_terminal_tail_shape, + )); + } + break; + } + if !block_is_exceptional(pred_block) + && !pred_block.cold + && !block_has_non_nop_real_instructions(pred_block) + { + stack.extend(predecessors[pred.idx()].iter().copied()); + } + } + } + + let mut visited = vec![false; self.blocks.len()]; + for (seed, direct_only) in seeds { + let mut stack = vec![seed]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + let block = &self.blocks[block_idx.idx()]; + if block_is_exceptional(block) + || block.cold + || block_has_protected_instructions(block) + { + continue; + } + visited[block_idx.idx()] = true; + let successors = normal_successors(&self.blocks[block_idx.idx()]); + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + if direct_only { + continue; + } + for successor in successors { + stack.push(successor); + } + } + } + } + + fn deoptimize_borrow_in_protected_method_call_after_terminal_except_tail(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_non_nop_real_instructions(block: &Block) -> bool { + block.instructions.iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop)) + }) + } + + fn handler_chain_has_exception_match(blocks: &[Block], handler: BlockIdx) -> bool { + let mut cursor = handler; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + if blocks[cursor.idx()].instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) { + return true; + } + cursor = blocks[cursor.idx()].next; + } + false + } + + fn block_has_protected_method_call(blocks: &[Block], block: &Block) -> bool { + if !block_has_protected_instructions(block) { + return false; + } + block.instructions.iter().any(|info| { + let is_method_load = matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) if namei.get(info.arg).is_method() + ); + is_method_load + && info.except_handler.is_some_and(|handler| { + handler_chain_has_exception_match(blocks, handler.handler_block) + }) + }) + } + + fn protected_method_call_handlers(blocks: &[Block], block: &Block) -> Vec { + let mut handlers = Vec::new(); + for info in &block.instructions { + if !matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) if namei.get(info.arg).is_method() + ) { + continue; + } + let Some(handler) = info.except_handler.map(|handler| handler.handler_block) else { + continue; + }; + if !handler_chain_has_exception_match(blocks, handler) { + continue; + } + if !handlers.contains(&handler) { + handlers.push(handler); + } + } + handlers + } + + fn protected_method_call_count(blocks: &[Block], block: &Block) -> usize { + block + .instructions + .iter() + .filter(|info| { + let is_method_load = matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) if namei.get(info.arg).is_method() + ); + is_method_load + && info.except_handler.is_some_and(|handler| { + handler_chain_has_exception_match(blocks, handler.handler_block) + }) + }) + .count() + } + + fn block_stores_fast(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + ) + ) + }) + } + + fn block_shares_handler(block: &Block, handlers: &[BlockIdx]) -> bool { + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| handlers.contains(&handler)) + } + + fn starts_with_inlined_comprehension_restore(block: &Block) -> bool { + if block.start_depth.is_none_or(|depth| depth == 0) { + return false; + } + let mut saw_store = false; + for info in &block.instructions { + match info.instr.real() { + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. }, + ) => saw_store = true, + Some(Instruction::Nop) => {} + Some(_) => return saw_store, + None => {} + } + } + false + } + + fn handler_chain_resumes_normally(blocks: &[Block], handler: BlockIdx) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut has_terminal_exit = false; + let mut stack = vec![handler]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let last_real_info = block + .instructions + .iter() + .rev() + .find(|info| info.instr.real().is_some()); + let last_real = last_real_info.and_then(|info| info.instr.real()); + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL + && info.instr.is_unconditional_jump() + && has_pop_except + }) { + return true; + } + if block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue + ) + ) + }) { + has_terminal_exit = true; + } + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if last_real_info.is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) { + let target = last_real_info.unwrap().target; + if !has_pop_except && target != BlockIdx::NULL { + stack.push(target); + } + } else if !matches!( + last_real, + Some( + Instruction::RaiseVarargs { .. } + | Instruction::Reraise { .. } + | Instruction::ReturnValue + ) + ) && block.next != BlockIdx::NULL + { + stack.push(block.next); + } + } + !has_terminal_exit + } + + fn protected_block_has_terminal_exception_handler(blocks: &[Block], block: &Block) -> bool { + let mut seen_handlers = Vec::new(); + for handler in block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + { + if seen_handlers.contains(&handler) { + continue; + } + seen_handlers.push(handler); + let mut cursor = handler; + let mut visited = vec![false; blocks.len()]; + let mut has_exception_match = false; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + if blocks[cursor.idx()].instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) { + has_exception_match = true; + break; + } + cursor = blocks[cursor.idx()].next; + } + if has_exception_match && !handler_chain_resumes_normally(blocks, handler) { + return true; + } + } + false + } + + fn protected_block_has_raising_exception_handler(blocks: &[Block], block: &Block) -> bool { + let mut seen_handlers = Vec::new(); + for handler in block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + { + if seen_handlers.contains(&handler) { + continue; + } + seen_handlers.push(handler); + let mut stack = Vec::new(); + let mut visited = vec![false; blocks.len()]; + let mut cursor = handler; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let handler_block = &blocks[cursor.idx()]; + if handler_block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) { + if handler_block.next != BlockIdx::NULL { + stack.push(handler_block.next); + } + break; + } + cursor = handler_block.next; + } + visited.fill(false); + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + let mut stop_path = false; + for info in &block.instructions { + match info.instr.real() { + Some( + Instruction::RaiseVarargs { .. } | Instruction::Reraise { .. }, + ) => { + return true; + } + Some(Instruction::ReturnValue | Instruction::PopExcept) => { + stop_path = true; + break; + } + _ => {} + } + } + if stop_path { + continue; + } + for info in &block.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + if info.instr.is_unconditional_jump() && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if block.next != BlockIdx::NULL { + stack.push(block.next); + } + } + } + false + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let mut to_deopt = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + if !block_is_exceptional(block) + && !block.cold + && !block_stores_fast(block) + && protected_method_call_count(&self.blocks, block) >= 2 + && protected_block_has_terminal_exception_handler(&self.blocks, block) + && protected_block_has_raising_exception_handler(&self.blocks, block) + { + to_deopt.push(BlockIdx::new(idx as u32)); + continue; + } + if block_is_exceptional(block) + || block.cold + || starts_with_inlined_comprehension_restore(block) + || !block_has_protected_method_call(&self.blocks, block) + || predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + !block_is_exceptional(pred_block) + && !pred_block.cold + && !block_has_protected_instructions(pred_block) + && block_has_non_nop_real_instructions(pred_block) + }) + { + continue; + } + + let method_handlers = protected_method_call_handlers(&self.blocks, block); + let mut seen = vec![false; self.blocks.len()]; + let mut stack = predecessors[idx].clone(); + while let Some(pred) = stack.pop() { + if pred == BlockIdx::NULL || seen[pred.idx()] { + continue; + } + seen[pred.idx()] = true; + let pred_block = &self.blocks[pred.idx()]; + if block_has_protected_instructions(pred_block) { + if protected_block_has_terminal_exception_handler(&self.blocks, pred_block) + && protected_block_has_raising_exception_handler(&self.blocks, pred_block) + && !block_shares_handler(pred_block, &method_handlers) + { + to_deopt.push(BlockIdx::new(idx as u32)); + } + break; + } + if !block_is_exceptional(pred_block) + && !pred_block.cold + && !block_has_non_nop_real_instructions(pred_block) + { + stack.extend(predecessors[pred.idx()].iter().copied()); + } + } + } + + for block_idx in to_deopt { + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + + fn deoptimize_borrow_after_except_star_try_tail(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_fast_load(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) + }) + } + + fn handler_chain_has_exception_group_match(blocks: &[Block], handler: BlockIdx) -> bool { + let mut cursor = handler; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + if blocks[cursor.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::CheckEgMatch))) + { + return true; + } + cursor = blocks[cursor.idx()].next; + } + false + } + + fn block_has_exception_group_handler(blocks: &[Block], block: &Block) -> bool { + block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .any(|handler| handler_chain_has_exception_group_match(blocks, handler)) + } + + fn normal_successors(block: &Block) -> Vec { + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (idx, block) in self.blocks.iter().enumerate() { + for successor in normal_successors(block) { + predecessors[successor.idx()].push(BlockIdx::new(idx as u32)); + } + } + + let mut to_deopt = Vec::new(); + for (idx, block) in self.blocks.iter().enumerate() { + if block_is_exceptional(block) + || block.cold + || block_has_exception_group_handler(&self.blocks, block) + || !block_has_fast_load(block) + { + continue; + } + + let mut visited = vec![false; self.blocks.len()]; + let mut stack = predecessors[idx].clone(); + while let Some(pred) = stack.pop() { + if pred == BlockIdx::NULL || visited[pred.idx()] { + continue; + } + visited[pred.idx()] = true; + let pred_block = &self.blocks[pred.idx()]; + if block_has_protected_instructions(pred_block) + && block_has_exception_group_handler(&self.blocks, pred_block) + { + to_deopt.push(BlockIdx::new(idx as u32)); + break; + } + if !block_is_exceptional(pred_block) && !pred_block.cold { + stack.extend(predecessors[pred.idx()].iter().copied()); + } + } + } + + to_deopt.sort_by_key(|idx| idx.idx()); + to_deopt.dedup(); + for block_idx in to_deopt { + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + + fn deoptimize_borrow_for_folded_nonliteral_exprs(&mut self) { + let mut starts_after_folded_nonliteral_expr = vec![false; self.blocks.len()]; + for block in &self.blocks { + let Some(last) = block + .instructions + .iter() + .rev() + .find(|info| !matches!(info.instr.real(), Some(Instruction::Nop))) + else { + continue; + }; + if !last.folded_from_nonliteral_expr { + continue; + } + if block.next != BlockIdx::NULL { + starts_after_folded_nonliteral_expr[block.next.idx()] = true; + } + if last.target != BlockIdx::NULL { + starts_after_folded_nonliteral_expr[last.target.idx()] = true; + } + } + + for (block_idx, block) in self.blocks.iter_mut().enumerate() { + let mut deopt_tail = false; + let mut prev_non_nop_folded = starts_after_folded_nonliteral_expr[block_idx]; + let mut prev_prev_non_nop_folded = false; + let mut prev_non_nop_was_unpack = false; + for info in &mut block.instructions { + let folded_from_nonliteral_expr = info.folded_from_nonliteral_expr; + let real = info.instr.real(); + let store_from_folded_nonliteral_expr = match real { + Some(Instruction::StoreFast { .. } | Instruction::StoreFastLoadFast { .. }) => { + folded_from_nonliteral_expr || prev_non_nop_folded + } + Some(Instruction::StoreFastStoreFast { .. }) => { + folded_from_nonliteral_expr + || prev_non_nop_folded + || (prev_non_nop_was_unpack && prev_prev_non_nop_folded) + } + _ => false, + }; + if store_from_folded_nonliteral_expr { + deopt_tail = true; + } + if !folded_from_nonliteral_expr && !deopt_tail { + if let Some(real) = real + && !matches!(real, Instruction::Nop | Instruction::Cache) + { + prev_prev_non_nop_folded = prev_non_nop_folded; + prev_non_nop_folded = folded_from_nonliteral_expr; + prev_non_nop_was_unpack = + matches!(real, Instruction::UnpackSequence { .. }); + } + continue; + } + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + if let Some(real) = real + && !matches!(real, Instruction::Nop | Instruction::Cache) + { + prev_prev_non_nop_folded = prev_non_nop_folded; + prev_non_nop_folded = folded_from_nonliteral_expr; + prev_non_nop_was_unpack = matches!(real, Instruction::UnpackSequence { .. }); + } + } + } + } + + fn deoptimize_borrow_after_terminal_except_before_with(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + has_pop_except && jumps_to_target + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_starts_with_with_setup(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::LoadSpecial { .. }))) + } + + fn block_starts_with_with_cleanup_handler(block: &Block) -> bool { + let mut reals = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .filter(|instr| !matches!(instr, Instruction::Nop)); + matches!( + (reals.next(), reals.next()), + ( + Some(Instruction::PushExcInfo), + Some(Instruction::WithExceptStart) + ) + ) + } + + fn block_has_exception_match(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + } + + fn next_chain_reaches_exception_match_before_with_cleanup( + blocks: &[Block], + start: BlockIdx, + ) -> bool { + let mut cursor = start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + if block_starts_with_with_cleanup_handler(block) { + return false; + } + if block_has_exception_match(block) { + return true; + } + cursor = block.next; + } + false + } + + fn normal_successors(block: &Block) -> Vec { + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + for successor in normal_successors(block) { + predecessors[successor.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + + let seeds: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + if block_is_exceptional(block) + || !block_has_protected_instructions(block) + || !block_starts_with_with_setup(block) + || !next_chain_reaches_exception_match_before_with_cleanup( + &self.blocks, + block.next, + ) + { + return None; + } + let has_terminal_except_predecessor = predecessors[idx].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + pred_block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + && block_has_exception_match_handler(&self.blocks, pred_block) + }); + let has_handler_resume_predecessor = predecessors[idx].iter().any(|pred| { + is_handler_resume_predecessor( + &self.blocks[pred.idx()], + BlockIdx::new(idx as u32), + ) + }); + (has_terminal_except_predecessor && !has_handler_resume_predecessor) + .then_some(BlockIdx::new(idx as u32)) + }) + .collect(); + + let mut to_deopt = Vec::new(); + let mut visited = vec![false; self.blocks.len()]; + for seed in seeds { + let mut stack = vec![seed]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + let block = &self.blocks[block_idx.idx()]; + if block_is_exceptional(block) || !block_has_protected_instructions(block) { + continue; + } + visited[block_idx.idx()] = true; + to_deopt.push(block_idx); + for successor in normal_successors(block) { + stack.push(successor); + } + } + } + + to_deopt.sort_by_key(|idx| idx.idx()); + to_deopt.dedup(); + for block_idx in to_deopt { + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + + fn deoptimize_borrow_in_async_finally_early_return_tail(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn block_has_send(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Send { .. }))) + } + + fn block_has_get_anext(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::GetANext))) + } + + fn block_has_return(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))) + } + + fn normal_successors(block: &Block) -> Vec { + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + if target != BlockIdx::NULL { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (idx, block) in self.blocks.iter().enumerate() { + for successor in normal_successors(block) { + predecessors[successor.idx()].push(BlockIdx::new(idx as u32)); + } + } + let relevant_send_blocks: Vec<_> = self + .blocks + .iter() + .enumerate() + .map(|(idx, block)| { + block_has_send(block) + && !block_has_get_anext(block) + && !predecessors[idx] + .iter() + .any(|pred| block_has_get_anext(&self.blocks[pred.idx()])) + }) + .collect(); + if !relevant_send_blocks.iter().any(|has_send| *has_send) { + return; + } + + fn has_early_return_before_send(blocks: &[Block], relevant_send_blocks: &[bool]) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![BlockIdx(0)]; + while let Some(block_idx) = stack.pop() { + if block_idx == BlockIdx::NULL || visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + let block = &blocks[block_idx.idx()]; + if block.cold || block_is_exceptional(block) { + continue; + } + if relevant_send_blocks[block_idx.idx()] { + continue; + } + if block_has_return(block) { + return true; + } + stack.extend(normal_successors(block)); + } + false + } + + let has_early_return = has_early_return_before_send(&self.blocks, &relevant_send_blocks); + if !has_early_return { + return; + } + + for (idx, block) in self.blocks.iter_mut().enumerate() { + if idx == 0 || block.cold || block_is_exceptional(block) { + continue; + } + deoptimize_block_borrows(block); + } + } + + fn deoptimize_borrow_after_handler_resume_loop_tail(&mut self) { + fn deoptimize_block_borrows(block: &mut Block) { + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + has_pop_except && jumps_to_target + } + + fn is_suppressing_with_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + if !block + .instructions + .last() + .is_some_and(|info| info.target == target && info.instr.is_unconditional_jump()) + { + return false; + } + let mut reals = block + .instructions + .iter() + .rev() + .filter_map(|info| info.instr.real()); + matches!( + (reals.next(), reals.next(), reals.next(), reals.next(), reals.next()), + ( + Some(instr), + Some(Instruction::PopTop), + Some(Instruction::PopTop), + Some(Instruction::PopTop), + Some(Instruction::PopExcept), + ) if instr.is_unconditional_jump() + ) + } + + fn block_has_check_exc_match(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + } + + fn predecessor_chain_has_check_exc_match( + blocks: &[Block], + predecessors: &[Vec], + start: BlockIdx, + stop: BlockIdx, + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack = vec![start]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || cursor == stop || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + if block_has_check_exc_match(&blocks[cursor.idx()]) { + return true; + } + for pred in &predecessors[cursor.idx()] { + stack.push(*pred); + } + } + false + } + + fn has_exception_match_resume_predecessor( + blocks: &[Block], + predecessors: &[Vec], + target: BlockIdx, + ) -> bool { + predecessors[target.idx()].iter().any(|pred| { + let block = &blocks[pred.idx()]; + is_handler_resume_predecessor(block, target) + && !is_suppressing_with_resume_predecessor(block, target) + && predecessor_chain_has_check_exc_match(blocks, predecessors, *pred, target) + }) + } + + fn starts_with_bool_guard(block: &Block) -> bool { + let infos: Vec<_> = block + .instructions + .iter() + .filter(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .take(3) + .collect(); + matches!( + infos.as_slice(), + [ + first, + second, + third, + .. + ] if matches!( + first.instr.real(), + Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) + ) && matches!(second.instr.real(), Some(Instruction::ToBool)) + && matches!( + third.instr.real(), + Some( + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + ) + ) + ) + } + + fn block_has_loop_back(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) + } + + fn block_has_for_iter(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) + } + + fn block_has_get_iter(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::GetIter))) + } + + fn block_has_fast_load(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastLoadFast { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) + ) + }) + } + + fn block_has_method_load(block: &Block) -> bool { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) if namei.get(info.arg).is_method() + ) + }) + } + + fn block_is_calling_finally_cleanup(block: &Block) -> bool { + let has_call = block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx + ) + ) + }); + has_call + && block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::Reraise { .. }))) + && !block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + } + + fn has_jump_back_predecessor_to( + blocks: &[Block], + predecessors: &[Vec], + target: BlockIdx, + ) -> bool { + predecessors[target.idx()].iter().any(|pred| { + let pred_block = &blocks[pred.idx()]; + if pred_block.cold || block_is_exceptional(pred_block) { + return false; + } + blocks[pred.idx()].instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }) + }) + } + + fn tail_successors( + blocks: &[Block], + predecessors: &[Vec], + block_idx: BlockIdx, + ) -> Vec { + let block = &blocks[block_idx.idx()]; + if block_has_loop_back(block) { + return Vec::new(); + } + if let Some(for_iter) = block + .instructions + .iter() + .find(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) + { + let mut successors = Vec::with_capacity(3); + if for_iter.target != BlockIdx::NULL { + successors.push(for_iter.target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + if let Some(cond_idx) = trailing_conditional_jump_index(block) { + let mut successors = Vec::with_capacity(2); + let target = block.instructions[cond_idx].target; + let is_loop_header = has_jump_back_predecessor_to(blocks, predecessors, block_idx); + if target != BlockIdx::NULL && !is_loop_header { + successors.push(target); + } + if block.next != BlockIdx::NULL { + successors.push(block.next); + } + return successors; + } + let Some(last) = block.instructions.last() else { + return (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect(); + }; + if last.instr.is_scope_exit() { + return Vec::new(); + } + if last.instr.is_unconditional_jump() { + return (last.target != BlockIdx::NULL) + .then_some(last.target) + .into_iter() + .collect(); + } + (block.next != BlockIdx::NULL) + .then_some(block.next) + .into_iter() + .collect() + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + let mut is_handler_resume_block = vec![false; self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + && block.instructions.last().is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) + { + is_handler_resume_block[pred_idx] = true; + } + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let has_calling_finally_cleanup = self.blocks.iter().any(block_is_calling_finally_cleanup); + let has_exception_match_handler = self.blocks.iter().any(|block| { + block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + }); + let has_exception_group_match_handler = self.blocks.iter().any(|block| { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::CheckEgMatch))) + }); + + let suppressing_exception_match_method_tails: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + if block_is_exceptional(block) + || !has_exception_group_match_handler + || !block_has_fast_load(block) + || !block_has_method_load(block) + { + return None; + } + predecessors[idx] + .iter() + .any(|pred| { + is_suppressing_with_resume_predecessor( + &self.blocks[pred.idx()], + BlockIdx::new(idx as u32), + ) + }) + .then_some(BlockIdx::new(idx as u32)) + }) + .collect(); + for block_idx in suppressing_exception_match_method_tails { + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + + let seeds: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + let has_bool_guard_tail = starts_with_bool_guard(block); + let has_loop_tail = block_has_for_iter(block) || block_has_get_iter(block); + let has_finally_except_loop_tail = + has_calling_finally_cleanup && has_exception_match_handler && has_loop_tail; + let has_protected_predecessor = predecessors[idx].iter().any(|pred| { + self.blocks[pred.idx()] + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + }); + let has_handler_resume_predecessor = predecessors[idx].iter().any(|pred| { + is_handler_resume_block[pred.idx()] + || is_handler_resume_predecessor( + &self.blocks[pred.idx()], + BlockIdx::new(idx as u32), + ) + }); + let has_suppressing_with_resume_predecessor = + predecessors[idx].iter().any(|pred| { + is_suppressing_with_resume_predecessor( + &self.blocks[pred.idx()], + BlockIdx::new(idx as u32), + ) + }); + let has_exception_match_resume_predecessor = has_exception_match_resume_predecessor( + &self.blocks, + &predecessors, + BlockIdx::new(idx as u32), + ); + let has_handler_resume_loop_tail = block_has_get_iter(block) + && has_suppressing_with_resume_predecessor + && has_exception_match_resume_predecessor; + let has_supported_tail = has_bool_guard_tail + || has_finally_except_loop_tail + || has_handler_resume_loop_tail; + if block_is_exceptional(block) || !has_supported_tail { + return None; + } + let should_seed = (has_protected_predecessor && has_finally_except_loop_tail) + || (has_bool_guard_tail && has_handler_resume_predecessor) + || has_handler_resume_loop_tail; + should_seed.then_some((BlockIdx::new(idx as u32), has_handler_resume_loop_tail)) + }) + .collect(); + + let mut visited = vec![false; self.blocks.len()]; + for (seed, include_join_tail) in seeds { + let mut segment = Vec::new(); + let mut found_loop_back = false; + let mut seen = vec![false; self.blocks.len()]; + let mut stack = vec![seed]; + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || seen[cursor.idx()] { + continue; + } + seen[cursor.idx()] = true; + let block = &self.blocks[cursor.idx()]; + if block_is_exceptional(block) { + continue; + } + segment.push(cursor); + if block_has_loop_back(block) { + found_loop_back = true; + continue; + } + for successor in tail_successors(&self.blocks, &predecessors, cursor) { + stack.push(successor); + } + } + if !found_loop_back { + continue; + } + + let segment_ops: Vec<_> = segment + .iter() + .flat_map(|block_idx| { + self.blocks[block_idx.idx()] + .instructions + .iter() + .filter_map(|info| info.instr.real()) + }) + .collect(); + let has_call = segment_ops.iter().any(|instr| { + matches!(instr, Instruction::Call { .. } | Instruction::CallKw { .. }) + }); + let has_store_fast = segment_ops.iter().any(|instr| { + matches!( + instr, + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + ) + }); + if !has_call || !has_store_fast { + continue; + } + + let mut in_segment = vec![false; self.blocks.len()]; + for block_idx in &segment { + in_segment[block_idx.idx()] = true; + } + + for block_idx in segment { + if visited[block_idx.idx()] { + continue; + } + if !include_join_tail + && block_idx != seed + && predecessors[block_idx.idx()] + .iter() + .any(|pred| !in_segment[pred.idx()] && !is_handler_resume_block[pred.idx()]) + { + continue; + } + visited[block_idx.idx()] = true; + deoptimize_block_borrows(&mut self.blocks[block_idx.idx()]); + } + } + } + + fn deoptimize_borrow_after_push_exc_info(&mut self) { + for block in &mut self.blocks { + let mut in_exception_state = false; + for info in &mut block.instructions { + match info.instr.real() { + Some(Instruction::PushExcInfo) => { + in_exception_state = true; + } + Some(Instruction::PopExcept | Instruction::Reraise { .. }) => { + in_exception_state = false; + } + Some(Instruction::LoadFastBorrow { .. }) if in_exception_state => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) + if in_exception_state => + { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + } + + fn deoptimize_borrow_after_protected_import(&mut self) { + fn deoptimize_borrow(info: &mut InstructionInfo) { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + + fn deoptimize_block_borrows_from(block: &mut Block, start: usize) { + for info in block.instructions.iter_mut().skip(start) { + deoptimize_borrow(info); + } + } + + fn deoptimize_protected_block_borrows_from( + block: &mut Block, + start: usize, + protected_store_locals: &[bool], + ) { + for info in block.instructions.iter_mut().skip(start) { + if info.except_handler.is_none() { + break; + } + match info.instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { + let local = usize::from(var_num.get(info.arg)); + if protected_store_locals.get(local).copied().unwrap_or(false) { + continue; + } + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { var_nums }) => { + let (left, right) = var_nums.get(info.arg).indexes(); + let skip_left = protected_store_locals + .get(usize::from(left)) + .copied() + .unwrap_or(false); + let skip_right = protected_store_locals + .get(usize::from(right)) + .copied() + .unwrap_or(false); + if skip_left && skip_right { + continue; + } + } + _ => {} + } + deoptimize_borrow(info); + } + } + + fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + has_pop_except && jumps_to_target + } + + fn handler_chain_returns(blocks: &[Block], handler_block: BlockIdx) -> bool { + let mut cursor = handler_block; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let mut saw_pop_except = false; + for info in &blocks[cursor.idx()].instructions { + match info.instr.real() { + Some(Instruction::ReturnValue) => return true, + Some(Instruction::PopExcept) => saw_pop_except = true, + Some(instr) + if saw_pop_except + && (instr.is_unconditional_jump() || instr.is_scope_exit()) => + { + return false; + } + _ => {} + } + } + if saw_pop_except { + return false; + } + cursor = blocks[cursor.idx()].next; + } + false + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let seeds: Vec<_> = + self.blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + if block_is_exceptional(block) { + return None; + } + let import_idx = block.instructions.iter().position(|info| { + info.except_handler.is_some() + && matches!(info.instr.real(), Some(Instruction::ImportName { .. })) + })?; + let handler_returns = block.instructions[import_idx] + .except_handler + .is_some_and(|handler| { + handler_chain_returns(&self.blocks, handler.handler_block) + }); + if !handler_returns { + return None; + } + Some((BlockIdx::new(idx as u32), import_idx)) + }) + .collect(); + + let mut block_order = vec![u32::MAX; self.blocks.len()]; + let mut cursor = BlockIdx(0); + let mut pos = 0u32; + while cursor != BlockIdx::NULL { + block_order[cursor.idx()] = pos; + pos += 1; + cursor = self.blocks[cursor.idx()].next; + } + + let mut visited = vec![false; self.blocks.len()]; + for (seed, import_idx) in seeds { + let mut protected_store_locals = vec![false; self.metadata.varnames.len()]; + for info in self.blocks[seed.idx()] + .instructions + .iter() + .skip(import_idx + 1) + { + if info.except_handler.is_none() { + break; + } + if let Some(Instruction::StoreFast { var_num }) = info.instr.real() { + let local = usize::from(var_num.get(info.arg)); + if let Some(slot) = protected_store_locals.get_mut(local) { + *slot = true; + } + } + } + + let mut in_segment = vec![false; self.blocks.len()]; + in_segment[seed.idx()] = true; + let mut segment = vec![(seed, import_idx + 1)]; + let mut cursor = self.blocks[seed.idx()].next; + while cursor != BlockIdx::NULL && !block_is_exceptional(&self.blocks[cursor.idx()]) { + if block_has_protected_instructions(&self.blocks[cursor.idx()]) { + break; + } + if predecessors[cursor.idx()].iter().any(|pred| { + !in_segment[pred.idx()] + && block_order[pred.idx()] < block_order[cursor.idx()] + && !is_handler_resume_predecessor(&self.blocks[pred.idx()], cursor) + }) { + break; + } + in_segment[cursor.idx()] = true; + segment.push((cursor, 0)); + if self.blocks[cursor.idx()] + .instructions + .iter() + .any(|info| info.instr.real().is_some_and(|instr| instr.is_scope_exit())) + { + break; + } + cursor = self.blocks[cursor.idx()].next; + } + + for (block_idx, start) in segment { + if visited[block_idx.idx()] { + continue; + } + visited[block_idx.idx()] = true; + if block_idx == seed { + deoptimize_protected_block_borrows_from( + &mut self.blocks[block_idx.idx()], + start, + &protected_store_locals, + ); + } else { + deoptimize_block_borrows_from(&mut self.blocks[block_idx.idx()], start); + } + } + } + } + + fn deoptimize_borrow_after_protected_store_tail(&mut self) { + fn deoptimize_block_borrows_from(block: &mut Block, start: usize) { + for info in block.instructions.iter_mut().skip(start) { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + + fn is_handler_resume_predecessor(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + has_pop_except && jumps_to_target + } + + fn handler_chain_can_resume_to_segment( + blocks: &[Block], + block: &Block, + in_segment: &[bool], + ) -> bool { + let mut visited = vec![false; blocks.len()]; + let handler_blocks: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .collect(); + for handler_block in handler_blocks { + let mut cursor = handler_block; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let handler = &blocks[cursor.idx()]; + let mut after_pop_except = false; + for info in &handler.instructions { + if matches!(info.instr.real(), Some(Instruction::PopExcept)) { + after_pop_except = true; + continue; + } + if after_pop_except + && info.target != BlockIdx::NULL + && in_segment[info.target.idx()] + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + { + return true; + } + } + cursor = handler.next; + } + } + false + } + + fn handler_chain_has_explicit_raise(blocks: &[Block], block: &Block) -> bool { + let mut visited = vec![false; blocks.len()]; + let mut stack: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .collect(); + while let Some(cursor) = stack.pop() { + if cursor == BlockIdx::NULL || visited[cursor.idx()] { + continue; + } + visited[cursor.idx()] = true; + let handler = &blocks[cursor.idx()]; + if handler + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))) + { + return true; + } + for info in &handler.instructions { + if is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL { + stack.push(info.target); + } + if info.instr.is_unconditional_jump() && info.target != BlockIdx::NULL { + stack.push(info.target); + } + } + if handler.next != BlockIdx::NULL { + stack.push(handler.next); + } + } + false + } + + fn block_has_tail_deopt_trigger_from(block: &Block, start: usize) -> bool { + block.instructions.iter().skip(start).any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx + | Instruction::StoreAttr { .. } + | Instruction::StoreSubscr + ) + ) + }) + } + + fn block_has_attr_named(block: &Block, names: &IndexSet, attr: &str) -> bool { + block.instructions.iter().any(|info| { + let raw = u32::from(info.arg) as usize; + matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) + if names[usize::try_from(namei.get(info.arg).name_idx()).unwrap()].as_str() + == attr + || names + .get_index(raw) + .is_some_and(|name| name.as_str() == attr) + || names + .get_index(raw >> 1) + .is_some_and(|name| name.as_str() == attr) + ) + }) + } + + fn block_has_protected_instructions(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + } + + fn block_has_non_nop_real_instructions(block: &Block) -> bool { + block.instructions.iter().any(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop)) + }) + } + + fn first_unprotected_suffix(block: &Block) -> Option { + let mut saw_protected = false; + for (idx, info) in block.instructions.iter().enumerate() { + if info.except_handler.is_some() { + saw_protected = true; + } else if saw_protected { + return Some(idx); + } + } + None + } + + fn collect_stored_fast_locals_until(block: &Block, end: usize) -> Vec { + let mut locals = Vec::new(); + for info in block.instructions.iter().take(end) { + match info.instr.real() { + Some(Instruction::StoreFast { var_num }) => { + locals.push(usize::from(var_num.get(info.arg))); + } + Some(Instruction::StoreFastLoadFast { var_nums }) => { + let (store_idx, _) = var_nums.get(info.arg).indexes(); + locals.push(usize::from(store_idx)); + } + Some(Instruction::StoreFastStoreFast { var_nums }) => { + let (idx1, idx2) = var_nums.get(info.arg).indexes(); + locals.push(usize::from(idx1)); + locals.push(usize::from(idx2)); + } + _ => {} + } + } + locals + } + + fn borrows_any_local_from(block: &Block, locals: &[usize], start: usize) -> bool { + block + .instructions + .iter() + .skip(start) + .any(|info| match info.instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { + locals.contains(&usize::from(var_num.get(info.arg))) + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { var_nums }) => { + let (idx1, idx2) = var_nums.get(info.arg).indexes(); + locals.contains(&usize::from(idx1)) || locals.contains(&usize::from(idx2)) + } + _ => false, + }) + } + + fn starts_with_borrowed_local_bool_guard(block: &Block, locals: &[usize]) -> bool { + let mut reals = block + .instructions + .iter() + .filter(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .take(3); + let Some(first) = reals.next() else { + return false; + }; + let Some(second) = reals.next() else { + return false; + }; + let Some(third) = reals.next() else { + return false; + }; + let borrows_stored_local = match first.instr.real() { + Some(Instruction::LoadFastBorrow { var_num }) => { + locals.contains(&usize::from(var_num.get(first.arg))) + } + _ => false, + }; + borrows_stored_local + && matches!(second.instr.real(), Some(Instruction::ToBool)) + && matches!( + third.instr.real(), + Some(Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. }) + ) + } + + fn conditional_target(block: &Block) -> Option { + block + .instructions + .iter() + .find(|info| is_conditional_jump(&info.instr) && info.target != BlockIdx::NULL) + .map(|info| info.target) + } + + fn block_is_simple_exit_branch(block: &Block) -> bool { + let last_real = block + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()); + last_real.is_some_and(|instr| { + instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() + }) + } + + fn contains_debug_four_guard(block: &Block, names: &IndexSet) -> bool { + let reals: Vec<_> = block + .instructions + .iter() + .filter(|info| { + info.instr.real().is_some_and(|instr| { + !matches!(instr, Instruction::Nop | Instruction::NotTaken) + }) + }) + .collect(); + if reals.len() < 5 { + return false; + } + reals.windows(5).any(|window| { + let loads_debug_attr = window.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadAttr { namei }) + if names[usize::try_from(namei.get(info.arg).name_idx()).unwrap()].as_str() + == "debug" + ) + }); + let compares_with_four = window.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::LoadSmallInt { i }) if i.get(info.arg) == 4 + ) + }) && window + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::CompareOp { .. }))); + let has_conditional = window.iter().any(|info| is_conditional_jump(&info.instr)); + loads_debug_attr && compares_with_four && has_conditional + }) + } + + fn marker_only_block(block: &Block) -> bool { + block.instructions.iter().all(|info| { + info.instr + .real() + .is_none_or(|instr| matches!(instr, Instruction::Nop | Instruction::NotTaken)) + }) + } + + fn predecessor_chain_contains_debug_four_guard( + blocks: &[Block], + predecessors: &[Vec], + block_idx: BlockIdx, + names: &IndexSet, + ) -> bool { + predecessors[block_idx.idx()].iter().any(|pred| { + contains_debug_four_guard(&blocks[pred.idx()], names) + || (marker_only_block(&blocks[pred.idx()]) + && predecessors[pred.idx()].iter().any(|pred_pred| { + contains_debug_four_guard(&blocks[pred_pred.idx()], names) + })) + }) + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + let mut to_deopt = Vec::new(); + for block in &self.blocks { + if block_is_exceptional(block) + || !block + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + || !block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some( + Instruction::Call { .. } + | Instruction::CallKw { .. } + | Instruction::CallFunctionEx + ) + ) + }) + || !block_has_exception_match_handler(&self.blocks, block) + { + continue; + } + let same_block_tail_start = first_unprotected_suffix(block); + if same_block_tail_start.is_some() { + continue; + } + let stored_locals = collect_stored_fast_locals_until(block, block.instructions.len()); + if stored_locals.is_empty() { + continue; + } + let handler_has_explicit_raise = handler_chain_has_explicit_raise(&self.blocks, block); + let tail = next_nonempty_block(&self.blocks, block.next); + if tail != BlockIdx::NULL + && !block_is_exceptional(&self.blocks[tail.idx()]) + && !block_has_protected_instructions(&self.blocks[tail.idx()]) + && starts_with_borrowed_local_bool_guard(&self.blocks[tail.idx()], &stored_locals) + && !handler_has_explicit_raise + { + let jump_target = conditional_target(&self.blocks[tail.idx()]); + let fallthrough = next_nonempty_block(&self.blocks, self.blocks[tail.idx()].next); + if let Some(jump_target) = jump_target { + let branches = [(jump_target, fallthrough), (fallthrough, jump_target)]; + for (work, exit) in branches { + if work == BlockIdx::NULL || exit == BlockIdx::NULL { + continue; + } + let work_block = &self.blocks[work.idx()]; + let exit_block = &self.blocks[exit.idx()]; + if !block_is_exceptional(work_block) + && !block_has_protected_instructions(work_block) + && block_has_tail_deopt_trigger_from(work_block, 0) + && borrows_any_local_from(work_block, &stored_locals, 0) + && block_is_simple_exit_branch(exit_block) + { + to_deopt.push((tail, 0)); + to_deopt.push((work, 0)); + } + } + } + } + let mut in_segment = vec![false; self.blocks.len()]; + let mut segment = Vec::new(); + let mut cursor = { + if tail == BlockIdx::NULL + || block_is_exceptional(&self.blocks[tail.idx()]) + || block_has_protected_instructions(&self.blocks[tail.idx()]) + { + continue; + } + tail + }; + while cursor != BlockIdx::NULL { + let segment_block = &self.blocks[cursor.idx()]; + if block_is_exceptional(segment_block) + || block_has_protected_instructions(segment_block) + { + break; + } + segment.push((cursor, 0)); + in_segment[cursor.idx()] = true; + let last_real = segment_block + .instructions + .iter() + .rev() + .find_map(|info| info.instr.real()); + if last_real.is_some_and(|instr| { + instr.is_scope_exit() || AnyInstruction::Real(instr).is_unconditional_jump() + }) { + break; + } + cursor = next_nonempty_block(&self.blocks, segment_block.next); + } + if segment.is_empty() + || segment.iter().any(|(block_idx, _)| { + contains_debug_four_guard(&self.blocks[block_idx.idx()], &self.metadata.names) + }) + || predecessor_chain_contains_debug_four_guard( + &self.blocks, + &predecessors, + segment[0].0, + &self.metadata.names, + ) + || !segment.iter().any(|(block_idx, start)| { + block_has_tail_deopt_trigger_from(&self.blocks[block_idx.idx()], *start) + }) + || handler_chain_can_resume_to_segment(&self.blocks, block, &in_segment) + || segment.iter().any(|(block_idx, _)| { + predecessors[block_idx.idx()].iter().any(|pred| { + let pred_block = &self.blocks[pred.idx()]; + !in_segment[pred.idx()] + && !block_is_exceptional(pred_block) + && !pred_block.cold + && !block_has_protected_instructions(pred_block) + && block_has_non_nop_real_instructions(pred_block) + }) + }) + || !segment.iter().any(|(block_idx, start)| { + borrows_any_local_from(&self.blocks[block_idx.idx()], &stored_locals, *start) + }) + { + continue; + } + for (block_idx, start) in segment { + if predecessors[block_idx.idx()] + .iter() + .any(|pred| is_handler_resume_predecessor(&self.blocks[pred.idx()], block_idx)) + { + continue; + } + to_deopt.push((block_idx, start)); + } + } + + let mut continue_targets = Vec::new(); + for (handler_idx, block) in self.blocks.iter().enumerate() { + if !block.cold + || !block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) + { + continue; + } + let mut visited = vec![false; self.blocks.len()]; + let mut cursor = BlockIdx::new(handler_idx as u32); + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let handler = &self.blocks[cursor.idx()]; + let has_pop_except = handler + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + if has_pop_except { + for info in &handler.instructions { + if info.target != BlockIdx::NULL + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + { + continue_targets.push(info.target); + } + } + } + cursor = handler.next; + } + } + + continue_targets.sort_by_key(|idx| idx.idx()); + continue_targets.dedup(); + for target in continue_targets { let block = &self.blocks[target.idx()]; if block.cold || block_is_exceptional(block) @@ -4121,43 +8263,200 @@ impl CodeInfo { if stored_locals.is_empty() { continue; } - let tail = next_nonempty_block(&self.blocks, block.next); - if tail == BlockIdx::NULL - || block_is_exceptional(&self.blocks[tail.idx()]) - || !block_has_tail_deopt_trigger_from(&self.blocks[tail.idx()], 0) - || !borrows_any_local_from(&self.blocks[tail.idx()], &stored_locals, 0) + let tail = next_nonempty_block(&self.blocks, block.next); + if tail == BlockIdx::NULL + || block_is_exceptional(&self.blocks[tail.idx()]) + || !block_has_tail_deopt_trigger_from(&self.blocks[tail.idx()], 0) + || !borrows_any_local_from(&self.blocks[tail.idx()], &stored_locals, 0) + { + continue; + } + let tail_jumps_back_to_target = + self.blocks[tail.idx()].instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + if tail_jumps_back_to_target { + to_deopt.push((tail, 0)); + } + } + + to_deopt.sort_by_key(|(idx, start)| (idx.idx(), *start)); + let mut merged: Vec<(BlockIdx, usize)> = Vec::new(); + for (idx, start) in to_deopt { + match merged.last_mut() { + Some((last_idx, last_start)) if *last_idx == idx => { + *last_start = (*last_start).min(start); + } + _ => merged.push((idx, start)), + } + } + for (block_idx, start) in merged { + if block_has_attr_named(&self.blocks[block_idx.idx()], &self.metadata.names, "_mesg") { + continue; + } + if contains_debug_four_guard(&self.blocks[block_idx.idx()], &self.metadata.names) + || predecessor_chain_contains_debug_four_guard( + &self.blocks, + &predecessors, + block_idx, + &self.metadata.names, + ) + { + continue; + } + deoptimize_block_borrows_from(&mut self.blocks[block_idx.idx()], start); + } + } + + fn reborrow_after_suppressing_handler_resume_cleanup(&mut self) { + fn is_suppressing_handler_resume_to(block: &Block, target: BlockIdx) -> bool { + let has_pop_except = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))); + let clears_exception_name = block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::StoreFast { .. }))) + && block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::DeleteFast { .. }))); + let jumps_to_target = block.instructions.iter().any(|info| { + info.target == target + && matches!( + info.instr.real(), + Some( + Instruction::JumpForward { .. } + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }); + let reraises = block.instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::RaiseVarargs { .. } | Instruction::Reraise { .. }) + ) + }); + has_pop_except && clears_exception_name && jumps_to_target && !reraises + } + + fn block_enters_with_context(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::LoadSpecial { .. }))) + } + + let mut predecessors = vec![Vec::new(); self.blocks.len()]; + for (pred_idx, block) in self.blocks.iter().enumerate() { + if block.next != BlockIdx::NULL { + predecessors[block.next.idx()].push(BlockIdx::new(pred_idx as u32)); + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); + } + } + } + + for (block_idx, predecessors) in predecessors.iter().enumerate() { + let target = BlockIdx::new(block_idx as u32); + if self.blocks[block_idx].cold || block_is_exceptional(&self.blocks[block_idx]) { + continue; + } + if !predecessors + .iter() + .any(|pred| is_suppressing_handler_resume_to(&self.blocks[pred.idx()], target)) + { + continue; + } + if block_enters_with_context(&self.blocks[block_idx]) { + continue; + } + let starts_with_cleanup_call = self.blocks[block_idx] + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .take(5) + .any(|instr| matches!(instr, Instruction::Call { .. })); + if !starts_with_cleanup_call { + continue; + } + let starts_with_named_except_value_load = self.blocks[block_idx] + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .take(5) + .any(|instr| matches!(instr, Instruction::LoadFastCheck { .. })); + if starts_with_named_except_value_load { + continue; + } + if let Some(info) = self.blocks[block_idx].instructions.iter_mut().find(|info| { + info.instr + .real() + .is_some_and(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) + }) && matches!(info.instr.real(), Some(Instruction::LoadFast { .. })) + { + info.instr = Instruction::LoadFastBorrow { + var_num: Arg::marker(), + } + .into(); + } + } + } + + fn deoptimize_borrow_before_import_after_join_store(&mut self) { + let mut predecessor_count = vec![0usize; self.blocks.len()]; + for block in &self.blocks { + if block.next != BlockIdx::NULL { + predecessor_count[block.next.idx()] += 1; + } + for info in &block.instructions { + if info.target != BlockIdx::NULL { + predecessor_count[info.target.idx()] += 1; + } + } + } + + for (block_idx, block) in self.blocks.iter_mut().enumerate() { + if predecessor_count[block_idx] < 2 + || !block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ImportName { .. }))) { continue; } - let tail_jumps_back_to_target = - self.blocks[tail.idx()].instructions.iter().any(|info| { - info.target == target - && matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) - }); - if tail_jumps_back_to_target { - to_deopt.push((tail, 0)); - } - } - to_deopt.sort_by_key(|(idx, start)| (idx.idx(), *start)); - let mut merged: Vec<(BlockIdx, usize)> = Vec::new(); - for (idx, start) in to_deopt { - match merged.last_mut() { - Some((last_idx, last_start)) if *last_idx == idx => { - *last_start = (*last_start).min(start); + let len = block.instructions.len(); + for idx in 0..len.saturating_sub(1) { + if !matches!( + block.instructions[idx].instr.real(), + Some(Instruction::LoadFastBorrow { .. }) + ) { + continue; } - _ => merged.push((idx, start)), + if !matches!( + block.instructions[idx + 1].instr.real(), + Some(Instruction::StoreGlobal { .. }) + ) { + continue; + } + block.instructions[idx].instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); } } - for (block_idx, start) in merged { - deoptimize_block_borrows_from(&mut self.blocks[block_idx.idx()], start); - } } fn deoptimize_borrow_for_match_keys_attr(&mut self) { @@ -4170,10 +8469,10 @@ impl CodeInfo { let block = &self.blocks[block_idx]; let len = block.instructions.len(); for i in 0..len { - let Some(Opcode::LoadFastBorrow) = block.instructions[i].instr.real_opcode() else { + let Some(Instruction::LoadFastBorrow { .. }) = block.instructions[i].instr.real() + else { continue; }; - let Some(Instruction::LoadAttr { namei }) = block .instructions .get(i + 1) @@ -4193,17 +8492,17 @@ impl CodeInfo { loop { let scan_block = &self.blocks[scan_block_idx]; for info in scan_block.instructions.iter().skip(scan_start) { - match info.instr.real_opcode() { + match info.instr.real() { Some( - Opcode::LoadConst - | Opcode::LoadSmallInt - | Opcode::LoadFast - | Opcode::LoadFastBorrow - | Opcode::LoadAttr - | Opcode::Nop, + Instruction::LoadConst { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadAttr { .. } + | Instruction::Nop, ) => {} - Some(Opcode::BuildTuple) => saw_build_tuple = true, - Some(Opcode::MatchKeys) => { + Some(Instruction::BuildTuple { .. }) => saw_build_tuple = true, + Some(Instruction::MatchKeys) => { saw_match_keys = true; break; } @@ -4237,7 +8536,10 @@ impl CodeInfo { } for (block_idx, instr_idx) in to_deopt { - self.blocks[block_idx].instructions[instr_idx].instr = Opcode::LoadFast.into(); + self.blocks[block_idx].instructions[instr_idx].instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); } } @@ -4252,20 +8554,108 @@ impl CodeInfo { reals.next() } - fn deoptimize_borrow(info: &mut InstructionInfo) { - let Some(opcode) = info.instr.real_opcode() else { - return; + fn block_ends_with_suppressing_with_resume_jump(block: &Block) -> bool { + let mut reals = block + .instructions + .iter() + .rev() + .filter_map(|info| info.instr.real()); + let Some(last) = reals.next() else { + return false; }; + if !last.is_unconditional_jump() { + return false; + } + matches!( + (reals.next(), reals.next(), reals.next(), reals.next()), + ( + Some(Instruction::PopTop), + Some(Instruction::PopTop), + Some(Instruction::PopTop), + Some(Instruction::PopExcept) + ) + ) + } - info.instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast.into(), - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast.into(), - _ => info.instr, + fn block_ends_with_handler_resume_jump(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + && block.instructions.last().is_some_and(|info| { + info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() + }) + } + + fn handler_chain_returns_before_resume(blocks: &[Block], handler_block: BlockIdx) -> bool { + let mut cursor = handler_block; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + let block = &blocks[cursor.idx()]; + if block + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))) + { + return true; + } + if block_ends_with_handler_resume_jump(block) { + return false; + } + cursor = block.next; + } + false + } + + fn block_has_returning_exception_match_handler(blocks: &[Block], block: &Block) -> bool { + let mut visited = vec![false; blocks.len()]; + let handler_blocks: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .collect(); + for handler_block in handler_blocks { + let mut cursor = handler_block; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + if blocks[cursor.idx()].instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) { + return handler_chain_returns_before_resume(blocks, handler_block); + } + cursor = blocks[cursor.idx()].next; + } + } + false + } + + fn deoptimize_borrow(info: &mut InstructionInfo) { + match info.instr.real() { + Some(Instruction::LoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} } } - const fn is_attr_load(instr: Instruction) -> bool { - matches!(instr.opcode(), Opcode::LoadAttr | Opcode::LoadSuperAttr) + fn is_attr_load(instr: Instruction) -> bool { + matches!( + instr, + Instruction::LoadAttr { .. } | Instruction::LoadSuperAttr { .. } + ) } fn attr_load_is_method(info: InstructionInfo) -> bool { @@ -4276,21 +8666,26 @@ impl CodeInfo { } } - const fn is_subscript_index_setup(instr: Instruction) -> bool { + fn is_subscript_index_setup(instr: Instruction) -> bool { matches!( - instr.opcode(), - Opcode::LoadConst - | Opcode::LoadSmallInt - | Opcode::LoadFast - | Opcode::LoadFastBorrow - | Opcode::LoadFastCheck - | Opcode::Nop + instr, + Instruction::LoadConst { .. } + | Instruction::LoadSmallInt { .. } + | Instruction::LoadFast { .. } + | Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastCheck { .. } + | Instruction::Nop ) } enum DeoptKind { - ReturnIter { tail_start_idx: usize }, - Subscript { binary_op_idx: usize }, + ReturnIter { + tail_start_idx: usize, + }, + Subscript { + binary_op_idx: usize, + direct_root: bool, + }, } fn should_deopt_borrowed_attr_chain( @@ -4298,12 +8693,6 @@ impl CodeInfo { load_idx: usize, ) -> Option { let mut cursor = load_idx + 1; - if !real_instrs - .get(cursor) - .is_some_and(|(_, info)| info.instr.real().is_some_and(is_attr_load)) - { - return None; - } let mut last_attr_is_method = false; while let Some((_, info)) = real_instrs.get(cursor) { if !info.instr.real().is_some_and(is_attr_load) { @@ -4312,14 +8701,22 @@ impl CodeInfo { last_attr_is_method = attr_load_is_method(*info); cursor += 1; } + let direct_root = cursor == load_idx + 1; + if direct_root + && !real_instrs.get(cursor).is_some_and(|(_, info)| { + info.instr.real().is_some_and(is_subscript_index_setup) + }) + { + return None; + } let (_, next_info) = real_instrs.get(cursor)?; - match next_info.instr.real_opcode() { - Some(Opcode::GetIter) => Some(DeoptKind::ReturnIter { + match next_info.instr.real() { + Some(Instruction::GetIter) => Some(DeoptKind::ReturnIter { tail_start_idx: cursor + 1, }), - Some(Opcode::Call | Opcode::CallKw) => real_instrs + Some(Instruction::Call { .. } | Instruction::CallKw { .. }) => real_instrs .get(cursor + 1) .and_then(|(_, info)| info.instr.real()) .and_then(|instr| { @@ -4344,6 +8741,7 @@ impl CodeInfo { ) .then_some(DeoptKind::Subscript { binary_op_idx: cursor, + direct_root, }) }) } @@ -4364,21 +8762,18 @@ impl CodeInfo { } let block = &blocks[block_idx.idx()]; for info in block.instructions.iter().skip(current_start) { - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; - - match opcode { - Opcode::ReturnValue => return true, - Opcode::StoreFast - | Opcode::StoreFastLoadFast - | Opcode::StoreFastStoreFast - | Opcode::DeleteFast - | Opcode::LoadFastAndClear => return false, + match info.instr.real() { + Some(Instruction::ReturnValue) => return true, + Some( + Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + | Instruction::StoreFastStoreFast { .. } + | Instruction::DeleteFast { .. } + | Instruction::LoadFastAndClear { .. }, + ) => return false, _ => {} } } - block_idx = block.next; current_start = 0; } @@ -4412,7 +8807,10 @@ impl CodeInfo { } continue; } - if !matches!(second_last_real_instr(block), Some(Instruction::PopExcept)) { + let is_handler_resume_jump = + matches!(second_last_real_instr(block), Some(Instruction::PopExcept)) + || block_ends_with_suppressing_with_resume_jump(block); + if !is_handler_resume_jump { for info in &block.instructions { if info.target != BlockIdx::NULL { predecessors[info.target.idx()].push(BlockIdx::new(pred_idx as u32)); @@ -4440,18 +8838,42 @@ impl CodeInfo { for &block_idx in &order[..first_handler_pos] { is_pre_handler[block_idx.idx()] = true; } + let mut is_protected_source = vec![false; self.blocks.len()]; + let mut reachable_from_protected_predecessor = vec![false; self.blocks.len()]; let mut reachable_from_protected = vec![false; self.blocks.len()]; + let mut direct_subscript_from_returning_protected = vec![false; self.blocks.len()]; for &block_idx in &order[..first_handler_pos] { let idx = block_idx.idx(); let block = &self.blocks[idx]; - let is_protected_source = block_has_exception_match_handler(&self.blocks, block); - reachable_from_protected[idx] = predecessors[idx].iter().any(|pred| { - self.blocks[pred.idx()] - .instructions - .iter() - .any(|info| info.except_handler.is_some()) - || reachable_from_protected[pred.idx()] - }) || is_protected_source; + is_protected_source[idx] = block_has_exception_match_handler(&self.blocks, block); + let has_direct_normal_protected_predecessor = predecessors[idx].iter().any(|pred| { + !block_is_exceptional(&self.blocks[pred.idx()]) + && self.blocks[pred.idx()] + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + }); + let has_direct_returning_protected_predecessor = predecessors[idx].iter().any(|pred| { + !block_is_exceptional(&self.blocks[pred.idx()]) + && block_has_returning_exception_match_handler( + &self.blocks, + &self.blocks[pred.idx()], + ) + }); + let has_unprotected_normal_predecessor = predecessors[idx].iter().any(|pred| { + !block_is_exceptional(&self.blocks[pred.idx()]) + && !self.blocks[pred.idx()] + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + }); + reachable_from_protected_predecessor[idx] = + has_direct_normal_protected_predecessor && !has_unprotected_normal_predecessor; + reachable_from_protected[idx] = (is_protected_source[idx] + || has_direct_normal_protected_predecessor) + && !has_unprotected_normal_predecessor; + direct_subscript_from_returning_protected[idx] = + has_direct_returning_protected_predecessor && !has_unprotected_normal_predecessor; } let mut cross_block_deopts = Vec::new(); @@ -4469,9 +8891,12 @@ impl CodeInfo { .collect(); let mut to_deopt = Vec::new(); for (real_idx, (instr_idx, info)) in real_instrs.iter().enumerate() { + let has_prior_protected_instr = real_instrs[..real_idx] + .iter() + .any(|(_, info)| info.except_handler.is_some()); let is_attr_chain_root = matches!( - info.instr.real_opcode(), - Some(Opcode::LoadFast | Opcode::LoadFastBorrow) + info.instr.real(), + Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) ); if info.except_handler.is_some() || !is_attr_chain_root { continue; @@ -4494,39 +8919,49 @@ impl CodeInfo { continue; } } - if matches!(deopt_kind, DeoptKind::Subscript { .. }) - && !reachable_from_protected[block_idx.idx()] - { - continue; + if let DeoptKind::Subscript { direct_root, .. } = deopt_kind { + let should_deopt_subscript = if direct_root { + direct_subscript_from_returning_protected[block_idx.idx()] + } else { + reachable_from_protected_predecessor[block_idx.idx()] + || (is_protected_source[block_idx.idx()] && has_prior_protected_instr) + }; + if !should_deopt_subscript { + continue; + } } - - if matches!(info.instr.real_opcode(), Some(Opcode::LoadFastBorrow)) { + if matches!(info.instr.real(), Some(Instruction::LoadFastBorrow { .. })) { to_deopt.push(*instr_idx); } - - if let DeoptKind::Subscript { binary_op_idx } = deopt_kind { + if let DeoptKind::Subscript { binary_op_idx, .. } = deopt_kind { for (extra_instr_idx, extra_info) in real_instrs .iter() .skip(real_idx + 1) .take(binary_op_idx.saturating_sub(real_idx + 1)) .map(|(idx, info)| (*idx, *info)) { - if matches!(extra_info.instr.real_opcode(), Some(Opcode::LoadFastBorrow)) { + if matches!( + extra_info.instr.real(), + Some(Instruction::LoadFastBorrow { .. }) + ) { to_deopt.push(extra_instr_idx); } } if matches!( real_instrs .get(binary_op_idx + 1) - .and_then(|(_, info)| info.instr.real_opcode()), - Some(Opcode::StoreFast) + .and_then(|(_, info)| info.instr.real()), + Some(Instruction::StoreFast { .. }) ) { for (extra_instr_idx, extra_info) in real_instrs.iter().skip(binary_op_idx + 2) { if matches!( - extra_info.instr.real_opcode(), - Some(Opcode::LoadFastBorrow | Opcode::LoadFastBorrowLoadFastBorrow) + extra_info.instr.real(), + Some( + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) ) { to_deopt.push(*extra_instr_idx); } @@ -4553,10 +8988,10 @@ impl CodeInfo { .enumerate() { if matches!( - tail_info.instr.real_opcode(), + tail_info.instr.real(), Some( - Opcode::LoadFastBorrow - | Opcode::LoadFastBorrowLoadFastBorrow + Instruction::LoadFastBorrow { .. } + | Instruction::LoadFastBorrowLoadFastBorrow { .. } ) ) { cross_block_deopts.push((tail_block_idx, tail_instr_idx)); @@ -4572,19 +9007,26 @@ impl CodeInfo { } } for (block_idx, instr_idx) in cross_block_deopts { - let Some(opcode) = self.blocks[block_idx.idx()].instructions[instr_idx] + match self.blocks[block_idx.idx()].instructions[instr_idx] .instr - .real_opcode() - else { - continue; - }; - - self.blocks[block_idx.idx()].instructions[instr_idx].instr = match opcode { - Opcode::LoadFastBorrow => Opcode::LoadFast, - Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, - _ => continue, + .real() + { + Some(Instruction::LoadFastBorrow { .. }) => { + self.blocks[block_idx.idx()].instructions[instr_idx].instr = + Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { + self.blocks[block_idx.idx()].instructions[instr_idx].instr = + Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} } - .into(); } } @@ -4600,10 +9042,10 @@ impl CodeInfo { fn is_cleanup_restore_prefix(instructions: &[InstructionInfo]) -> bool { let mut saw_pop_iter = false; for info in instructions { - match info.instr.real_opcode() { - Some(Opcode::EndFor) if !saw_pop_iter => {} - Some(Opcode::PopIter) if !saw_pop_iter => saw_pop_iter = true, - Some(Opcode::Swap | Opcode::PopTop) if saw_pop_iter => {} + match info.instr.real() { + Some(Instruction::EndFor) if !saw_pop_iter => {} + Some(Instruction::PopIter) if !saw_pop_iter => saw_pop_iter = true, + Some(Instruction::Swap { .. } | Instruction::PopTop) if saw_pop_iter => {} _ => return false, } } @@ -4641,23 +9083,27 @@ impl CodeInfo { for (i, info) in block.instructions.iter().copied().enumerate() { if !in_restore_prefix && matches!( - info.instr.real_opcode(), - Some(Opcode::StoreFast | Opcode::StoreFastStoreFast) + info.instr.real(), + Some( + Instruction::StoreFast { .. } | Instruction::StoreFastStoreFast { .. } + ) ) && !new_instructions.is_empty() && (new_instructions.iter().all(|prev: &InstructionInfo| { matches!( - prev.instr.real_opcode(), - Some(Opcode::Swap | Opcode::PopTop) + prev.instr.real(), + Some(Instruction::Swap { .. } | Instruction::PopTop) ) }) || is_cleanup_restore_prefix(&new_instructions)) { in_restore_prefix = true; } - let expand = matches!(info.instr.real_opcode(), Some(Opcode::StoreFastStoreFast)) - && (is_cleanup_restore_prefix(&new_instructions) - || (i == 0 && starts_after_cleanup[block_idx]) - || in_restore_prefix); + let expand = matches!( + info.instr.real(), + Some(Instruction::StoreFastStoreFast { .. }) + ) && (is_cleanup_restore_prefix(&new_instructions) + || (i == 0 && starts_after_cleanup[block_idx]) + || in_restore_prefix); if expand { let Some(Instruction::StoreFastStoreFast { var_nums }) = info.instr.real() @@ -4668,18 +9114,25 @@ impl CodeInfo { let (idx1, idx2) = packed.indexes(); let mut first = info; - first.instr = Opcode::StoreFast.into(); + first.instr = Instruction::StoreFast { + var_num: Arg::marker(), + } + .into(); first.arg = OpArg::new(u32::from(idx1)); new_instructions.push(first); let mut second = info; - second.instr = Opcode::StoreFast.into(); + second.instr = Instruction::StoreFast { + var_num: Arg::marker(), + } + .into(); second.arg = OpArg::new(u32::from(idx2)); new_instructions.push(second); continue; } - in_restore_prefix &= matches!(info.instr.real_opcode(), Some(Opcode::StoreFast)); + in_restore_prefix &= + matches!(info.instr.real(), Some(Instruction::StoreFast { .. })); new_instructions.push(info); } block.instructions = new_instructions; @@ -4736,13 +9189,29 @@ impl CodeInfo { worklist.push(target); } } - if matches!(info.instr.real_opcode(), Some(Opcode::ForIter)) + if matches!(info.instr.real(), Some(Instruction::ForIter { .. })) && info.target != BlockIdx::NULL && merge_unsafe_mask(&mut in_masks[info.target.idx()], &unsafe_mask) { worklist.push(info.target); } match info.instr.real() { + None if matches!( + info.instr.pseudo(), + Some(PseudoInstruction::StoreFastMaybeNull { .. }) + ) => + { + let Some(PseudoInstruction::StoreFastMaybeNull { var_num }) = + info.instr.pseudo() + else { + unreachable!(); + }; + let var_idx = var_num.get(info.arg) as usize; + if var_idx < nlocals { + unsafe_mask[var_idx] = true; + } + new_instructions.push(info); + } Some(Instruction::DeleteFast { var_num }) => { let var_idx = usize::from(var_num.get(info.arg)); if var_idx < nlocals { @@ -4837,11 +9306,10 @@ impl CodeInfo { let mut second = info; second.instr = if needs_check_2 { - Opcode::LoadFastCheck + Opcode::LoadFastCheck.into() } else { - Opcode::LoadFast - } - .into(); + Opcode::LoadFast.into() + }; second.arg = OpArg::new(idx2 as u32); new_instructions.push(first); @@ -5038,7 +9506,9 @@ impl CodeInfo { self.fold_binop_constants(); self.fold_unary_constants(); self.fold_binop_constants(); + self.fold_unary_constants(); self.fold_tuple_constants(); + self.fold_binop_constants(); self.fold_list_constants(); self.fold_set_constants(); self.optimize_lists_and_sets(); @@ -5054,6 +9524,7 @@ impl CodeInfo { self.debug_block_dump(), )); self.fold_tuple_constants(); + self.fold_binop_constants(); self.fold_list_constants(); self.fold_set_constants(); self.optimize_lists_and_sets(); @@ -5095,6 +9566,7 @@ impl CodeInfo { self.debug_block_dump(), )); reorder_conditional_chain_and_jump_back_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); trace.push(( "after_push_cold_blocks_to_end".to_owned(), @@ -5105,7 +9577,17 @@ impl CodeInfo { trace.push(("after_normalize_jumps".to_owned(), self.debug_block_dump())); reorder_conditional_exit_and_jump_blocks(&mut self.blocks); reorder_conditional_jump_and_exit_blocks(&mut self.blocks); + reorder_conditional_break_continue_blocks(&mut self.blocks); + reorder_conditional_explicit_continue_scope_exit_blocks(&mut self.blocks); + reorder_conditional_implicit_continue_scope_exit_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); + deduplicate_adjacent_jump_back_blocks(&mut self.blocks); + reorder_conditional_body_and_implicit_continue_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); reorder_jump_over_exception_cleanup_blocks(&mut self.blocks); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); + reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, false); trace.push(("after_reorder".to_owned(), self.debug_block_dump())); self.dce(); @@ -5119,6 +9601,7 @@ impl CodeInfo { )); materialize_empty_conditional_exit_targets(&mut self.blocks); + materialize_exception_cleanup_jump_targets(&mut self.blocks); trace.push(( "after_materialize_empty_conditional_exit_targets".to_owned(), self.debug_block_dump(), @@ -5162,6 +9645,7 @@ impl CodeInfo { remove_redundant_nops_and_jumps(&mut self.blocks); inline_with_suppress_return_blocks(&mut self.blocks); inline_pop_except_return_blocks(&mut self.blocks); + inline_named_except_cleanup_normal_exit_jumps(&mut self.blocks); duplicate_named_except_cleanup_returns(&mut self.blocks, &self.metadata); self.eliminate_unreachable_blocks(); trace.push(( @@ -5184,6 +9668,9 @@ impl CodeInfo { redirect_empty_block_targets(&mut self.blocks); let _ = self.max_stackdepth()?; convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); + remove_redundant_nops_and_jumps(&mut self.blocks); + self.mark_assertion_success_tail_borrow_disabled(); + self.mark_unprotected_debug_four_tails_borrow_disabled(); trace.push(( "after_convert_pseudo_ops".to_owned(), self.debug_block_dump(), @@ -5198,11 +9685,22 @@ impl CodeInfo { "after_raw_optimize_load_fast_borrow".to_owned(), self.debug_block_dump(), )); + self.deoptimize_borrow_in_targeted_assert_message_blocks(); + trace.push(( + "after_deoptimize_borrow_in_targeted_assert_message_blocks".to_owned(), + self.debug_block_dump(), + )); self.deoptimize_borrow_for_folded_nonliteral_exprs(); trace.push(( "after_deoptimize_borrow_for_folded_nonliteral_exprs".to_owned(), self.debug_block_dump(), )); + self.deoptimize_borrow_after_generator_exception_return(); + self.deoptimize_borrow_after_async_for_cleanup_resume(); + trace.push(( + "after_deoptimize_borrow_after_generator_exception_return".to_owned(), + self.debug_block_dump(), + )); self.deoptimize_borrow_after_multi_handler_resume_join(); trace.push(( "after_deoptimize_borrow_after_multi_handler_resume_join".to_owned(), @@ -5213,13 +9711,63 @@ impl CodeInfo { "after_deoptimize_borrow_after_named_except_cleanup_join".to_owned(), self.debug_block_dump(), )); + self.deoptimize_borrow_after_reraising_except_handler(); + trace.push(( + "after_deoptimize_borrow_after_reraising_except_handler".to_owned(), + self.debug_block_dump(), + )); self.deoptimize_borrow_in_protected_conditional_tail(); trace.push(( "after_deoptimize_borrow_in_protected_conditional_tail".to_owned(), self.debug_block_dump(), )); + self.deoptimize_borrow_after_terminal_except_tail(); + trace.push(( + "after_deoptimize_borrow_after_terminal_except_tail".to_owned(), + self.debug_block_dump(), + )); + self.deoptimize_borrow_after_except_star_try_tail(); + trace.push(( + "after_deoptimize_borrow_after_except_star_try_tail".to_owned(), + self.debug_block_dump(), + )); + self.deoptimize_borrow_in_protected_method_call_after_terminal_except_tail(); + trace.push(( + "after_deoptimize_borrow_in_protected_method_call_after_terminal_except_tail" + .to_owned(), + self.debug_block_dump(), + )); + self.deoptimize_borrow_after_terminal_except_before_with(); + trace.push(( + "after_deoptimize_borrow_after_terminal_except_before_with".to_owned(), + self.debug_block_dump(), + )); + self.deoptimize_borrow_in_async_finally_early_return_tail(); + trace.push(( + "after_deoptimize_borrow_in_async_finally_early_return_tail".to_owned(), + self.debug_block_dump(), + )); + self.deoptimize_borrow_after_handler_resume_loop_tail(); + trace.push(( + "after_deoptimize_borrow_after_handler_resume_loop_tail".to_owned(), + self.debug_block_dump(), + )); self.deoptimize_borrow_after_protected_import(); + trace.push(( + "after_deoptimize_borrow_after_protected_import".to_owned(), + self.debug_block_dump(), + )); + self.deoptimize_borrow_before_import_after_join_store(); + trace.push(( + "after_deoptimize_borrow_before_import_after_join_store".to_owned(), + self.debug_block_dump(), + )); self.deoptimize_borrow_after_protected_store_tail(); + trace.push(( + "after_deoptimize_borrow_after_protected_store_tail".to_owned(), + self.debug_block_dump(), + )); + self.deoptimize_borrow_after_deoptimized_async_with_enter(); trace.push(( "after_optimize_load_fast_borrow".to_owned(), self.debug_block_dump(), @@ -5228,7 +9776,15 @@ impl CodeInfo { self.deoptimize_borrow_for_handler_return_paths(); self.deoptimize_borrow_for_match_keys_attr(); self.deoptimize_borrow_in_protected_attr_chain_tail(); + self.reborrow_after_suppressing_handler_resume_cleanup(); trace.push(("after_borrow_deopts".to_owned(), self.debug_block_dump())); + self.deoptimize_store_fast_store_fast_after_cleanup(); + self.apply_static_swaps(); + self.deoptimize_store_fast_store_fast_after_cleanup(); + self.optimize_load_global_push_null(); + self.reorder_entry_prefix_cell_setup(); + self.remove_unused_consts(); + deoptimize_borrow_after_push_exc_info_in_blocks(&mut self.blocks); Ok(trace) } @@ -5917,6 +10473,23 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { .get(final_target.idx()) .copied() .unwrap_or(u32::MAX); + let conditional = is_conditional_jump(&ins.instr); + if conditional && final_target_pos <= source_pos { + let mut scan = blocks[bi].next; + let mut chain_has_delete_subscr = false; + let mut seen = vec![false; blocks.len()]; + while scan != BlockIdx::NULL && scan != target && !seen[scan.idx()] { + seen[scan.idx()] = true; + chain_has_delete_subscr |= + blocks[scan.idx()].instructions.iter().any(|info| { + matches!(info.instr.real(), Some(Instruction::DeleteSubscr)) + }); + scan = blocks[scan.idx()].next; + } + if chain_has_delete_subscr { + continue; + } + } if !include_conditional && source_pos < target_pos && final_target_pos < target_pos { // Keep the forward hop when threading would turn it into a @@ -5938,7 +10511,6 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { // the line-anchored continue/break jump that follows. continue; } - let conditional = is_conditional_jump(&ins.instr); if conditional && !matches!( jump_thread_kind(target_ins.instr), @@ -5952,9 +10524,6 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { else { continue; }; - if conditional && final_target_pos <= source_pos { - continue; - } if ins.target == final_target { continue; } @@ -6137,21 +10706,20 @@ fn normalize_jumps(blocks: &mut Vec) { info.instr = match info.instr.into() { AnyOpcode::Pseudo(PseudoOpcode::Jump) => { if target_pos > source_pos { - Opcode::JumpForward + Opcode::JumpForward.into() } else { - Opcode::JumpBackward + Opcode::JumpBackward.into() } } AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { if target_pos > source_pos { - Opcode::JumpForward + Opcode::JumpForward.into() } else { - Opcode::JumpBackwardNoInterrupt + Opcode::JumpBackwardNoInterrupt.into() } } - _ => continue, - } - .into(); + _ => info.instr, + }; } } } @@ -6204,8 +10772,7 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { } let target = last.target; - if block_is_exceptional(&blocks[current.idx()]) - || block_is_exceptional(&blocks[target.idx()]) + if block_is_exceptional(&blocks[target.idx()]) || (is_named_except_cleanup_normal_exit_block(&blocks[current.idx()]) && target_pushes_handler(&blocks[target.idx()])) { @@ -6319,13 +10886,11 @@ fn inline_single_predecessor_artificial_expr_exit_blocks(blocks: &mut [Block]) { struct TargetPredecessorFlags { targeted: Vec, - jump: Vec, plain_jump: Vec, } fn compute_target_predecessor_flags(blocks: &[Block]) -> TargetPredecessorFlags { let mut targeted = vec![false; blocks.len()]; - let mut jump = vec![false; blocks.len()]; let mut plain_jump = vec![false; blocks.len()]; for block in blocks { for instr in &block.instructions { @@ -6338,9 +10903,6 @@ fn compute_target_predecessor_flags(blocks: &[Block]) -> TargetPredecessorFlags } let idx = target.idx(); targeted[idx] = true; - if is_jump_instruction(instr) { - jump[idx] = true; - } if matches!(jump_thread_kind(instr.instr), Some(JumpThreadKind::Plain)) { plain_jump[idx] = true; } @@ -6348,29 +10910,16 @@ fn compute_target_predecessor_flags(blocks: &[Block]) -> TargetPredecessorFlags } TargetPredecessorFlags { targeted, - jump, plain_jump, } } fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { - fn ends_with_for_cleanup(block: &Block) -> bool { - let mut reals = block - .instructions - .iter() - .rev() - .filter_map(|info| info.instr.real()); - matches!( - (reals.next(), reals.next()), - (Some(Instruction::PopIter), Some(Instruction::EndFor)) - ) - } - let mut changes = 0; let TargetPredecessorFlags { targeted: targeted_blocks, - jump: jump_targets, plain_jump: plain_jump_targets, + .. } = compute_target_predecessor_flags(blocks); let mut block_order = Vec::new(); let mut current = BlockIdx(0); @@ -6379,6 +10928,7 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { current = blocks[current.idx()].next; } let mut fallthrough_prev_lineno = vec![None; blocks.len()]; + let mut fallthrough_prev_pop_top_lineno = vec![None; blocks.len()]; let mut prev_nonempty = BlockIdx::NULL; for &block_idx in &block_order { if blocks[block_idx.idx()].instructions.is_empty() { @@ -6386,7 +10936,6 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { } if prev_nonempty != BlockIdx::NULL && !targeted_blocks[block_idx.idx()] - && ends_with_for_cleanup(&blocks[prev_nonempty.idx()]) && block_has_fallthrough(&blocks[prev_nonempty.idx()]) && next_nonempty_block(blocks, blocks[prev_nonempty.idx()].next) == block_idx { @@ -6395,6 +10944,14 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { .last() .map(instruction_lineno); } + if prev_nonempty != BlockIdx::NULL + && block_has_fallthrough(&blocks[prev_nonempty.idx()]) + && next_nonempty_block(blocks, blocks[prev_nonempty.idx()].next) == block_idx + && let Some(last) = blocks[prev_nonempty.idx()].instructions.last() + && matches!(last.instr.real(), Some(Instruction::PopTop)) + { + fallthrough_prev_pop_top_lineno[block_idx.idx()] = Some(instruction_lineno(last)); + } prev_nonempty = block_idx; } @@ -6410,13 +10967,16 @@ fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { let mut remove = false; if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if lineno < 0 { - remove = !(instr.remove_no_location_nop - && src == 0 - && jump_targets[block_idx.idx()]); - } else if src == 0 - && fallthrough_prev_lineno[block_idx.idx()] - .is_some_and(|prev_lineno| prev_lineno == lineno) + if instr.preserve_block_start_no_location_nop { + remove = false; + } else if lineno < 0 + || (src == 0 + && fallthrough_prev_lineno[block_idx.idx()] + .is_some_and(|prev_lineno| prev_lineno == lineno)) + || (src == 0 + && src_instructions.len() == 1 + && fallthrough_prev_pop_top_lineno[block_idx.idx()] + .is_some_and(|prev_lineno| prev_lineno == lineno)) { remove = true; } else if instr.remove_no_location_nop @@ -6530,7 +11090,13 @@ fn remove_redundant_jumps_in_blocks(blocks: &mut [Block]) -> usize { last_instr.preserve_redundant_jump_as_nop = false; } let last_instr = blocks[idx].instructions.last_mut().unwrap(); + let remove_no_location_nop = last_instr.remove_no_location_nop; + let preserve_block_start_no_location_nop = + last_instr.preserve_block_start_no_location_nop; set_to_nop(last_instr); + last_instr.remove_no_location_nop = remove_no_location_nop; + last_instr.preserve_block_start_no_location_nop = + preserve_block_start_no_location_nop; changes += 1; current = blocks[idx].next; continue; @@ -6589,7 +11155,17 @@ fn redirect_empty_unconditional_jump_targets(blocks: &mut [Block]) { if instr.target == BlockIdx::NULL || !instr.instr.is_unconditional_jump() { instr.target } else { - next_nonempty_block(blocks, instr.target) + let target = next_nonempty_block(blocks, instr.target); + if matches!( + jump_thread_kind(instr.instr), + Some(JumpThreadKind::NoInterrupt) + ) && (target == BlockIdx::NULL + || is_jump_back_only_block(blocks, target)) + { + instr.target + } else { + target + } } }) .collect() @@ -6674,6 +11250,92 @@ fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { } } +fn materialize_exception_cleanup_jump_targets(blocks: &mut [Block]) { + let mut inserts = Vec::new(); + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + let Some(last) = blocks[idx].instructions.last() else { + current = blocks[idx].next; + continue; + }; + if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { + current = blocks[idx].next; + continue; + } + + let mut cursor = blocks[idx].next; + let mut saw_cleanup = false; + while cursor != BlockIdx::NULL && cursor != last.target { + let block = &blocks[cursor.idx()]; + if !block_is_exceptional(block) && !is_exception_cleanup_block(block) { + break; + } + saw_cleanup = true; + cursor = block.next; + } + if saw_cleanup + && cursor == last.target + && !blocks[last.target.idx()].instructions.is_empty() + && !block_starts_with_with_exit_none_call(&blocks[last.target.idx()]) + && !matches!( + blocks[last.target.idx()] + .instructions + .first() + .and_then(|info| info.instr.real()), + Some(Instruction::Nop) + ) + { + inserts.push(last.target); + } + + current = blocks[idx].next; + } + + inserts.sort_by_key(|idx| idx.idx()); + inserts.dedup(); + for target in inserts { + let Some(first) = blocks[target.idx()].instructions.first().copied() else { + continue; + }; + blocks[target.idx()].instructions.insert( + 0, + InstructionInfo { + instr: Instruction::Nop.into(), + arg: OpArg::NULL, + target: BlockIdx::NULL, + location: first.location, + end_location: first.end_location, + except_handler: None, + folded_from_nonliteral_expr: false, + lineno_override: Some(-1), + cache_entries: 0, + preserve_redundant_jump_as_nop: false, + remove_no_location_nop: false, + preserve_block_start_no_location_nop: true, + }, + ); + } +} + +fn block_starts_with_with_exit_none_call(block: &Block) -> bool { + let real_instrs: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.instr.real()) + .take(4) + .collect(); + matches!( + real_instrs.as_slice(), + [ + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::LoadConst { .. }, + Instruction::Call { .. }, + ] + ) +} + fn merge_unsafe_mask(slot: &mut Option>, incoming: &[bool]) -> bool { match slot { Some(existing) => { @@ -6699,22 +11361,24 @@ fn deoptimize_borrow_after_push_exc_info_in_blocks(blocks: &mut [Block]) { while current != BlockIdx::NULL { let block = &mut blocks[current.idx()]; for info in &mut block.instructions { - let Some(opcode) = info.instr.real_opcode() else { - continue; - }; - - match opcode { - Opcode::PushExcInfo => { + match info.instr.real() { + Some(Instruction::PushExcInfo) => { in_exception_state = true; } - Opcode::PopExcept | Opcode::Reraise => { + Some(Instruction::PopExcept | Instruction::Reraise { .. }) => { in_exception_state = false; } - Opcode::LoadFastBorrow if in_exception_state => { - info.instr = Opcode::LoadFast.into(); + Some(Instruction::LoadFastBorrow { .. }) if in_exception_state => { + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); } - Opcode::LoadFastBorrowLoadFastBorrow if in_exception_state => { - info.instr = Opcode::LoadFastLoadFast.into(); + Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) if in_exception_state => { + info.instr = Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(); } _ => {} } @@ -6735,7 +11399,7 @@ fn next_nonempty_block(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { } fn is_load_const_none(instr: &InstructionInfo, metadata: &CodeUnitMetadata) -> bool { - matches!(instr.instr.real_opcode(), Some(Opcode::LoadConst)) + matches!(instr.instr.real(), Some(Instruction::LoadConst { .. })) && matches!( metadata.consts.get_index(u32::from(instr.arg) as usize), Some(ConstantData::None) @@ -6767,7 +11431,8 @@ fn is_jump_instruction(instr: &InstructionInfo) -> bool { instr.instr.is_unconditional_jump() || is_conditional_jump(&instr.instr) } -fn is_exit_without_lineno(block: &Block) -> bool { +fn is_exit_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { + let block = &blocks[block_idx.idx()]; let Some(first) = block.instructions.first() else { return false; }; @@ -6786,8 +11451,10 @@ fn is_exit_without_lineno(block: &Block) -> bool { // CPython duplicates no-lineno exit blocks before propagating locations. // RustPython's late CFG can inline the following synthetic jump-back block // into that exit block first, collapsing `POP_EXCEPT; JUMP_BACKWARD` into a - // single block. Treat that merged tail as exit-like so resolve_line_numbers() - // can still duplicate it per predecessor and recover CPython's structure. + // single block. Treat that merged tail as exit-like only for real loop + // backedges so resolve_line_numbers() can recover CPython's loop cleanup + // structure without duplicating cold exception-handler jumps to normal + // forward continuation code. let Some((last, prefix)) = block.instructions.split_last() else { return false; }; @@ -6801,6 +11468,7 @@ fn is_exit_without_lineno(block: &Block) -> bool { && prefix .iter() .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) + && has_non_exception_loop_backedge_to(blocks, block_idx, last.target) } fn block_has_no_lineno(block: &Block) -> bool { @@ -6832,6 +11500,29 @@ fn shared_jump_back_target(block: &Block) -> Option { Some(last.target) } +fn has_non_exception_loop_backedge_to( + blocks: &[Block], + cleanup_block: BlockIdx, + target: BlockIdx, +) -> bool { + let target = next_nonempty_block(blocks, target); + if target == BlockIdx::NULL { + return false; + } + + blocks.iter().enumerate().any(|(source_idx, block)| { + let source = BlockIdx(source_idx as u32); + source != cleanup_block + && !block_is_exceptional(block) + && comes_before(blocks, target, source) + && block.instructions.iter().any(|info| { + info.instr.is_unconditional_jump() + && info.target != BlockIdx::NULL + && next_nonempty_block(blocks, info.target) == target + }) + }) +} + fn is_jump_only_block(block: &Block) -> bool { let [instr] = block.instructions.as_slice() else { return false; @@ -6839,6 +11530,17 @@ fn is_jump_only_block(block: &Block) -> bool { instr.instr.is_unconditional_jump() && instr.target != BlockIdx::NULL } +fn is_jump_back_only_block(blocks: &[Block], block_idx: BlockIdx) -> bool { + if block_idx == BlockIdx::NULL || !is_jump_only_block(&blocks[block_idx.idx()]) { + return false; + } + comes_before( + blocks, + next_nonempty_block(blocks, blocks[block_idx.idx()].instructions[0].target), + block_idx, + ) +} + fn is_pop_top_jump_block(block: &Block) -> bool { let mut real_instrs = block .instructions @@ -6884,7 +11586,7 @@ fn is_exception_cleanup_block(block: &Block) -> bool { && block .instructions .last() - .is_some_and(|instr| matches!(instr.instr.real_opcode(), Some(Opcode::Reraise))) + .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::Reraise { .. }))) } fn is_with_suppress_exit_block(block: &Block) -> bool { @@ -6917,11 +11619,14 @@ fn block_contains_suspension_point(block: &Block) -> bool { block .instructions .iter() - .filter_map(|info| info.instr.real_opcode()) + .filter_map(|info| info.instr.real()) .any(|instr| { matches!( instr, - Opcode::YieldValue | Opcode::GetAwaitable | Opcode::GetANext | Opcode::EndAsyncFor + Instruction::YieldValue { .. } + | Instruction::GetAwaitable { .. } + | Instruction::GetANext + | Instruction::EndAsyncFor ) }) } @@ -6939,76 +11644,345 @@ fn is_stop_iteration_error_handler_block(block: &Block) -> bool { instr: AnyInstruction::Real(Instruction::Reraise { .. }), .. } - ] if matches!(func.get(*arg), oparg::IntrinsicFunction1::StopIterationError) - ) + ] if matches!(func.get(*arg), oparg::IntrinsicFunction1::StopIterationError) + ) +} + +fn block_has_only_stop_iteration_error_handlers(block: &Block, blocks: &[Block]) -> bool { + let mut saw_handler = false; + for info in &block.instructions { + let Some(handler) = info.except_handler else { + continue; + }; + saw_handler = true; + let target = next_nonempty_block(blocks, handler.handler_block); + if target == BlockIdx::NULL || !is_stop_iteration_error_handler_block(&blocks[target.idx()]) + { + return false; + } + } + saw_handler +} + +fn block_has_exception_match_handler(blocks: &[Block], block: &Block) -> bool { + let mut visited = vec![false; blocks.len()]; + let handler_blocks: Vec<_> = block + .instructions + .iter() + .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) + .collect(); + for handler_block in handler_blocks { + let mut cursor = handler_block; + while cursor != BlockIdx::NULL && !visited[cursor.idx()] { + visited[cursor.idx()] = true; + if blocks[cursor.idx()].instructions.iter().any(|info| { + matches!( + info.instr.real(), + Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) + ) + }) { + return true; + } + cursor = blocks[cursor.idx()].next; + } + } + false +} + +fn block_is_exceptional(block: &Block) -> bool { + block.except_handler || block.preserve_lasti || is_exception_cleanup_block(block) +} + +fn trailing_conditional_jump_index(block: &Block) -> Option { + let last_idx = block.instructions.len().checked_sub(1)?; + if is_conditional_jump(&block.instructions[last_idx].instr) + && block.instructions[last_idx].target != BlockIdx::NULL + { + return Some(last_idx); + } + + let cond_idx = last_idx.checked_sub(1)?; + if matches!( + block.instructions[last_idx].instr.real(), + Some(Instruction::NotTaken) + ) && is_conditional_jump(&block.instructions[cond_idx].instr) + && block.instructions[cond_idx].target != BlockIdx::NULL + { + Some(cond_idx) + } else { + None + } +} + +fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + let next = blocks[idx].next; + let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { + current = next; + continue; + }; + let last = blocks[idx].instructions[cond_idx]; + + let Some(reversed) = reversed_conditional(&last.instr) else { + current = next; + continue; + }; + + let exit_start = next; + let jump_start = last.target; + if exit_start == BlockIdx::NULL || jump_start == BlockIdx::NULL || exit_start == jump_start + { + current = next; + continue; + } + + let mut exit_end = BlockIdx::NULL; + let mut exit_block = BlockIdx::NULL; + let mut cursor = exit_start; + let mut exit_segment_valid = true; + while cursor != BlockIdx::NULL && cursor != jump_start { + if block_is_exceptional(&blocks[cursor.idx()]) { + exit_segment_valid = false; + break; + } + if !blocks[cursor.idx()].instructions.is_empty() { + if exit_block != BlockIdx::NULL { + exit_segment_valid = false; + break; + } + exit_block = cursor; + } + exit_end = cursor; + cursor = blocks[cursor.idx()].next; + } + if !exit_segment_valid + || cursor != jump_start + || exit_end == BlockIdx::NULL + || exit_block == BlockIdx::NULL + || !is_scope_exit_block(&blocks[exit_block.idx()]) + { + current = next; + continue; + } + + let mut jump_end = BlockIdx::NULL; + let mut jump_block = BlockIdx::NULL; + cursor = jump_start; + while cursor != BlockIdx::NULL { + if block_is_exceptional(&blocks[cursor.idx()]) + || block_is_protected(&blocks[cursor.idx()]) + { + jump_block = BlockIdx::NULL; + break; + } + jump_end = cursor; + if blocks[cursor.idx()].instructions.is_empty() { + cursor = blocks[cursor.idx()].next; + continue; + } + if is_jump_only_block(&blocks[cursor.idx()]) { + jump_block = cursor; + } + break; + } + if jump_block == BlockIdx::NULL { + current = next; + continue; + } + let jump_instr = blocks[jump_block.idx()].instructions[0]; + let jump_is_forward = matches!( + jump_instr.instr.real(), + Some(Instruction::JumpForward { .. }) + ); + let jump_is_backward = matches!( + jump_instr.instr.real(), + Some(Instruction::JumpBackward { .. }) + ); + let exit_is_reorderable = + blocks[exit_block.idx()] + .instructions + .last() + .is_some_and(|info| { + matches!( + info.instr.real(), + Some(Instruction::ReturnValue | Instruction::RaiseVarargs { .. }) + ) + }); + if !(jump_is_forward || jump_is_backward && exit_is_reorderable) { + current = next; + continue; + } + if jump_is_backward { + if block_is_protected(&blocks[idx]) + || block_is_protected(&blocks[exit_block.idx()]) + || block_is_protected(&blocks[jump_block.idx()]) + { + current = next; + continue; + } + let after_jump_start = blocks[jump_end.idx()].next; + let after_jump = next_nonempty_block(blocks, after_jump_start); + let after_jump_is_end_async_for = after_jump != BlockIdx::NULL + && blocks[after_jump.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::EndAsyncFor))); + if after_jump_is_end_async_for { + current = next; + continue; + } + let jump_target = jump_instr.target; + let jumps_to_for_iter = blocks[jump_target.idx()] + .instructions + .first() + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))); + let jump_exits_to_loop_exit = trailing_conditional_jump_index( + &blocks[jump_target.idx()], + ) + .is_some_and(|loop_cond_idx| { + let loop_exit_start = blocks[jump_target.idx()].instructions[loop_cond_idx].target; + loop_exit_start == after_jump_start + || next_nonempty_block(blocks, loop_exit_start) == after_jump + }); + if after_jump != BlockIdx::NULL + && !blocks[after_jump.idx()].cold + && !jumps_to_for_iter + && !jump_exits_to_loop_exit + { + current = next; + continue; + } + } + + let after_jump = blocks[jump_end.idx()].next; + blocks[idx].next = jump_start; + blocks[jump_end.idx()].next = exit_start; + blocks[exit_end.idx()].next = after_jump; + + let cond_mut = &mut blocks[idx].instructions[cond_idx]; + cond_mut.instr = reversed; + cond_mut.target = exit_start; + + current = after_jump; + } } -fn block_has_only_stop_iteration_error_handlers(block: &Block, blocks: &[Block]) -> bool { - let mut saw_handler = false; - for info in &block.instructions { - let Some(handler) = info.except_handler else { +fn reorder_conditional_jump_and_exit_blocks(blocks: &mut [Block]) { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + let next = blocks[idx].next; + let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { + current = next; continue; }; - saw_handler = true; - let target = next_nonempty_block(blocks, handler.handler_block); - if target == BlockIdx::NULL || !is_stop_iteration_error_handler_block(&blocks[target.idx()]) + let last = blocks[idx].instructions[cond_idx]; + + let Some(reversed) = reversed_conditional(&last.instr) else { + current = next; + continue; + }; + + let jump_start = next; + let exit_start = last.target; + if jump_start == BlockIdx::NULL || exit_start == BlockIdx::NULL || jump_start == exit_start { - return false; + current = next; + continue; } - } - saw_handler -} -fn block_has_exception_match_handler(blocks: &[Block], block: &Block) -> bool { - let mut visited = vec![false; blocks.len()]; - let handler_blocks: Vec<_> = block - .instructions - .iter() - .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) - .collect(); - for handler_block in handler_blocks { - let mut cursor = handler_block; - while cursor != BlockIdx::NULL && !visited[cursor.idx()] { - visited[cursor.idx()] = true; - if blocks[cursor.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::CheckExcMatch))) - { - return true; + let mut jump_end = BlockIdx::NULL; + let mut jump_block = BlockIdx::NULL; + let mut cursor = jump_start; + let mut jump_segment_valid = true; + while cursor != BlockIdx::NULL && cursor != exit_start { + if block_is_exceptional(&blocks[cursor.idx()]) { + jump_segment_valid = false; + break; + } + if !blocks[cursor.idx()].instructions.is_empty() { + if jump_block != BlockIdx::NULL || !is_jump_only_block(&blocks[cursor.idx()]) { + jump_segment_valid = false; + break; + } + jump_block = cursor; } + jump_end = cursor; cursor = blocks[cursor.idx()].next; } - } - false -} + if !jump_segment_valid || cursor != exit_start || jump_block == BlockIdx::NULL { + current = next; + continue; + } + let jump_instr = blocks[jump_block.idx()].instructions[0]; + if !matches!( + jump_instr.instr.real(), + Some(Instruction::JumpForward { .. }) + ) { + current = next; + continue; + } -fn block_is_exceptional(block: &Block) -> bool { - block.except_handler || block.preserve_lasti || is_exception_cleanup_block(block) -} + let mut exit_end = BlockIdx::NULL; + let mut exit_block = BlockIdx::NULL; + let after_exit = loop { + if cursor == BlockIdx::NULL { + break BlockIdx::NULL; + } + if block_is_exceptional(&blocks[cursor.idx()]) { + if exit_block != BlockIdx::NULL { + break cursor; + } + exit_block = BlockIdx::NULL; + break BlockIdx::NULL; + } + if !blocks[cursor.idx()].instructions.is_empty() { + if exit_block != BlockIdx::NULL { + break cursor; + } + if !is_scope_exit_block(&blocks[cursor.idx()]) { + exit_block = BlockIdx::NULL; + break BlockIdx::NULL; + } + exit_block = cursor; + } + exit_end = cursor; + cursor = blocks[cursor.idx()].next; + }; + if exit_block == BlockIdx::NULL || exit_end == BlockIdx::NULL { + current = next; + continue; + } -fn trailing_conditional_jump_index(block: &Block) -> Option { - let last_idx = block.instructions.len().checked_sub(1)?; - if is_conditional_jump(&block.instructions[last_idx].instr) - && block.instructions[last_idx].target != BlockIdx::NULL - { - return Some(last_idx); - } + blocks[idx].next = exit_start; + blocks[exit_end.idx()].next = jump_start; + blocks[jump_end.idx()].next = after_exit; - let cond_idx = last_idx.checked_sub(1)?; - if matches!( - block.instructions[last_idx].instr.real(), - Some(Instruction::NotTaken) - ) && is_conditional_jump(&block.instructions[cond_idx].instr) - && block.instructions[cond_idx].target != BlockIdx::NULL - { - Some(cond_idx) - } else { - None + let cond_mut = &mut blocks[idx].instructions[cond_idx]; + cond_mut.instr = reversed; + cond_mut.target = jump_start; + + current = after_exit; } } -fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { +fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { + let target_comes_before = |target: BlockIdx, block: BlockIdx, blocks: &[Block]| -> bool { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + if current == target { + return true; + } + if current == block { + return false; + } + current = blocks[current.idx()].next; + } + false + }; + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -7024,39 +11998,120 @@ fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { continue; }; - let exit_start = next; + let chain_start = next; let jump_start = last.target; - if exit_start == BlockIdx::NULL || jump_start == BlockIdx::NULL || exit_start == jump_start + if chain_start == BlockIdx::NULL + || jump_start == BlockIdx::NULL + || chain_start == jump_start { current = next; continue; } + let mut chain_has_suspension_point = false; + let mut scan = chain_start; + while scan != BlockIdx::NULL && scan != jump_start { + if block_contains_suspension_point(&blocks[scan.idx()]) { + chain_has_suspension_point = true; + break; + } + scan = blocks[scan.idx()].next; + } + let chain_starts_with_false_path_jump = trailing_conditional_jump_index( + &blocks[chain_start.idx()], + ) + .is_some_and(|chain_cond_idx| { + is_false_path_conditional_jump( + &blocks[chain_start.idx()].instructions[chain_cond_idx].instr, + ) + }); + let chain_is_single_exit_block = is_scope_exit_block(&blocks[chain_start.idx()]) + && next_nonempty_block(blocks, blocks[chain_start.idx()].next) == jump_start; + let chain_is_jump_only_exit_block = is_jump_only_block(&blocks[chain_start.idx()]) + && !target_comes_before( + blocks[chain_start.idx()].instructions[0].target, + chain_start, + blocks, + ); + let chain_is_jump_back_exit_block = is_jump_only_block(&blocks[chain_start.idx()]) + && target_comes_before( + blocks[chain_start.idx()].instructions[0].target, + chain_start, + blocks, + ); + let allow_true_path_jump_back_reorder = + matches!(last.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) + && (chain_has_suspension_point + || chain_starts_with_false_path_jump + || chain_is_single_exit_block + || chain_is_jump_only_exit_block + || chain_is_jump_back_exit_block); + let is_generic_false_path_reorder = !allow_true_path_jump_back_reorder; + if !is_false_path_conditional_jump(&last.instr) && !allow_true_path_jump_back_reorder { + current = next; + continue; + } + if is_generic_false_path_reorder && is_scope_exit_block(&blocks[chain_start.idx()]) { + current = next; + continue; + } + if block_is_protected(&blocks[idx]) && block_contains_suspension_point(&blocks[idx]) { + current = next; + continue; + } + if let Some(chain_cond_idx) = trailing_conditional_jump_index(&blocks[chain_start.idx()]) { + let chain_cond = blocks[chain_start.idx()].instructions[chain_cond_idx]; + if matches!( + chain_cond.instr.real().map(Into::into), + Some(Opcode::PopJumpIfTrue) + ) { + let chain_true_target = next_nonempty_block(blocks, chain_cond.target); + if chain_true_target != BlockIdx::NULL + && !is_scope_exit_block(&blocks[chain_true_target.idx()]) + && !is_jump_only_block(&blocks[chain_true_target.idx()]) + && !is_pop_top_jump_block(&blocks[chain_true_target.idx()]) + { + current = next; + continue; + } + } + } - let mut exit_end = BlockIdx::NULL; - let mut exit_block = BlockIdx::NULL; - let mut cursor = exit_start; - let mut exit_segment_valid = true; + let mut chain_end = BlockIdx::NULL; + let mut saw_nonempty = false; + let mut nonempty_blocks = 0usize; + let mut real_instr_count = 0usize; + let mut cursor = chain_start; + let mut chain_valid = true; while cursor != BlockIdx::NULL && cursor != jump_start { - if block_is_exceptional(&blocks[cursor.idx()]) { - exit_segment_valid = false; + if block_is_exceptional(&blocks[cursor.idx()]) + || (block_is_protected(&blocks[cursor.idx()]) + && block_contains_suspension_point(&blocks[cursor.idx()]) + && !block_has_only_stop_iteration_error_handlers(&blocks[cursor.idx()], blocks)) + { + chain_valid = false; break; } - if !blocks[cursor.idx()].instructions.is_empty() { - if exit_block != BlockIdx::NULL { - exit_segment_valid = false; - break; - } - exit_block = cursor; + if !blocks[cursor.idx()].instructions.is_empty() { + saw_nonempty = true; + nonempty_blocks += 1; + real_instr_count += blocks[cursor.idx()] + .instructions + .iter() + .filter(|info| info.instr.real().is_some()) + .count(); } - exit_end = cursor; + chain_end = cursor; cursor = blocks[cursor.idx()].next; } - if !exit_segment_valid - || cursor != jump_start - || exit_end == BlockIdx::NULL - || exit_block == BlockIdx::NULL - || !is_scope_exit_block(&blocks[exit_block.idx()]) - { + if !chain_valid || !saw_nonempty || chain_end == BlockIdx::NULL || cursor != jump_start { + current = next; + continue; + } + if is_generic_false_path_reorder && nonempty_blocks > 1 { + current = next; + continue; + } + if !is_generic_false_path_reorder && (nonempty_blocks > 8 || real_instr_count > 80) { current = next; continue; } @@ -7065,9 +12120,7 @@ fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { let mut jump_block = BlockIdx::NULL; cursor = jump_start; while cursor != BlockIdx::NULL { - if block_is_exceptional(&blocks[cursor.idx()]) - || block_is_protected(&blocks[cursor.idx()]) - { + if block_is_exceptional(&blocks[cursor.idx()]) { jump_block = BlockIdx::NULL; break; } @@ -7076,37 +12129,94 @@ fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { cursor = blocks[cursor.idx()].next; continue; } - if is_jump_only_block(&blocks[cursor.idx()]) { + if !is_jump_only_block(&blocks[cursor.idx()]) + || !target_comes_before(blocks[cursor.idx()].instructions[0].target, cursor, blocks) + { + jump_block = BlockIdx::NULL; + } else { jump_block = cursor; } break; } - if jump_block == BlockIdx::NULL { + if jump_block == BlockIdx::NULL || jump_end == BlockIdx::NULL { current = next; continue; } - if !matches!( - blocks[jump_block.idx()].instructions[0].instr.real_opcode(), - Some(Opcode::JumpForward) - ) { + + let after_jump = next_nonempty_block(blocks, blocks[jump_block.idx()].next); + if nonempty_blocks == 1 + && !is_jump_only_block(&blocks[chain_start.idx()]) + && after_jump != BlockIdx::NULL + && !blocks[after_jump.idx()].cold + && !block_is_exceptional(&blocks[after_jump.idx()]) + && !is_scope_exit_block(&blocks[after_jump.idx()]) + && !is_loop_cleanup_block(&blocks[after_jump.idx()]) + { current = next; continue; } - let after_jump = blocks[jump_end.idx()].next; - blocks[idx].next = jump_start; - blocks[jump_end.idx()].next = exit_start; - blocks[exit_end.idx()].next = after_jump; - + let mut cloned_jump = blocks[jump_block.idx()].clone(); + cloned_jump.next = chain_start; + cloned_jump.start_depth = None; + let cloned_idx = BlockIdx::new(blocks.len() as u32); + blocks.push(cloned_jump); + blocks[idx].next = cloned_idx; let cond_mut = &mut blocks[idx].instructions[cond_idx]; cond_mut.instr = reversed; - cond_mut.target = exit_start; + cond_mut.target = chain_start; - current = after_jump; + current = next; } } -fn reorder_conditional_jump_and_exit_blocks(blocks: &mut [Block]) { +fn reorder_conditional_scope_exit_and_jump_back_blocks( + blocks: &mut [Block], + allow_for_iter_jump_targets: bool, + allow_true_scope_exit_reorder: bool, +) { + fn jump_targets_for_iter(blocks: &[Block], jump_block: BlockIdx) -> bool { + if jump_block == BlockIdx::NULL { + return false; + } + let Some(info) = blocks[jump_block.idx()].instructions.first() else { + return false; + }; + let target = next_nonempty_block(blocks, info.target); + target != BlockIdx::NULL + && blocks[target.idx()] + .instructions + .first() + .is_some_and(|target_info| { + matches!(target_info.instr.real(), Some(Instruction::ForIter { .. })) + }) + } + + fn is_explicit_continue_to_for_iter(blocks: &[Block], jump_block: BlockIdx) -> bool { + if jump_block == BlockIdx::NULL { + return false; + } + let Some(info) = blocks[jump_block.idx()].instructions.first() else { + return false; + }; + matches!( + info.instr.real(), + Some(Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. }) + ) && !matches!(info.lineno_override, Some(line) if line < 0) + && jump_targets_for_iter(blocks, jump_block) + } + + fn is_explicit_non_for_jump_back(blocks: &[Block], jump_block: BlockIdx) -> bool { + if jump_block == BlockIdx::NULL || jump_targets_for_iter(blocks, jump_block) { + return false; + } + let Some(info) = blocks[jump_block.idx()].instructions.first() else { + return false; + }; + matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) + && info.lineno_override.is_some_and(|line| line >= 0) + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -7115,108 +12225,308 @@ fn reorder_conditional_jump_and_exit_blocks(blocks: &mut [Block]) { current = next; continue; }; - let last = blocks[idx].instructions[cond_idx]; + let cond = blocks[idx].instructions[cond_idx]; + if matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })) { + let exit_start = next; + let exit_block = next_nonempty_block(blocks, exit_start); + let jump_start = cond.target; + let jump_block = next_nonempty_block(blocks, jump_start); + let after_jump = if jump_block != BlockIdx::NULL { + next_nonempty_block(blocks, blocks[jump_block.idx()].next) + } else { + BlockIdx::NULL + }; + if exit_start == BlockIdx::NULL + || exit_block == BlockIdx::NULL + || jump_start == BlockIdx::NULL + || jump_block == BlockIdx::NULL + || jump_block == exit_block + || block_is_exceptional(&blocks[idx]) + || block_is_exceptional(&blocks[jump_block.idx()]) + || block_is_exceptional(&blocks[exit_block.idx()]) + || block_is_protected(&blocks[idx]) + || block_is_protected(&blocks[jump_block.idx()]) + || block_is_protected(&blocks[exit_block.idx()]) + || !is_scope_exit_block(&blocks[exit_block.idx()]) + || !is_jump_back_only_block(blocks, jump_block) + || (!allow_for_iter_jump_targets + && is_explicit_continue_to_for_iter(blocks, jump_block)) + && blocks[exit_block.idx()].instructions.iter().any(|info| { + matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. })) + }) + || (!allow_for_iter_jump_targets + && is_explicit_non_for_jump_back(blocks, jump_block)) + || (!allow_for_iter_jump_targets + && after_jump != BlockIdx::NULL + && !blocks[after_jump.idx()].cold + && !is_scope_exit_block(&blocks[after_jump.idx()]) + && !is_loop_cleanup_block(&blocks[after_jump.idx()])) + || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block + { + current = next; + continue; + } - let Some(reversed) = reversed_conditional(&last.instr) else { + let after_jump = blocks[jump_block.idx()].next; + blocks[idx].instructions[cond_idx].instr = reversed_conditional(&cond.instr) + .expect("PopJumpIfFalse has a reversed conditional"); + blocks[idx].instructions[cond_idx].target = exit_start; + blocks[idx].next = jump_start; + blocks[jump_block.idx()].next = exit_start; + blocks[exit_block.idx()].next = after_jump; current = next; continue; - }; - - let jump_start = next; - let exit_start = last.target; - if jump_start == BlockIdx::NULL || exit_start == BlockIdx::NULL || jump_start == exit_start + } + if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { + current = next; + continue; + } + if !allow_true_scope_exit_reorder { + current = next; + continue; + } + let exit_block = next_nonempty_block(blocks, cond.target); + let jump_block = next_nonempty_block(blocks, next); + if exit_block == BlockIdx::NULL + || jump_block == BlockIdx::NULL + || !is_scope_exit_block(&blocks[exit_block.idx()]) + || !is_jump_only_block(&blocks[jump_block.idx()]) + || (jump_targets_for_iter(blocks, jump_block) + && !is_explicit_continue_to_for_iter(blocks, jump_block)) + || next_nonempty_block(blocks, blocks[jump_block.idx()].next) != exit_block + || !comes_before( + blocks, + next_nonempty_block(blocks, blocks[jump_block.idx()].instructions[0].target), + jump_block, + ) { current = next; continue; } - let mut jump_end = BlockIdx::NULL; - let mut jump_block = BlockIdx::NULL; - let mut cursor = jump_start; - let mut jump_segment_valid = true; - while cursor != BlockIdx::NULL && cursor != exit_start { - if block_is_exceptional(&blocks[cursor.idx()]) { - jump_segment_valid = false; - break; - } - if !blocks[cursor.idx()].instructions.is_empty() { - if jump_block != BlockIdx::NULL || !is_jump_only_block(&blocks[cursor.idx()]) { - jump_segment_valid = false; - break; - } - jump_block = cursor; - } - jump_end = cursor; - cursor = blocks[cursor.idx()].next; + let after_exit = blocks[exit_block.idx()].next; + blocks[idx].instructions[cond_idx].instr = + reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); + blocks[idx].instructions[cond_idx].target = jump_block; + blocks[idx].next = exit_block; + blocks[exit_block.idx()].next = jump_block; + blocks[jump_block.idx()].next = after_exit; + current = next; + } +} + +fn reorder_conditional_break_continue_blocks(blocks: &mut [Block]) { + fn is_break_exit_like(blocks: &[Block], block_idx: BlockIdx) -> bool { + let block = &blocks[block_idx.idx()]; + if !is_jump_only_block(block) { + return false; } - if !jump_segment_valid || cursor != exit_start || jump_block == BlockIdx::NULL { + matches!( + block.instructions[0].instr.real(), + Some(Instruction::JumpForward { .. }) + ) && !comes_before( + blocks, + next_nonempty_block(blocks, block.instructions[0].target), + block_idx, + ) + } + + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + let next = blocks[idx].next; + let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { current = next; continue; - } - let jump_instr = blocks[jump_block.idx()].instructions[0]; - if !matches!(jump_instr.instr.real_opcode(), Some(Opcode::JumpForward)) { + }; + let cond = blocks[idx].instructions[cond_idx]; + if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { current = next; continue; } - let mut exit_end = BlockIdx::NULL; - let mut exit_block = BlockIdx::NULL; - let after_exit = loop { - if cursor == BlockIdx::NULL { - break BlockIdx::NULL; - } - if block_is_exceptional(&blocks[cursor.idx()]) { - if exit_block != BlockIdx::NULL { - break cursor; - } - exit_block = BlockIdx::NULL; - break BlockIdx::NULL; - } - if !blocks[cursor.idx()].instructions.is_empty() { - if exit_block != BlockIdx::NULL { - break cursor; - } - if !is_scope_exit_block(&blocks[cursor.idx()]) { - exit_block = BlockIdx::NULL; - break BlockIdx::NULL; - } - exit_block = cursor; - } - exit_end = cursor; - cursor = blocks[cursor.idx()].next; - }; - if exit_block == BlockIdx::NULL || exit_end == BlockIdx::NULL { + let jump_start = next; + let jump_block = next_nonempty_block(blocks, jump_start); + let exit_start = cond.target; + let exit_block = next_nonempty_block(blocks, exit_start); + if jump_start == BlockIdx::NULL + || jump_block == BlockIdx::NULL + || exit_start == BlockIdx::NULL + || exit_block == BlockIdx::NULL + || jump_block == exit_block + || block_is_exceptional(&blocks[idx]) + || block_is_exceptional(&blocks[jump_block.idx()]) + || block_is_exceptional(&blocks[exit_block.idx()]) + || block_is_protected(&blocks[idx]) + || block_is_protected(&blocks[jump_block.idx()]) + || block_is_protected(&blocks[exit_block.idx()]) + || !is_jump_back_only_block(blocks, jump_block) + || !is_break_exit_like(blocks, exit_block) + || next_nonempty_block(blocks, blocks[jump_block.idx()].next) != exit_block + { current = next; continue; } + let after_exit = blocks[exit_block.idx()].next; + blocks[idx].instructions[cond_idx].instr = + reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); + blocks[idx].instructions[cond_idx].target = jump_start; blocks[idx].next = exit_start; - blocks[exit_end.idx()].next = jump_start; - blocks[jump_end.idx()].next = after_exit; + blocks[exit_block.idx()].next = jump_start; + blocks[jump_block.idx()].next = after_exit; + current = next; + } +} - let cond_mut = &mut blocks[idx].instructions[cond_idx]; - cond_mut.instr = reversed; - cond_mut.target = jump_start; +fn reorder_conditional_explicit_continue_scope_exit_blocks(blocks: &mut [Block]) { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + let next = blocks[idx].next; + let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { + current = next; + continue; + }; + let cond = blocks[idx].instructions[cond_idx]; + if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { + current = next; + continue; + } - current = after_exit; + let jump_start = next; + let jump_block = next_nonempty_block(blocks, jump_start); + let exit_start = cond.target; + let exit_block = next_nonempty_block(blocks, exit_start); + if jump_start == BlockIdx::NULL + || jump_block == BlockIdx::NULL + || exit_start == BlockIdx::NULL + || exit_block == BlockIdx::NULL + || jump_block == exit_block + || block_is_exceptional(&blocks[idx]) + || block_is_exceptional(&blocks[jump_block.idx()]) + || block_is_exceptional(&blocks[exit_block.idx()]) + || block_is_protected(&blocks[idx]) + || block_is_protected(&blocks[jump_block.idx()]) + || block_is_protected(&blocks[exit_block.idx()]) + || !is_scope_exit_block(&blocks[exit_block.idx()]) + || !is_jump_back_only_block(blocks, jump_block) + || next_nonempty_block(blocks, blocks[jump_block.idx()].next) != exit_block + || blocks[jump_block.idx()] + .instructions + .first() + .is_none_or(|info| info.lineno_override.is_none_or(|lineno| lineno < 0)) + { + current = next; + continue; + } + + let after_exit = blocks[exit_block.idx()].next; + blocks[idx].instructions[cond_idx].instr = + reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); + blocks[idx].instructions[cond_idx].target = jump_start; + blocks[idx].next = exit_start; + blocks[exit_block.idx()].next = jump_start; + blocks[jump_block.idx()].next = after_exit; + current = next; } } -fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { - let target_comes_before = |target: BlockIdx, block: BlockIdx, blocks: &[Block]| -> bool { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if current == target { - return true; - } - if current == block { - return false; - } - current = blocks[current.idx()].next; +fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) { + fn jump_back_target(block: &Block) -> Option { + let [info] = block.instructions.as_slice() else { + return None; + }; + if matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) + && info.target != BlockIdx::NULL + { + Some(info.target) + } else { + None + } + } + + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + let next = blocks[idx].next; + let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { + current = next; + continue; + }; + let cond = blocks[idx].instructions[cond_idx]; + if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })) { + current = next; + continue; + } + + let exit_start = next; + let exit_block = next_nonempty_block(blocks, exit_start); + let jump_start = cond.target; + let jump_block = next_nonempty_block(blocks, jump_start); + let jump_target = if jump_block != BlockIdx::NULL { + jump_back_target(&blocks[jump_block.idx()]) + } else { + None + }; + let jumps_to_for_iter = jump_target.is_some_and(|target| { + blocks[target.idx()] + .instructions + .first() + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) + }); + let after_jump_start = if jump_block != BlockIdx::NULL { + blocks[jump_block.idx()].next + } else { + BlockIdx::NULL + }; + let after_jump = next_nonempty_block(blocks, after_jump_start); + let jump_exits_to_loop_exit = jump_target.is_some_and(|target| { + trailing_conditional_jump_index(&blocks[target.idx()]).is_some_and(|cond_idx| { + let loop_exit_start = blocks[target.idx()].instructions[cond_idx].target; + loop_exit_start == after_jump_start + || next_nonempty_block(blocks, loop_exit_start) == after_jump + }) + }); + if exit_start == BlockIdx::NULL + || exit_block == BlockIdx::NULL + || jump_start == BlockIdx::NULL + || jump_block == BlockIdx::NULL + || exit_block == jump_block + || block_is_exceptional(&blocks[idx]) + || block_is_exceptional(&blocks[exit_block.idx()]) + || block_is_exceptional(&blocks[jump_block.idx()]) + || block_is_protected(&blocks[idx]) + || block_is_protected(&blocks[exit_block.idx()]) + || block_is_protected(&blocks[jump_block.idx()]) + || !is_scope_exit_block(&blocks[exit_block.idx()]) + || !is_jump_back_only_block(blocks, jump_block) + || blocks[jump_block.idx()] + .instructions + .first() + .is_some_and(|info| info.lineno_override.is_some_and(|lineno| lineno >= 0)) + || jumps_to_for_iter + || (after_jump != BlockIdx::NULL + && !blocks[after_jump.idx()].cold + && !jump_exits_to_loop_exit) + || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block + { + current = next; + continue; } - false - }; + let after_jump = blocks[jump_block.idx()].next; + blocks[idx].instructions[cond_idx].instr = + reversed_conditional(&cond.instr).expect("PopJumpIfFalse has a reversed conditional"); + blocks[idx].instructions[cond_idx].target = exit_start; + blocks[idx].next = jump_start; + blocks[jump_block.idx()].next = exit_start; + blocks[exit_block.idx()].next = after_jump; + current = next; + } +} + +fn reorder_exception_handler_conditional_continue_raise_blocks(blocks: &mut [Block]) { let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); @@ -7225,167 +12535,257 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { current = next; continue; }; - let last = blocks[idx].instructions[cond_idx]; - - let Some(reversed) = reversed_conditional(&last.instr) else { + let cond = blocks[idx].instructions[cond_idx]; + if !matches!( + cond.instr.real(), + Some(Instruction::PopJumpIfNotNone { .. } | Instruction::PopJumpIfFalse { .. }) + ) || !(block_is_exceptional(&blocks[idx]) || blocks[idx].cold) + { current = next; continue; - }; + } - let chain_start = next; - let jump_start = last.target; - if chain_start == BlockIdx::NULL + let exit_start = next; + let exit_block = next_nonempty_block(blocks, exit_start); + let jump_start = cond.target; + let jump_block = next_nonempty_block(blocks, jump_start); + let jump_target = if jump_block != BlockIdx::NULL { + next_nonempty_block(blocks, blocks[jump_block.idx()].instructions[0].target) + } else { + BlockIdx::NULL + }; + if exit_start == BlockIdx::NULL + || exit_block == BlockIdx::NULL || jump_start == BlockIdx::NULL - || chain_start == jump_start + || jump_block == BlockIdx::NULL + || jump_target == BlockIdx::NULL + || exit_block == jump_block + || !is_scope_exit_block(&blocks[exit_block.idx()]) + || !is_jump_back_only_block(blocks, jump_block) + || !blocks[exit_block.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))) + || !blocks[jump_target.idx()] + .instructions + .first() + .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) + || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block { current = next; continue; } - let mut chain_has_suspension_point = false; - let mut scan = chain_start; - while scan != BlockIdx::NULL && scan != jump_start { - if block_contains_suspension_point(&blocks[scan.idx()]) { - chain_has_suspension_point = true; - break; - } - scan = blocks[scan.idx()].next; + + let after_jump = blocks[jump_block.idx()].next; + blocks[idx].instructions[cond_idx].instr = reversed_conditional(&cond.instr) + .expect("handler conditional has a reversed conditional"); + blocks[idx].instructions[cond_idx].target = exit_start; + blocks[idx].next = jump_start; + blocks[jump_block.idx()].next = exit_start; + blocks[exit_block.idx()].next = after_jump; + current = next; + } +} + +fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { + fn jump_back_target(block: &Block) -> Option { + let [info] = block.instructions.as_slice() else { + return None; + }; + if matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) + && info.target != BlockIdx::NULL + { + Some(info.target) + } else { + None } - let chain_starts_with_false_path_jump = trailing_conditional_jump_index( - &blocks[chain_start.idx()], - ) - .is_some_and(|chain_cond_idx| { - is_false_path_conditional_jump( - &blocks[chain_start.idx()].instructions[chain_cond_idx].instr, - ) - }); - let chain_is_single_exit_block = is_scope_exit_block(&blocks[chain_start.idx()]) - && next_nonempty_block(blocks, blocks[chain_start.idx()].next) == jump_start; - let chain_is_jump_only_exit_block = is_jump_only_block(&blocks[chain_start.idx()]) - && !target_comes_before( - blocks[chain_start.idx()].instructions[0].target, - chain_start, - blocks, - ); - let allow_true_path_jump_back_reorder = - matches!(last.instr.real_opcode(), Some(Opcode::PopJumpIfTrue)) - && (chain_has_suspension_point - || chain_starts_with_false_path_jump - || chain_is_single_exit_block - || chain_is_jump_only_exit_block); - let is_generic_false_path_reorder = !allow_true_path_jump_back_reorder; - if !is_false_path_conditional_jump(&last.instr) && !allow_true_path_jump_back_reorder { - current = next; + } + + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let Some(target) = jump_back_target(&blocks[current.idx()]) else { + current = blocks[current.idx()].next; + continue; + }; + if block_is_exceptional(&blocks[current.idx()]) + || block_is_protected(&blocks[current.idx()]) + { + current = blocks[current.idx()].next; continue; } - if block_is_protected(&blocks[idx]) && block_contains_suspension_point(&blocks[idx]) { - current = next; + + let duplicate = next_nonempty_block(blocks, blocks[current.idx()].next); + if duplicate == BlockIdx::NULL + || duplicate == current + || block_is_exceptional(&blocks[duplicate.idx()]) + || block_is_protected(&blocks[duplicate.idx()]) + || jump_back_target(&blocks[duplicate.idx()]) != Some(target) + { + current = blocks[current.idx()].next; continue; } - if let Some(chain_cond_idx) = trailing_conditional_jump_index(&blocks[chain_start.idx()]) { - let chain_cond = blocks[chain_start.idx()].instructions[chain_cond_idx]; - if matches!( - chain_cond.instr.real().map(Into::into), - Some(Opcode::PopJumpIfTrue) - ) { - let chain_true_target = next_nonempty_block(blocks, chain_cond.target); - if chain_true_target != BlockIdx::NULL - && !is_scope_exit_block(&blocks[chain_true_target.idx()]) - && !is_jump_only_block(&blocks[chain_true_target.idx()]) - && !is_pop_top_jump_block(&blocks[chain_true_target.idx()]) - { - current = next; - continue; + + for block in blocks.iter_mut() { + for info in &mut block.instructions { + if info.target == duplicate { + info.target = current; } } } + current = blocks[current.idx()].next; + } +} - let mut chain_end = BlockIdx::NULL; - let mut saw_nonempty = false; - let mut nonempty_blocks = 0usize; - let mut real_instr_count = 0usize; - let mut cursor = chain_start; - let mut chain_valid = true; - while cursor != BlockIdx::NULL && cursor != jump_start { - if block_is_exceptional(&blocks[cursor.idx()]) - || (block_is_protected(&blocks[cursor.idx()]) - && block_contains_suspension_point(&blocks[cursor.idx()]) - && !block_has_only_stop_iteration_error_handlers(&blocks[cursor.idx()], blocks)) +fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut [Block]) { + fn jump_back_target(blocks: &[Block], block_idx: BlockIdx) -> Option { + if block_idx == BlockIdx::NULL { + return None; + } + let instructions = blocks[block_idx.idx()].instructions.as_slice(); + let jump = match instructions { + [jump] if jump.instr.is_unconditional_jump() => jump, + [not_taken, jump] + if matches!(not_taken.instr.real(), Some(Instruction::NotTaken)) + && jump.instr.is_unconditional_jump() => { - chain_valid = false; - break; + jump } - if !blocks[cursor.idx()].instructions.is_empty() { - saw_nonempty = true; - nonempty_blocks += 1; - real_instr_count += blocks[cursor.idx()] - .instructions - .iter() - .filter(|info| info.instr.real().is_some()) - .count(); + _ => return None, + }; + if jump.target == BlockIdx::NULL + || !comes_before(blocks, next_nonempty_block(blocks, jump.target), block_idx) + { + return None; + } + Some(jump.target) + } + + fn find_body_tail( + blocks: &[Block], + body_start: BlockIdx, + false_jump: BlockIdx, + target: BlockIdx, + ) -> Option { + let mut cursor = body_start; + let mut visited = vec![false; blocks.len()]; + while cursor != BlockIdx::NULL && cursor != false_jump { + if visited[cursor.idx()] { + return None; + } + visited[cursor.idx()] = true; + if block_is_exceptional(&blocks[cursor.idx()]) { + return None; + } + if jump_back_target(blocks, cursor) == Some(target) { + return Some(cursor); } - chain_end = cursor; cursor = blocks[cursor.idx()].next; } - if !chain_valid || !saw_nonempty || chain_end == BlockIdx::NULL || cursor != jump_start { - current = next; - continue; - } - if !is_generic_false_path_reorder && (nonempty_blocks > 8 || real_instr_count > 80) { - current = next; - continue; - } + None + } - let mut jump_end = BlockIdx::NULL; - let mut jump_block = BlockIdx::NULL; - cursor = jump_start; + fn body_segment_contains_for_iter( + blocks: &[Block], + body_start: BlockIdx, + body_tail: BlockIdx, + ) -> bool { + let mut cursor = body_start; + let mut visited = vec![false; blocks.len()]; while cursor != BlockIdx::NULL { - if block_is_exceptional(&blocks[cursor.idx()]) { - jump_block = BlockIdx::NULL; - break; - } - jump_end = cursor; - if blocks[cursor.idx()].instructions.is_empty() { - cursor = blocks[cursor.idx()].next; - continue; + if visited[cursor.idx()] { + return false; } - if !is_jump_only_block(&blocks[cursor.idx()]) - || !target_comes_before(blocks[cursor.idx()].instructions[0].target, cursor, blocks) + visited[cursor.idx()] = true; + if blocks[cursor.idx()] + .instructions + .iter() + .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) { - jump_block = BlockIdx::NULL; - } else { - jump_block = cursor; + return true; } - break; + if cursor == body_tail { + return false; + } + cursor = blocks[cursor.idx()].next; } - if jump_block == BlockIdx::NULL || jump_end == BlockIdx::NULL { + false + } + + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + let next = blocks[idx].next; + let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { + current = next; + continue; + }; + let cond = blocks[idx].instructions[cond_idx]; + if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { current = next; continue; } - let after_jump = next_nonempty_block(blocks, blocks[jump_block.idx()].next); - if nonempty_blocks == 1 - && !is_jump_only_block(&blocks[chain_start.idx()]) - && after_jump != BlockIdx::NULL - && !blocks[after_jump.idx()].cold - && !block_is_exceptional(&blocks[after_jump.idx()]) - && !is_scope_exit_block(&blocks[after_jump.idx()]) - && !is_loop_cleanup_block(&blocks[after_jump.idx()]) + let false_jump_start = next; + let false_jump = next_nonempty_block(blocks, false_jump_start); + let body_start = cond.target; + let body = next_nonempty_block(blocks, body_start); + let Some(loop_target) = jump_back_target(blocks, false_jump) else { + current = next; + continue; + }; + if false_jump_start == BlockIdx::NULL + || false_jump == BlockIdx::NULL + || body_start == BlockIdx::NULL + || body == BlockIdx::NULL + || false_jump == body + || block_is_exceptional(&blocks[idx]) + || block_is_exceptional(&blocks[false_jump.idx()]) + || block_is_exceptional(&blocks[body.idx()]) + || block_is_protected(&blocks[idx]) + || block_is_protected(&blocks[false_jump.idx()]) + || block_is_protected(&blocks[body.idx()]) + || next_nonempty_block(blocks, blocks[false_jump.idx()].next) != body { current = next; continue; } - let mut cloned_jump = blocks[jump_block.idx()].clone(); - cloned_jump.next = chain_start; - cloned_jump.start_depth = None; - let cloned_idx = BlockIdx::new(blocks.len() as u32); - blocks.push(cloned_jump); - blocks[idx].next = cloned_idx; - let cond_mut = &mut blocks[idx].instructions[cond_idx]; - cond_mut.instr = reversed; - cond_mut.target = chain_start; + let Some(body_tail) = find_body_tail(blocks, body, false_jump, loop_target) else { + current = next; + continue; + }; + if !body_segment_contains_for_iter(blocks, body, body_tail) { + current = next; + continue; + } + let after_body = blocks[body_tail.idx()].next; + if after_body == BlockIdx::NULL || after_body == false_jump_start { + current = next; + continue; + } - current = next; + if blocks[idx] + .instructions + .get(cond_idx + 1) + .is_none_or(|info| !matches!(info.instr.real(), Some(Instruction::NotTaken))) + && matches!( + blocks[false_jump.idx()] + .instructions + .first() + .and_then(|info| info.instr.real()), + Some(Instruction::NotTaken) + ) + { + let not_taken = blocks[false_jump.idx()].instructions.remove(0); + blocks[idx].instructions.insert(cond_idx + 1, not_taken); + } + blocks[idx].instructions[cond_idx].instr = + reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); + blocks[idx].instructions[cond_idx].target = false_jump_start; + blocks[idx].next = body_start; + blocks[body_tail.idx()].next = false_jump_start; + blocks[false_jump.idx()].next = after_body; + current = blocks[idx].next; } } @@ -7403,7 +12803,7 @@ fn reorder_jump_over_exception_cleanup_blocks(blocks: &mut [Block]) { current = next; continue; }; - if !matches!(last.instr.real_opcode(), Some(Opcode::JumpForward)) + if !matches!(last.instr.real(), Some(Instruction::JumpForward { .. })) || last.target == BlockIdx::NULL { current = next; @@ -7634,7 +13034,7 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve }; let target = next_nonempty_block(blocks, last.target); - if target == BlockIdx::NULL || !is_exit_without_lineno(&blocks[target.idx()]) { + if target == BlockIdx::NULL || !is_exit_without_lineno(blocks, target) { current = blocks[current.idx()].next; continue; } @@ -7683,7 +13083,7 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve current, target, )) - && is_exit_without_lineno(&blocks[target.idx()]) + && is_exit_without_lineno(blocks, target) && let Some((location, end_location)) = propagation_location(last) && let Some(first) = blocks[target.idx()].instructions.first_mut() { @@ -7841,6 +13241,9 @@ fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { if jump_target == BlockIdx::NULL || !comes_before(blocks, jump_target, target) { continue; } + if !has_non_exception_loop_backedge_to(blocks, target, jump_target) { + continue; + } let layout_pred = find_layout_predecessor(blocks, target); if layout_pred == BlockIdx::NULL @@ -7913,9 +13316,15 @@ fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { let last_insts = &blocks[last_block.idx()].instructions; // Only apply when the last block is EXACTLY a return-None epilogue. let is_return_block = last_insts.len() == 2 - && matches!(last_insts[0].instr.real_opcode(), Some(Opcode::LoadConst)) + && matches!( + last_insts[0].instr, + AnyInstruction::Real(Instruction::LoadConst { .. }) + ) && is_load_const_none(&last_insts[0], metadata) - && matches!(last_insts[1].instr.real(), Some(Instruction::ReturnValue)); + && matches!( + last_insts[1].instr, + AnyInstruction::Real(Instruction::ReturnValue) + ); if !is_return_block { return; } @@ -7942,11 +13351,11 @@ fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { let already_has_return = block.instructions.len() >= 2 && { let n = block.instructions.len(); matches!( - block.instructions[n - 2].instr.real_opcode(), - Some(Opcode::LoadConst) + block.instructions[n - 2].instr, + AnyInstruction::Real(Instruction::LoadConst { .. }) ) && matches!( - block.instructions[n - 1].instr.real(), - Some(Instruction::ReturnValue) + block.instructions[n - 1].instr, + AnyInstruction::Real(Instruction::ReturnValue) ) }; if !block.except_handler @@ -8079,12 +13488,12 @@ fn is_named_except_cleanup_return_block(block: &Block, metadata: &CodeUnitMetada if matches!(pop_except.instr.real(), Some(Instruction::PopExcept)) && is_load_const_none(load_none1, metadata) && matches!( - store.instr.real_opcode(), - Some(Opcode::StoreFast | Opcode::StoreName ) + store.instr.real(), + Some(Instruction::StoreFast { .. } | Instruction::StoreName { .. }) ) && matches!( - delete.instr.real_opcode(), - Some(Opcode::DeleteFast | Opcode::DeleteName ) + delete.instr.real(), + Some(Instruction::DeleteFast { .. } | Instruction::DeleteName { .. }) ) && is_load_const_none(load_none2, metadata) && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) @@ -8158,8 +13567,8 @@ fn duplicate_named_except_cleanup_returns(blocks: &mut Vec, metadata: &Co fn is_const_return_block(block: &Block) -> bool { block.instructions.len() == 2 && matches!( - block.instructions[0].instr.real_opcode(), - Some(Opcode::LoadConst) + block.instructions[0].instr.real(), + Some(Instruction::LoadConst { .. }) ) && matches!( block.instructions[1].instr.real(), @@ -8202,6 +13611,30 @@ fn inline_pop_except_return_blocks(blocks: &mut [Block]) { } } +fn inline_named_except_cleanup_normal_exit_jumps(blocks: &mut [Block]) { + for block_idx in 0..blocks.len() { + let Some(jump_idx) = blocks[block_idx].instructions.len().checked_sub(1) else { + continue; + }; + let jump = blocks[block_idx].instructions[jump_idx]; + if !jump.instr.is_unconditional_jump() || jump.target == BlockIdx::NULL { + continue; + } + + let target = next_nonempty_block(blocks, jump.target); + if target == BlockIdx::NULL + || target == BlockIdx(block_idx as u32) + || !is_named_except_cleanup_normal_exit_block(&blocks[target.idx()]) + { + continue; + } + + let cloned_cleanup = blocks[target.idx()].instructions.clone(); + blocks[block_idx].instructions.pop(); + blocks[block_idx].instructions.extend(cloned_cleanup); + } +} + /// Label exception targets: walk CFG with except stack, set per-instruction /// handler info and block preserve_lasti flag. Converts POP_BLOCK to NOP. /// flowgraph.c label_exception_targets + push_except_block @@ -8242,8 +13675,11 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { if is_push { // Determine preserve_lasti from instruction type (push_except_block) let preserve_lasti = matches!( - blocks[bi].instructions[i].instr.pseudo_opcode(), - Some(PseudoOpcode::SetupWith | PseudoOpcode::SetupCleanup) + blocks[bi].instructions[i].instr.pseudo(), + Some( + PseudoInstruction::SetupWith { .. } + | PseudoInstruction::SetupCleanup { .. } + ) ); // Set preserve_lasti on handler block @@ -8293,7 +13729,9 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { // - plain yield outside try: depth=1 → DEPTH1 set // - yield inside try: depth=2+ → no DEPTH1 // - yield-from/await: has internal SETUP_FINALLY → depth=2+ → no DEPTH1 - if let Some(Opcode::YieldValue) = blocks[bi].instructions[i].instr.real_opcode() { + if let Some(Instruction::YieldValue { .. }) = + blocks[bi].instructions[i].instr.real() + { last_yield_except_depth = stack.len() as i32; } @@ -8375,11 +13813,16 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) } // Jump pseudo ops are resolved during block linearization PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. } => {} + PseudoInstruction::StoreFastMaybeNull { .. } => { + info.instr = Instruction::StoreFast { + var_num: Arg::marker(), + } + .into(); + } // These should have been resolved earlier PseudoInstruction::AnnotationsPlaceholder | PseudoInstruction::JumpIfFalse { .. } - | PseudoInstruction::JumpIfTrue { .. } - | PseudoInstruction::StoreFastMaybeNull { .. } => { + | PseudoInstruction::JumpIfTrue { .. } => { unreachable!("Unexpected pseudo instruction in convert_pseudo_ops: {pseudo:?}") } } diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index e67f729e1fc..448e22d256e 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -591,6 +591,14 @@ impl SymbolTableAnalyzer { } // Analyze symbols in current scope + let remove_owned_cells_from_free = matches!( + symbol_table.typ, + CompilerScope::Function + | CompilerScope::AsyncFunction + | CompilerScope::Lambda + | CompilerScope::Comprehension + | CompilerScope::Annotation + ); for symbol in symbol_table.symbols.values_mut() { self.analyze_symbol( symbol, @@ -600,6 +608,13 @@ impl SymbolTableAnalyzer { class_entry, )?; + // CPython analyze_cells(): once a function-like scope owns a + // child-requested name as a cell, that name is no longer free in + // the enclosing scope. + if remove_owned_cells_from_free && symbol.scope == SymbolScope::Cell { + newfree.shift_remove(symbol.name.as_str()); + } + // Collect free variables from this scope if symbol.scope == SymbolScope::Free || symbol.flags.contains(SymbolFlags::FREE_CLASS) { newfree.insert(symbol.name.clone()); @@ -2128,6 +2143,7 @@ impl SymbolTableBuilder { false, // lambdas are never async false, // don't skip defaults )?; + self.tables.last_mut().unwrap().typ = CompilerScope::Lambda; } else { self.enter_scope( "lambda", diff --git a/scripts/compare_bytecode.py b/scripts/compare_bytecode.py index 56ad77ffc1d..8b0ba7c3745 100644 --- a/scripts/compare_bytecode.py +++ b/scripts/compare_bytecode.py @@ -74,10 +74,16 @@ def _start_one(interpreter, targets, base_dir): output_file = None try: files_file = tempfile.NamedTemporaryFile( - mode="w", encoding="utf-8", delete=False, dir=PROJECT_ROOT + mode="w", + encoding="utf-8", + delete=False, + prefix="compare-bytecode-files-", ) output_file = tempfile.NamedTemporaryFile( - mode="w", encoding="utf-8", delete=False, dir=PROJECT_ROOT + mode="w", + encoding="utf-8", + delete=False, + prefix="compare-bytecode-output-", ) for _, path in targets: files_file.write(path)