From 2eab3ab3c74fba36891ad97fbfddf1f3bab23f41 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 19:08:48 +0900 Subject: [PATCH 001/131] Share marshal ref table between code object and its internals read_marshal_bytes, _str, _str_vec, _name_tuple, and _const_tuple now take a shared ref table and resolve TYPE_REF / register FLAG_REF entries. deserialize_code is split into a public wrapper and an inner function that receives the ref table; deserialize_value_depth opens a fresh inner ref space when it hits Type::Code, mirroring CPython's behaviour of putting the code object itself at ref slot 0. Nested code objects inside const tuples reuse the surrounding code's ref space via the new read_const_value helper. --- crates/compiler-core/src/marshal.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 503369a983e..d7bd4f85a1a 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -337,8 +337,9 @@ fn read_marshal_bytes( let len = rdr.read_u32()?; let bytes = rdr.read_slice(len)?.to_vec(); if let Some(idx) = slot { - refs[idx] = - Some(bag.make_constant::(BorrowedConstant::Bytes { value: &bytes })); + refs[idx] = Some(bag.make_constant::(BorrowedConstant::Bytes { + value: &bytes, + })); } Ok(bytes) } @@ -482,8 +483,9 @@ fn read_marshal_const_tuple( .map(|_| read_const_value(rdr, bag, child_depth, refs)) .collect::>()?; if let Some(idx) = slot { - refs[idx] = - Some(bag.make_constant::(BorrowedConstant::Tuple { elements: &items })); + refs[idx] = Some(bag.make_constant::(BorrowedConstant::Tuple { + elements: &items, + })); } Ok(items.into_iter().collect()) } @@ -730,8 +732,8 @@ fn deserialize_value_after_header( // Code-objects keep their own inner ref table because Bag::Value (the // outer marshal value) and the constant-bag's Constant type are not // in general the same. When the outer header carried FLAG_REF, the - // code object occupies slot 0 of the single global ref space, so we - // mirror that by reserving slot 0 of the inner table. + // code object occupies slot 0 of CPython's single global ref space, + // so we mirror that by reserving slot 0 of the inner table. let value = if matches!(typ, Type::Code) { let mut inner_refs: Vec::Constant>> = Vec::new(); if flag { From 286b667e513cf385fc9a87c1fc7916ccc64fa973 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 19:12:22 +0900 Subject: [PATCH 002/131] Align PYC magic number, FORMAT_VERSION, and header check with CPython 3.14 PYC_MAGIC_NUMBER changes from 2994 to 3627, matching CPython 3.14's pyc_magic_number_token (0x0a0d0e2b). marshal FORMAT_VERSION drops from 5 to 4 (the encoder/marshal.version value; the decoder already accepts both). check_pyc_magic_number_bytes now compares all four magic bytes instead of the first two. --- crates/compiler-core/src/marshal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index d7bd4f85a1a..6ac7fa9556f 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -5,7 +5,7 @@ use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use rustpython_wtf8::Wtf8; -pub const FORMAT_VERSION: u32 = 5; +pub const FORMAT_VERSION: u32 = 4; #[derive(Clone, Copy, Debug)] pub enum MarshalError { From 5d1ec596139ebb02326d4c730417cfaf34c17873 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 20:55:38 +0900 Subject: [PATCH 003/131] Accept CPython-tagged .pyc as read-only bytecode source SourceFileLoader.get_code now also looks for .pyc files using _RP_FALLBACK_CACHE_TAGS (currently ('cpython-314',)) in addition to sys.implementation.cache_tag. The matched .pyc is only used for reading; recompilation still writes to the RustPython-tagged path, so CPython's .pyc is never overwritten. Source-stat / hash / timestamp validation logic is unchanged. --- Lib/importlib/_bootstrap_external.py | 46 +++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 95ce14b2c39..1eba51bfbb1 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -236,6 +236,13 @@ def _write_atomic(path, data, mode=0o666): # Deprecated. DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES +# RustPython: additional cache tags to try when looking up bytecode files. +# RustPython itself writes .pyc with its own cache_tag, but reading .pyc +# generated by CPython (e.g. via `python3 -m compileall`) requires accepting +# the CPython tag too. These act as read-only fallbacks; bytecode writes +# always use sys.implementation.cache_tag. +_RP_FALLBACK_CACHE_TAGS = ('cpython-314',) + def cache_from_source(path, debug_override=None, *, optimization=None): """Given the path to a .py file, return the path to its .pyc file. @@ -841,20 +848,43 @@ def get_code(self, fullname): except NotImplementedError: bytecode_path = None else: + # RustPython: include CPython-tagged variants as read-only + # fallbacks so .pyc generated by stock CPython can be reused. + _rp_primary_tag = sys.implementation.cache_tag + _rp_candidate_paths = [bytecode_path] + if _rp_primary_tag: + _rp_marker = f'.{_rp_primary_tag}.' + for _rp_alt_tag in _RP_FALLBACK_CACHE_TAGS: + if _rp_alt_tag and _rp_alt_tag != _rp_primary_tag: + _rp_alt = bytecode_path.replace( + _rp_marker, f'.{_rp_alt_tag}.', 1 + ) + if _rp_alt != bytecode_path: + _rp_candidate_paths.append(_rp_alt) + try: st = self.path_stats(source_path) except OSError: pass else: source_mtime = int(st['mtime']) - try: - data = self.get_data(bytecode_path) - except OSError: - pass - else: + # bytecode_path stays as the write target (primary RustPython + # tag); _rp_read_path tracks where the actual .pyc was found + # for verbose/error messages. + data = None + _rp_read_path = bytecode_path + for _rp_candidate in _rp_candidate_paths: + try: + data = self.get_data(_rp_candidate) + except OSError: + data = None + continue + _rp_read_path = _rp_candidate + break + if data is not None: exc_details = { 'name': fullname, - 'path': bytecode_path, + 'path': _rp_read_path, } try: flags = _classify_pyc(data, fullname, exc_details) @@ -883,10 +913,10 @@ def get_code(self, fullname): except (ImportError, EOFError): pass else: - _bootstrap._verbose_message('{} matches {}', bytecode_path, + _bootstrap._verbose_message('{} matches {}', _rp_read_path, source_path) return _compile_bytecode(bytes_data, name=fullname, - bytecode_path=bytecode_path, + bytecode_path=_rp_read_path, source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) From e28c03720706a9b07ceb37be2439afa9889094c3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 21:22:40 +0900 Subject: [PATCH 004/131] Apply rustfmt to marshal helpers --- crates/compiler-core/src/marshal.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 6ac7fa9556f..7ff62c80931 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -337,9 +337,8 @@ fn read_marshal_bytes( let len = rdr.read_u32()?; let bytes = rdr.read_slice(len)?.to_vec(); if let Some(idx) = slot { - refs[idx] = Some(bag.make_constant::(BorrowedConstant::Bytes { - value: &bytes, - })); + refs[idx] = + Some(bag.make_constant::(BorrowedConstant::Bytes { value: &bytes })); } Ok(bytes) } @@ -483,9 +482,8 @@ fn read_marshal_const_tuple( .map(|_| read_const_value(rdr, bag, child_depth, refs)) .collect::>()?; if let Some(idx) = slot { - refs[idx] = Some(bag.make_constant::(BorrowedConstant::Tuple { - elements: &items, - })); + refs[idx] = + Some(bag.make_constant::(BorrowedConstant::Tuple { elements: &items })); } Ok(items.into_iter().collect()) } From 1d5054bed4ca8e6a2793657efb6e9f258515e537 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 21:22:43 +0900 Subject: [PATCH 005/131] Marshal PySlice from format version 4 instead of 5 CPython's marshal supports TYPE_SLICE from format version 4 onwards and that is the default version. Rejecting slice dumps below version 5 made marshal.dumps(slice(...)) fail with the default version and broke test.test_marshal.SliceTestCase.test_slice. --- crates/vm/src/stdlib/marshal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index 6e0fc4e7f5d..c825ee056ea 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -324,7 +324,7 @@ mod decl { buf.write_u8(b'c'); marshal::serialize_code(buf, &co.code); } else if let Some(sl) = obj.downcast_ref::() { - if version < 5 { + if version < 4 { return Err(vm.new_value_error("unmarshallable object".to_string())); } buf.write_u8(b':'); From 0fbf8498f7de7642ab5df52d03d3dc3ba0c1186c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 22:42:55 +0900 Subject: [PATCH 006/131] Revert "Accept CPython-tagged .pyc as read-only bytecode source" Lib/importlib/_bootstrap_external.py is CPython's own code copied verbatim; local patches here defeat compatibility tracking. The cpython-XX cache_tag fallback needs to live on the RustPython side (Rust code or sys.implementation.cache_tag policy), not as edits to the imported standard library. This reverts commit 1fc426d0fb5fcdb50d35cad13bbb43e8f6ce1c7f. --- Lib/importlib/_bootstrap_external.py | 46 +++++----------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 1eba51bfbb1..95ce14b2c39 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -236,13 +236,6 @@ def _write_atomic(path, data, mode=0o666): # Deprecated. DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES -# RustPython: additional cache tags to try when looking up bytecode files. -# RustPython itself writes .pyc with its own cache_tag, but reading .pyc -# generated by CPython (e.g. via `python3 -m compileall`) requires accepting -# the CPython tag too. These act as read-only fallbacks; bytecode writes -# always use sys.implementation.cache_tag. -_RP_FALLBACK_CACHE_TAGS = ('cpython-314',) - def cache_from_source(path, debug_override=None, *, optimization=None): """Given the path to a .py file, return the path to its .pyc file. @@ -848,43 +841,20 @@ def get_code(self, fullname): except NotImplementedError: bytecode_path = None else: - # RustPython: include CPython-tagged variants as read-only - # fallbacks so .pyc generated by stock CPython can be reused. - _rp_primary_tag = sys.implementation.cache_tag - _rp_candidate_paths = [bytecode_path] - if _rp_primary_tag: - _rp_marker = f'.{_rp_primary_tag}.' - for _rp_alt_tag in _RP_FALLBACK_CACHE_TAGS: - if _rp_alt_tag and _rp_alt_tag != _rp_primary_tag: - _rp_alt = bytecode_path.replace( - _rp_marker, f'.{_rp_alt_tag}.', 1 - ) - if _rp_alt != bytecode_path: - _rp_candidate_paths.append(_rp_alt) - try: st = self.path_stats(source_path) except OSError: pass else: source_mtime = int(st['mtime']) - # bytecode_path stays as the write target (primary RustPython - # tag); _rp_read_path tracks where the actual .pyc was found - # for verbose/error messages. - data = None - _rp_read_path = bytecode_path - for _rp_candidate in _rp_candidate_paths: - try: - data = self.get_data(_rp_candidate) - except OSError: - data = None - continue - _rp_read_path = _rp_candidate - break - if data is not None: + try: + data = self.get_data(bytecode_path) + except OSError: + pass + else: exc_details = { 'name': fullname, - 'path': _rp_read_path, + 'path': bytecode_path, } try: flags = _classify_pyc(data, fullname, exc_details) @@ -913,10 +883,10 @@ def get_code(self, fullname): except (ImportError, EOFError): pass else: - _bootstrap._verbose_message('{} matches {}', _rp_read_path, + _bootstrap._verbose_message('{} matches {}', bytecode_path, source_path) return _compile_bytecode(bytes_data, name=fullname, - bytecode_path=_rp_read_path, + bytecode_path=bytecode_path, source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) From ce0ac6172db4f85e16af163dac3ab7c54c3cc0e6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 24 May 2026 16:54:47 +0900 Subject: [PATCH 007/131] Set marshal FORMAT_VERSION to 5 to match CPython 3.14.5 Py_MARSHAL_VERSION is 5 in CPython 3.14.5 (Include/marshal.h:16) and TYPE_SLICE serialization rejects version < 5 (Python/marshal.c:720). Restore the same threshold and constant so marshal.version and the slice-marshal gate match CPython. --- crates/compiler-core/src/marshal.rs | 2 +- crates/vm/src/stdlib/marshal.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 7ff62c80931..ffe8b871852 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -5,7 +5,7 @@ use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use rustpython_wtf8::Wtf8; -pub const FORMAT_VERSION: u32 = 4; +pub const FORMAT_VERSION: u32 = 5; #[derive(Clone, Copy, Debug)] pub enum MarshalError { diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index c825ee056ea..6e0fc4e7f5d 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -324,7 +324,7 @@ mod decl { buf.write_u8(b'c'); marshal::serialize_code(buf, &co.code); } else if let Some(sl) = obj.downcast_ref::() { - if version < 4 { + if version < 5 { return Err(vm.new_value_error("unmarshallable object".to_string())); } buf.write_u8(b':'); From 93657463add6af6e84a2b89bb52a9d11a343df74 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 24 May 2026 16:59:28 +0900 Subject: [PATCH 008/131] Thread marshal recursion depth through nested code objects Code objects embedded in const-tuples reset the depth budget on each recursion, so a hostile or pathological marshal stream of code-in-tuple- in-code can blow the stack despite MAX_MARSHAL_STACK_DEPTH. Pass the current depth through deserialize_code_inner and read_marshal_const_tuple and decrement at each code-object/tuple boundary. Also route dict keys through deserialize_value_after_header so TYPE_CODE keys decode instead of failing with BadType. --- crates/codegen/src/compile.rs | 4425 +----- crates/codegen/src/error.rs | 4 + crates/codegen/src/ir.rs | 11789 ++++------------ .../compiler-core/src/bytecode/instruction.rs | 223 + crates/compiler-core/src/marshal.rs | 66 +- 5 files changed, 3668 insertions(+), 12839 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index caa40571dda..5c9e13dbeb8 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -27,10 +27,10 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use rustpython_compiler_core::{ Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, bytecode::{ - self, AnyInstruction, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, CodeObject, - ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, IntrinsicFunction1, - Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, SpecialMethod, - UnpackExArgs, oparg, + self, AnyInstruction, AnyOpcode, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, + CodeObject, ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, + IntrinsicFunction1, Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, + SpecialMethod, UnpackExArgs, oparg, }, }; use rustpython_wtf8::Wtf8Buf; @@ -82,7 +82,7 @@ impl ExprExt for ast::Expr { const MAXBLOCKS: usize = 20; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FBlockType { WhileLoop, ForLoop, @@ -132,8 +132,10 @@ enum BuiltinGeneratorCallKind { #[derive(Debug, Clone)] pub struct FBlockInfo { pub fb_type: FBlockType, - pub fb_block: BlockIdx, - pub fb_exit: BlockIdx, + // CPython _PyCompile_FBlockInfo stores jump_target_label values here. + pub(crate) fb_block: Option, + // CPython's optional type-specific exit or cleanup jump_target_label. + pub(crate) fb_exit: Option, pub fb_range: TextRange, // additional data for fblock unwinding pub fb_datum: FBlockDatum, @@ -173,16 +175,6 @@ struct Compiler { /// Disable constant tuple/list/set collection folding in contexts where /// CPython keeps the builder form for later assignment lowering. disable_const_collection_folding: bool, - split_next_for_normal_exit_from_break: bool, - fallthrough_has_statement_successor: bool, - fallthrough_has_local_statement_successor: bool, - fallthrough_next_statement_is_if: bool, - fallthrough_next_statement_is_try: bool, - fallthrough_next_statement_is_function_def: bool, - fallthrough_next_statement_has_empty_test_prefix: bool, - fallthrough_successor_stack: Vec<(bool, bool, bool, bool, bool, bool)>, - try_else_orelse_conditional_base_stack: Vec, - in_finally_normal_body: bool, } #[derive(Clone, Copy)] @@ -212,7 +204,6 @@ impl Default for CompileOpts { #[derive(Debug, Clone, Copy)] struct CompileContext { - loop_data: Option<(BlockIdx, BlockIdx)>, in_class: bool, func: FunctionContext, /// True if we're anywhere inside an async function (even inside nested comprehensions) @@ -499,178 +490,18 @@ impl Compiler { ) } - fn constant_expr_truthiness(&mut self, expr: &ast::Expr) -> CompileResult> { - Ok(self - .try_fold_constant_expr(expr)? - .and_then(|constant| match constant { - ConstantData::Tuple { .. } => None, - constant => Some(Self::constant_truthiness(&constant)), - })) - } - - fn current_block_stack_depth(&self) -> Option { - let code = self.code_stack.last()?; - let block = &code.blocks[code.current_block.idx()]; - let mut depth = 0u32; - for info in &block.instructions { - if info.instr.is_block_push() { - continue; - } - let effect = info.instr.stack_effect_info(u32::from(info.arg)); - let popped = effect.popped(); - let pushed = effect.pushed(); - depth = depth.checked_sub(popped)?.checked_add(pushed)?; - if info.instr.is_scope_exit() || info.instr.is_unconditional_jump() { - return None; - } - } - Some(depth) - } - - fn has_always_taken_jump_in_test( - &mut self, - expr: &ast::Expr, - condition: bool, - ) -> CompileResult { - Ok(match expr { - ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) => { - let (last_value, prefix_values) = values.split_last().unwrap(); - let cond2 = matches!(op, ast::BoolOp::Or); - for value in prefix_values { - if self.has_always_taken_jump_in_test(value, cond2)? { - return Ok(true); - } - } - self.has_always_taken_jump_in_test(last_value, condition)? - } - ast::Expr::UnaryOp(ast::ExprUnaryOp { - op: ast::UnaryOp::Not, - operand, - .. - }) => self.has_always_taken_jump_in_test(operand, !condition)?, - ast::Expr::If(ast::ExprIf { - test, body, orelse, .. - }) => { - self.has_always_taken_jump_in_test(test, false)? - || self.has_always_taken_jump_in_test(body, condition)? - || self.has_always_taken_jump_in_test(orelse, condition)? - } - _ => matches!(self.constant_expr_truthiness(expr)?, Some(value) if value == condition), - }) - } - - fn jump_test_starts_with_empty_prefix( - &mut self, - expr: &ast::Expr, - condition: bool, - ) -> CompileResult { - Ok(match expr { - ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) => { - let (last_value, prefix_values) = values.split_last().unwrap(); - let cond2 = matches!(op, ast::BoolOp::Or); - for value in prefix_values { - if self.jump_test_starts_with_empty_prefix(value, cond2)? - || matches!( - self.constant_expr_truthiness(value)?, - Some(value) if value != cond2 - ) - { - return Ok(true); - } - } - self.jump_test_starts_with_empty_prefix(last_value, condition)? - } - ast::Expr::UnaryOp(ast::ExprUnaryOp { - op: ast::UnaryOp::Not, - operand, - .. - }) => self.jump_test_starts_with_empty_prefix(operand, !condition)?, - _ => matches!(self.constant_expr_truthiness(expr)?, Some(value) if value != condition), - }) - } - - fn statement_starts_with_empty_if_test_prefix( - &mut self, - stmt: &ast::Stmt, - ) -> CompileResult { - match stmt { - ast::Stmt::If(ast::StmtIf { test, .. }) => { - self.jump_test_starts_with_empty_prefix(test, false) - } - _ => Ok(false), - } - } - - fn disable_load_fast_borrow_for_block(&mut self, block: BlockIdx) { - if block != BlockIdx::NULL { - self.current_code_info().blocks[block.idx()].disable_load_fast_borrow = true; - } - } - - fn mark_try_else_orelse_entry_block(&mut self, block: BlockIdx) { - if block != BlockIdx::NULL { - self.current_code_info().blocks[block.idx()].try_else_orelse_entry = true; - } - } - - fn mark_label_block(&mut self, block: BlockIdx) { - if block != BlockIdx::NULL { - self.current_code_info().blocks[block.idx()].label = true; - } - } - - fn mark_load_fast_barrier_block(&mut self, block: BlockIdx) { - if block != BlockIdx::NULL { - let block = &mut self.current_code_info().blocks[block.idx()]; - block.label = true; - block.load_fast_barrier = true; - } - } - - fn mark_load_fast_passthrough_block(&mut self, block: BlockIdx) { - if block != BlockIdx::NULL { - self.current_code_info().blocks[block.idx()].load_fast_passthrough = true; - } - } - - fn mark_load_fast_label_reuse_passthrough_block(&mut self, block: BlockIdx) { - if block != BlockIdx::NULL { - self.current_code_info().blocks[block.idx()].load_fast_label_reuse_passthrough = true; - } - } - - fn mark_conditional_ifexp_orelse_entry_block(&mut self, block: BlockIdx) { + /// Mark a direct-CFG block as carrying CPython's `basicblock.b_label`. + /// + /// In CPython, codegen records labels in the instruction sequence with + /// `USE_LABEL()`, then `_PyCfg_FromInstructionSequence()` emits + /// `_PyCfgBuilder_UseLabel()` only for instruction offsets reached by + /// `HAS_TARGET` operands. The direct-CFG backend uses this marker where the + /// CPython CFG builder would retain a materialized label boundary. + fn mark_cpython_cfg_label_block(&mut self, block: BlockIdx) { + let code = self.current_code_info(); + let block = code.resolve_instr_sequence_label(block); if block != BlockIdx::NULL { - self.current_code_info().blocks[block.idx()].conditional_ifexp_orelse_entry = true; - } - } - - fn instruction_count_snapshot(&mut self) -> Vec { - self.current_code_info() - .blocks - .iter() - .map(|block| block.instructions.len()) - .collect() - } - - fn mark_new_conditional_jump_locations_since( - &mut self, - snapshot: &[usize], - target: BlockIdx, - range: TextRange, - ) { - let source = self.source_file.to_source_code(); - let location = source.source_location(range.start(), PositionEncoding::Utf8); - let end_location = source.source_location(range.end(), PositionEncoding::Utf8); - for (idx, block) in self.current_code_info().blocks.iter_mut().enumerate() { - let start = snapshot.get(idx).copied().unwrap_or(0); - for instr in block.instructions.iter_mut().skip(start) { - if instr.target == target && ir::is_conditional_jump(&instr.instr) { - instr.location = location; - instr.end_location = end_location; - instr.preserve_tobool_jump_location = true; - } - } + code.mark_cpython_cfg_label(block); } } @@ -689,7 +520,9 @@ impl Compiler { private: None, blocks: vec![ir::Block::default()], current_block: BlockIdx::new(0), - annotations_blocks: None, + instr_sequence: ir::InstructionSequence::new(), + instr_sequence_label_map: ir::InstructionSequenceLabelMap::new(), + annotations_instr_sequence: None, metadata: ir::CodeUnitMetadata { name: code_name.to_string(), qualname: Some(code_name.to_string()), @@ -710,8 +543,6 @@ impl Compiler { fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: 0, // Module is always the first symbol table in_conditional_block: 0, - in_final_with_cleanup_statement: 0, - in_try_else_orelse: 0, next_conditional_annotation_index: 0, }; Self { @@ -723,7 +554,6 @@ impl Compiler { done_with_future_stmts: DoneWithFuture::No, future_annotations: false, ctx: CompileContext { - loop_data: None, in_class: false, func: FunctionContext::NoFunction, in_async_scope: false, @@ -734,16 +564,6 @@ impl Compiler { do_not_emit_bytecode: 0, disable_const_boolop_folding: false, disable_const_collection_folding: false, - split_next_for_normal_exit_from_break: false, - fallthrough_has_statement_successor: false, - fallthrough_has_local_statement_successor: false, - fallthrough_next_statement_is_if: false, - fallthrough_next_statement_is_try: false, - fallthrough_next_statement_is_function_def: false, - fallthrough_next_statement_has_empty_test_prefix: false, - fallthrough_successor_stack: Vec::new(), - try_else_orelse_conditional_base_stack: Vec::new(), - in_finally_normal_body: false, } } @@ -773,919 +593,43 @@ impl Compiler { matches!(target, ast::Expr::List(_) | ast::Expr::Tuple(_)) } - fn statements_end_with_scope_exit(body: &[ast::Stmt]) -> bool { - body.last() - .is_some_and(Self::statement_ends_with_scope_exit) - } - - fn statements_end_with_finally_entry_scope_exit(body: &[ast::Stmt]) -> bool { - body.last() - .is_some_and(Self::statement_ends_with_finally_entry_scope_exit) - } - - fn statement_ends_with_finally_entry_scope_exit(stmt: &ast::Stmt) -> bool { - match stmt { - ast::Stmt::Return(_) - | ast::Stmt::Raise(_) - | ast::Stmt::Break(_) - | ast::Stmt::Continue(_) => true, - ast::Stmt::If(ast::StmtIf { body, .. }) => { - Self::statements_end_with_finally_entry_scope_exit(body) - } - _ => false, - } - } - - fn statement_ends_with_scope_exit(stmt: &ast::Stmt) -> bool { - match stmt { - ast::Stmt::Return(_) | ast::Stmt::Raise(_) => true, - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - let has_else = elif_else_clauses - .last() - .is_some_and(|clause| clause.test.is_none()); - has_else - && Self::statements_end_with_scope_exit(body) - && elif_else_clauses - .iter() - .all(|clause| Self::statements_end_with_scope_exit(&clause.body)) - } - _ => false, - } - } - - fn statements_end_with_with_cleanup_scope_exit(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::With(ast::StmtWith { body, .. }) => { - Self::statements_end_with_scope_exit(body) - || Self::statements_end_with_with_cleanup_scope_exit(body) - } - _ => false, - }) - } - - fn statements_end_with_nonterminal_with_cleanup(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::With(ast::StmtWith { body, .. }) => { - !Self::statements_end_with_scope_exit(body) - && !Self::statements_end_with_with_cleanup_scope_exit(body) - } - _ => false, - }) - } - - fn statements_contain_with(body: &[ast::Stmt]) -> bool { - body.iter().any(|stmt| match stmt { - ast::Stmt::With(_) => true, - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_contain_with(body) - || elif_else_clauses - .iter() - .any(|clause| Self::statements_contain_with(&clause.body)) - } - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) => { - Self::statements_contain_with(body) - || handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_contain_with(&handler.body) - }) - || Self::statements_contain_with(orelse) - || Self::statements_contain_with(finalbody) - } - ast::Stmt::For(ast::StmtFor { body, orelse, .. }) - | ast::Stmt::While(ast::StmtWhile { body, orelse, .. }) => { - Self::statements_contain_with(body) || Self::statements_contain_with(orelse) - } - ast::Stmt::Match(ast::StmtMatch { cases, .. }) => cases - .iter() - .any(|case| Self::statements_contain_with(&case.body)), - _ => false, - }) - } - - fn statements_contain_nested_with(body: &[ast::Stmt]) -> bool { - body.iter().any(|stmt| match stmt { - ast::Stmt::With(ast::StmtWith { body, .. }) => Self::statements_contain_with(body), - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_contain_nested_with(body) - || elif_else_clauses - .iter() - .any(|clause| Self::statements_contain_nested_with(&clause.body)) - } - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) => { - Self::statements_contain_nested_with(body) - || handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_contain_nested_with(&handler.body) - }) - || Self::statements_contain_nested_with(orelse) - || Self::statements_contain_nested_with(finalbody) - } - ast::Stmt::For(ast::StmtFor { body, orelse, .. }) - | ast::Stmt::While(ast::StmtWhile { body, orelse, .. }) => { - Self::statements_contain_nested_with(body) - || Self::statements_contain_nested_with(orelse) - } - ast::Stmt::Match(ast::StmtMatch { cases, .. }) => cases - .iter() - .any(|case| Self::statements_contain_nested_with(&case.body)), - _ => false, - }) - } - - fn statements_contain_async_comprehension(body: &[ast::Stmt]) -> bool { - use ast::visitor::Visitor; - - #[derive(Default)] - struct AsyncComprehensionVisitor { - found: bool, - } - - impl ast::visitor::Visitor<'_> for AsyncComprehensionVisitor { - fn visit_expr(&mut self, expr: &ast::Expr) { - if self.found { - return; - } - match expr { - ast::Expr::ListComp(ast::ExprListComp { generators, .. }) - | ast::Expr::SetComp(ast::ExprSetComp { generators, .. }) - | ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) - | ast::Expr::Generator(ast::ExprGenerator { generators, .. }) - if generators.iter().any(|generator| generator.is_async) => - { - self.found = true; - } - _ => ast::visitor::walk_expr(self, expr), - } - } - } - - let mut visitor = AsyncComprehensionVisitor::default(); - for stmt in body { - visitor.visit_stmt(stmt); - if visitor.found { - return true; - } - } - false - } - - fn statements_are_single_with(body: &[ast::Stmt]) -> bool { - matches!(body, [ast::Stmt::With(_)]) - } - - fn statements_end_with_try_finally(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| { - matches!( - stmt, - ast::Stmt::Try(ast::StmtTry { finalbody, .. }) if !finalbody.is_empty() - ) - }) - } - - fn statements_end_with_try_finally_finalbody_with(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { finalbody, .. }) if !finalbody.is_empty() => { - Self::statements_contain_with(finalbody) - } - _ => false, - }) - } - - fn statements_end_with_try_finally_conditional_finalbody(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { finalbody, .. }) if !finalbody.is_empty() => { - Self::statements_end_with_conditional(finalbody) - } - _ => false, - }) - } - - fn statements_contain_for_with_conditional_body(body: &[ast::Stmt]) -> bool { - body.iter().any(|stmt| match stmt { - ast::Stmt::For(ast::StmtFor { body, orelse, .. }) => { - body.iter().any(|stmt| matches!(stmt, ast::Stmt::If(_))) - || Self::statements_contain_for_with_conditional_body(body) - || Self::statements_contain_for_with_conditional_body(orelse) - } - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_contain_for_with_conditional_body(body) - || elif_else_clauses.iter().any(|clause| { - Self::statements_contain_for_with_conditional_body(&clause.body) - }) - } - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) => { - Self::statements_contain_for_with_conditional_body(body) - || handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_contain_for_with_conditional_body(&handler.body) - }) - || Self::statements_contain_for_with_conditional_body(orelse) - || Self::statements_contain_for_with_conditional_body(finalbody) - } - ast::Stmt::With(ast::StmtWith { body, .. }) => { - Self::statements_contain_for_with_conditional_body(body) - } - _ => false, - }) - } - - fn statements_end_with_nested_finalbody_try_finally(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { finalbody, .. }) if !finalbody.is_empty() => { - Self::statements_end_with_try_finally(finalbody) - } - _ => false, - }) - } - - fn statements_end_with_try_star_except(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| { - matches!( - stmt, - ast::Stmt::Try(ast::StmtTry { - handlers, - finalbody, - is_star: true, - .. - }) if !handlers.is_empty() && finalbody.is_empty() - ) - }) - } - - fn statements_end_with_try_except_no_finally(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| { - matches!( - stmt, - ast::Stmt::Try(ast::StmtTry { - handlers, - finalbody, - is_star: false, - .. - }) if !handlers.is_empty() && finalbody.is_empty() - ) - }) - } - - fn statements_end_with_try_except_scope_exit_handlers(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - handlers, - finalbody, - is_star: false, - .. - }) if !handlers.is_empty() && finalbody.is_empty() => { - Self::handlers_end_with_scope_exit(handlers) - } - _ => false, - }) - } - - fn statements_end_with_try_except_scope_exit_handlers_nonconditional_body( - body: &[ast::Stmt], - ) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - finalbody, - is_star: false, - .. - }) if !handlers.is_empty() && finalbody.is_empty() => { - Self::handlers_end_with_scope_exit(handlers) - && !Self::statements_end_with_conditional(body) - } - _ => false, - }) - } - - fn statements_end_with_successor_join(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - elif_else_clauses - .last() - .is_some_and(|clause| clause.test.is_none()) - || Self::statements_end_with_try_except_no_finally(body) - || elif_else_clauses - .iter() - .any(|clause| Self::statements_end_with_try_except_no_finally(&clause.body)) - } - _ => false, - }) - } - - fn statements_end_with_import(body: &[ast::Stmt]) -> bool { - body.last() - .is_some_and(|stmt| matches!(stmt, ast::Stmt::Import(_) | ast::Stmt::ImportFrom(_))) - } - - fn statements_are_single_name_store_from_attr_call_or_subscript(body: &[ast::Stmt]) -> bool { - let [ast::Stmt::Assign(ast::StmtAssign { targets, value, .. })] = body else { - return false; - }; - targets.len() == 1 - && matches!(targets[0], ast::Expr::Name(_)) - // CPython codegen_try_except() treats the try body as a single - // VISIT_SEQ before the no-location POP_BLOCK/JUMP_NO_INTERRUPT/end - // label sequence. A single local store from an attribute load is - // the same protected-body shape as the existing call/subscript - // cases here (for example cached_property.__get__). - && matches!( - value.as_ref(), - ast::Expr::Attribute(_) | ast::Expr::Call(_) | ast::Expr::Subscript(_) - ) - } - - fn statements_are_name_store_then_same_name_method_call(body: &[ast::Stmt]) -> bool { - let [ - ast::Stmt::Assign(ast::StmtAssign { targets, .. }), - ast::Stmt::Expr(ast::StmtExpr { value, .. }), - ] = body - else { - return false; - }; - let [ast::Expr::Name(ast::ExprName { id: stored, .. })] = targets.as_slice() else { - return false; - }; - matches!( - value.as_ref(), - ast::Expr::Call(ast::ExprCall { func, .. }) - if matches!( - func.as_ref(), - ast::Expr::Attribute(ast::ExprAttribute { value, .. }) - if matches!( - value.as_ref(), - ast::Expr::Name(ast::ExprName { id, .. }) if id == stored - ) - ) - ) - } - - fn statements_are_single_unpack_store_from_call(body: &[ast::Stmt]) -> bool { - let [ast::Stmt::Assign(ast::StmtAssign { targets, value, .. })] = body else { - return false; - }; - let [target] = targets.as_slice() else { - return false; - }; - let elts = match target { - ast::Expr::Tuple(ast::ExprTuple { elts, .. }) - | ast::Expr::List(ast::ExprList { elts, .. }) => elts, - _ => return false, - }; - elts.iter().all(|elt| matches!(elt, ast::Expr::Name(_))) - && matches!(value.as_ref(), ast::Expr::Call(_)) - } - - fn statements_are_name_stores_from_calls(body: &[ast::Stmt]) -> bool { - !body.is_empty() - && body.iter().all(|stmt| { - let ast::Stmt::Assign(ast::StmtAssign { targets, value, .. }) = stmt else { - return false; - }; - targets.len() == 1 - && matches!(targets[0], ast::Expr::Name(_)) - && matches!(value.as_ref(), ast::Expr::Call(_)) - }) - } - - fn statements_are_single_bound_method_call_expr(body: &[ast::Stmt]) -> bool { - matches!( - body, - [ast::Stmt::Expr(ast::StmtExpr { value, .. })] - if matches!( - value.as_ref(), - ast::Expr::Call(ast::ExprCall { func, .. }) - if matches!(func.as_ref(), ast::Expr::Attribute(_)) - ) - ) - } - - fn statements_contain_nonterminal_try_except(body: &[ast::Stmt]) -> bool { - body.iter() - .take(body.len().saturating_sub(1)) - .any(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - handlers, - finalbody, - is_star: false, - .. - }) => !handlers.is_empty() && finalbody.is_empty(), - _ => false, - }) - } - - fn statements_contain_try_except(body: &[ast::Stmt]) -> bool { - body.iter().any(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - handlers, - finalbody, - is_star: false, - .. - }) => !handlers.is_empty() && finalbody.is_empty(), - _ => false, - }) - } - - fn statements_contain_try_finally(body: &[ast::Stmt]) -> bool { - body.iter().any(|stmt| { - matches!( - stmt, - ast::Stmt::Try(ast::StmtTry { finalbody, .. }) if !finalbody.is_empty() - ) - }) - } - - fn statements_end_with_try_except_handler_fallthrough(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - finalbody, - is_star: false, - .. - }) => { - finalbody.is_empty() - && !handlers.is_empty() - && Self::statements_end_with_scope_exit(body) - && handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - !Self::statements_end_with_scope_exit(&handler.body) - }) - } - _ => false, - }) - } - - fn statements_end_with_try_except_else_handler_scope_exit(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - handlers, - orelse, - finalbody, - is_star: false, - .. - }) => { - !orelse.is_empty() - && finalbody.is_empty() - && handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_end_with_finally_entry_scope_exit(&handler.body) - }) - } - _ => false, - }) - } - - fn statements_end_with_open_conditional_fallthrough(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::If(ast::StmtIf { - elif_else_clauses, .. - }) => elif_else_clauses - .last() - .is_none_or(|clause| clause.test.is_some()), - _ => false, - }) - } - - fn statements_end_with_conditional(body: &[ast::Stmt]) -> bool { - body.last() - .is_some_and(Self::statement_ends_with_conditional) - } - - fn statement_ends_with_conditional(stmt: &ast::Stmt) -> bool { - match stmt { - ast::Stmt::If(_) | ast::Stmt::Match(_) => true, - ast::Stmt::With(ast::StmtWith { body, .. }) => { - Self::statements_end_with_conditional(body) - } - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) => { - if !finalbody.is_empty() { - return Self::statements_end_with_conditional(finalbody); - } - let normal_body = if orelse.is_empty() { body } else { orelse }; - Self::statements_end_with_conditional(normal_body) - || handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_end_with_conditional(&handler.body) - }) - } - _ => false, - } - } - - fn statements_end_with_conditional_scope_exit(&self, body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Assert(_) => self.opts.optimize == 0, - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_end_with_scope_exit(body) - || elif_else_clauses - .iter() - .any(|clause| Self::statements_end_with_scope_exit(&clause.body)) - } - _ => false, - }) - } - - fn statements_contain_conditional_scope_exit(body: &[ast::Stmt]) -> bool { - body.iter().any(|stmt| match stmt { - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_end_with_scope_exit(body) - || elif_else_clauses - .iter() - .any(|clause| Self::statements_end_with_scope_exit(&clause.body)) - } - _ => false, - }) - } - - fn statement_is_name_store_from_attribute(stmt: &ast::Stmt) -> bool { - matches!( - stmt, - ast::Stmt::Assign(ast::StmtAssign { targets, value, .. }) - if targets.len() == 1 - && matches!(targets[0], ast::Expr::Name(_)) - && matches!(value.as_ref(), ast::Expr::Attribute(_)) - ) - } - - fn statement_is_name_store_from_call(stmt: &ast::Stmt) -> bool { - matches!( - stmt, - ast::Stmt::Assign(ast::StmtAssign { targets, value, .. }) - if targets.len() == 1 - && matches!(targets[0], ast::Expr::Name(_)) - && matches!(value.as_ref(), ast::Expr::Call(_)) - ) - } - - fn is_attribute_error_handler(handler: &ast::ExceptHandler) -> bool { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { type_, .. }) = - handler; - matches!( - type_.as_deref(), - Some(ast::Expr::Name(ast::ExprName { id, .. })) if id.as_str() == "AttributeError" - ) - } - - fn has_cpython_try_else_attribute_probe_end_barrier( - body: &[ast::Stmt], - handlers: &[ast::ExceptHandler], - orelse: &[ast::Stmt], - ) -> bool { - handlers.len() == 1 - && Self::is_attribute_error_handler(&handlers[0]) - && body - .last() - .is_some_and(Self::statement_is_name_store_from_attribute) - && orelse - .last() - .is_some_and(Self::statement_is_name_store_from_call) - } - - fn statements_end_with_loop_fallthrough(&mut self, body: &[ast::Stmt]) -> CompileResult { - match body.last() { - Some(ast::Stmt::While(ast::StmtWhile { test, body, .. })) => { - Ok(!matches!(test.as_ref(), ast::Expr::BoolOp(_)) - && !matches!(self.constant_expr_truthiness(test)?, Some(true)) - && !Self::statements_contain_direct_break(body)) - } - _ => Ok(false), - } - } - - fn statements_end_with_while_true_direct_break( - &mut self, - body: &[ast::Stmt], - ) -> CompileResult { - match body.last() { - Some(ast::Stmt::While(ast::StmtWhile { test, body, .. })) => { - Ok(matches!(self.constant_expr_truthiness(test)?, Some(true)) - && Self::statements_contain_direct_break(body)) - } - _ => Ok(false), - } - } - - fn statements_end_with_while_true_without_break( - &mut self, - body: &[ast::Stmt], - ) -> CompileResult { - match body.last() { - Some(ast::Stmt::While(ast::StmtWhile { test, body, .. })) => { - Ok(matches!(self.constant_expr_truthiness(test)?, Some(true)) - && !Self::statements_contain_direct_break(body)) - } - _ => Ok(false), - } - } - - fn statements_are_single_with_while_true_without_break( - &mut self, - body: &[ast::Stmt], - ) -> CompileResult { - let [ast::Stmt::With(ast::StmtWith { body, is_async, .. })] = body else { - return Ok(false); - }; - if *is_async { - return Ok(false); - } - match body.last() { - Some(ast::Stmt::While(ast::StmtWhile { test, body, .. })) => { - Ok(matches!(self.constant_expr_truthiness(test)?, Some(true)) - && !Self::statements_contain_direct_break(body)) - } - _ => Ok(false), - } - } - - fn statements_end_with_while_true_tail_direct_break( - &mut self, - body: &[ast::Stmt], - ) -> CompileResult { - match body.last() { - Some(ast::Stmt::While(ast::StmtWhile { test, body, .. })) => { - Ok(matches!(self.constant_expr_truthiness(test)?, Some(true)) - && Self::statements_end_with_direct_break(body)) - } - _ => Ok(false), - } - } - - fn statements_contain_direct_break(body: &[ast::Stmt]) -> bool { - body.iter().any(Self::statement_contains_direct_break) - } - - fn statements_contain_try_except_orelse_direct_break(body: &[ast::Stmt]) -> bool { - body.iter().any(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - is_star: false, - .. - }) if !handlers.is_empty() && finalbody.is_empty() => { - (Self::statements_contain_direct_break(body) - || Self::statements_contain_direct_break(orelse)) - && handlers.iter().all(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - !Self::statements_contain_direct_break(&handler.body) - }) - } - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_contain_try_except_orelse_direct_break(body) - || elif_else_clauses.iter().any(|clause| { - Self::statements_contain_try_except_orelse_direct_break(&clause.body) - }) - } - ast::Stmt::With(ast::StmtWith { body, .. }) => { - Self::statements_contain_try_except_orelse_direct_break(body) - } - ast::Stmt::Match(ast::StmtMatch { cases, .. }) => cases - .iter() - .any(|case| Self::statements_contain_try_except_orelse_direct_break(&case.body)), - ast::Stmt::For(_) | ast::Stmt::While(_) => false, - _ => false, - }) - } - - fn statements_are_single_for_direct_break(body: &[ast::Stmt]) -> bool { - matches!( - body, - [ast::Stmt::For(ast::StmtFor { body, is_async, .. })] - if !is_async && Self::statements_contain_direct_break(body) - ) - } - - fn statements_are_single_for_loop(body: &[ast::Stmt]) -> bool { - matches!( - body, - [ast::Stmt::For(ast::StmtFor { - is_async: false, - .. - })] - ) - } - - fn statements_end_with_for_loop(body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| { - matches!( - stmt, - ast::Stmt::For(ast::StmtFor { - is_async: false, - .. - }) - ) - }) - } - - fn statements_end_with_direct_break(body: &[ast::Stmt]) -> bool { - body.last() - .is_some_and(Self::statement_ends_with_direct_break) - } - - fn handlers_end_with_scope_exit(handlers: &[ast::ExceptHandler]) -> bool { - handlers.iter().all(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_end_with_scope_exit(&handler.body) - }) - } - - fn statement_ends_with_direct_break(stmt: &ast::Stmt) -> bool { - match stmt { - ast::Stmt::Break(_) => true, - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_end_with_direct_break(body) - || elif_else_clauses - .iter() - .any(|clause| Self::statements_end_with_direct_break(&clause.body)) - } - ast::Stmt::With(ast::StmtWith { body, .. }) => { - Self::statements_end_with_direct_break(body) - } - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) => { - Self::statements_end_with_direct_break(body) - || handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_end_with_direct_break(&handler.body) - }) - || Self::statements_end_with_direct_break(orelse) - || Self::statements_end_with_direct_break(finalbody) - } - ast::Stmt::Match(ast::StmtMatch { cases, .. }) => cases - .iter() - .any(|case| Self::statements_end_with_direct_break(&case.body)), - ast::Stmt::For(_) | ast::Stmt::While(_) => false, - _ => false, - } - } - - fn statement_contains_direct_break(stmt: &ast::Stmt) -> bool { - match stmt { - ast::Stmt::Break(_) => true, - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - Self::statements_contain_direct_break(body) - || elif_else_clauses - .iter() - .any(|clause| Self::statements_contain_direct_break(&clause.body)) - } - ast::Stmt::With(ast::StmtWith { body, .. }) => { - Self::statements_contain_direct_break(body) - } - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) => { - Self::statements_contain_direct_break(body) - || handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - Self::statements_contain_direct_break(&handler.body) - }) - || Self::statements_contain_direct_break(orelse) - || Self::statements_contain_direct_break(finalbody) - } - ast::Stmt::Match(ast::StmtMatch { cases, .. }) => cases - .iter() - .any(|case| Self::statements_contain_direct_break(&case.body)), - ast::Stmt::For(_) | ast::Stmt::While(_) => false, - _ => false, - } - } - - fn statements_end_with_optimized_finally_entry_scope_exit(&self, body: &[ast::Stmt]) -> bool { - body.last() - .is_some_and(|stmt| self.statement_ends_with_optimized_finally_entry_scope_exit(stmt)) - } - - fn statement_ends_with_optimized_finally_entry_scope_exit(&self, stmt: &ast::Stmt) -> bool { - match stmt { - ast::Stmt::Assert(_) => self.opts.optimize == 0, - ast::Stmt::If(ast::StmtIf { body, .. }) => { - self.statements_end_with_optimized_finally_entry_scope_exit(body) - } - _ => Self::statement_ends_with_finally_entry_scope_exit(stmt), - } - } - - fn preserves_finally_entry_nop(&self, body: &[ast::Stmt]) -> bool { - body.last().is_some_and(|stmt| match stmt { - ast::Stmt::Try(ast::StmtTry { - body, - handlers, - finalbody, - .. - }) => { - !finalbody.is_empty() && !Self::statements_end_with_conditional(finalbody) - || (!handlers.is_empty() && Self::statements_end_with_scope_exit(body)) - } - ast::Stmt::If(ast::StmtIf { - body, - elif_else_clauses, - .. - }) => { - elif_else_clauses.is_empty() - && self.statements_end_with_optimized_finally_entry_scope_exit(body) - } - ast::Stmt::Assert(_) => self.opts.optimize == 0, - _ => false, - }) - } - fn compile_module_annotation_setup_sequence( &mut self, body: &[ast::Stmt], loc: TextRange, ) -> CompileResult<()> { - let (saved_blocks, saved_current_block) = { + let ( + saved_blocks, + saved_current_block, + saved_instr_sequence, + saved_instr_sequence_label_map, + saved_annotations_instr_sequence, + ) = { let code = self.current_code_info(); ( mem::replace(&mut code.blocks, vec![ir::Block::default()]), mem::replace(&mut code.current_block, BlockIdx::new(0)), + mem::replace(&mut code.instr_sequence, ir::InstructionSequence::new()), + mem::replace( + &mut code.instr_sequence_label_map, + ir::InstructionSequenceLabelMap::new(), + ), + code.annotations_instr_sequence.take(), ) }; let result = self.compile_module_annotate(body, Some(loc)); - let annotations_blocks = { + { let code = self.current_code_info(); - let annotations_blocks = mem::replace(&mut code.blocks, saved_blocks); + code.blocks = saved_blocks; + let annotations_instr_sequence = + mem::replace(&mut code.instr_sequence, saved_instr_sequence); code.current_block = saved_current_block; - annotations_blocks + code.instr_sequence_label_map = saved_instr_sequence_label_map; + code.annotations_instr_sequence = Some(annotations_instr_sequence); + debug_assert!(saved_annotations_instr_sequence.is_none()); }; - self.current_code_info().annotations_blocks = Some(annotations_blocks); result.map(|_| ()) } @@ -2466,7 +1410,9 @@ impl Compiler { private, blocks: vec![ir::Block::default()], current_block: BlockIdx::new(0), - annotations_blocks: None, + instr_sequence: ir::InstructionSequence::new(), + instr_sequence_label_map: ir::InstructionSequenceLabelMap::new(), + annotations_instr_sequence: None, metadata: ir::CodeUnitMetadata { name: name.to_owned(), qualname: None, // Will be set below @@ -2491,27 +1437,11 @@ impl Compiler { fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: key, in_conditional_block: 0, - in_final_with_cleanup_statement: 0, - in_try_else_orelse: 0, next_conditional_annotation_index: 0, }; // Push the old compiler unit on the stack (like PyCapsule) // This happens before setting qualname - self.fallthrough_successor_stack.push(( - self.fallthrough_has_statement_successor, - self.fallthrough_has_local_statement_successor, - self.fallthrough_next_statement_is_if, - self.fallthrough_next_statement_is_try, - self.fallthrough_next_statement_is_function_def, - self.fallthrough_next_statement_has_empty_test_prefix, - )); - self.fallthrough_has_statement_successor = false; - self.fallthrough_has_local_statement_successor = false; - self.fallthrough_next_statement_is_if = false; - self.fallthrough_next_statement_is_try = false; - self.fallthrough_next_statement_is_function_def = false; - self.fallthrough_next_statement_has_empty_test_prefix = false; self.code_stack.push(code_info); // Set qualname after pushing (uses compiler_set_qualname logic) @@ -2519,8 +1449,6 @@ impl Compiler { self.set_qualname(); } - self.emit_prefix_cell_setup(); - // Emit RESUME (handles async preamble and module lineno 0) // CPython: LOCATION(lineno, lineno, 0, 0), then loc.lineno = 0 for module self.emit_resume_for_scope(scope_type, lineno); @@ -2528,19 +1456,9 @@ impl Compiler { Ok(()) } - /// Emit RESUME instruction with proper handling for async preamble and module lineno. + /// Emit RESUME instruction with proper handling for module lineno. /// codegen_enter_scope equivalent for RESUME emission. fn emit_resume_for_scope(&mut self, scope_type: CompilerScope, lineno: u32) { - // For generators and async functions, emit RETURN_GENERATOR + POP_TOP before RESUME - let is_gen = - scope_type == CompilerScope::AsyncFunction || self.current_symbol_table().is_generator; - if is_gen { - emit!(self, Instruction::ReturnGenerator); - self.mark_last_line_only_location(lineno); - emit!(self, Instruction::PopTop); - self.mark_last_line_only_location(lineno); - } - // CPython: LOCATION(lineno, lineno, 0, 0) // Module scope: loc.lineno = 0 (before the first line) let lineno_override = if scope_type == CompilerScope::Module { @@ -2557,7 +1475,7 @@ impl Compiler { let end_location = location; // end_lineno = lineno, end_col = 0 let except_handler = None; - self.current_block().instructions.push(ir::InstructionInfo { + self.cpython_cfg_builder_addop(ir::InstructionInfo { instr: Instruction::Resume { context: OpArgMarker::marker(), } @@ -2567,54 +1485,11 @@ impl Compiler { location, end_location, except_handler, - folded_from_nonliteral_expr: false, lineno_override, cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, }); } - fn emit_prefix_cell_setup(&mut self) { - let metadata = &self.code_stack.last().unwrap().metadata; - let varnames = metadata.varnames.clone(); - let cellvars = metadata.cellvars.clone(); - let freevars = metadata.freevars.clone(); - let ncells = cellvars.len(); - if ncells > 0 { - let cellfixedoffsets = ir::build_cellfixedoffsets(&varnames, &cellvars, &freevars); - let mut sorted = vec![None; varnames.len() + ncells]; - for (oldindex, fixed) in cellfixedoffsets.iter().copied().take(ncells).enumerate() { - sorted[fixed as usize] = Some(oldindex); - } - for oldindex in sorted.into_iter().flatten() { - let i_varnum: oparg::VarNum = - u32::try_from(oldindex).expect("too many cellvars").into(); - emit!(self, Instruction::MakeCell { i: i_varnum }); - self.set_no_location(); - } - } - - let nfrees = freevars.len(); - if nfrees > 0 { - emit!( - self, - Instruction::CopyFreeVars { - n: u32::try_from(nfrees).expect("too many freevars"), - } - ); - self.set_no_location(); - } - } - fn push_output( &mut self, flags: bytecode::CodeFlags, @@ -2656,24 +1531,6 @@ impl Compiler { // compiler_exit_scope fn exit_scope(&mut self) -> CodeObject { let _table = self.pop_symbol_table(); - if let Some(( - previous, - previous_local, - previous_next_is_if, - previous_next_is_try, - previous_next_is_function_def, - previous_next_has_empty_test_prefix, - )) = self.fallthrough_successor_stack.pop() - { - self.fallthrough_has_statement_successor = previous; - self.fallthrough_has_local_statement_successor = previous_local; - self.fallthrough_next_statement_is_if = previous_next_is_if; - self.fallthrough_next_statement_is_try = previous_next_is_try; - self.fallthrough_next_statement_is_function_def = previous_next_is_function_def; - self.fallthrough_next_statement_has_empty_test_prefix = - previous_next_has_empty_test_prefix; - } - // Various scopes can have sub_tables: // - ast::TypeParams scope can have sub_tables (the function body's symbol table) // - Module scope can have sub_tables (for TypeAlias scopes, nested functions, classes) @@ -2690,24 +1547,6 @@ impl Compiler { fn exit_annotation_scope(&mut self, saved_ctx: CompileContext) -> CodeObject { self.pop_annotation_symbol_table(); self.ctx = saved_ctx; - if let Some(( - previous, - previous_local, - previous_next_is_if, - previous_next_is_try, - previous_next_is_function_def, - previous_next_has_empty_test_prefix, - )) = self.fallthrough_successor_stack.pop() - { - self.fallthrough_has_statement_successor = previous; - self.fallthrough_has_local_statement_successor = previous_local; - self.fallthrough_next_statement_is_if = previous_next_is_if; - self.fallthrough_next_statement_is_try = previous_next_is_try; - self.fallthrough_next_statement_is_function_def = previous_next_is_function_def; - self.fallthrough_next_statement_has_empty_test_prefix = - previous_next_has_empty_test_prefix; - } - let pop = self.code_stack.pop(); let stack_top = compiler_unwrap_option(self, pop); unwrap_internal(self, stack_top.finalize_code(&self.opts)) @@ -2728,7 +1567,6 @@ impl Compiler { // Annotation scopes are never async (even inside async functions) let saved_ctx = self.ctx; self.ctx = CompileContext { - loop_data: None, in_class: saved_ctx.in_class, func: FunctionContext::Function, in_async_scope: false, @@ -2799,7 +1637,7 @@ impl Compiler { ); // Body label - continue with annotation evaluation - self.switch_to_block(body_block); + self.use_cpython_transient_label_block(body_block); } /// Push a new fblock @@ -2807,18 +1645,33 @@ impl Compiler { fn push_fblock( &mut self, fb_type: FBlockType, - fb_block: BlockIdx, - fb_exit: BlockIdx, + fb_block_direct: BlockIdx, + fb_exit_direct: BlockIdx, ) -> CompileResult<()> { - self.push_fblock_full(fb_type, fb_block, fb_exit, FBlockDatum::None) + self.push_fblock_full(fb_type, fb_block_direct, fb_exit_direct, FBlockDatum::None) } - /// Push an fblock with all parameters including fb_datum + /// Direct-CFG bridge for `_PyCompile_PushFBlock()`. fn push_fblock_full( &mut self, fb_type: FBlockType, - fb_block: BlockIdx, - fb_exit: BlockIdx, + fb_block_direct: BlockIdx, + fb_exit_direct: BlockIdx, + fb_datum: FBlockDatum, + ) -> CompileResult<()> { + let code = self.current_code_info(); + let fb_block = code.instr_sequence_label_for_block(fb_block_direct); + let fb_exit = code.instr_sequence_label_for_block(fb_exit_direct); + self.push_fblock_labels(fb_type, fb_block, fb_exit, fb_datum) + } + + /// CPython `_PyCompile_PushFBlock()`: store the active label targets on the + /// fblock stack. + fn push_fblock_labels( + &mut self, + fb_type: FBlockType, + fb_block: Option, + fb_exit: Option, fb_datum: FBlockDatum, ) -> CompileResult<()> { let fb_range = self.current_source_range; @@ -2840,11 +1693,31 @@ impl Compiler { /// Pop an fblock // = compiler_pop_fblock - fn pop_fblock(&mut self, _expected_type: FBlockType) -> FBlockInfo { + fn pop_fblock( + &mut self, + expected_type: FBlockType, + expected_block_direct: BlockIdx, + ) -> FBlockInfo { + let expected_block = self + .current_code_info() + .instr_sequence_label_for_block(expected_block_direct); + self.pop_fblock_label(expected_type, expected_block) + } + + /// CPython `_PyCompile_PopFBlock()`: assert the popped type and label. + fn pop_fblock_label( + &mut self, + expected_type: FBlockType, + expected_block: Option, + ) -> FBlockInfo { let code = self.current_code_info(); - // TODO: Add assertion to check expected type matches - // assert!(matches!(fblock.fb_type, expected_type)); - code.fblock.pop().expect("fblock stack underflow") + let fblock = code.fblock.pop().expect("fblock stack underflow"); + debug_assert_eq!(fblock.fb_type, expected_type); + debug_assert_eq!( + fblock.fb_block, expected_block, + "CPython _PyCompile_PopFBlock asserts the popped fb_block label" + ); + fblock } fn set_unwind_source_range(&mut self, loc: Option) { @@ -2943,15 +1816,8 @@ impl Compiler { emit!(self, Instruction::Swap { i: 2 }); // [value, exit_func, self_exit] } - // Call exit_func(self_exit, None, None, None) - self.set_unwind_source_range(*loc); - self.emit_load_const(ConstantData::None); - self.set_unwind_source_range(*loc); - self.emit_load_const(ConstantData::None); - self.set_unwind_source_range(*loc); - self.emit_load_const(ConstantData::None); self.set_unwind_source_range(*loc); - emit!(self, Instruction::Call { argc: 3 }); + self.compile_call_exit_with_nones(); // For async with, await the result if matches!(info.fb_type, FBlockType::AsyncWith) { @@ -2959,7 +1825,7 @@ impl Compiler { emit!(self, Instruction::GetAwaitable { r#where: 2 }); self.set_unwind_source_range(*loc); self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(true)?; + let _ = self.compile_yield_from_sequence(true); } // Pop the __exit__ result @@ -3025,6 +1891,19 @@ impl Compiler { preserve_tos: bool, stop_at_loop: bool, ) -> CompileResult> { + let (unwind_loc, _loop_fblock) = + self.unwind_fblock_stack_with_loop(preserve_tos, stop_at_loop)?; + Ok(unwind_loc) + } + + /// CPython `codegen_unwind_fblock_stack()`: unwind frame blocks and, when + /// requested by break/continue codegen, return the first loop fblock instead + /// of unwinding it. + fn unwind_fblock_stack_with_loop( + &mut self, + preserve_tos: bool, + stop_at_loop: bool, + ) -> CompileResult<(Option, Option)> { // Collect the info we need, with indices for FinallyTry blocks #[derive(Clone)] enum UnwindInfo { @@ -3035,6 +1914,7 @@ impl Compiler { }, } let mut unwind_infos = Vec::new(); + let mut loop_fblock = None; { let code = self.current_code_info(); @@ -3051,6 +1931,7 @@ impl Compiler { FBlockType::WhileLoop | FBlockType::ForLoop ) { + loop_fblock = Some(code.fblock[i].clone()); break; } @@ -3087,18 +1968,14 @@ impl Compiler { // Push PopValue fblock if preserving tos if preserve_tos { - self.push_fblock( - FBlockType::PopValue, - saved_fblock.fb_block, - saved_fblock.fb_block, - )?; + self.push_fblock(FBlockType::PopValue, BlockIdx::NULL, BlockIdx::NULL)?; } self.compile_statements(&body)?; unwind_loc = None; if preserve_tos { - self.pop_fblock(FBlockType::PopValue); + self.pop_fblock(FBlockType::PopValue, BlockIdx::NULL); } // Restore the fblock @@ -3108,7 +1985,7 @@ impl Compiler { } } - Ok(unwind_loc) + Ok((unwind_loc, loop_fblock)) } // could take impl Into>, but everything is borrowed from ast structs; we never @@ -3282,12 +2159,6 @@ impl Compiler { self.symbol_table_stack.push(symbol_table); - // Match flowgraph.c insert_prefix_instructions() for module-level - // synthetic cells before RESUME. - if has_module_cond_ann { - self.emit_prefix_cell_setup(); - } - self.emit_resume_for_scope(CompilerScope::Module, 1); emit!(self, PseudoInstruction::AnnotationsPlaceholder); @@ -3431,14 +2302,14 @@ impl Compiler { if !self.interactive && Self::is_const_expression(value) { self.compile_expression(value)?; } else { - self.current_block().instructions.pop(); // pop Instruction::PopTop + self.pop_last_emitted_instruction(); // pop Instruction::PopTop } } ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_) => { - let pop_instructions = self.current_block().instructions.pop(); + let pop_instructions = self.pop_last_emitted_instruction(); let store_inst = compiler_unwrap_option(self, pop_instructions); // pop Instruction::Store emit!(self, Instruction::Copy { i: 1 }); - self.current_block().instructions.push(store_inst); + self.push_emitted_instruction(store_inst); } _ => self.emit_load_const(ConstantData::None), } @@ -3463,135 +2334,78 @@ impl Compiler { } fn compile_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { - let inherited_successor = self.fallthrough_has_statement_successor; - for (idx, statement) in statements.iter().enumerate() { - let previous_successor = self.fallthrough_has_statement_successor; - let previous_local_successor = self.fallthrough_has_local_statement_successor; - let previous_next_is_if = self.fallthrough_next_statement_is_if; - let previous_next_is_try = self.fallthrough_next_statement_is_try; - let previous_next_is_function_def = self.fallthrough_next_statement_is_function_def; - let previous_next_has_empty_test_prefix = - self.fallthrough_next_statement_has_empty_test_prefix; - self.fallthrough_has_statement_successor = - inherited_successor || idx + 1 < statements.len(); - self.fallthrough_has_local_statement_successor = idx + 1 < statements.len(); - self.fallthrough_next_statement_is_if = statements - .get(idx + 1) - .is_some_and(|stmt| matches!(stmt, ast::Stmt::If(_))); - self.fallthrough_next_statement_is_try = statements.get(idx + 1).is_some_and(|stmt| { - matches!( - stmt, - ast::Stmt::Try(ast::StmtTry { - handlers, - finalbody, - .. - }) if !handlers.is_empty() && finalbody.is_empty() - ) - }); - self.fallthrough_next_statement_is_function_def = statements - .get(idx + 1) - .is_some_and(|stmt| matches!(stmt, ast::Stmt::FunctionDef(_))); - self.fallthrough_next_statement_has_empty_test_prefix = statements - .get(idx + 1) - .map(|stmt| self.statement_starts_with_empty_if_test_prefix(stmt)) - .transpose()? - .unwrap_or(false); - let result = self.compile_statement(statement); - self.fallthrough_has_statement_successor = previous_successor; - self.fallthrough_has_local_statement_successor = previous_local_successor; - self.fallthrough_next_statement_is_if = previous_next_is_if; - self.fallthrough_next_statement_is_try = previous_next_is_try; - self.fallthrough_next_statement_is_function_def = previous_next_is_function_def; - self.fallthrough_next_statement_has_empty_test_prefix = - previous_next_has_empty_test_prefix; - result?; + for statement in statements { + self.compile_statement(statement)?; } Ok(()) } fn compile_with_body_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { - let inherited_successor = self.fallthrough_has_statement_successor; - for (idx, statement) in statements.iter().enumerate() { - let previous_successor = self.fallthrough_has_statement_successor; - let previous_local_successor = self.fallthrough_has_local_statement_successor; - let previous_next_is_if = self.fallthrough_next_statement_is_if; - let previous_next_is_try = self.fallthrough_next_statement_is_try; - let previous_next_is_function_def = self.fallthrough_next_statement_is_function_def; - let previous_next_has_empty_test_prefix = - self.fallthrough_next_statement_has_empty_test_prefix; - self.fallthrough_has_statement_successor = - inherited_successor || idx + 1 < statements.len(); - self.fallthrough_has_local_statement_successor = idx + 1 < statements.len(); - self.fallthrough_next_statement_is_if = statements - .get(idx + 1) - .is_some_and(|stmt| matches!(stmt, ast::Stmt::If(_))); - self.fallthrough_next_statement_is_try = statements.get(idx + 1).is_some_and(|stmt| { - matches!( - stmt, - ast::Stmt::Try(ast::StmtTry { - handlers, - finalbody, - .. - }) if !handlers.is_empty() && finalbody.is_empty() - ) - }); - self.fallthrough_next_statement_is_function_def = statements - .get(idx + 1) - .is_some_and(|stmt| matches!(stmt, ast::Stmt::FunctionDef(_))); - self.fallthrough_next_statement_has_empty_test_prefix = statements - .get(idx + 1) - .map(|stmt| self.statement_starts_with_empty_if_test_prefix(stmt)) - .transpose()? - .unwrap_or(false); - if idx + 1 == statements.len() && matches!(statement, ast::Stmt::Try(_)) { - self.current_code_info().in_final_with_cleanup_statement += 1; - let result = self.compile_statement(statement); - self.current_code_info().in_final_with_cleanup_statement -= 1; - self.fallthrough_has_statement_successor = previous_successor; - self.fallthrough_has_local_statement_successor = previous_local_successor; - self.fallthrough_next_statement_is_if = previous_next_is_if; - self.fallthrough_next_statement_is_try = previous_next_is_try; - self.fallthrough_next_statement_is_function_def = previous_next_is_function_def; - self.fallthrough_next_statement_has_empty_test_prefix = - previous_next_has_empty_test_prefix; - result?; - } else { - let result = self.compile_statement(statement); - self.fallthrough_has_statement_successor = previous_successor; - self.fallthrough_has_local_statement_successor = previous_local_successor; - self.fallthrough_next_statement_is_if = previous_next_is_if; - self.fallthrough_next_statement_is_try = previous_next_is_try; - self.fallthrough_next_statement_is_function_def = previous_next_is_function_def; - self.fallthrough_next_statement_has_empty_test_prefix = - previous_next_has_empty_test_prefix; - result?; - } + for statement in statements { + self.compile_statement(statement)?; } Ok(()) } + /// CPython `codegen_call_exit_with_nones()`. + fn compile_call_exit_with_nones(&mut self) { + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::Call { argc: 3 }); + } + + /// CPython `codegen_with_except_finish()`. + /// + /// CPython creates a helper-local `exit` label followed immediately by the + /// caller's `exit` label. `_PyInstructionSequence_ApplyLabelMap()` resolves + /// both labels to the same next instruction, so the direct-CFG equivalent is + /// to target the caller exit block. + fn compile_with_except_finish(&mut self, cleanup_block: BlockIdx, exit_block: BlockIdx) { + let suppress_block = self.new_block(); + + emit!(self, Instruction::ToBool); + self.set_no_location(); + emit!( + self, + Instruction::PopJumpIfTrue { + delta: suppress_block + } + ); + self.set_no_location(); + emit!(self, Instruction::Reraise { depth: 2 }); + self.set_no_location(); + + self.use_cpython_label_block(suppress_block); + emit!(self, Instruction::PopTop); + self.set_no_location(); + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + emit!(self, Instruction::PopExcept); + self.set_no_location(); + emit!(self, Instruction::PopTop); + self.set_no_location(); + emit!(self, Instruction::PopTop); + self.set_no_location(); + emit!(self, Instruction::PopTop); + self.set_no_location(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: exit_block } + ); + self.set_no_location(); + + self.use_cpython_label_block(cleanup_block); + emit!(self, Instruction::Copy { i: 3 }); + self.set_no_location(); + emit!(self, Instruction::PopExcept); + self.set_no_location(); + emit!(self, Instruction::Reraise { depth: 1 }); + self.set_no_location(); + } + fn compile_loop_body_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { - let previous_successor = self.fallthrough_has_statement_successor; - let previous_local_successor = self.fallthrough_has_local_statement_successor; - let previous_next_is_if = self.fallthrough_next_statement_is_if; - let previous_next_is_try = self.fallthrough_next_statement_is_try; - let previous_next_is_function_def = self.fallthrough_next_statement_is_function_def; - let previous_next_has_empty_test_prefix = - self.fallthrough_next_statement_has_empty_test_prefix; - self.fallthrough_has_statement_successor = false; - self.fallthrough_has_local_statement_successor = false; - self.fallthrough_next_statement_is_if = false; - self.fallthrough_next_statement_is_try = false; - self.fallthrough_next_statement_is_function_def = false; - self.fallthrough_next_statement_has_empty_test_prefix = false; - let result = self.compile_statements(statements); - self.fallthrough_has_statement_successor = previous_successor; - self.fallthrough_has_local_statement_successor = previous_local_successor; - self.fallthrough_next_statement_is_if = previous_next_is_if; - self.fallthrough_next_statement_is_try = previous_next_is_try; - self.fallthrough_next_statement_is_function_def = previous_next_is_function_def; - self.fallthrough_next_statement_has_empty_test_prefix = previous_next_has_empty_test_prefix; - result + self.compile_statements(statements) } fn scope_needs_conditional_annotations_cell(symbol_table: &SymbolTable) -> bool { @@ -3611,6 +2425,19 @@ impl Compiler { self.compile_name(name, NameUsage::Store) } + fn emit_no_location_exception_name_cleanup(&mut self, name: &str) -> CompileResult<()> { + // CPython codegen_try_except() emits `name = None; del name` + // with NO_LOCATION for `except ... as name` cleanup. + self.set_no_location(); + self.emit_load_const(ConstantData::None); + self.set_no_location(); + self.store_name(name)?; + self.set_no_location(); + self.compile_name(name, NameUsage::Delete)?; + self.set_no_location(); + Ok(()) + } + fn mangle<'a>(&self, name: &'a str) -> Cow<'a, str> { // Use private from current code unit for name mangling let private = self @@ -4171,11 +2998,6 @@ impl Compiler { } ); self.switch_to_block(after_block); - if self.fallthrough_next_statement_has_empty_test_prefix { - self.mark_load_fast_barrier_block(after_block); - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } } else { // Optimized-out asserts still need to consume any nested // scope symbol tables they contain so later nested scopes @@ -4187,14 +3009,12 @@ impl Compiler { } } ast::Stmt::Break(_) => { - emit!(self, Instruction::Nop); // NOP for line tracing // Unwind fblock stack until we find a loop, emitting cleanup for each fblock self.compile_break_continue(statement.range(), true)?; let dead = self.new_block(); self.switch_to_block(dead); } ast::Stmt::Continue(_) => { - emit!(self, Instruction::Nop); // NOP for line tracing // Unwind fblock stack until we find a loop, emitting cleanup for each fblock self.compile_break_continue(statement.range(), false)?; let dead = self.new_block(); @@ -4286,35 +3106,12 @@ impl Compiler { range, .. }) => { - let folded_ifexp_assignment = matches!( - value.as_ref(), - ast::Expr::If(ast::ExprIf { test, .. }) - if self.try_fold_constant_expr(test)?.is_some() - ); if targets.len() == 1 && Self::is_unpack_assignment_target(&targets[0]) { self.compile_expression_without_const_collection_folding(value)?; } else { self.compile_expression(value)?; } - if folded_ifexp_assignment { - let current = self.current_code_info().current_block; - if self.current_code_info().blocks[current.idx()] - .instructions - .is_empty() - { - // CPython codegen_ifexp() always emits USE_LABEL(end). - // For a folded top-level if-expression assignment, - // flowgraph.c keeps that empty end block before the - // STORE_FAST. optimize_load_fast() does not push - // fallthrough successors from empty blocks, so later - // LOAD_FAST instructions remain strong. - self.mark_load_fast_barrier_block(current); - let continuation = self.new_block(); - self.switch_to_block(continuation); - } - } - for (i, target) in targets.iter().enumerate() { if i + 1 != targets.len() { self.set_source_range(*range); @@ -4379,7 +3176,6 @@ impl Compiler { // TypeParams scope is function-like let prev_ctx = self.ctx; self.ctx = CompileContext { - loop_data: None, in_class: prev_ctx.in_class, func: FunctionContext::Function, in_async_scope: false, @@ -4559,7 +3355,6 @@ impl Compiler { // TypeParams scope is function-like let prev_ctx = self.ctx; self.ctx = CompileContext { - loop_data: None, in_class: prev_ctx.in_class, func: FunctionContext::Function, in_async_scope: false, @@ -4616,7 +3411,6 @@ impl Compiler { let prev_ctx = self.ctx; self.ctx = CompileContext { - loop_data: None, in_class: prev_ctx.in_class, func: FunctionContext::Function, in_async_scope: false, @@ -4791,407 +3585,69 @@ impl Compiler { return self.compile_try_except_no_finally(body, handlers, orelse); } - let in_try_except_body = self - .current_code_info() - .fblock - .iter() - .any(|info| matches!(info.fb_type, FBlockType::TryExcept)); - let preserve_async_try_except_finally_scope_exit = - in_try_except_body && self.ctx.func == FunctionContext::AsyncFunction; - let preserve_finally_exit_empty_label = (self.fallthrough_has_statement_successor - || preserve_async_try_except_finally_scope_exit) - && ((!handlers.is_empty() && Self::handlers_end_with_scope_exit(handlers)) - || Self::statements_are_single_with(finalbody) - || Self::statements_contain_for_with_conditional_body(finalbody) - || Self::statements_end_with_open_conditional_fallthrough(finalbody) - || (preserve_async_try_except_finally_scope_exit - && Self::statements_contain_conditional_scope_exit(finalbody))); - let preserve_finally_exit_empty_barrier = preserve_async_try_except_finally_scope_exit - && Self::statements_contain_conditional_scope_exit(finalbody); - let preserve_finally_exit_try_except_barrier = self.fallthrough_has_statement_successor - && Self::statements_end_with_try_except_no_finally(finalbody); - let handlers_have_bare_except = handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - handler.type_.is_none() - }); - let preserve_finally_bare_handler_exit_label = - self.fallthrough_has_statement_successor && handlers_have_bare_except; - let preserve_finally_bare_scope_exit_handler_barrier = self - .fallthrough_has_statement_successor - && handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(handler) = handler; - handler.type_.is_none() && Self::statements_end_with_scope_exit(&handler.body) - }); - let handler_block = self.new_block(); - let try_except_end_block = (!handlers.is_empty()).then(|| self.new_block()); - let finally_block = self.new_block(); - - // finally needs TWO blocks: - // - finally_block: normal path (no exception active) - // - finally_except_block: exception path (PUSH_EXC_INFO -> body -> RERAISE) - let finally_except_block = if !finalbody.is_empty() { - Some(self.new_block()) - } else { - None - }; - let finally_cleanup_block = if finally_except_block.is_some() { - Some(self.new_block()) - } else { - None - }; - // End block - continuation point after try-finally - // Normal path jumps here to skip exception path blocks - let end_block = self.new_block(); - // Emit NOP at the try: line so LINE events fire for it - emit!(self, Instruction::Nop); + let body_block = self.new_block(); + let finally_except_block = self.new_block(); + let exit_block = self.new_block(); + let finally_cleanup_block = self.new_block(); - // Setup a finally block if we have a finally statement. - // Push fblock with handler info for exception table generation - // IMPORTANT: handler goes to finally_except_block (exception path), not finally_block - if !finalbody.is_empty() { - // SETUP_FINALLY doesn't push lasti for try body handler - // Exception table: L1 to L2 -> L4 [1] (no lasti) - let setup_target = finally_except_block.unwrap_or(finally_block); - emit!( - self, - PseudoInstruction::SetupFinally { - delta: setup_target - } - ); - // Store finally body in fb_datum for unwind_fblock to compile inline - self.push_fblock_full( - FBlockType::FinallyTry, - finally_block, - finally_block, - FBlockDatum::FinallyBody(finalbody.to_vec()), // Clone finally body for unwind - )?; - } + emit!( + self, + PseudoInstruction::SetupFinally { + delta: finally_except_block + } + ); + self.use_cpython_transient_label_block(body_block); + self.push_fblock_full( + FBlockType::FinallyTry, + body_block, + finally_except_block, + FBlockDatum::FinallyBody(finalbody.to_vec()), + )?; - // if handlers is empty, compile body directly - // without wrapping in TryExcept (only FinallyTry is needed) if handlers.is_empty() { - let preserve_finally_entry_nop = self.preserves_finally_entry_nop(body) - || self.statements_end_with_loop_fallthrough(body)?; - let force_remove_finally_entry_nop = - self.statements_are_single_with_while_true_without_break(body)?; - let preserve_while_break_end_label_before_finally = - self.statements_end_with_while_true_tail_direct_break(body)?; - - // Just compile body with FinallyTry fblock active (if finalbody exists) self.compile_statements(body)?; - - // Pop FinallyTry fblock BEFORE compiling orelse/finally (normal path) - // This prevents exception table from covering the normal path - if !finalbody.is_empty() { - emit!(self, PseudoInstruction::PopBlock); - if preserve_finally_entry_nop { - self.preserve_last_redundant_nop(); - } else if force_remove_finally_entry_nop { - self.set_no_location(); - self.force_remove_last_no_location_nop(); - } else { - self.set_no_location(); - self.remove_last_no_location_nop(); - } - self.pop_fblock(FBlockType::FinallyTry); - } - - // Compile orelse (usually empty for try-finally without except) self.compile_statements(orelse)?; - - let current_block_only_pop_block = { - let block = self.current_block(); - block.instructions.iter().all(|info| { - matches!( - info.instr, - AnyInstruction::Pseudo(PseudoInstruction::PopBlock) - ) - }) - }; - if preserve_while_break_end_label_before_finally - && !finalbody.is_empty() - && current_block_only_pop_block - { - // CPython codegen_while() emits USE_LABEL(end) for the break - // target. When codegen_try_finally() immediately emits the - // normal finally body after that empty end label, flowgraph.c's - // label builder keeps the empty labeled block as a b_next - // barrier, so optimize_load_fast() does not visit the finally - // body through fallthrough. - let end_label = self.current_code_info().current_block; - self.mark_load_fast_barrier_block(end_label); - let finally_body_block = self.new_block(); - self.disable_load_fast_borrow_for_block(finally_body_block); - self.switch_to_block(finally_body_block); - } - - // Snapshot sub_tables before first finally compilation - // This allows us to restore them for the second compilation (exception path) - let sub_table_cursor = if !finalbody.is_empty() && finally_except_block.is_some() { - self.symbol_table_stack.last().map(|t| t.next_sub_table) - } else { - None - }; - - // Compile finally body inline for normal path - if !finalbody.is_empty() { - let previous = self.in_finally_normal_body; - self.in_finally_normal_body = true; - let result = self.compile_statements(finalbody); - self.in_finally_normal_body = previous; - result?; - } - - // Jump to end (skip exception path blocks) - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: end_block } - ); - self.set_no_location(); - - if let Some(finally_except) = finally_except_block { - // Restore sub_tables for exception path compilation - if let Some(cursor) = sub_table_cursor - && let Some(current_table) = self.symbol_table_stack.last_mut() - { - current_table.next_sub_table = cursor; - } - - self.switch_to_block(finally_except); - // SETUP_CLEANUP before PUSH_EXC_INFO - if let Some(cleanup) = finally_cleanup_block { - emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup }); - self.set_no_location(); - } - emit!(self, Instruction::PushExcInfo); - self.set_no_location(); - if let Some(cleanup) = finally_cleanup_block { - self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; - } - self.compile_statements(finalbody)?; - - // RERAISE must be inside the cleanup handler's exception table - // range. When RERAISE re-raises the exception, the cleanup - // handler (COPY 3, POP_EXCEPT, RERAISE 1) runs POP_EXCEPT to - // restore exc_info before the exception reaches the outer handler. - emit!(self, Instruction::Reraise { depth: 0 }); - self.set_no_location(); - - // PopBlock after RERAISE (dead code, but marks the exception - // table range end so the cleanup covers RERAISE). - if finally_cleanup_block.is_some() { - emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::FinallyEnd); - } - } - - if let Some(cleanup) = finally_cleanup_block { - self.switch_to_block(cleanup); - emit!(self, Instruction::Copy { i: 3 }); - self.set_no_location(); - emit!(self, Instruction::PopExcept); - self.set_no_location(); - emit!(self, Instruction::Reraise { depth: 1 }); - self.set_no_location(); - } - - if preserve_finally_exit_empty_label - || preserve_finally_exit_empty_barrier - || preserve_finally_exit_try_except_barrier - || preserve_finally_bare_handler_exit_label - || preserve_finally_bare_scope_exit_handler_barrier - { - self.mark_load_fast_barrier_block(end_block); - } else { - self.mark_label_block(end_block); - } - self.switch_to_block(end_block); - if preserve_finally_exit_empty_label - || preserve_finally_exit_try_except_barrier - || preserve_finally_bare_handler_exit_label - || preserve_finally_bare_scope_exit_handler_barrier - { - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } else { - self.mark_load_fast_passthrough_block(end_block); - self.mark_load_fast_label_reuse_passthrough_block(end_block); - } - return Ok(()); + } else { + let try_except_end = self.new_block(); + self.compile_try_except_no_finally_with_end(body, handlers, orelse, try_except_end)?; } - // try: - let preserve_try_else_after_try_finally_conditional_finalbody = - !orelse.is_empty() && Self::statements_end_with_try_finally_conditional_finalbody(body); - emit!( - self, - PseudoInstruction::SetupFinally { - delta: handler_block - } - ); - self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; - self.compile_statements(body)?; emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::TryExcept); + self.pop_fblock(FBlockType::FinallyTry, body_block); - let cleanup_block = self.new_block(); - // We successfully ran the try block: - // else: - if !orelse.is_empty() { - let current = self.current_code_info().current_block; - self.mark_try_else_orelse_entry_block(current); - if preserve_try_else_after_try_finally_conditional_finalbody { - // CPython codegen_try_finally() compiles try/except/else by - // calling codegen_try_except() inside the active finally try. - // If the protected body ends with another try/finally whose - // finalbody exits through a conditional, codegen_try_except() - // starts the else suite from that inner exit label state, so - // flowgraph.c::optimize_load_fast() keeps the store-attr - // source pair strong. - self.disable_load_fast_borrow_for_block(current); - } - } - self.compile_statements(orelse)?; + let sub_table_cursor = self.symbol_table_stack.last().map(|t| t.next_sub_table); - let normal_finally_entry = try_except_end_block.unwrap_or(finally_block); + self.compile_statements(finalbody)?; emit!( self, - PseudoInstruction::JumpNoInterrupt { - delta: normal_finally_entry, - } + PseudoInstruction::JumpNoInterrupt { delta: exit_block } ); self.set_no_location(); - // except handlers: - self.switch_to_block(handler_block); + if let Some(cursor) = sub_table_cursor + && let Some(current_table) = self.symbol_table_stack.last_mut() + { + current_table.next_sub_table = cursor; + } - // SETUP_CLEANUP(cleanup) for except block - // This handles exceptions during exception matching - // Exception table: L2 to L3 -> L5 [1] lasti - // After PUSH_EXC_INFO, stack is [prev_exc, exc] - // depth=1 means keep prev_exc on stack when routing to cleanup + self.use_cpython_label_block(finally_except_block); emit!( self, PseudoInstruction::SetupCleanup { - delta: cleanup_block + delta: finally_cleanup_block } ); self.set_no_location(); - self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; - - // Exception is on top of stack now, pushed by unwind_blocks - // PUSH_EXC_INFO transforms [exc] -> [prev_exc, exc] for PopExcept emit!(self, Instruction::PushExcInfo); self.set_no_location(); - for handler in handlers { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { - type_, - name, - body, - range: handler_range, - .. - }) = &handler; - self.set_source_range(*handler_range); - let next_handler = self.new_block(); - - if let Some(exc_type) = type_ { - self.compile_expression(exc_type)?; - self.set_source_range(*handler_range); - emit!(self, Instruction::CheckExcMatch); - emit!( - self, - Instruction::PopJumpIfFalse { - delta: next_handler - } - ); - - if let Some(alias) = name { - self.store_name(alias.as_str())? - } else { - emit!(self, Instruction::PopTop); - } - } else { - emit!(self, Instruction::PopTop); - } - - let handler_cleanup_block = if name.is_some() { - let cleanup_end = self.new_block(); - emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup_end }); - self.push_fblock_full( - FBlockType::HandlerCleanup, - cleanup_end, - cleanup_end, - FBlockDatum::ExceptionName(name.as_ref().unwrap().as_str().to_owned()), - )?; - Some(cleanup_end) - } else { - self.push_fblock( - FBlockType::HandlerCleanup, - normal_finally_entry, - normal_finally_entry, - )?; - None - }; - - self.compile_statements(body)?; - - self.pop_fblock(FBlockType::HandlerCleanup); - if handler_cleanup_block.is_some() { - emit!(self, PseudoInstruction::PopBlock); - } - - if let Some(cleanup_end) = handler_cleanup_block { - let handler_normal_exit = self.new_block(); - emit!( - self, - PseudoInstruction::JumpNoInterrupt { - delta: handler_normal_exit, - } - ); - - self.switch_to_block(cleanup_end); - self.set_no_location(); - if let Some(alias) = name { - self.emit_load_const(ConstantData::None); - self.set_no_location(); - self.store_name(alias.as_str())?; - self.set_no_location(); - self.compile_name(alias.as_str(), NameUsage::Delete)?; - self.set_no_location(); - } - emit!(self, Instruction::Reraise { depth: 1 }); - self.set_no_location(); - self.switch_to_block(handler_normal_exit); - } - - emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::ExceptionHandler); - emit!(self, Instruction::PopExcept); - - if let Some(alias) = name { - self.emit_load_const(ConstantData::None); - self.store_name(alias.as_str())?; - self.compile_name(alias.as_str(), NameUsage::Delete)?; - } - - emit!( - self, - PseudoInstruction::JumpNoInterrupt { - delta: normal_finally_entry, - } - ); - self.set_no_location(); - - self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; - self.switch_to_block(next_handler); - } - + self.push_fblock(FBlockType::FinallyEnd, finally_except_block, BlockIdx::NULL)?; + self.compile_statements(finalbody)?; + self.pop_fblock(FBlockType::FinallyEnd, finally_except_block); emit!(self, Instruction::Reraise { depth: 0 }); self.set_no_location(); - self.pop_fblock(FBlockType::ExceptionHandler); - self.switch_to_block(cleanup_block); + self.use_cpython_label_block(finally_cleanup_block); emit!(self, Instruction::Copy { i: 3 }); self.set_no_location(); emit!(self, Instruction::PopExcept); @@ -5199,150 +3655,7 @@ impl Compiler { emit!(self, Instruction::Reraise { depth: 1 }); self.set_no_location(); - // finally (normal path): - // CPython's codegen_try_finally emits the wrapped try/except first and - // places the outer finally body at the inner try/except end label. Keep - // the FinallyTry fblock active through exception-handler normal exits so - // the CFG and exception-table ranges match that structure. - if let Some(try_except_end) = try_except_end_block { - if preserve_finally_bare_scope_exit_handler_barrier { - self.mark_load_fast_barrier_block(try_except_end); - } else { - self.mark_label_block(try_except_end); - } - self.switch_to_block(try_except_end); - if preserve_finally_bare_scope_exit_handler_barrier { - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } - self.use_label_block(finally_block); - } else { - self.switch_to_block(finally_block); - } - if !finalbody.is_empty() { - let preserve_finally_normal_pop_block_nop = orelse.is_empty() - && !Self::statements_end_with_scope_exit(body) - && (!Self::statements_end_with_open_conditional_fallthrough(body) - || Self::statements_end_with_finally_entry_scope_exit(body)) - && handlers.iter().all(|handler| match handler { - ast::ExceptHandler::ExceptHandler(handler) => { - Self::statements_end_with_scope_exit(&handler.body) - } - }); - if preserve_finally_normal_pop_block_nop && let Some(last_body_stmt) = body.last() { - self.set_source_range(last_body_stmt.range()); - } - emit!(self, PseudoInstruction::PopBlock); - if preserve_finally_normal_pop_block_nop { - self.preserve_last_redundant_nop(); - } else { - self.set_no_location(); - } - self.pop_fblock(FBlockType::FinallyTry); - - // Snapshot sub_tables before first finally compilation (for double compilation issue) - let sub_table_cursor = if finally_except_block.is_some() { - self.symbol_table_stack.last().map(|t| t.next_sub_table) - } else { - None - }; - - let previous = self.in_finally_normal_body; - self.in_finally_normal_body = true; - let result = self.compile_statements(finalbody); - self.in_finally_normal_body = previous; - result?; - // Jump to end_block to skip exception path blocks - // This prevents fall-through to finally_except_block - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: end_block } - ); - self.set_no_location(); - - // finally (exception path) - // This is where exceptions go to run finally before reraise - // Stack at entry: [lasti, exc] (from exception table with preserve_lasti=true) - let finally_except = finally_except_block.expect("finally except block"); - // Restore sub_tables for exception path compilation - if let Some(cursor) = sub_table_cursor - && let Some(current_table) = self.symbol_table_stack.last_mut() - { - current_table.next_sub_table = cursor; - } - - self.switch_to_block(finally_except); - - // SETUP_CLEANUP for finally body - // Exceptions during finally body need to go to cleanup block - if let Some(cleanup) = finally_cleanup_block { - emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup }); - self.set_no_location(); - } - emit!(self, Instruction::PushExcInfo); - self.set_no_location(); - if let Some(cleanup) = finally_cleanup_block { - self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; - } - - // Run finally body - self.compile_statements(finalbody)?; - - // RERAISE must be inside the cleanup handler's exception table - // range. The cleanup handler (COPY 3, POP_EXCEPT, RERAISE 1) - // runs POP_EXCEPT to restore exc_info before re-raising to - // the outer handler. - emit!(self, Instruction::Reraise { depth: 0 }); - self.set_no_location(); - - // PopBlock after RERAISE (dead code, but marks the exception - // table range end so the cleanup covers RERAISE). - if finally_cleanup_block.is_some() { - emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::FinallyEnd); - } - } - - // finally cleanup block - // This handles exceptions that occur during the finally body itself - // Stack at entry: [lasti, prev_exc, lasti2, exc2] after exception table routing - if let Some(cleanup) = finally_cleanup_block { - self.switch_to_block(cleanup); - // COPY 3: copy the exception from position 3 - emit!(self, Instruction::Copy { i: 3 }); - self.set_no_location(); - // POP_EXCEPT: restore prev_exc as current exception - emit!(self, Instruction::PopExcept); - self.set_no_location(); - // RERAISE 1: reraise with lasti from stack - emit!(self, Instruction::Reraise { depth: 1 }); - self.set_no_location(); - } - - // End block - continuation point after try-finally - // Normal execution continues here after the finally block - if preserve_finally_exit_empty_label - || preserve_finally_exit_empty_barrier - || preserve_finally_exit_try_except_barrier - || preserve_finally_bare_handler_exit_label - || preserve_finally_bare_scope_exit_handler_barrier - { - self.mark_load_fast_barrier_block(end_block); - } else { - self.mark_label_block(end_block); - } - self.switch_to_block(end_block); - if preserve_finally_exit_empty_label - || preserve_finally_exit_try_except_barrier - || preserve_finally_bare_handler_exit_label - || preserve_finally_bare_scope_exit_handler_barrier - { - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } else { - self.mark_load_fast_passthrough_block(end_block); - self.mark_load_fast_label_reuse_passthrough_block(end_block); - } + self.use_cpython_label_block(exit_block); Ok(()) } @@ -5353,175 +3666,40 @@ impl Compiler { handlers: &[ast::ExceptHandler], orelse: &[ast::Stmt], ) -> CompileResult<()> { + let end_block = self.new_block(); + self.compile_try_except_no_finally_with_end(body, handlers, orelse, end_block) + } + + fn compile_try_except_no_finally_with_end( + &mut self, + body: &[ast::Stmt], + handlers: &[ast::ExceptHandler], + orelse: &[ast::Stmt], + end_block: BlockIdx, + ) -> CompileResult<()> { + let body_block = self.new_block(); let handler_block = self.new_block(); let cleanup_block = self.new_block(); - let end_block = self.new_block(); - let has_terminal_raise_handlers = handlers.iter().all(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = - handler; - body.last() - .is_some_and(|stmt| matches!(stmt, ast::Stmt::Raise(_))) - }); - let has_terminal_bare_raise_handler = handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { - type_, - body, - .. - }) = handler; - type_.is_none() - && body - .last() - .is_some_and(|stmt| matches!(stmt, ast::Stmt::Raise(_))) - }); - let has_terminal_bare_reraise_handler = handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = - handler; - body.last().is_some_and(|stmt| { - matches!(stmt, ast::Stmt::Raise(ast::StmtRaise { exc: None, .. })) - }) - }); - let handlers_end_with_scope_exit = handlers.iter().all(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = - handler; - Self::statements_end_with_scope_exit(body) - }); - let handlers_have_fallthrough = handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = - handler; - !Self::statements_end_with_scope_exit(body) - }); - let handlers_are_typed = handlers.iter().all(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { - type_, .. - }) = handler; - type_.is_some() - }); - let handlers_have_alias = handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { name, .. }) = - handler; - name.is_some() - }); - let handlers_end_with_continue = handlers.iter().all(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { body, .. }) = - handler; - body.last() - .is_some_and(|stmt| matches!(stmt, ast::Stmt::Continue(_))) - }); - let handlers_stop_before_try_end = handlers_end_with_scope_exit - || (handlers_end_with_continue && self.ctx.loop_data.is_some()); - let body_exits_scope = Self::statements_end_with_scope_exit(body); - let starts_in_load_fast_barrier_block = { - let code_info = self.current_code_info(); - code_info.blocks[code_info.current_block.idx()].disable_load_fast_borrow - }; emit!( self, PseudoInstruction::SetupFinally { delta: handler_block } ); - self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; - let split_for_normal_exit_from_break = orelse.is_empty() - && self.fallthrough_has_statement_successor - && Self::statements_are_single_for_direct_break(body) - && !self - .current_code_info() - .fblock - .iter() - .any(|info| matches!(info.fb_type, FBlockType::With | FBlockType::AsyncWith)); - let preserve_try_else_attribute_probe_end = !orelse.is_empty() - && self.fallthrough_next_statement_is_if - && Self::has_cpython_try_else_attribute_probe_end_barrier(body, handlers, orelse); - let preserve_try_else_after_nested_handler_exit = - !orelse.is_empty() && Self::statements_end_with_try_except_scope_exit_handlers(body); - let preserve_try_else_after_inherited_barrier = - !orelse.is_empty() && starts_in_load_fast_barrier_block; - let preserve_try_else_after_try_finally_conditional_finalbody = - !orelse.is_empty() && Self::statements_end_with_try_finally_conditional_finalbody(body); - let previous_split_for_normal_exit_from_break = self.split_next_for_normal_exit_from_break; - self.split_next_for_normal_exit_from_break = - previous_split_for_normal_exit_from_break || split_for_normal_exit_from_break; - let compile_body_result = self.compile_statements(body); - self.split_next_for_normal_exit_from_break = previous_split_for_normal_exit_from_break; - compile_body_result?; - self.pop_fblock(FBlockType::TryExcept); + self.use_cpython_transient_label_block(body_block); + self.push_fblock(FBlockType::TryExcept, body_block, BlockIdx::NULL)?; + self.compile_statements(body)?; + self.pop_fblock(FBlockType::TryExcept, body_block); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - let exits_directly_to_with_cleanup = { - let code_info = self.current_code_info(); - code_info.in_final_with_cleanup_statement > 0 - && code_info.fblock.last().is_some_and(|info| { - matches!(info.fb_type, FBlockType::With | FBlockType::AsyncWith) - }) - }; - if !orelse.is_empty() && self.statements_end_with_conditional_scope_exit(body) { - self.preserve_last_redundant_nop(); - } else { - self.remove_last_no_location_nop(); - } - if !orelse.is_empty() { - if has_terminal_raise_handlers { - let orelse_block = self.new_block(); - self.switch_to_block(orelse_block); - } - let current = self.current_code_info().current_block; - self.mark_try_else_orelse_entry_block(current); - if preserve_try_else_after_nested_handler_exit - || preserve_try_else_after_inherited_barrier - || preserve_try_else_after_try_finally_conditional_finalbody - { - // CPython codegen_try_except() emits USE_LABEL(end) for the - // nested try/except before the outer POP_BLOCK and orelse. - // It can also start this try from a preceding empty end label, - // such as an earlier loop try whose handlers continue, or - // from codegen_try_finally()'s exit label after a conditional - // finalbody. - // flowgraph.c::optimize_load_fast() then starts the orelse - // entry from that label state instead of borrowing through the - // preceding protected body. - self.disable_load_fast_borrow_for_block(current); - } - } - let try_else_orelse_conditional_base = self.current_code_info().in_conditional_block; - self.current_code_info().in_try_else_orelse += 1; - self.try_else_orelse_conditional_base_stack - .push(try_else_orelse_conditional_base); - let compile_orelse_result = self.compile_statements(orelse); - self.try_else_orelse_conditional_base_stack.pop(); - self.current_code_info().in_try_else_orelse -= 1; - compile_orelse_result?; + self.compile_statements(orelse)?; emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - if orelse.is_empty() { - let current = self.current_code_info().current_block; - if self.fallthrough_has_statement_successor - && Self::statements_end_with_open_conditional_fallthrough(body) - { - self.mark_load_fast_barrier_block(current); - } else if !Self::statements_end_with_try_finally(body) { - self.mark_load_fast_passthrough_block(current); - } - } - if (!orelse.is_empty() && self.statements_end_with_loop_fallthrough(orelse)?) - || (exits_directly_to_with_cleanup - && handlers_end_with_scope_exit - && !Self::statements_end_with_nonterminal_with_cleanup(body)) - { - self.preserve_last_redundant_jump_as_nop(); - } else { - self.remove_last_no_location_nop(); - } - - if orelse.is_empty() && handlers_end_with_continue { - let fallthrough_block = self.new_block(); - self.mark_label_block(fallthrough_block); - self.switch_to_block(fallthrough_block); - } - self.switch_to_block(handler_block); + self.use_cpython_label_block(handler_block); emit!( self, PseudoInstruction::SetupCleanup { @@ -5531,9 +3709,9 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::PushExcInfo); self.set_no_location(); - self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + self.push_fblock(FBlockType::ExceptionHandler, BlockIdx::NULL, BlockIdx::NULL)?; - for handler in handlers { + for (i, handler) in handlers.iter().enumerate() { let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { type_, name, @@ -5542,6 +3720,11 @@ impl Compiler { .. }) = handler; self.set_source_range(*handler_range); + if type_.is_none() && i < handlers.len() - 1 { + return Err(self.error(CodegenErrorType::SyntaxError( + "default 'except:' must be last".to_owned(), + ))); + } let next_handler = self.new_block(); if let Some(exc_type) = type_ { @@ -5562,17 +3745,17 @@ impl Compiler { let cleanup_end = self.new_block(); emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup_end }); let cleanup_body = self.new_block(); - self.switch_to_block(cleanup_body); + self.use_cpython_label_block(cleanup_body); self.push_fblock_full( FBlockType::HandlerCleanup, cleanup_body, - cleanup_end, + BlockIdx::NULL, FBlockDatum::ExceptionName(alias.as_str().to_owned()), )?; self.compile_statements(body)?; - self.pop_fblock(FBlockType::HandlerCleanup); + self.pop_fblock(FBlockType::HandlerCleanup, cleanup_body); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); emit!(self, PseudoInstruction::PopBlock); @@ -5580,12 +3763,7 @@ impl Compiler { emit!(self, Instruction::PopExcept); self.set_no_location(); - self.emit_load_const(ConstantData::None); - self.set_no_location(); - self.store_name(alias.as_str())?; - self.set_no_location(); - self.compile_name(alias.as_str(), NameUsage::Delete)?; - self.set_no_location(); + self.emit_no_location_exception_name_cleanup(alias.as_str())?; emit!( self, @@ -5593,24 +3771,19 @@ impl Compiler { ); self.set_no_location(); - self.switch_to_block(cleanup_end); - self.emit_load_const(ConstantData::None); - self.set_no_location(); - self.store_name(alias.as_str())?; - self.set_no_location(); - self.compile_name(alias.as_str(), NameUsage::Delete)?; - self.set_no_location(); + self.use_cpython_label_block(cleanup_end); + self.emit_no_location_exception_name_cleanup(alias.as_str())?; emit!(self, Instruction::Reraise { depth: 1 }); self.set_no_location(); } else { emit!(self, Instruction::PopTop); let cleanup_body = self.new_block(); - self.switch_to_block(cleanup_body); - self.push_fblock(FBlockType::HandlerCleanup, cleanup_body, end_block)?; + self.use_cpython_label_block(cleanup_body); + self.push_fblock(FBlockType::HandlerCleanup, cleanup_body, BlockIdx::NULL)?; self.compile_statements(body)?; - self.pop_fblock(FBlockType::HandlerCleanup); + self.pop_fblock(FBlockType::HandlerCleanup, cleanup_body); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); emit!(self, Instruction::PopExcept); @@ -5622,14 +3795,14 @@ impl Compiler { self.set_no_location(); } - self.switch_to_block(next_handler); + self.use_cpython_label_block(next_handler); } emit!(self, Instruction::Reraise { depth: 0 }); self.set_no_location(); - self.pop_fblock(FBlockType::ExceptionHandler); + self.pop_fblock(FBlockType::ExceptionHandler, BlockIdx::NULL); - self.switch_to_block(cleanup_block); + self.use_cpython_label_block(cleanup_block); emit!(self, Instruction::Copy { i: 3 }); self.set_no_location(); emit!(self, Instruction::PopExcept); @@ -5637,218 +3810,7 @@ impl Compiler { emit!(self, Instruction::Reraise { depth: 1 }); self.set_no_location(); - let preserve_terminal_raise_end = has_terminal_raise_handlers - && (has_terminal_bare_raise_handler || has_terminal_bare_reraise_handler); - let passthrough_loop_terminal_if_end = self.ctx.loop_data.is_some() - && self.fallthrough_next_statement_is_if - && orelse.is_empty() - && has_terminal_raise_handlers - && handlers_are_typed - && handlers_end_with_scope_exit; - let preserve_loop_terminal_raise_end = has_terminal_raise_handlers - && self.ctx.loop_data.is_some() - && !passthrough_loop_terminal_if_end; - let terminal_raise_end_has_successor_join = preserve_terminal_raise_end - && orelse.is_empty() - && Self::statements_end_with_successor_join(body); - let preserve_post_nested_try_end = self.fallthrough_has_statement_successor - && orelse.is_empty() - && handlers_are_typed - && handlers_end_with_scope_exit - && (body_exits_scope - || (!passthrough_loop_terminal_if_end - && (self.statements_end_with_conditional_scope_exit(body) - || Self::statements_contain_conditional_scope_exit(body))) - || Self::statements_contain_nonterminal_try_except(body)); - let preserve_handler_nested_try_end = self.fallthrough_has_statement_successor - && orelse.is_empty() - && handlers_are_typed - // CPython codegen_try_except() uses USE_LABEL(end). Inside a - // function, a following FunctionDef starts with - // codegen_make_closure() in the same continuation block, so do - // not keep a Rust-only empty try-end barrier before that sequence. - && !(self.ctx.in_func() && self.fallthrough_next_statement_is_function_def) - && handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { - body, .. - }) = handler; - Self::statements_contain_try_except(body) - || Self::statements_contain_try_finally(body) - }); - let preserve_try_else_end = self.fallthrough_has_statement_successor - && !orelse.is_empty() - && handlers_end_with_scope_exit; - let preserve_handler_scope_exit_end = self.fallthrough_has_statement_successor - && orelse.is_empty() - && handlers_are_typed - && handlers_stop_before_try_end - && (Self::statements_end_with_import(body) - // In CPython codegen_try_except(), named handlers get a - // nested SETUP_CLEANUP cleanup path. When every named - // handler body terminates with raise, flowgraph.c removes - // that normal cleanup-to-end path as unreachable, so the - // following end label must not become a load-fast barrier. - // Inside a loop, CPython's block order still keeps the try - // successor as a separate optimize_load_fast() state, as in - // smtplib.SMTP.getreply(). - // A following try also starts a new SETUP_FINALLY state, as - // in asyncio's _sock_sendfile_native(). - || ((!handlers_have_alias - || !has_terminal_raise_handlers - || self.ctx.loop_data.is_some() - || self.fallthrough_next_statement_is_try) - && Self::statements_are_single_name_store_from_attr_call_or_subscript(body)) - || (self.fallthrough_next_statement_is_try - && Self::statements_are_name_stores_from_calls(body)) - || (self.fallthrough_next_statement_is_try - && Self::statements_are_name_store_then_same_name_method_call(body)) - || ((!handlers_have_alias - || !has_terminal_raise_handlers - || self.ctx.loop_data.is_some() - || self.fallthrough_next_statement_is_try) - && Self::statements_are_single_unpack_store_from_call(body)) - || (has_terminal_raise_handlers - && self.fallthrough_has_local_statement_successor - && Self::statements_are_single_bound_method_call_expr(body)) - || (handlers_end_with_scope_exit - && !has_terminal_raise_handlers - && self.fallthrough_has_local_statement_successor - && Self::statements_are_single_bound_method_call_expr(body)) - || (handlers_end_with_continue - && self.ctx.loop_data.is_some() - && Self::statements_are_single_for_loop(body)) - || (handlers_end_with_scope_exit - && !has_terminal_raise_handlers - && Self::statements_end_with_for_loop(body))); - let preserve_conditional_method_probe_end = self.current_code_info().in_conditional_block - > 0 - && orelse.is_empty() - && handlers_are_typed - && handlers_end_with_scope_exit - && !has_terminal_raise_handlers - && Self::statements_are_single_bound_method_call_expr(body); - let preserve_next_try_after_handler_fallthrough_end = - self.fallthrough_next_statement_is_try - && orelse.is_empty() - && handlers_have_fallthrough - && (Self::statements_end_with_open_conditional_fallthrough(body) - || (self.ctx.loop_data.is_some() - && handlers.iter().any(|handler| { - let ast::ExceptHandler::ExceptHandler( - ast::ExceptHandlerExceptHandler { body, .. }, - ) = handler; - Self::statements_end_with_finally_entry_scope_exit(body) - }))); - // CPython codegen_try_except() sends fallthrough handler exits through - // JUMP_NO_INTERRUPT to USE_LABEL(end). A bare handler followed by - // another try keeps that end label as the next try's entry barrier in - // flowgraph.c::optimize_load_fast(). - let preserve_next_try_after_bare_handler_end = self.fallthrough_next_statement_is_try - && handlers_have_fallthrough - && !handlers_are_typed; - let preserve_next_if_after_bare_handler_end = self.fallthrough_next_statement_is_if - && orelse.is_empty() - && handlers_have_fallthrough - && !handlers_are_typed; - let preserve_loop_next_try_after_nested_try_orelse_end = self.ctx.loop_data.is_some() - && self.fallthrough_next_statement_is_try - && !orelse.is_empty() - && Self::statements_contain_try_except(orelse); - let preserve_next_try_after_with_orelse_end = self.fallthrough_next_statement_is_try - && !orelse.is_empty() - && Self::statements_contain_nested_with(orelse); - let preserve_final_with_try_except_end = - orelse.is_empty() && Self::statements_end_with_nonterminal_with_cleanup(body); - let preserve_inherited_load_fast_barrier_end = starts_in_load_fast_barrier_block - && self.fallthrough_has_statement_successor - && orelse.is_empty(); - // CPython codegen_try_except() emits USE_LABEL(end) after the normal - // try path even when every handler continues to the loop header. - // flowgraph.c::optimize_load_fast() stops at that empty label before - // the following statement. - let preserve_continue_handler_end = self.fallthrough_has_statement_successor - && orelse.is_empty() - && handlers_end_with_continue - && self.ctx.loop_data.is_some(); - if preserve_terminal_raise_end - || preserve_loop_terminal_raise_end - || preserve_post_nested_try_end - || preserve_handler_nested_try_end - || preserve_try_else_attribute_probe_end - || preserve_try_else_end - || preserve_handler_scope_exit_end - || preserve_conditional_method_probe_end - || preserve_next_try_after_handler_fallthrough_end - || preserve_next_try_after_bare_handler_end - || preserve_next_if_after_bare_handler_end - || preserve_loop_next_try_after_nested_try_orelse_end - || preserve_next_try_after_with_orelse_end - || preserve_final_with_try_except_end - || preserve_inherited_load_fast_barrier_end - || preserve_continue_handler_end - { - let end_is_load_fast_barrier = preserve_post_nested_try_end - || preserve_handler_nested_try_end - || preserve_try_else_attribute_probe_end - || preserve_try_else_end - || preserve_handler_scope_exit_end - || preserve_conditional_method_probe_end - || preserve_next_try_after_handler_fallthrough_end - || preserve_next_try_after_bare_handler_end - || preserve_next_if_after_bare_handler_end - || preserve_loop_next_try_after_nested_try_orelse_end - || preserve_next_try_after_with_orelse_end - || preserve_final_with_try_except_end - || preserve_inherited_load_fast_barrier_end - || preserve_continue_handler_end; - let in_active_finally_try = self - .current_code_info() - .fblock - .iter() - .any(|info| matches!(info.fb_type, FBlockType::FinallyTry)); - let nested_handler_end_in_finally_try = - in_active_finally_try && preserve_handler_nested_try_end; - if end_is_load_fast_barrier && !nested_handler_end_in_finally_try { - self.mark_load_fast_barrier_block(end_block); - } else if nested_handler_end_in_finally_try { - // CPython codegen_try_except() emits USE_LABEL(end) inside the - // surrounding codegen_try_finally() normal body. This label - // does not create an optimize_load_fast() barrier before the - // following statement. - self.mark_load_fast_passthrough_block(end_block); - } else { - self.mark_label_block(end_block); - } - self.switch_to_block(end_block); - let continuation_block = self.new_block(); - if (preserve_terminal_raise_end || preserve_loop_terminal_raise_end) - && !self.in_finally_normal_body - && !in_active_finally_try - && !terminal_raise_end_has_successor_join - { - self.disable_load_fast_borrow_for_block(continuation_block); - } - if end_is_load_fast_barrier && !nested_handler_end_in_finally_try { - // CPython codegen_try_except() reaches this continuation through - // USE_LABEL(end). flowgraph.c::optimize_load_fast() visits the - // empty end block reached by JUMP_NO_INTERRUPT, then stops - // before the following block because basicblock_last_instr() is - // NULL. When this try starts in a block that already models - // such a barrier, keep the same state across this try's own - // USE_LABEL(end). - self.disable_load_fast_borrow_for_block(continuation_block); - } - self.switch_to_block(continuation_block); - } else { - if passthrough_loop_terminal_if_end { - // CPython codegen_try_except() emits USE_LABEL(end) directly - // before the successor statement. For the loop + continue-if - // shape used by configparser, that end label must not become a - // separate empty Rust-only barrier before optimize_load_fast(). - self.mark_load_fast_passthrough_block(end_block); - } - self.use_label_block(end_block); - } + self.use_cpython_label_block(end_block); Ok(()) } @@ -5860,14 +3822,13 @@ impl Compiler { finalbody: &[ast::Stmt], ) -> CompileResult<()> { if finalbody.is_empty() { - return self.compile_try_star_except(body, handlers, orelse, finalbody); + return self.compile_try_star_except(body, handlers, orelse); } - let preserve_finally_exit_empty_label = - self.fallthrough_has_statement_successor && !handlers.is_empty(); + let body_block = self.new_block(); let finally_except_block = self.new_block(); - let finally_cleanup_block = self.new_block(); let exit_block = self.new_block(); + let finally_cleanup_block = self.new_block(); emit!( self, @@ -5875,9 +3836,10 @@ impl Compiler { delta: finally_except_block } ); + self.use_cpython_transient_label_block(body_block); self.push_fblock_full( FBlockType::FinallyTry, - finally_except_block, + body_block, finally_except_block, FBlockDatum::FinallyBody(finalbody.to_vec()), )?; @@ -5885,49 +3847,15 @@ impl Compiler { if handlers.is_empty() { self.compile_statements(body)?; } else { - let previous_successor = self.fallthrough_has_statement_successor; - let previous_local_successor = self.fallthrough_has_local_statement_successor; - let previous_next_is_if = self.fallthrough_next_statement_is_if; - let previous_next_is_try = self.fallthrough_next_statement_is_try; - let previous_next_is_function_def = self.fallthrough_next_statement_is_function_def; - let previous_next_has_empty_test_prefix = - self.fallthrough_next_statement_has_empty_test_prefix; - self.fallthrough_has_statement_successor = false; - self.fallthrough_has_local_statement_successor = false; - self.fallthrough_next_statement_is_if = false; - self.fallthrough_next_statement_is_try = false; - self.fallthrough_next_statement_is_function_def = false; - self.fallthrough_next_statement_has_empty_test_prefix = false; - let result = self.compile_try_star_except(body, handlers, orelse, &[]); - self.fallthrough_has_statement_successor = previous_successor; - self.fallthrough_has_local_statement_successor = previous_local_successor; - self.fallthrough_next_statement_is_if = previous_next_is_if; - self.fallthrough_next_statement_is_try = previous_next_is_try; - self.fallthrough_next_statement_is_function_def = previous_next_is_function_def; - self.fallthrough_next_statement_has_empty_test_prefix = - previous_next_has_empty_test_prefix; - result?; - if self - .current_block() - .instructions - .last() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - { - self.force_remove_last_no_location_nop(); - } + self.compile_try_star_except(body, handlers, orelse)?; } emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.force_remove_last_no_location_nop(); - self.pop_fblock(FBlockType::FinallyTry); + self.pop_fblock(FBlockType::FinallyTry, body_block); let sub_table_cursor = self.symbol_table_stack.last().map(|t| t.next_sub_table); - let previous = self.in_finally_normal_body; - self.in_finally_normal_body = true; - let result = self.compile_statements(finalbody); - self.in_finally_normal_body = previous; - result?; + self.compile_statements(finalbody)?; emit!( self, @@ -5941,7 +3869,7 @@ impl Compiler { current_table.next_sub_table = cursor; } - self.switch_to_block(finally_except_block); + self.use_cpython_label_block(finally_except_block); emit!( self, PseudoInstruction::SetupCleanup { @@ -5951,17 +3879,13 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::PushExcInfo); self.set_no_location(); - self.push_fblock( - FBlockType::FinallyEnd, - finally_cleanup_block, - finally_cleanup_block, - )?; + self.push_fblock(FBlockType::FinallyEnd, finally_except_block, BlockIdx::NULL)?; self.compile_statements(finalbody)?; - self.pop_fblock(FBlockType::FinallyEnd); + self.pop_fblock(FBlockType::FinallyEnd, finally_except_block); emit!(self, Instruction::Reraise { depth: 0 }); self.set_no_location(); - self.switch_to_block(finally_cleanup_block); + self.use_cpython_label_block(finally_cleanup_block); emit!(self, Instruction::Copy { i: 3 }); self.set_no_location(); emit!(self, Instruction::PopExcept); @@ -5969,14 +3893,7 @@ impl Compiler { emit!(self, Instruction::Reraise { depth: 1 }); self.set_no_location(); - self.switch_to_block(exit_block); - if preserve_finally_exit_empty_label { - self.mark_load_fast_barrier_block(exit_block); - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } else { - self.mark_load_fast_passthrough_block(exit_block); - } + self.use_cpython_label_block(exit_block); Ok(()) } @@ -5986,56 +3903,16 @@ impl Compiler { body: &[ast::Stmt], handlers: &[ast::ExceptHandler], orelse: &[ast::Stmt], - finalbody: &[ast::Stmt], ) -> CompileResult<()> { // compiler_try_star_except // Stack layout during handler processing: [prev_exc, orig, list, rest] - let preserve_finally_exit_empty_label = self.fallthrough_has_statement_successor - && !handlers.is_empty() - && !finalbody.is_empty(); + let body_block = self.new_block(); let handler_block = self.new_block(); - let finally_block = self.new_block(); - let cleanup_block = self.new_block(); + let else_block = self.new_block(); let end_block = self.new_block(); + let cleanup_block = self.new_block(); let reraise_star_block = self.new_block(); let reraise_block = self.new_block(); - let finally_cleanup_block = if !finalbody.is_empty() { - Some(self.new_block()) - } else { - None - }; - let exit_block = self.new_block(); - let continuation_block = end_block; - let else_block = if orelse.is_empty() && finalbody.is_empty() { - continuation_block - } else { - self.new_block() - }; - if !handlers.is_empty() { - self.disable_load_fast_borrow_for_block(end_block); - if !finalbody.is_empty() { - self.disable_load_fast_borrow_for_block(exit_block); - } - } - - // Emit NOP at the try: line so LINE events fire for it - emit!(self, Instruction::Nop); - - // Push fblock with handler info for exception table generation - if !finalbody.is_empty() { - emit!( - self, - PseudoInstruction::SetupFinally { - delta: finally_block - } - ); - self.push_fblock_full( - FBlockType::FinallyTry, - finally_block, - finally_block, - FBlockDatum::FinallyBody(finalbody.to_vec()), - )?; - } // SETUP_FINALLY for try body emit!( @@ -6044,21 +3921,20 @@ impl Compiler { delta: handler_block } ); - self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; + self.use_cpython_transient_label_block(body_block); + self.push_fblock(FBlockType::TryExcept, body_block, BlockIdx::NULL)?; self.compile_statements(body)?; emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.remove_last_no_location_nop(); - self.pop_fblock(FBlockType::TryExcept); + self.pop_fblock(FBlockType::TryExcept, body_block); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: else_block } ); self.set_no_location(); - self.remove_last_no_location_nop(); // Exception handler entry - self.switch_to_block(handler_block); + self.use_cpython_label_block(handler_block); // Stack: [exc] (from exception table) emit!( @@ -6067,39 +3943,21 @@ impl Compiler { delta: cleanup_block } ); + self.set_no_location(); // PUSH_EXC_INFO emit!(self, Instruction::PushExcInfo); + self.set_no_location(); // Stack: [prev_exc, exc] // Push EXCEPTION_GROUP_HANDLER fblock self.push_fblock( FBlockType::ExceptionGroupHandler, - cleanup_block, - cleanup_block, + BlockIdx::NULL, + BlockIdx::NULL, )?; - // Initialize handler stack before the loop - // BUILD_LIST 0 + COPY 2 to set up [prev_exc, orig, list, rest] - emit!(self, Instruction::BuildList { count: 0 }); - // Stack: [prev_exc, exc, []] - emit!(self, Instruction::Copy { i: 2 }); - // Stack: [prev_exc, orig, list, rest] - let n = handlers.len(); - if n == 0 { - // Empty handlers (invalid AST) - append rest to list and proceed - // Stack: [prev_exc, orig, list, rest] - emit!(self, Instruction::ListAppend { i: 1 }); - // Stack: [prev_exc, orig, list] - emit!( - self, - PseudoInstruction::JumpNoInterrupt { - delta: reraise_star_block - } - ); - self.set_no_location(); - } for (i, handler) in handlers.iter().enumerate() { let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { type_, @@ -6109,17 +3967,23 @@ impl Compiler { .. }) = handler; let is_last_handler = i == n - 1; + self.set_source_range(*handler_range); + let next_handler_block = self.new_block(); + let except_with_error_block = self.new_block(); let no_match_block = self.new_block(); - let next_handler_block = if is_last_handler { - reraise_star_block - } else { - self.new_block() - }; + + if i == 0 { + // CPython initializes the except* work stack inside the first + // handler iteration in codegen_try_star_except(). + emit!(self, Instruction::BuildList { count: 0 }); + emit!(self, Instruction::Copy { i: 2 }); + } // Compile exception type if let Some(exc_type) = type_ { self.compile_expression(exc_type)?; + self.set_source_range(*handler_range); } else { return Err(self.error(CodegenErrorType::SyntaxError( "except* must specify an exception type".to_owned(), @@ -6155,16 +4019,18 @@ impl Compiler { // Stack: [prev_exc, orig, list, new_rest] // HANDLER_CLEANUP fblock for handler body + let cleanup_body_block = self.new_block(); emit!( self, PseudoInstruction::SetupCleanup { delta: handler_except_block } ); + self.use_cpython_label_block(cleanup_body_block); self.push_fblock_full( FBlockType::HandlerCleanup, - next_handler_block, - end_block, + cleanup_body_block, + BlockIdx::NULL, if let Some(alias) = name { FBlockDatum::ExceptionName(alias.as_str().to_owned()) } else { @@ -6178,36 +4044,29 @@ impl Compiler { // Handler body completed normally emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::HandlerCleanup); + self.pop_fblock(FBlockType::HandlerCleanup, cleanup_body_block); // Cleanup name binding if let Some(alias) = name { - self.emit_load_const(ConstantData::None); - self.store_name(alias.as_str())?; - self.compile_name(alias.as_str(), NameUsage::Delete)?; + self.emit_no_location_exception_name_cleanup(alias.as_str())?; } - if is_last_handler { - emit!(self, Instruction::ListAppend { i: 1 }); - } emit!( self, PseudoInstruction::JumpNoInterrupt { delta: next_handler_block } ); + self.set_no_location(); // Handler raised an exception (cleanup_end label) - self.switch_to_block(handler_except_block); + self.use_cpython_label_block(handler_except_block); // Stack: [prev_exc, orig, list, new_rest, lasti, raised_exc] // (lasti is pushed because push_lasti=true in HANDLER_CLEANUP fblock) // Cleanup name binding - self.set_no_location(); if let Some(alias) = name { - self.emit_load_const(ConstantData::None); - self.store_name(alias.as_str())?; - self.compile_name(alias.as_str(), NameUsage::Delete)?; + self.emit_no_location_exception_name_cleanup(alias.as_str())?; } // LIST_APPEND(3) - append raised_exc to list @@ -6216,59 +4075,59 @@ impl Compiler { // nth_value(i) = stack[len - i - 1], we need stack[2] = list // stack[5 - i - 1] = 2 -> i = 2 emit!(self, Instruction::ListAppend { i: 3 }); + self.set_no_location(); // Stack: [prev_exc, orig, list, new_rest, lasti] // POP_TOP - pop lasti emit!(self, Instruction::PopTop); + self.set_no_location(); // Stack: [prev_exc, orig, list, new_rest] - if is_last_handler { - emit!(self, Instruction::ListAppend { i: 1 }); - emit!( - self, - PseudoInstruction::JumpNoInterrupt { - delta: reraise_star_block - } - ); - } else { - emit!( - self, - PseudoInstruction::JumpNoInterrupt { - delta: next_handler_block - } - ); - } + emit!( + self, + PseudoInstruction::JumpNoInterrupt { + delta: except_with_error_block + } + ); + self.set_no_location(); - if is_last_handler { - self.switch_to_block(no_match_block); - self.set_source_range(*handler_range); - emit!(self, Instruction::PopTop); // pop match (None) - // Stack: [prev_exc, orig, list, new_rest] + self.use_cpython_label_block(next_handler_block); + emit!(self, Instruction::Nop); + self.set_no_location(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { + delta: except_with_error_block + } + ); + self.set_no_location(); - self.set_no_location(); + self.use_cpython_label_block(no_match_block); + self.set_source_range(*handler_range); + emit!(self, Instruction::PopTop); // pop match (None) + // Stack: [prev_exc, orig, list, new_rest] + + self.use_cpython_label_block(except_with_error_block); + + if is_last_handler { emit!(self, Instruction::ListAppend { i: 1 }); + self.set_no_location(); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: reraise_star_block } ); - } else { - self.switch_to_block(no_match_block); - self.set_source_range(*handler_range); - emit!(self, Instruction::PopTop); // pop match (None) - // Stack: [prev_exc, orig, list, new_rest] - self.switch_to_block(next_handler_block); + self.set_no_location(); } } // Pop EXCEPTION_GROUP_HANDLER fblock - self.pop_fblock(FBlockType::ExceptionGroupHandler); + self.pop_fblock(FBlockType::ExceptionGroupHandler, BlockIdx::NULL); // Reraise star block - self.switch_to_block(reraise_star_block); + self.use_cpython_label_block(reraise_star_block); // Stack: [prev_exc, orig, list] - self.set_no_location(); // CALL_INTRINSIC_2 PREP_RERAISE_STAR // Takes 2 args (orig, list) and produces result @@ -6278,10 +4137,12 @@ impl Compiler { func: bytecode::IntrinsicFunction2::PrepReraiseStar } ); + self.set_no_location(); // Stack: [prev_exc, result] // COPY 1 emit!(self, Instruction::Copy { i: 1 }); + self.set_no_location(); // Stack: [prev_exc, result, result] // POP_JUMP_IF_NOT_NONE reraise @@ -6291,131 +4152,59 @@ impl Compiler { delta: reraise_block } ); + self.set_no_location(); // Stack: [prev_exc, result] // Nothing to reraise // POP_TOP - pop result (None) emit!(self, Instruction::PopTop); + self.set_no_location(); // Stack: [prev_exc] emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); // POP_EXCEPT - restore previous exception context emit!(self, Instruction::PopExcept); + self.set_no_location(); // Stack: [] emit!( self, - PseudoInstruction::JumpNoInterrupt { - delta: continuation_block - } + PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - self.force_remove_last_no_location_nop(); // Reraise the result - self.switch_to_block(reraise_block); + self.use_cpython_label_block(reraise_block); // Stack: [prev_exc, result] emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); emit!(self, Instruction::Swap { i: 2 }); + self.set_no_location(); // Stack: [result, prev_exc] // POP_EXCEPT emit!(self, Instruction::PopExcept); + self.set_no_location(); // Stack: [result] // RERAISE 0 emit!(self, Instruction::Reraise { depth: 0 }); - - self.switch_to_block(cleanup_block); self.set_no_location(); + + self.use_cpython_label_block(cleanup_block); emit!(self, Instruction::Copy { i: 3 }); + self.set_no_location(); emit!(self, Instruction::PopExcept); + self.set_no_location(); emit!(self, Instruction::Reraise { depth: 1 }); + self.set_no_location(); - // try-else path - if else_block != continuation_block { - self.switch_to_block(else_block); - self.compile_statements(orelse)?; - - emit!( - self, - PseudoInstruction::JumpNoInterrupt { - delta: continuation_block - } - ); - self.set_no_location(); - } - - if !finalbody.is_empty() { - self.switch_to_block(end_block); - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.remove_last_no_location_nop(); - self.pop_fblock(FBlockType::FinallyTry); - - // Snapshot sub_tables before first finally compilation - let sub_table_cursor = self.symbol_table_stack.last().map(|t| t.next_sub_table); - - // Compile finally body inline for normal path - self.compile_statements(finalbody)?; - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: exit_block } - ); - self.set_no_location(); - - // Restore sub_tables for exception path compilation - if let Some(cursor) = sub_table_cursor - && let Some(current_table) = self.symbol_table_stack.last_mut() - { - current_table.next_sub_table = cursor; - } - - // Exception handler path - self.switch_to_block(finally_block); - emit!(self, Instruction::PushExcInfo); - self.set_no_location(); - - if let Some(cleanup) = finally_cleanup_block { - emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup }); - self.set_no_location(); - self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; - } - - self.compile_statements(finalbody)?; - - if finally_cleanup_block.is_some() { - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.pop_fblock(FBlockType::FinallyEnd); - } - - emit!(self, Instruction::Reraise { depth: 0 }); - self.set_no_location(); - - if let Some(cleanup) = finally_cleanup_block { - self.switch_to_block(cleanup); - emit!(self, Instruction::Copy { i: 3 }); - self.set_no_location(); - emit!(self, Instruction::PopExcept); - self.set_no_location(); - emit!(self, Instruction::Reraise { depth: 1 }); - self.set_no_location(); - } - } + self.use_cpython_label_block(else_block); + self.compile_statements(orelse)?; - self.switch_to_block(if finalbody.is_empty() { - end_block - } else { - exit_block - }); - if preserve_finally_exit_empty_label { - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } + self.use_cpython_label_block(end_block); Ok(()) } @@ -6501,7 +4290,6 @@ impl Compiler { // Set up context let prev_ctx = self.ctx; self.ctx = CompileContext { - loop_data: None, in_class: prev_ctx.in_class, func: if is_async { FunctionContext::AsyncFunction @@ -6519,18 +4307,7 @@ impl Compiler { let is_gen = is_async || self.current_symbol_table().is_generator; let stop_iteration_block = if is_gen { let handler_block = self.new_block(); - emit!( - self, - PseudoInstruction::SetupCleanup { - delta: handler_block - } - ); - self.set_no_location(); - // CPython's codegen_wrap_in_stopiteration_handler() inserts - // SETUP_CLEANUP at instruction-sequence index 0, so after the - // generator prefix is inserted the protected range begins at the - // function-start RESUME. - self.move_last_instruction_before_scope_start_resume(); + self.insert_cpython_stopiteration_setup_cleanup(handler_block); self.push_fblock(FBlockType::StopIteration, handler_block, handler_block)?; Some(handler_block) } else { @@ -6572,8 +4349,8 @@ impl Compiler { if let Some(handler_block) = stop_iteration_block { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::StopIteration); - self.switch_to_block(handler_block); + self.pop_fblock(FBlockType::StopIteration, handler_block); + self.use_cpython_label_block(handler_block); emit!( self, Instruction::CallIntrinsic1 { @@ -6757,14 +4534,14 @@ impl Compiler { let code_info = self.current_code_info(); let saved_blocks = code_info.blocks.clone(); let saved_current_block = code_info.current_block; - let saved_annotations_blocks = code_info.annotations_blocks.clone(); + let saved_instr_sequence = code_info.instr_sequence.clone(); + let saved_instr_sequence_label_map = code_info.instr_sequence_label_map.clone(); + let saved_annotations_instr_sequence = code_info.annotations_instr_sequence.clone(); let saved_metadata = code_info.metadata.clone(); let saved_static_attributes = code_info.static_attributes.clone(); let saved_in_inlined_comp = code_info.in_inlined_comp; let saved_fblock = code_info.fblock.clone(); let saved_in_conditional_block = code_info.in_conditional_block; - let saved_in_final_with_cleanup_statement = code_info.in_final_with_cleanup_statement; - let saved_in_try_else_orelse = code_info.in_try_else_orelse; let saved_next_conditional_annotation_index = code_info.next_conditional_annotation_index; let saved_source_range = self.current_source_range; @@ -6776,14 +4553,14 @@ impl Compiler { let code_info = self.current_code_info(); code_info.blocks = saved_blocks; code_info.current_block = saved_current_block; - code_info.annotations_blocks = saved_annotations_blocks; + code_info.instr_sequence = saved_instr_sequence; + code_info.instr_sequence_label_map = saved_instr_sequence_label_map; + code_info.annotations_instr_sequence = saved_annotations_instr_sequence; code_info.metadata = saved_metadata; code_info.static_attributes = saved_static_attributes; code_info.in_inlined_comp = saved_in_inlined_comp; code_info.fblock = saved_fblock; code_info.in_conditional_block = saved_in_conditional_block; - code_info.in_final_with_cleanup_statement = saved_in_final_with_cleanup_statement; - code_info.in_try_else_orelse = saved_in_try_else_orelse; code_info.next_conditional_annotation_index = saved_next_conditional_annotation_index; self.current_source_range = saved_source_range; @@ -6821,7 +4598,6 @@ impl Compiler { // Annotation scopes are never async (even inside async functions) let saved_ctx = self.ctx; self.ctx = CompileContext { - loop_data: None, in_class: saved_ctx.in_class, func: FunctionContext::Function, in_async_scope: false, @@ -7014,7 +4790,6 @@ impl Compiler { // TypeParams scope is function-like self.ctx = CompileContext { - loop_data: None, in_class: saved_ctx.in_class, func: FunctionContext::Function, in_async_scope: false, @@ -7565,7 +5340,6 @@ impl Compiler { // TypeParams scope is function-like self.ctx = CompileContext { - loop_data: None, in_class: saved_ctx.in_class, func: FunctionContext::Function, in_async_scope: false, @@ -7583,7 +5357,6 @@ impl Compiler { self.ctx = CompileContext { func: FunctionContext::NoFunction, in_class: true, - loop_data: None, in_async_scope: false, }; let class_code = self.compile_class_body(name, body, type_params, firstlineno)?; @@ -7793,46 +5566,11 @@ impl Compiler { self.new_block() }; - if self.has_always_taken_jump_in_test(test, false)? { - self.disable_load_fast_borrow_for_block(next_block); - self.disable_load_fast_borrow_for_block(end_block); - } - let in_direct_try_else_orelse_conditional = { - let base = self.try_else_orelse_conditional_base_stack.last().copied(); - let code_info = self.current_code_info(); - code_info.in_try_else_orelse > 0 - && base.is_some_and(|base| code_info.in_conditional_block == base + 1) - }; - let preserve_try_else_scope_exit_target_nop = elif_else_clauses.is_empty() - && in_direct_try_else_orelse_conditional - && !self.fallthrough_has_local_statement_successor - && Self::statements_end_with_scope_exit(body); self.compile_jump_if(test, false, next_block)?; - let body_starts_with_empty_test_prefix = body - .first() - .map(|stmt| self.statement_starts_with_empty_if_test_prefix(stmt)) - .transpose()? - .unwrap_or(false); - if body_starts_with_empty_test_prefix { - let body_block = self.new_block(); - self.mark_load_fast_barrier_block(body_block); - self.switch_to_block(body_block); - } self.compile_statements(body)?; let Some((clause, rest)) = elif_else_clauses.split_first() else { - if Self::statements_end_with_scope_exit(body) - && self.fallthrough_next_statement_has_empty_test_prefix - { - self.mark_load_fast_barrier_block(end_block); - } else { - self.mark_load_fast_label_reuse_passthrough_block(end_block); - } - self.use_label_block(end_block); - if preserve_try_else_scope_exit_target_nop { - self.set_source_range(test.range()); - emit!(self, Instruction::Nop); - } + self.use_cpython_label_block(end_block); return Ok(()); }; @@ -7841,7 +5579,7 @@ impl Compiler { PseudoInstruction::JumpNoInterrupt { delta: end_block } ); self.set_no_location(); - self.switch_to_block(next_block); + self.use_cpython_label_block(next_block); if let Some(test) = &clause.test { self.compile_if(test, &clause.body, rest, test.range())?; @@ -7849,8 +5587,7 @@ impl Compiler { debug_assert!(rest.is_empty()); self.compile_statements(&clause.body)?; } - self.mark_load_fast_label_reuse_passthrough_block(end_block); - self.use_label_block(end_block); + self.use_cpython_label_block(end_block); Ok(()) } @@ -7862,81 +5599,20 @@ impl Compiler { ) -> CompileResult<()> { self.enter_conditional_block(); - let while_block = self.switch_to_new_or_reuse_empty(); - let after_block = self.new_block(); - let else_block = self.new_block(); - self.push_fblock(FBlockType::WhileLoop, while_block, after_block)?; - let test_truthiness = self.constant_expr_truthiness(test)?; - let preserve_while_true_break_end = matches!(test_truthiness, Some(true)) - && self.fallthrough_has_statement_successor - && Self::statements_contain_try_except_orelse_direct_break(body); - let preserve_next_empty_test_prefix_end = self.fallthrough_has_statement_successor - && self.fallthrough_next_statement_has_empty_test_prefix; - if matches!(test_truthiness, Some(true)) { - let prev_source_range = self.current_source_range; - self.set_source_range(test.range()); - emit!(self, Instruction::Nop); - self.preserve_last_redundant_nop(); - self.set_source_range(prev_source_range); - } - if matches!(test_truthiness, Some(false)) { - self.disable_load_fast_borrow_for_block(else_block); - self.disable_load_fast_borrow_for_block(after_block); - } - let preserve_one_line_protected_infinite_loop_body = matches!(test_truthiness, Some(true)) - && self - .current_code_info() - .fblock - .iter() - .any(|info| matches!(info.fb_type, FBlockType::TryExcept)) - && body.first().is_some_and(|stmt| { - let source = self.source_file.to_source_code(); - source.line_index(test.range().start()) == source.line_index(stmt.range().start()) - }) - && !Self::statements_are_single_bound_method_call_expr(body); - self.compile_jump_if(test, false, else_block)?; - if preserve_one_line_protected_infinite_loop_body { - self.disable_load_fast_borrow_for_block(while_block); - } + let loop_block = self.start_cpython_label_block(); + let end_block = self.new_block(); + let anchor_block = self.new_block(); + self.push_fblock(FBlockType::WhileLoop, loop_block, end_block)?; + self.compile_jump_if(test, false, anchor_block)?; - let was_in_loop = self.ctx.loop_data.replace((while_block, after_block)); self.compile_loop_body_statements(body)?; - self.ctx.loop_data = was_in_loop; - emit!(self, PseudoInstruction::Jump { delta: while_block }); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); self.set_no_location(); - self.pop_fblock(FBlockType::WhileLoop); - self.switch_to_block(else_block); + self.pop_fblock(FBlockType::WhileLoop, loop_block); + self.use_cpython_label_block(anchor_block); self.compile_statements(orelse)?; - let reuse_empty_while_end_label = - orelse.is_empty() && !matches!(body.last(), Some(ast::Stmt::Break(_))); - let effective_after_block = if reuse_empty_while_end_label { - // CPython codegen_while() emits USE_LABEL(anchor), then immediately - // USE_LABEL(end) when there is no orelse. flowgraph.c reuses that - // current empty anchor block for the end label when the loop body - // still has a normal backedge path, so the function epilogue is - // not duplicated once more for the loop-false edge. - self.use_label_block(after_block); - else_block - } else { - self.switch_to_block(after_block); - after_block - }; - if (preserve_while_true_break_end || preserve_next_empty_test_prefix_end) - && self.current_code_info().blocks[effective_after_block.idx()] - .instructions - .is_empty() - { - // CPython codegen_while() emits USE_LABEL(end) as the break target. - // When the next statement starts with a folded constant BoolOp - // prefix, flowgraph.c::basicblock_optimize_load_const() can leave - // that path as an empty block, and optimize_load_fast() stops at - // basicblock_last_instr(block) == NULL before the following - // statement. - self.mark_load_fast_barrier_block(effective_after_block); - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } + self.use_cpython_label_block(end_block); self.leave_conditional_block(); Ok(()) @@ -8014,7 +5690,7 @@ impl Compiler { emit!(self, Instruction::Call { argc: 0 }); // [aexit_func, self_ae, awaitable] emit!(self, Instruction::GetAwaitable { r#where: 1 }); self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(true)?; + let _ = self.compile_yield_from_sequence(true); } else { // Load __exit__ and __enter__, then call __enter__ emit!( @@ -8043,14 +5719,16 @@ impl Compiler { delta: exc_handler_block } ); + let body_block = self.new_block(); + self.use_cpython_transient_label_block(body_block); self.push_fblock( if is_async { FBlockType::AsyncWith } else { FBlockType::With }, - exc_handler_block, // block start (will become exit target after store) - after_block, + body_block, + exc_handler_block, )?; // Store or pop the enter result @@ -8076,116 +5754,40 @@ impl Compiler { self.compile_with(items, body, is_async)?; } - let nested_multiline_with_cleanup_target_nop = !is_async - && !self.fallthrough_has_local_statement_successor - && Self::statements_end_with_scope_exit(body) - && { - let parent_with_ranges: Vec<_> = self - .current_code_info() - .fblock - .iter() - .rev() - .skip(1) - .filter_map(|info| { - matches!(info.fb_type, FBlockType::With).then_some(info.fb_range) - }) - .collect(); - let source = self.source_file.to_source_code(); - let current_line = source.line_index(with_range.start()); - parent_with_ranges - .iter() - .any(|range| source.line_index(range.start()) != current_line) - }; - let preserve_outer_cleanup_target_nop = !is_async - && (Self::statements_end_with_with_cleanup_scope_exit(body) - || self.statements_end_with_conditional_scope_exit(body) - || Self::statements_end_with_try_except_handler_fallthrough(body) - || Self::statements_end_with_try_except_else_handler_scope_exit(body) - || Self::statements_end_with_try_finally(body) - || self.statements_end_with_loop_fallthrough(body)?); - let remove_while_true_break_cleanup_target_nop = !is_async - && (self.statements_end_with_while_true_direct_break(body)? - || self.statements_end_with_while_true_without_break(body)?); - let preserve_while_true_tail_break_successor_load_fast = - !is_async && self.statements_end_with_while_true_tail_direct_break(body)?; - let preserve_try_except_tail_successor_load_fast = !is_async - // CPython codegen_try_except() leaves a USE_LABEL(end) before the - // with normal __exit__ cleanup when scope-exiting handlers are the - // only shape at the tail. If the protected body itself ends with - // a conditional, that inner end label lets flowgraph.c continue - // borrowing through the with successor, as in xmlrpc.client. - && Self::statements_end_with_try_except_scope_exit_handlers_nonconditional_body(body); - let preserve_try_finally_with_finalbody_successor_load_fast = - Self::statements_end_with_try_finally_finalbody_with(body); - let materialize_async_with_outer_cleanup_target_nop = is_async - && Self::statements_end_with_nested_finalbody_try_finally(body) - && self - .current_code_info() - .fblock - .iter() - .any(|info| matches!(info.fb_type, FBlockType::With)); - - // Pop fblock before normal exit. CPython emits this POP_BLOCK with - // no location for sync with, but with the with-item location for - // async with. + let fblock_type = if is_async { + FBlockType::AsyncWith + } else { + FBlockType::With + }; + // CPython pops the async-with fblock before emitting POP_BLOCK, but + // sync with emits the artificial POP_BLOCK before popping the fblock. if is_async { + self.pop_fblock(fblock_type, body_block); self.set_source_range(with_range); - } - emit!(self, PseudoInstruction::PopBlock); - if !is_async { + emit!(self, PseudoInstruction::PopBlock); + } else { + emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - if preserve_outer_cleanup_target_nop { - self.preserve_last_redundant_nop(); - } else if remove_while_true_break_cleanup_target_nop - || Self::statements_end_with_try_star_except(body) - { - self.force_remove_last_no_location_nop(); - } else { - self.remove_last_no_location_nop(); - } self.set_source_range(with_range); + self.pop_fblock(fblock_type, body_block); } - self.pop_fblock(if is_async { - FBlockType::AsyncWith - } else { - FBlockType::With - }); - // ===== Normal exit path ===== - // Stack: [..., exit_func, self_exit] - // Call exit_func(self_exit, None, None, None) - self.emit_load_const(ConstantData::None); - self.emit_load_const(ConstantData::None); - self.emit_load_const(ConstantData::None); - emit!(self, Instruction::Call { argc: 3 }); + self.compile_call_exit_with_nones(); if is_async { emit!(self, Instruction::GetAwaitable { r#where: 2 }); self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(true)?; + let _ = self.compile_yield_from_sequence(true); } emit!(self, Instruction::PopTop); // Pop __exit__ result emit!(self, PseudoInstruction::Jump { delta: after_block }); self.set_no_location(); - if preserve_while_true_tail_break_successor_load_fast - || preserve_try_except_tail_successor_load_fast - || preserve_try_finally_with_finalbody_successor_load_fast - { - // CPython codegen_while()/codegen_try_except() emit USE_LABEL(end) - // before codegen_with_inner() emits the normal __exit__ cleanup. - // codegen_try_finally() also inlines finalbody before its exit - // label, so a nested with finalbody leaves the same empty-block - // boundary. flowgraph.c::optimize_load_fast() therefore does not - // borrow the successor loads after this with statement. - self.disable_load_fast_borrow_for_block(after_block); - } // ===== Exception handler path ===== // Stack at entry: [..., exit_func, self_exit, lasti, exc] // PUSH_EXC_INFO -> [..., exit_func, self_exit, lasti, prev_exc, exc] - self.switch_to_block(exc_handler_block); + self.use_cpython_label_block(exc_handler_block); let cleanup_block = self.new_block(); - let suppress_block = self.new_block(); emit!( self, @@ -8193,7 +5795,6 @@ impl Compiler { delta: cleanup_block } ); - self.push_fblock(FBlockType::ExceptionHandler, exc_handler_block, after_block)?; emit!(self, Instruction::PushExcInfo); @@ -8204,62 +5805,13 @@ impl Compiler { if is_async { emit!(self, Instruction::GetAwaitable { r#where: 2 }); self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(true)?; + let _ = self.compile_yield_from_sequence(true); } - emit!(self, Instruction::ToBool); - emit!( - self, - Instruction::PopJumpIfTrue { - delta: suppress_block - } - ); - - emit!(self, Instruction::Reraise { depth: 2 }); - - // ===== Suppress block ===== - // Stack: [..., exit_func, self_exit, lasti, prev_exc, exc, True] - self.switch_to_block(suppress_block); - emit!(self, Instruction::PopTop); // pop True - emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::ExceptionHandler); - emit!(self, Instruction::PopExcept); // pop exc, restore prev_exc - emit!(self, Instruction::PopTop); // pop lasti - emit!(self, Instruction::PopTop); // pop self_exit - emit!(self, Instruction::PopTop); // pop exit_func - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: after_block } - ); - - // ===== Cleanup block (for nested exception during __exit__) ===== - // Stack: [..., __exit__, lasti, prev_exc, lasti2, exc2] - // COPY 3: copy prev_exc to TOS - // POP_EXCEPT: restore exception state - // RERAISE 1: re-raise with lasti - // - // NOTE: We DON'T clear the fblock stack here because we want - // outer exception handlers (e.g., try-except wrapping this with statement) - // to be in the exception table for these instructions. - // If we cleared fblock, exceptions here would propagate uncaught. - self.switch_to_block(cleanup_block); - // CPython codegen_with_except_finish() emits POP_EXCEPT_AND_RERAISE - // with NO_LOCATION at this cleanup label. - emit!(self, Instruction::Copy { i: 3 }); - self.set_no_location(); - emit!(self, Instruction::PopExcept); - self.set_no_location(); - emit!(self, Instruction::Reraise { depth: 1 }); - self.set_no_location(); + self.compile_with_except_finish(cleanup_block, after_block); // ===== After block ===== self.switch_to_block(after_block); - if materialize_async_with_outer_cleanup_target_nop - || nested_multiline_with_cleanup_target_nop - { - self.set_source_range(with_range); - emit!(self, Instruction::Nop); - } self.leave_conditional_block(); Ok(()) @@ -8280,24 +5832,14 @@ impl Compiler { let for_block = self.new_block(); let else_block = self.new_block(); let after_block = self.new_block(); - let body_contains_direct_break = Self::statements_contain_direct_break(body); - let split_normal_exit_from_break = !is_async - && self.split_next_for_normal_exit_from_break - && body_contains_direct_break - && self - .current_code_info() - .fblock - .iter() - .any(|info| matches!(info.fb_type, FBlockType::TryExcept)); - let preserve_async_for_break_exit_empty_label = - is_async && self.fallthrough_has_statement_successor && body_contains_direct_break; - let normal_exit_block = if split_normal_exit_from_break { - self.new_block() - } else { - after_block - }; let mut end_async_for_target = BlockIdx::NULL; + if !is_async { + // CPython codegen_for() pushes the loop fblock before compiling + // the iterable expression. + self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + } + // The thing iterated: self.compile_for_iterable_expression(iter, is_async)?; @@ -8308,7 +5850,7 @@ impl Compiler { self.set_source_range(iter.range()); emit!(self, Instruction::GetAiter); - self.switch_to_block(for_block); + self.use_cpython_label_block(for_block); self.set_source_range(for_range); // codegen_async_for: push fblock BEFORE SETUP_FINALLY @@ -8318,7 +5860,7 @@ impl Compiler { emit!(self, PseudoInstruction::SetupFinally { delta: else_block }); emit!(self, Instruction::GetAnext); self.emit_load_const(ConstantData::None); - end_async_for_target = self.compile_yield_from_sequence(true)?; + end_async_for_target = self.compile_yield_from_sequence(true); // POP_BLOCK for SETUP_FINALLY - only GetANext/yield_from are protected emit!(self, PseudoInstruction::PopBlock); emit!(self, Instruction::NotTaken); @@ -8329,96 +5871,47 @@ impl Compiler { // Retrieve Iterator emit!(self, Instruction::GetIter); - self.switch_to_block(for_block); - - // Push fblock for for loop - self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + self.use_cpython_label_block(for_block); emit!(self, Instruction::ForIter { delta: else_block }); - // Match CPython's line attribution by compiling the loop target on - // the target range directly instead of leaving a synthetic anchor - // NOP between FOR_ITER and the unpack/store sequence. let saved_range = self.current_source_range; self.set_source_range(target.range()); + emit!(self, Instruction::Nop); self.compile_store(target)?; self.set_source_range(saved_range); }; - let was_in_loop = self.ctx.loop_data.replace((for_block, after_block)); self.compile_loop_body_statements(body)?; - self.ctx.loop_data = was_in_loop; emit!(self, PseudoInstruction::Jump { delta: for_block }); self.set_no_location(); - self.switch_to_block(else_block); + self.use_cpython_label_block(else_block); // Except block for __anext__ / end of sync for - // No PopBlock here - for async, POP_BLOCK is already in for_block - self.pop_fblock(FBlockType::ForLoop); - - // End-of-loop instructions are on the `for` line, not the body's last line - let saved_range = self.current_source_range; - self.set_source_range(iter.range()); if is_async { + // codegen_async_for emits END_ASYNC_FOR at the iterator location, + // then pops the for-loop fblock before the else block. + let saved_range = self.current_source_range; + self.set_source_range(iter.range()); self.emit_end_async_for(end_async_for_target); + self.set_source_range(saved_range); } else { + // codegen_for emits END_FOR/POP_ITER with NO_LOCATION. Line numbers + // are propagated later by flowgraph.c::resolve_line_numbers(). emit!(self, Instruction::EndFor); + self.set_no_location(); emit!(self, Instruction::PopIter); + self.set_no_location(); } - self.set_source_range(saved_range); + // No PopBlock here - for async, POP_BLOCK is already in for_block + self.pop_fblock(FBlockType::ForLoop, for_block); self.compile_statements(orelse)?; - let fallthrough_into_finally_body = !is_async - && self.in_finally_normal_body - && !body_contains_direct_break - && orelse.is_empty(); - let in_active_finally_try = self - .current_code_info() - .fblock - .iter() - .any(|info| matches!(info.fb_type, FBlockType::FinallyTry)); - let preserve_break_exit_empty_label = !is_async - && in_active_finally_try - && body_contains_direct_break - && !orelse.is_empty() - && Self::statements_end_with_scope_exit(orelse); - if preserve_break_exit_empty_label { - // CPython codegen_for() places USE_LABEL(end) after the orelse, - // while codegen_break() jumps to loop->fb_exit. When the orelse - // exits the scope, optimize_load_fast() does not carry borrowed - // refs from the break jump into the successor statement. - self.mark_load_fast_barrier_block(normal_exit_block); - self.switch_to_block(normal_exit_block); - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } else if !fallthrough_into_finally_body { - self.mark_label_block(normal_exit_block); - self.mark_load_fast_label_reuse_passthrough_block(normal_exit_block); - self.switch_to_block(normal_exit_block); - } - if preserve_async_for_break_exit_empty_label { - let continuation_block = self.new_block(); - self.switch_to_block(continuation_block); - } + self.use_cpython_label_block(after_block); // Implicit return after for-loop should be attributed to the `for` line self.set_source_range(iter.range()); - if split_normal_exit_from_break { - emit!(self, Instruction::Nop); - self.set_no_location(); - self.preserve_last_redundant_nop(); - self.preserve_last_redundant_jump_as_nop(); - self.mark_last_no_location_exit(); - emit!( - self, - PseudoInstruction::JumpNoInterrupt { delta: after_block } - ); - self.set_no_location(); - self.remove_last_no_location_nop(); - self.switch_to_block(after_block); - self.set_source_range(iter.range()); - } self.leave_conditional_block(); Ok(()) @@ -8548,13 +6041,13 @@ impl Compiler { // Iterate over the fail_pop vector in reverse order, skipping the first label. for &label in pc.fail_pop.iter().skip(1).rev() { // CPython emit_and_reset_fail_pop() uses USE_LABEL here. - self.use_label_block(label); + self.use_cpython_label_block(label); // Emit the POP instruction. emit!(self, Instruction::PopTop); } // Finally, use the first label. // CPython emit_and_reset_fail_pop() uses USE_LABEL here too. - self.use_label_block(pc.fail_pop[0]); + self.use_cpython_label_block(pc.fail_pop[0]); pc.fail_pop.clear(); // Free the memory used by the vector. pc.fail_pop.shrink_to_fit(); @@ -9213,9 +6706,6 @@ impl Compiler { // Emit a jump to the common end label and reset any failure jump targets. self.set_source_range(alt.range()); emit!(self, PseudoInstruction::Jump { delta: end }); - if let Some(last) = self.current_block().instructions.last_mut() { - last.match_success_jump = true; - } self.set_source_range(alt.range()); self.emit_and_reset_fail_pop(pc); } @@ -9234,7 +6724,7 @@ impl Compiler { // Use the label "end". // CPython codegen_pattern_or() emits USE_LABEL(c, end). - self.use_label_block(end); + self.use_cpython_label_block(end); // Adjust the final captures. let n_stores = control.as_ref().unwrap().len(); @@ -9421,32 +6911,6 @@ impl Compiler { } } - fn contains_match_or(pattern: &ast::Pattern) -> bool { - match pattern { - ast::Pattern::MatchSequence(match_sequence) => { - match_sequence.patterns.iter().any(contains_match_or) - } - ast::Pattern::MatchMapping(match_mapping) => { - match_mapping.patterns.iter().any(contains_match_or) - } - ast::Pattern::MatchClass(match_class) => { - match_class.arguments.patterns.iter().any(contains_match_or) - || match_class - .arguments - .keywords - .iter() - .any(|keyword| contains_match_or(&keyword.pattern)) - } - ast::Pattern::MatchAs(match_as) => { - match_as.pattern.as_deref().is_some_and(contains_match_or) - } - ast::Pattern::MatchOr(_) => true, - ast::Pattern::MatchValue(_) - | ast::Pattern::MatchSingleton(_) - | ast::Pattern::MatchStar(_) => false, - } - } - self.compile_expression(subject)?; let end = self.new_block(); @@ -9456,17 +6920,6 @@ impl Compiler { num_cases > 1 && is_trailing_wildcard_default(&cases.last().unwrap().pattern); let case_count = num_cases - if has_default { 1 } else { 0 }; - let has_preceding_match_or_case = case_count > 1 - && cases - .iter() - .take(case_count - 1) - .any(|m| contains_match_or(&m.pattern)); - let last_match_or_has_immediate_scope_exit_body = case_count > 0 - && contains_match_or(&cases[case_count - 1].pattern) - && cases[case_count - 1] - .body - .first() - .is_some_and(Self::statement_ends_with_scope_exit); for (i, m) in cases.iter().enumerate().take(case_count) { // Only copy the subject if not on the last case if i != case_count - 1 { @@ -9495,10 +6948,6 @@ impl Compiler { pattern_context.fail_pop[0], Some(m.pattern.range()), )?; - if self.current_block_ends_with_conditional_jump() { - let success = self.new_block(); - self.switch_to_block(success); - } } if i != case_count - 1 { @@ -9506,45 +6955,20 @@ impl Compiler { self.set_source_range(first_stmt.range()); } emit!(self, Instruction::PopTop); - if let Some(last) = self.current_block().instructions.last_mut() { - // CPython emits NEXT_LOCATION here; resolve it after - // redundant NOP removal so a following pass NOP survives. - last.lineno_override = Some(-2); - } - if matches!(m.body.first(), Some(ast::Stmt::Try(_))) { - let body_block = self.new_block(); - self.switch_to_block(body_block); - } + // CPython emits NEXT_LOCATION here; resolve it after redundant + // NOP removal so a following pass NOP survives. + self.set_last_emitted_lineno_override(ir::NEXT_LOCATION_OVERRIDE); } - let body_contains_async_comprehension = - Self::statements_contain_async_comprehension(&m.body); self.compile_statements(&m.body)?; - if body_contains_async_comprehension { - emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); - } else { - emit!(self, PseudoInstruction::Jump { delta: end }); - } + emit!(self, PseudoInstruction::Jump { delta: end }); self.set_no_location(); - if let Some(last) = self.current_block().instructions.last_mut() { - last.match_success_jump = true; - } self.set_source_range(m.pattern.range()); self.emit_and_reset_fail_pop(pattern_context); } if has_default { let m = &cases[num_cases - 1]; - if has_preceding_match_or_case || last_match_or_has_immediate_scope_exit_body { - // CPython codegen_match_inner() emits COPY 1 for all but the - // last non-default case, while codegen_pattern_or() only pops - // the OR-pattern copy on failure. A trailing default reached - // after such an OR-pattern, or after a terminal OR-pattern - // body that CPython keeps adjacent to each alternative, keeps - // CPython's load-fast analysis from borrowing loads here. - let current = self.current_code_info().current_block; - self.disable_load_fast_borrow_for_block(current); - } self.set_source_range(m.pattern.range()); if num_cases == 1 { emit!(self, Instruction::PopTop); @@ -9557,7 +6981,7 @@ impl Compiler { self.compile_statements(&m.body)?; } // CPython codegen_match_inner() emits USE_LABEL(c, end). - self.use_label_block(end); + self.use_cpython_label_block(end); Ok(()) } @@ -9681,11 +7105,11 @@ impl Compiler { self.set_no_location(); // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. - self.switch_to_block(cleanup); + self.use_cpython_label_block(cleanup); emit!(self, Instruction::Swap { i: 2 }); emit!(self, Instruction::PopTop); - self.switch_to_block(end); + self.use_cpython_label_block(end); Ok(()) } @@ -9732,7 +7156,7 @@ impl Compiler { emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); self.set_no_location(); - self.switch_to_block(cleanup); + self.use_cpython_label_block(cleanup); emit!(self, Instruction::PopTop); if !condition { self.set_no_location(); @@ -9744,7 +7168,7 @@ impl Compiler { ); } - self.switch_to_block(end); + self.use_cpython_label_block(end); Ok(()) } @@ -10166,7 +7590,7 @@ impl Compiler { self.compile_jump_if_inner(last_value, condition, target_block, source_range)?; if next2 != target_block { - self.switch_to_block(next2); + self.use_cpython_label_block(next2); } Ok(()) } @@ -10184,15 +7608,11 @@ impl Compiler { self.compile_jump_if_inner(body, condition, target_block, source_range)?; emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); self.set_no_location(); - // CPython emits this jump with NO_LOCATION in codegen_jump_if() - // and flowgraph.c::propagate_line_numbers() copies the previous - // body-expression location onto it. - self.copy_previous_location_to_last_instruction(); - self.switch_to_block(next2); + self.use_cpython_label_block(next2); self.compile_jump_if_inner(orelse, condition, target_block, source_range)?; - self.switch_to_block(end); + self.use_cpython_label_block(end); Ok(()) } ast::Expr::Compare(ast::ExprCompare { @@ -10203,62 +7623,8 @@ impl Compiler { }) if ops.len() > 1 => { self.compile_jump_if_compare(left, ops, comparators, condition, target_block) } - // `x is None` / `x is not None` → POP_JUMP_IF_NONE / POP_JUMP_IF_NOT_NONE - ast::Expr::Compare(ast::ExprCompare { - left, - ops, - comparators, - .. - }) if ops.len() == 1 - && matches!(ops[0], ast::CmpOp::Is | ast::CmpOp::IsNot) - && comparators.len() == 1 - && matches!(&comparators[0], ast::Expr::NoneLiteral(_)) => - { - self.compile_expression(left)?; - // CPython codegen first emits LOAD_CONST None; IS_OP; POP_JUMP... - // and flowgraph.c::basicblock_optimize_load_const folds it into - // POP_JUMP_IF_NONE / POP_JUMP_IF_NOT_NONE. Register None here - // to preserve CPython's co_consts ordering even though we emit - // the folded jump directly. - self.arg_constant(ConstantData::None); - let source = self.source_file.to_source_code(); - let comparator_line = source.line_index(comparators[0].range().start()); - let left_line = source.line_index(left.range().start()); - if comparator_line != left_line { - self.set_source_range(comparators[0].range()); - emit!(self, Instruction::Nop); - self.set_source_range(source_range.unwrap_or_else(|| expression.range())); - } - let is_not = matches!(ops[0], ast::CmpOp::IsNot); - // is None + jump_if_false → POP_JUMP_IF_NOT_NONE - // is None + jump_if_true → POP_JUMP_IF_NONE - // is not None + jump_if_false → POP_JUMP_IF_NONE - // is not None + jump_if_true → POP_JUMP_IF_NOT_NONE - let jump_if_none = condition != is_not; - self.set_source_range(source_range.unwrap_or_else(|| expression.range())); - if jump_if_none { - emit!( - self, - Instruction::PopJumpIfNone { - delta: target_block, - } - ); - } else { - emit!( - self, - Instruction::PopJumpIfNotNone { - delta: target_block, - } - ); - } - Ok(()) - } _ => { // Fall back case which always will work! - if matches!(self.constant_expr_truthiness(expression)?, Some(value) if value == condition) - { - self.disable_load_fast_borrow_for_block(target_block); - } self.compile_expression(expression)?; self.set_source_range(expression.range()); emit!(self, Instruction::ToBool); @@ -10298,66 +7664,33 @@ impl Compiler { /// This means, that the last value remains on the stack. fn compile_bool_op(&mut self, op: &ast::BoolOp, values: &[ast::Expr]) -> CompileResult<()> { let boolop_range = self.current_source_range; - fn flatten_same_boolop_values<'a>( - op: &ast::BoolOp, - values: &'a [ast::Expr], - current_range: TextRange, - outer_pop_range: Option, - out: &mut Vec<(&'a ast::Expr, Option)>, - ) { - for (idx, value) in values.iter().enumerate() { - let is_last = idx + 1 == values.len(); - let pop_range = if is_last { - outer_pop_range - } else { - Some(current_range) - }; - if let ast::Expr::BoolOp(ast::ExprBoolOp { - op: inner_op, - values, - .. - }) = value - && inner_op == op - { - flatten_same_boolop_values(op, values, value.range(), pop_range, out); - } else { - out.push((value, pop_range)); - } - } - } - - let mut flattened = Vec::with_capacity(values.len()); - flatten_same_boolop_values(op, values, boolop_range, None, &mut flattened); - let after_block = self.new_block(); - let ((last_value, _), prefix_values) = flattened.split_last().unwrap(); + let (last_value, prefix_values) = values.split_last().unwrap(); - for &(value, pop_range) in prefix_values { + for value in prefix_values { let continue_block = self.new_block(); self.compile_expression(value)?; self.set_source_range(boolop_range); self.emit_short_circuit_test(op, after_block); self.switch_to_block(continue_block); - self.set_source_range(pop_range.expect("prefix boolop value must have pop range")); + self.set_source_range(boolop_range); emit!(self, Instruction::PopTop); } self.compile_expression(last_value)?; - self.switch_to_block(after_block); + self.use_cpython_label_block(after_block); Ok(()) } - /// Emit `Copy 1` + conditional jump for short-circuit evaluation. - /// For `And`, emits `PopJumpIfFalse`; for `Or`, emits `PopJumpIfTrue`. + /// Emit CPython-style pseudo conditional jump for short-circuit evaluation. + /// flowgraph.c lowers it to `COPY 1; TO_BOOL; POP_JUMP_IF_*`. fn emit_short_circuit_test(&mut self, op: &ast::BoolOp, target: BlockIdx) { - emit!(self, Instruction::Copy { i: 1 }); - emit!(self, Instruction::ToBool); match op { ast::BoolOp::And => { - emit!(self, Instruction::PopJumpIfFalse { delta: target }); + emit!(self, PseudoInstruction::JumpIfFalse { delta: target }); } ast::BoolOp::Or => { - emit!(self, Instruction::PopJumpIfTrue { delta: target }); + emit!(self, PseudoInstruction::JumpIfTrue { delta: target }); } } } @@ -10521,29 +7854,23 @@ impl Compiler { /// JUMP exit /// exit: /// END_SEND - fn compile_yield_from_sequence(&mut self, is_await: bool) -> CompileResult { + fn compile_yield_from_sequence(&mut self, is_await: bool) -> BlockIdx { let send_block = self.new_block(); let fail_block = self.new_block(); let exit_block = self.new_block(); // send: - self.switch_to_block(send_block); + self.use_cpython_label_block(send_block); emit!(self, Instruction::Send { delta: exit_block }); // SETUP_FINALLY fail - set up exception handler for YIELD_VALUE emit!(self, PseudoInstruction::SetupFinally { delta: fail_block }); - self.push_fblock( - FBlockType::TryExcept, // Use TryExcept for exception handler - send_block, - exit_block, - )?; // YIELD_VALUE with arg=1 (yield-from/await mode - not wrapped for async gen) emit!(self, Instruction::YieldValue { arg: 1 }); // POP_BLOCK before RESUME emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::TryExcept); // RESUME emit!( @@ -10569,16 +7896,16 @@ impl Compiler { // CPython lets this block fall through to END_SEND during codegen; // push_cold_blocks_to_end later inserts the no-interrupt jump after // moving the cold fail block behind the warm exit path. - self.switch_to_block(fail_block); + self.use_cpython_label_block(fail_block); emit!(self, Instruction::CleanupThrow); // exit: END_SEND // Stack: [receiver, value] (from SEND) or [None, value] (from CLEANUP_THROW) // END_SEND: [receiver/None, value] -> [value] - self.switch_to_block(exit_block); + self.use_cpython_label_block(exit_block); emit!(self, Instruction::EndSend); - Ok(send_block) + send_block } /// Returns true if the expression is a constant with no side effects. @@ -10631,13 +7958,11 @@ impl Compiler { ast::BoolOp::Or if is_truthy => { self.set_source_range(last_constant_range.expect("missing boolop range")); self.emit_load_const(last_constant.expect("missing boolop constant")); - self.mark_last_instruction_folded_from_nonliteral_expr(); return Ok(()); } ast::BoolOp::And if !is_truthy => { self.set_source_range(last_constant_range.expect("missing boolop range")); self.emit_load_const(last_constant.expect("missing boolop constant")); - self.mark_last_instruction_folded_from_nonliteral_expr(); return Ok(()); } ast::BoolOp::Or | ast::BoolOp::And => { @@ -10649,17 +7974,10 @@ impl Compiler { if simplified_prefix == values.len() { self.set_source_range(last_constant_range.expect("missing boolop range")); self.emit_load_const(last_constant.expect("missing folded boolop constant")); - self.mark_last_instruction_folded_from_nonliteral_expr(); return Ok(()); } if simplified_prefix > 0 { - let tail = &values[simplified_prefix..]; - if let [value] = tail { - self.compile_expression(value)?; - } else { - self.compile_bool_op(op, tail)?; - } - self.mark_last_instruction_folded_from_nonliteral_expr(); + self.compile_bool_op(op, values)?; return Ok(()); } } @@ -10851,7 +8169,7 @@ impl Compiler { self.set_source_range(range); emit!(self, Instruction::GetAwaitable { r#where: 0 }); self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(true)?; + let _ = self.compile_yield_from_sequence(true); } ast::Expr::YieldFrom(ast::ExprYieldFrom { value, .. }) => { match self.ctx.func { @@ -10868,7 +8186,7 @@ impl Compiler { self.set_source_range(range); emit!(self, Instruction::GetYieldFromIter); self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(false)?; + let _ = self.compile_yield_from_sequence(false); } ast::Expr::Name(ast::ExprName { id, .. }) => self.load_name(id.as_str())?, ast::Expr::Lambda(ast::ExprLambda { @@ -10941,7 +8259,6 @@ impl Compiler { self.set_qualname(); self.ctx = CompileContext { - loop_data: Option::None, in_class: prev_ctx.in_class, func: FunctionContext::Function, // Lambda is never async, so new scope is not async @@ -11097,25 +8414,12 @@ impl Compiler { ast::Expr::If(ast::ExprIf { test, body, orelse, .. }) => { - let starts_with_stack_values = self - .current_block_stack_depth() - .is_some_and(|depth| depth > 0); - let folded_test_truthiness = self - .try_fold_constant_expr(test)? - .as_ref() - .map(Self::constant_truthiness); let else_block = self.new_block(); let after_block = self.new_block(); - if self.current_code_info().in_conditional_block > 0 { - self.mark_conditional_ifexp_orelse_entry_block(else_block); - } self.compile_jump_if(test, false, else_block)?; // True case self.compile_expression(body)?; - if folded_test_truthiness == Some(true) { - self.mark_last_instruction_folded_from_nonliteral_expr(); - } emit!( self, PseudoInstruction::JumpNoInterrupt { delta: after_block } @@ -11123,25 +8427,11 @@ impl Compiler { self.set_no_location(); // False case - self.switch_to_block(else_block); + self.use_cpython_label_block(else_block); self.compile_expression(orelse)?; - if folded_test_truthiness == Some(false) { - self.mark_last_instruction_folded_from_nonliteral_expr(); - } // End - self.switch_to_block(after_block); - if folded_test_truthiness.is_some() && starts_with_stack_values { - // CPython codegen_ifexp() always emits USE_LABEL(end). - // When the folded conditional expression is nested inside - // a larger stack expression, flowgraph.c keeps that end - // label as an empty block. optimize_load_fast() does not - // push fallthrough successors from empty blocks, so later - // LOAD_FAST instructions remain strong. - self.mark_load_fast_barrier_block(after_block); - let continuation = self.new_block(); - self.switch_to_block(continuation); - } + self.use_cpython_label_block(after_block); } ast::Expr::Named(ast::ExprNamed { @@ -11283,7 +8573,7 @@ impl Compiler { { current_table.next_sub_table = cursor; } - self.switch_to_block(loop_block); + self.use_cpython_label_block(loop_block); self.set_source_range(loc); emit!(self, Instruction::ForIter { delta: cleanup }); @@ -11318,7 +8608,7 @@ impl Compiler { } } - self.switch_to_block(cleanup); + self.use_cpython_label_block(cleanup); self.set_source_range(loc); emit!(self, Instruction::EndFor); self.set_source_range(loc); @@ -11345,7 +8635,7 @@ impl Compiler { self.set_source_range(loc); emit!(self, PseudoInstruction::Jump { delta: end }); - self.switch_to_block(fallback); + self.use_cpython_label_block(fallback); Ok(()) } @@ -11456,7 +8746,7 @@ impl Compiler { self.set_source_range(func.range()); emit!(self, Instruction::PushNull); self.codegen_call_helper(0, args, call_range, None)?; - self.switch_to_block(end); + self.use_cpython_label_block(end); } else { // Regular call: push func, then NULL for self_or_null slot // Stack layout: [func, NULL, args...] - same as method call [func, self, args...] @@ -12075,7 +9365,6 @@ impl Compiler { // Non-inlined path: create a new code object (generator expressions, etc.) self.ctx = CompileContext { - loop_data: None, in_class: prev_ctx.in_class, func: if is_async { FunctionContext::AsyncFunction @@ -12113,18 +9402,7 @@ impl Compiler { let is_gen_scope = self.current_symbol_table().is_generator || is_async; let stop_iteration_block = if is_gen_scope { let handler_block = self.new_block(); - emit!( - self, - PseudoInstruction::SetupCleanup { - delta: handler_block - } - ); - self.set_no_location(); - // CPython's codegen_wrap_in_stopiteration_handler() inserts - // SETUP_CLEANUP at instruction-sequence index 0, so after the - // generator prefix is inserted the protected range begins at the - // comprehension-start RESUME. - self.move_last_instruction_before_scope_start_resume(); + self.insert_cpython_stopiteration_setup_cleanup(handler_block); self.push_fblock(FBlockType::StopIteration, handler_block, handler_block)?; Some(handler_block) } else { @@ -12150,13 +9428,7 @@ impl Compiler { if !generator.ifs.is_empty() { let if_cleanup_block = self.new_block(); for if_condition in &generator.ifs { - let snapshot = self.instruction_count_snapshot(); self.compile_jump_if(if_condition, false, if_cleanup_block)?; - self.mark_new_conditional_jump_locations_since( - &snapshot, - if_cleanup_block, - element_range, - ); } let body_block = self.new_block(); self.switch_to_block(body_block); @@ -12177,7 +9449,7 @@ impl Compiler { self.compile_comprehension_iter(generator)?; } - self.switch_to_block(loop_block); + self.use_cpython_label_block(loop_block); let mut end_async_for_target = BlockIdx::NULL; if generator.is_async { emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); @@ -12188,11 +9460,11 @@ impl Compiler { after_block, )?; self.emit_load_const(ConstantData::None); - end_async_for_target = self.compile_yield_from_sequence(true)?; + end_async_for_target = self.compile_yield_from_sequence(true); // POP_BLOCK before store: only __anext__/yield_from are // protected by SetupFinally targeting END_ASYNC_FOR. emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::AsyncComprehensionGenerator); + self.pop_fblock(FBlockType::AsyncComprehensionGenerator, loop_block); self.compile_store(&generator.target)?; } else { let saved_range = self.current_source_range; @@ -12220,13 +9492,7 @@ impl Compiler { // CPython always lowers comprehension guards through codegen_jump_if // and leaves constant-folding to later CFG optimization passes. for if_condition in &generator.ifs { - let snapshot = self.instruction_count_snapshot(); self.compile_jump_if(if_condition, false, if_cleanup_block)?; - self.mark_new_conditional_jump_locations_since( - &snapshot, - if_cleanup_block, - element_range, - ); } if !generator.ifs.is_empty() { let body_block = self.new_block(); @@ -12250,11 +9516,11 @@ impl Compiler { self.set_source_range(backedge_range); emit!(self, PseudoInstruction::Jump { delta: loop_block }); - self.switch_to_block(if_cleanup_block); + self.use_cpython_label_block(if_cleanup_block); self.set_source_range(backedge_range); emit!(self, PseudoInstruction::Jump { delta: loop_block }); - self.switch_to_block(after_block); + self.use_cpython_label_block(after_block); if is_async { self.set_source_range(comprehension_range); // EndAsyncFor pops both the exception and the aiter @@ -12268,7 +9534,7 @@ impl Compiler { } } ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block } => { - self.switch_to_block(if_cleanup_block); + self.use_cpython_label_block(if_cleanup_block); } } } @@ -12283,8 +9549,8 @@ impl Compiler { if let Some(handler_block) = stop_iteration_block { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::StopIteration); - self.switch_to_block(handler_block); + self.pop_fblock(FBlockType::StopIteration, handler_block); + self.use_cpython_label_block(handler_block); emit!( self, Instruction::CallIntrinsic1 { @@ -12317,7 +9583,7 @@ impl Compiler { if is_async_list_set_dict_comprehension { emit!(self, Instruction::GetAwaitable { r#where: 0 }); self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(true)?; + let _ = self.compile_yield_from_sequence(true); } Ok(()) @@ -12509,7 +9775,6 @@ impl Compiler { delta: cleanup_block } ); - self.push_fblock(FBlockType::TryExcept, cleanup_block, end_block)?; Some((cleanup_block, end_block)) } else { None @@ -12537,13 +9802,7 @@ impl Compiler { if !generator.ifs.is_empty() { let if_cleanup_block = self.new_block(); for if_condition in &generator.ifs { - let snapshot = self.instruction_count_snapshot(); self.compile_jump_if(if_condition, false, if_cleanup_block)?; - self.mark_new_conditional_jump_locations_since( - &snapshot, - if_cleanup_block, - element_range, - ); } let body_block = self.new_block(); self.switch_to_block(body_block); @@ -12561,7 +9820,7 @@ impl Compiler { self.compile_comprehension_iter(generator)?; } - self.switch_to_block(loop_block); + self.use_cpython_label_block(loop_block); let mut end_async_for_target = BlockIdx::NULL; if generator.is_async { @@ -12573,9 +9832,9 @@ impl Compiler { after_block, )?; self.emit_load_const(ConstantData::None); - end_async_for_target = self.compile_yield_from_sequence(true)?; + end_async_for_target = self.compile_yield_from_sequence(true); emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::AsyncComprehensionGenerator); + self.pop_fblock(FBlockType::AsyncComprehensionGenerator, loop_block); self.compile_store(&generator.target)?; } else { let saved_range = self.current_source_range; @@ -12604,13 +9863,7 @@ impl Compiler { // CPython always lowers comprehension guards through codegen_jump_if // and leaves constant-folding to later CFG optimization passes. for if_condition in &generator.ifs { - let snapshot = self.instruction_count_snapshot(); self.compile_jump_if(if_condition, false, if_cleanup_block)?; - self.mark_new_conditional_jump_locations_since( - &snapshot, - if_cleanup_block, - element_range, - ); } } @@ -12629,14 +9882,11 @@ impl Compiler { is_async, end_async_for_target, } => { + self.use_cpython_label_block(if_cleanup_block); self.set_source_range(backedge_range); emit!(self, PseudoInstruction::Jump { delta: loop_block }); - self.switch_to_block(if_cleanup_block); - self.set_source_range(backedge_range); - emit!(self, PseudoInstruction::Jump { delta: loop_block }); - - self.switch_to_block(after_block); + self.use_cpython_label_block(after_block); if is_async { self.set_source_range(comprehension_range); self.emit_end_async_for(end_async_for_target); @@ -12647,7 +9897,7 @@ impl Compiler { } } ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block } => { - self.switch_to_block(if_cleanup_block); + self.use_cpython_label_block(if_cleanup_block); } } } @@ -12657,7 +9907,6 @@ impl Compiler { if let Some((cleanup_block, end_block)) = cleanup_blocks { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::TryExcept); // Match CPython codegen_pop_inlined_comprehension_locals(): // the synthetic jump that skips the exception cleanup uses @@ -12670,7 +9919,7 @@ impl Compiler { self.set_no_location(); // Exception cleanup path - self.switch_to_block(cleanup_block); + self.use_cpython_label_block(cleanup_block); // Stack: [saved_values..., collection, exception] emit!(self, Instruction::Swap { i: 2 }); self.set_no_location(); @@ -12694,7 +9943,7 @@ impl Compiler { self.set_no_location(); // Normal end path - self.switch_to_block(end_block); + self.use_cpython_label_block(end_block); self.set_source_range(comprehension_range); } @@ -12755,145 +10004,185 @@ impl Compiler { } // Low level helper functions: + /// CPython `_PyCfgBuilder_Addop()`: start a new basic block if the current + /// one is terminated, then append the instruction to the current block. + fn cpython_cfg_builder_addop(&mut self, info: ir::InstructionInfo) { + self.maybe_start_cpython_cfg_addop_block(); + self.push_emitted_instruction(info); + } + + fn push_emitted_instruction(&mut self, info: ir::InstructionInfo) { + self.current_code_info() + .addop_to_instr_sequence(info) + .expect("malformed instruction sequence emission"); + self.current_block().instructions.push(info); + } + + fn push_emitted_instruction_with_target_label( + &mut self, + info: ir::InstructionInfo, + target_label: ir::InstructionSequenceLabel, + ) { + self.current_code_info() + .addop_to_instr_sequence_with_target_label(info, target_label) + .expect("malformed instruction sequence emission"); + self.current_block().instructions.push(info); + } + + fn pop_last_emitted_instruction(&mut self) -> Option { + let sequence_info = self.current_code_info().pop_instr_sequence(); + let block_info = self.current_block().instructions.pop(); + match (sequence_info, block_info) { + (Some(sequence_info), Some(block_info)) => { + debug_assert_eq!( + format!("{:?}", sequence_info.instr), + format!("{:?}", block_info.instr), + "direct-CFG shadow must pop the same opcode as the CPython instruction sequence" + ); + debug_assert_eq!(u32::from(sequence_info.arg), u32::from(block_info.arg)); + debug_assert_eq!(sequence_info.target, block_info.target); + Some(block_info) + } + (None, None) => None, + _ => { + debug_assert!( + false, + "direct-CFG shadow and CPython instruction sequence must pop together" + ); + block_info + } + } + } + + fn last_emitted_instruction_mut(&mut self) -> Option<&mut ir::InstructionInfo> { + self.current_block().instructions.last_mut() + } + + fn set_last_emitted_lineno_override(&mut self, lineno_override: i32) { + self.current_code_info() + .set_last_instr_sequence_lineno_override(lineno_override); + if let Some(last) = self.last_emitted_instruction_mut() { + last.lineno_override = Some(lineno_override); + } + } + fn _emit>(&mut self, instr: I, arg: OpArg, target: BlockIdx) { if self.do_not_emit_bytecode > 0 { return; } + let instr = instr.into(); + let opcode = AnyOpcode::from(instr); + debug_assert!( + !instr.is_assembler(), + "CPython codegen_addop_* must not emit assembler-only opcodes" + ); + debug_assert!( + opcode.has_arg() || instr.has_target() || u32::from(arg) == 0, + "CPython _PyInstructionSequence_Addop requires either OPCODE_HAS_ARG, HAS_TARGET, or oparg == 0" + ); + debug_assert!( + target == BlockIdx::NULL || instr.has_target(), + "CPython codegen_addop_j only accepts HAS_TARGET opcodes" + ); let range = self.current_source_range; let source = self.source_file.to_source_code(); let location = source.source_location(range.start(), PositionEncoding::Utf8); let end_location = source.source_location(range.end(), PositionEncoding::Utf8); let except_handler = None; - self.current_block().instructions.push(ir::InstructionInfo { - instr: instr.into(), + self.cpython_cfg_builder_addop(ir::InstructionInfo { + instr, arg, target, location, end_location, except_handler, - folded_from_nonliteral_expr: false, lineno_override: None, cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, }); } - fn mark_last_instruction_folded_from_nonliteral_expr(&mut self) { - if let Some(info) = self.current_block().instructions.last_mut() { - info.folded_from_nonliteral_expr = true; - } - } - - fn preserve_last_redundant_jump_as_nop(&mut self) { - if let Some(info) = self.current_block().instructions.last_mut() { - info.preserve_redundant_jump_as_nop = true; - } - } - - fn preserve_last_redundant_nop(&mut self) { - if let Some(info) = self.current_block().instructions.last_mut() { - info.preserve_block_start_no_location_nop = true; - } - } - - fn remove_last_no_location_nop(&mut self) { - if let Some(info) = self.current_block().instructions.last_mut() { - info.remove_no_location_nop = true; - } - } - - fn copy_previous_location_to_last_instruction(&mut self) { - let instructions = &mut self.current_block().instructions; - let Some(last_idx) = instructions.len().checked_sub(1) else { - return; - }; - let Some(previous_idx) = last_idx.checked_sub(1) else { + /// CPython `codegen_addop_j()`: emit a HAS_TARGET instruction with a + /// jump_target_label oparg. The direct target is only the shadow CFG bridge. + fn emit_jump_label>( + &mut self, + instr: I, + target_label: ir::InstructionSequenceLabel, + target_direct: BlockIdx, + ) { + if self.do_not_emit_bytecode > 0 { return; - }; - let previous = instructions[previous_idx]; - let last = &mut instructions[last_idx]; - last.location = previous.location; - last.end_location = previous.end_location; - last.lineno_override = previous.lineno_override; - } - - fn force_remove_last_no_location_nop(&mut self) { - if let Some(info) = self.current_block().instructions.last_mut() { - info.remove_no_location_nop = true; - info.folded_operand_nop = true; } + let instr = instr.into(); + debug_assert!( + instr.has_target(), + "CPython codegen_addop_j only accepts HAS_TARGET opcodes" + ); + debug_assert!( + !instr.is_assembler(), + "CPython codegen_addop_j must not emit assembler-only opcodes" + ); + let range = self.current_source_range; + let source = self.source_file.to_source_code(); + let location = source.source_location(range.start(), PositionEncoding::Utf8); + let end_location = source.source_location(range.end(), PositionEncoding::Utf8); + self.maybe_start_cpython_cfg_addop_block(); + self.push_emitted_instruction_with_target_label( + ir::InstructionInfo { + instr, + arg: OpArg::NULL, + target: target_direct, + location, + end_location, + except_handler: None, + lineno_override: None, + cache_entries: 0, + }, + target_label, + ); } /// Mark the last emitted instruction as having no source location. /// Prevents it from triggering LINE events in sys.monitoring. fn set_no_location(&mut self) { - if let Some(last) = self.current_block().instructions.last_mut() { - last.lineno_override = Some(-1); - } + self.set_last_emitted_lineno_override(-1); } - fn move_last_instruction_before_scope_start_resume(&mut self) { - let instructions = &mut self.current_block().instructions; - let Some(last_idx) = instructions.len().checked_sub(1) else { - return; - }; - let Some(resume_idx) = - instructions[..last_idx] - .iter() - .rposition(|info| match info.instr.real() { - Some(Instruction::Resume { context }) => { - matches!( - context.get(info.arg).location(), - oparg::ResumeLocation::AtFuncStart - ) - } + /// CPython `codegen_wrap_in_stopiteration_handler()` inserts + /// `SETUP_CLEANUP` into the instruction sequence at index 0 with + /// `_PyInstructionSequence_InsertInstruction()`. Generator and cell/free + /// prefixes are inserted later by `flowgraph.c::insert_prefix_instructions()`. + fn insert_cpython_stopiteration_setup_cleanup(&mut self, handler_block: BlockIdx) { + let code = self.current_code_info(); + let entry = code + .blocks + .first_mut() + .expect("code unit must have an entry block"); + debug_assert!( + entry + .instructions + .first() + .is_some_and(|info| match info.instr.real() { + Some(Instruction::Resume { context }) => matches!( + context.get(info.arg).location(), + oparg::ResumeLocation::AtFuncStart + ), _ => false, - }) - else { - return; - }; - - let instruction = instructions.remove(last_idx); - instructions.insert(resume_idx, instruction); - } - - fn mark_last_no_location_exit(&mut self) { - if let Some(last) = self.current_block().instructions.last_mut() { - last.no_location_exit = true; - } - } - - fn mark_last_line_only_location(&mut self, lineno: u32) { - if let Some(last) = self.current_block().instructions.last_mut() { - let location = SourceLocation { - line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), - character_offset: OneIndexed::MIN, - }; - last.location = location; - last.end_location = location; - last.lineno_override = Some(ir::LINE_ONLY_LOCATION_OVERRIDE); - } - } - - fn mark_last_break_continue_cleanup_jump(&mut self) { - if let Some(last) = self.current_block().instructions.last_mut() { - last.break_continue_cleanup_jump = true; - } - } + }), + "scope entry must start with a function-start RESUME" + ); + debug_assert!( + !entry.instructions.iter().any(|info| matches!( + info.instr.real(), + Some( + Instruction::ReturnGenerator + | Instruction::MakeCell { .. } + | Instruction::CopyFreeVars { .. } + ) + )), + "CPython inserts StopIteration cleanup before CFG prefix instructions" + ); - fn mark_last_for_loop_break_cleanup_jump(&mut self) { - if let Some(last) = self.current_block().instructions.last_mut() { - last.for_loop_break_cleanup_jump = true; - } + code.insert_start_setup_cleanup(handler_block); } fn emit_no_arg>(&mut self, ins: I) { @@ -13489,10 +10778,8 @@ impl Compiler { fn emit_return_const_no_location(&mut self, constant: ConstantData) { self.emit_load_const(constant); self.set_no_location(); - self.mark_last_no_location_exit(); emit!(self, Instruction::ReturnValue); self.set_no_location(); - self.mark_last_no_location_exit(); } fn emit_end_async_for(&mut self, send_target: BlockIdx) { @@ -13605,217 +10892,55 @@ impl Compiler { return Ok(()); } - // unwind_fblock_stack - // We need to unwind fblocks and compile cleanup code. For FinallyTry blocks, - // we need to compile the finally body inline, but we must temporarily pop - // the fblock so that nested break/continue in the finally body don't see it. - - // First, find the loop - let code = self.current_code_info(); - let mut loop_idx = None; - let mut is_for_loop = false; - - for i in (0..code.fblock.len()).rev() { - match code.fblock[i].fb_type { - FBlockType::WhileLoop => { - loop_idx = Some(i); - is_for_loop = false; - break; - } - FBlockType::ForLoop => { - loop_idx = Some(i); - is_for_loop = true; - break; - } - FBlockType::ExceptionGroupHandler => { - return Err( - self.error_ranged(CodegenErrorType::BreakContinueReturnInExceptStar, range) - ); - } - _ => {} - } - } + let prev_source_range = self.current_source_range; + self.set_source_range(range); + emit!(self, Instruction::Nop); - let Some(loop_idx) = loop_idx else { + let (mut unwind_loc, loop_fblock) = self.unwind_fblock_stack_with_loop(false, true)?; + let Some(loop_fblock) = loop_fblock else { + self.set_source_range(prev_source_range); if is_break { return Err(self.error_ranged(CodegenErrorType::InvalidBreak, range)); } return Err(self.error_ranged(CodegenErrorType::InvalidContinue, range)); }; - let loop_block = code.fblock[loop_idx].fb_block; - let exit_block = code.fblock[loop_idx].fb_exit; - - let prev_source_range = self.current_source_range; - self.set_source_range(range); - emit!(self, Instruction::Nop); - self.force_remove_last_no_location_nop(); - self.set_source_range(prev_source_range); - - // Collect the fblocks we need to unwind through, from top down to (but not including) the loop - #[derive(Clone)] - enum UnwindAction { - With { - is_async: bool, - range: TextRange, - }, - HandlerCleanup { - name: Option, - }, - TryExcept, - FinallyTry { - body: Vec, - fblock_idx: usize, - }, - FinallyEnd, - PopValue, // Pop return value when continue/break cancels a return - } - let mut unwind_actions = Vec::new(); - - { - let code = self.current_code_info(); - for i in (loop_idx + 1..code.fblock.len()).rev() { - match code.fblock[i].fb_type { - FBlockType::With => { - unwind_actions.push(UnwindAction::With { - is_async: false, - range: code.fblock[i].fb_range, - }); - } - FBlockType::AsyncWith => { - unwind_actions.push(UnwindAction::With { - is_async: true, - range: code.fblock[i].fb_range, - }); - } - FBlockType::HandlerCleanup => { - let name = match &code.fblock[i].fb_datum { - FBlockDatum::ExceptionName(name) => Some(name.clone()), - _ => None, - }; - unwind_actions.push(UnwindAction::HandlerCleanup { name }); - } - FBlockType::TryExcept => { - unwind_actions.push(UnwindAction::TryExcept); - } - FBlockType::FinallyTry => { - // Need to execute finally body before break/continue - if let FBlockDatum::FinallyBody(ref body) = code.fblock[i].fb_datum { - unwind_actions.push(UnwindAction::FinallyTry { - body: body.clone(), - fblock_idx: i, - }); - } - } - FBlockType::FinallyEnd => { - // Inside finally block reached via exception - need to pop exception - unwind_actions.push(UnwindAction::FinallyEnd); - } - FBlockType::PopValue => { - // Pop the return value that was saved on stack - unwind_actions.push(UnwindAction::PopValue); - } - _ => {} - } - } - } - - // Emit cleanup for each fblock - let mut jump_no_location = false; - for action in unwind_actions { - match action { - UnwindAction::With { is_async, range } => { - // Stack: [..., exit_func, self_exit] - let saved_range = self.current_source_range; - self.set_source_range(range); - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.emit_load_const(ConstantData::None); - self.emit_load_const(ConstantData::None); - self.emit_load_const(ConstantData::None); - emit!(self, Instruction::Call { argc: 3 }); - - if is_async { - emit!(self, Instruction::GetAwaitable { r#where: 2 }); - self.emit_load_const(ConstantData::None); - let _ = self.compile_yield_from_sequence(true)?; - } - - emit!(self, Instruction::PopTop); - self.set_source_range(saved_range); - jump_no_location = true; - } - UnwindAction::HandlerCleanup { ref name } => { - // codegen_unwind_fblock(HANDLER_CLEANUP) - if name.is_some() { - // Named handler: PopBlock for inner SETUP_CLEANUP - emit!(self, PseudoInstruction::PopBlock); - } - // PopBlock for outer SETUP_CLEANUP (ExceptionHandler) - emit!(self, PseudoInstruction::PopBlock); - emit!(self, Instruction::PopExcept); - if let Some(name) = name { - self.emit_load_const(ConstantData::None); - self.store_name(name)?; - self.compile_name(name, NameUsage::Delete)?; - } - } - UnwindAction::TryExcept => { - // codegen_unwind_fblock(TRY_EXCEPT) - emit!(self, PseudoInstruction::PopBlock); - } - UnwindAction::FinallyTry { body, fblock_idx } => { - // codegen_unwind_fblock(FINALLY_TRY) - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - - // compile finally body inline - // Temporarily pop the FinallyTry fblock so nested break/continue - // in the finally body won't see it again. - let code = self.current_code_info(); - let saved_fblock = code.fblock.remove(fblock_idx); - - self.compile_statements(&body)?; - - // Restore the fblock (though this break/continue will jump away, - // this keeps the fblock stack consistent for error checking) - let code = self.current_code_info(); - code.fblock.insert(fblock_idx, saved_fblock); - jump_no_location = true; - } - UnwindAction::FinallyEnd => { - // codegen_unwind_fblock(FINALLY_END) - emit!(self, Instruction::PopTop); // exc_value - emit!(self, PseudoInstruction::PopBlock); - emit!(self, Instruction::PopExcept); - } - UnwindAction::PopValue => { - // Pop the return value - continue/break cancels the pending return - emit!(self, Instruction::PopTop); - } - } - } - - // CPython unwinds a for-loop break with POP_TOP rather than POP_ITER. - if is_break && is_for_loop { - emit!(self, Instruction::PopTop); + // CPython `codegen_break()` unwinds the loop fblock itself after + // `codegen_unwind_fblock_stack()` returns it. `codegen_continue()` + // jumps directly to the loop body label. + if is_break { + self.unwind_fblock(&loop_fblock, false, &mut unwind_loc)?; } // Jump to target - let target = if is_break { exit_block } else { loop_block }; - let saved_range = self.current_source_range; - self.set_source_range(range); - emit!(self, PseudoInstruction::Jump { delta: target }); - self.mark_last_break_continue_cleanup_jump(); - if is_break && is_for_loop { - self.mark_last_for_loop_break_cleanup_jump(); - self.mark_load_fast_barrier_block(exit_block); - } - if jump_no_location { - self.mark_last_no_location_exit(); + let target_label = if is_break { + loop_fblock + .fb_exit + .expect("loop fblock must have a CPython exit label") + } else { + loop_fblock + .fb_block + .expect("loop fblock must have a CPython block label") + }; + let target_direct = self + .current_code_info() + .block_for_instr_sequence_label(Some(target_label)); + if let Some(loc) = unwind_loc { + self.set_source_range(loc); + } else { + self.set_source_range(range); + }; + self.emit_jump_label( + PseudoInstruction::Jump { + delta: OpArgMarker::marker(), + }, + target_label, + target_direct, + ); + if unwind_loc.is_none() { self.set_no_location(); } - self.set_source_range(saved_range); + self.set_source_range(prev_source_range); Ok(()) } @@ -13825,40 +10950,59 @@ impl Compiler { &mut info.blocks[info.current_block] } - fn current_block_ends_with_conditional_jump(&mut self) -> bool { - let info = self.current_code_info(); - info.blocks[info.current_block] + /// CPython `_PyCfgBuilder_Addop()` calls + /// `cfg_builder_maybe_start_new_block()` before appending each instruction. + /// That keeps any `IS_TERMINATOR_OPCODE` as the final instruction in its + /// basicblock before `flowgraph.c::check_cfg()`. + fn maybe_start_cpython_cfg_addop_block(&mut self) { + let code = self.current_code_info(); + let cur = code.current_block; + if !code.blocks[cur.idx()] .instructions .last() - .is_some_and(|info| ir::is_conditional_jump(&info.instr)) - } - - /// Switch to a fresh block, but reuse the current block when it is empty - /// and unlinked, mirroring CPython's USE_LABEL behavior in - /// `cfg_builder_maybe_start_new_block` (Python/flowgraph.c): when the - /// current block has no instructions and no existing label/next pointer, - /// CPython attaches the new label to the current block instead of creating - /// a new one. RustPython's plain `switch_to_block(new_block())` always - /// creates a fresh block, which leaves a stray empty block in the b_next - /// chain (e.g. after compile_try_except's switch_to_block(end_block)). - /// That stray empty block then causes optimize_load_fast_borrow to stop - /// fall-through propagation at the wrong place. Use this helper for - /// "entry to construct" labels (while loop header, for loop header, etc.) - /// where reusing the empty current block is semantically safe. - fn switch_to_new_or_reuse_empty(&mut self) -> BlockIdx { + .is_some_and(|instr| instr.instr.is_terminator()) + { + return; + } + + debug_assert_eq!(code.blocks[cur.idx()].next, BlockIdx::NULL); + let block = self.new_block(); + self.switch_to_block(block); + } + + fn cpython_cfg_builder_current_block_is_terminated_for_label(block: &ir::Block) -> bool { + !block.instructions.is_empty() || block.has_cpython_cfg_label() + } + + /// Start a label block using CPython's `cfg_builder_current_block_is_terminated()` + /// rule: if the current block is empty and unlabeled, `_PyCfgBuilder_UseLabel()` + /// attaches the pending label to that block instead of allocating a new one. + fn start_cpython_label_block(&mut self) -> BlockIdx { let cur = self.current_code_info().current_block; let block = &self.current_code_info().blocks[cur.idx()]; - if block.instructions.is_empty() && block.next == BlockIdx::NULL { - cur - } else { + let block = if Self::cpython_cfg_builder_current_block_is_terminated_for_label(block) { let b = self.new_block(); self.switch_to_block(b); b - } + } else { + debug_assert_eq!(block.next, BlockIdx::NULL); + cur + }; + self.mark_cpython_cfg_label_block(block); + self.current_code_info().use_instr_sequence_label(block); + block } - fn use_label_block(&mut self, block: BlockIdx) { + /// Switch to a block as CPython instruction-sequence labels would resolve. + /// + /// Consecutive `USE_LABEL()` calls can map multiple labels to the same + /// instruction offset before `_PyCfg_FromInstructionSequence()` builds a + /// CFG. Reuse an empty current block even if it already carries a label so + /// direct CFG codegen preserves that aliasing. + fn use_cpython_label_block(&mut self, block: BlockIdx) { let code = self.current_code_info(); + code.use_instr_sequence_label(block); + let block = code.resolve_instr_sequence_label(block); let cur = code.current_block; let can_reuse_current = cur != block && code.blocks[cur.idx()].instructions.is_empty() @@ -13867,80 +11011,65 @@ impl Compiler { && code.blocks[block.idx()].next == BlockIdx::NULL; if !can_reuse_current { + code.mark_cpython_cfg_label(block); self.switch_to_block(block); return; } - let target_flags = { - let target = &code.blocks[block.idx()]; - ( - target.except_handler, - target.preserve_lasti, - target.start_depth, - target.cold, - target.disable_load_fast_borrow, - target.try_else_orelse_entry, - target.label, - target.load_fast_passthrough, - target.load_fast_label_reuse_passthrough, - ) - }; { - let current = &mut code.blocks[cur.idx()]; - current.except_handler |= target_flags.0; - current.preserve_lasti |= target_flags.1; - current.start_depth = current.start_depth.or(target_flags.2); - current.cold |= target_flags.3; - current.disable_load_fast_borrow = target_flags.4; - current.try_else_orelse_entry |= target_flags.5; - current.label |= target_flags.6; - current.load_fast_passthrough |= target_flags.7; - current.load_fast_label_reuse_passthrough |= target_flags.8; + let target = &code.blocks[block.idx()]; + debug_assert!(!target.except_handler); + debug_assert!(!target.preserve_lasti); + debug_assert!(target.start_depth.is_none()); + debug_assert!(!target.cold); } + code.mark_cpython_cfg_label(cur); - for candidate in &mut code.blocks { - if candidate.next == block { - candidate.next = cur; - } - for instr in &mut candidate.instructions { - if instr.target == block { - instr.target = cur; - } - } - } - for fblock in &mut code.fblock { - if fblock.fb_block == block { - fblock.fb_block = cur; - } - if fblock.fb_exit == block { - fblock.fb_exit = cur; - } - } - if let Some((loop_block, exit_block)) = &mut self.ctx.loop_data { - if *loop_block == block { - *loop_block = cur; - } - if *exit_block == block { - *exit_block = cur; - } + self.current_code_info() + .use_instr_sequence_label_at_block(block, cur); + } + + /// Enter a compile-time-only `USE_LABEL()` site that CPython drops before + /// the CFG exists. + /// + /// Labels such as try-body entries can matter while codegen records fblock + /// targets, but `_PyInstructionSequence_ApplyLabelMap()` and + /// `_PyCfg_FromInstructionSequence()` only keep labels that are reached by + /// `HAS_TARGET` operands. Consecutive dropped labels at an already-retained + /// label offset still alias to the same next instruction, so reusing a + /// labeled empty current block is valid here. + fn use_cpython_transient_label_block(&mut self, block: BlockIdx) { + let code = self.current_code_info(); + code.use_instr_sequence_label(block); + let block = code.resolve_instr_sequence_label(block); + let cur = code.current_block; + let can_reuse_current = cur != block + && code.blocks[cur.idx()].instructions.is_empty() + && code.blocks[cur.idx()].next == BlockIdx::NULL + && code.blocks[block.idx()].instructions.is_empty() + && code.blocks[block.idx()].next == BlockIdx::NULL; + + if !can_reuse_current { + self.switch_to_block(block); + return; } + + self.current_code_info() + .use_instr_sequence_label_at_block(block, cur); } fn new_block(&mut self) -> BlockIdx { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); - let inherited_disable_load_fast_borrow = - code.blocks[code.current_block].disable_load_fast_borrow; - let block = ir::Block { - disable_load_fast_borrow: inherited_disable_load_fast_borrow, - ..ir::Block::default() - }; - code.blocks.push(block); + code.blocks.push(ir::Block::default()); + code.instr_sequence_label_map.push_unmapped_label(); idx } fn switch_to_block(&mut self, block: BlockIdx) { let code = self.current_code_info(); + code.use_instr_sequence_label(block); + let block = code.resolve_instr_sequence_label(block); let prev = code.current_block; assert_ne!(prev, block, "recursive switching {prev:?} -> {block:?}"); assert_eq!( @@ -15273,7 +12402,6 @@ def f(x, y, z): let prev_ctx = compiler.ctx; compiler.ctx = CompileContext { - loop_data: None, in_class: prev_ctx.in_class, func: if is_async { FunctionContext::AsyncFunction @@ -15286,14 +12414,7 @@ def f(x, y, z): let is_gen = is_async || compiler.current_symbol_table().is_generator; let stop_iteration_block = if is_gen { let handler_block = compiler.new_block(); - emit!( - compiler, - PseudoInstruction::SetupCleanup { - delta: handler_block - } - ); - compiler.set_no_location(); - compiler.move_last_instruction_before_scope_start_resume(); + compiler.insert_cpython_stopiteration_setup_cleanup(handler_block); compiler .push_fblock(FBlockType::StopIteration, handler_block, handler_block) .unwrap(); @@ -15313,8 +12434,8 @@ def f(x, y, z): if let Some(handler_block) = stop_iteration_block { emit!(compiler, PseudoInstruction::PopBlock); compiler.set_no_location(); - compiler.pop_fblock(FBlockType::StopIteration); - compiler.switch_to_block(handler_block); + compiler.pop_fblock(FBlockType::StopIteration, handler_block); + compiler.use_cpython_label_block(handler_block); emit!( compiler, Instruction::CallIntrinsic1 { @@ -16569,7 +13690,6 @@ def f(self): ops.iter().map(|unit| unit.op).collect::>() ) }); - assert!( matches!(else_close_load[0].op, Instruction::LoadFastBorrow { .. }), "CPython codegen_try_except() emits Try.orelse directly with VISIT_SEQ(), so flowgraph.c::optimize_load_fast() reaches the else self.close receiver: {:?}", @@ -18097,8 +15217,8 @@ def f(format, other): assert!( loads .iter() - .any(|op| matches!(op, Instruction::LoadFast { .. })), - "CPython optimize_load_fast() keeps trailing nested OR-pattern default loads strong, got {loads:?}", + .all(|op| matches!(op, Instruction::LoadFastBorrow { .. })), + "CPython 3.14 optimize_load_fast() borrows nested OR-pattern default loads, got {loads:?}", ); } @@ -21511,27 +18631,17 @@ def f({params}): assert!( f.instructions.iter().any(|unit| matches!( unit.op, - Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } + Instruction::LoadFastCheck { var_num } if f.varnames [usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))))] == "p64" )), - "expected high-index parameter p64 to use LOAD_FAST, got ops={:?}", + "CPython 3.14 fast_scan_many_locals() checks high-index parameters per block; expected p64 to use LOAD_FAST_CHECK, got ops={:?}", f.instructions .iter() .map(|unit| unit.op) .collect::>() ); - assert!( - !f.instructions.iter().any(|unit| matches!( - unit.op, - Instruction::LoadFastCheck { var_num } - if f.varnames - [usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg)))))] - == "p64" - )), - "high-index parameter p64 should not use LOAD_FAST_CHECK before deletion" - ); } #[test] @@ -36569,7 +33679,7 @@ async def f(self, asyncio, sys, task, timeout_handle, sleep): } #[test] - fn try_else_attribute_probe_end_keeps_following_loads_strong() { + fn test_try_else_attribute_probe_end_allows_following_loads_borrow() { let code = compile_exec( "\ def f(self): @@ -36608,18 +33718,11 @@ def f(self): .collect(); assert!( - ops.iter().any( - |unit| matches!(unit.op, Instruction::LoadFastLoadFast { .. }) - && u8::from(unit.arg) == pair_arg - ), - "CPython leaves an empty try/else attribute-probe end block before the following if, so args/dict return pair stays strong, got ops={ops:?}", - ); - assert!( - ops.iter().all(|unit| { - !matches!(unit.op, Instruction::LoadFastBorrowLoadFastBorrow { .. }) - || u8::from(unit.arg) != pair_arg - }), - "try/else attribute-probe end should not borrow args/dict return pair, got ops={ops:?}", + ops.iter().any(|unit| matches!( + unit.op, + Instruction::LoadFastBorrowLoadFastBorrow { .. } + ) && u8::from(unit.arg) == pair_arg), + "CPython 3.14 optimize_load_fast() borrows args/dict after the try/else attribute-probe end; got ops={ops:?}", ); } @@ -41027,8 +38130,8 @@ def f(self): assert!( instructions .iter() - .any(|unit| matches!(unit.op, Instruction::LoadFastBorrow { .. })), - "one-line protected bound-method while body should keep CPython borrowed receiver, got instructions={instructions:?}" + .any(|unit| matches!(unit.op, Instruction::LoadFast { .. })), + "CPython 3.14 keeps the one-line protected bound-method receiver strong, got instructions={instructions:?}" ); } diff --git a/crates/codegen/src/error.rs b/crates/codegen/src/error.rs index 86ac86244ad..5e66d6cf1b8 100644 --- a/crates/codegen/src/error.rs +++ b/crates/codegen/src/error.rs @@ -40,6 +40,8 @@ impl fmt::Display for CodegenError { pub enum InternalError { StackOverflow, StackUnderflow, + InconsistentStackDepth, + MalformedControlFlowGraph, MissingSymbol(String), } @@ -48,6 +50,8 @@ impl Display for InternalError { match self { Self::StackOverflow => write!(f, "stack overflow"), Self::StackUnderflow => write!(f, "stack underflow"), + Self::InconsistentStackDepth => write!(f, "inconsistent stack depth"), + Self::MalformedControlFlowGraph => write!(f, "malformed control flow graph."), Self::MissingSymbol(s) => write!( f, "The symbol '{s}' must be present in the symbol table, even when it is undefined in python." diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index db1119873b2..a2f5b224be8 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1,5 +1,3 @@ -use alloc::collections::VecDeque; -use core::cmp::Reverse; use core::ops; use crate::{IndexMap, IndexSet, error::InternalError}; @@ -13,8 +11,8 @@ use rustpython_compiler_core::{ bytecode::{ AnyInstruction, AnyOpcode, Arg, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, ExceptionTableEntry, - InstrDisplayContext, Instruction, IntrinsicFunction1, Label, OpArg, Opcode, - PseudoInstruction, PseudoOpcode, PyCodeLocationInfoKind, encode_exception_table, oparg, + InstrDisplayContext, Instruction, IntrinsicFunction1, OpArg, Opcode, PseudoInstruction, + PseudoOpcode, PyCodeLocationInfoKind, encode_exception_table, oparg, }, varint::{write_signed_varint, write_varint}, }; @@ -29,6 +27,7 @@ struct LineTableLocation { } pub(crate) const LINE_ONLY_LOCATION_OVERRIDE: i32 = -4; +pub(crate) const NEXT_LOCATION_OVERRIDE: i32 = -2; const MAX_INT_SIZE_BITS: u64 = 128; const MAX_COLLECTION_SIZE: usize = 256; @@ -197,36 +196,10 @@ pub struct InstructionInfo { pub location: SourceLocation, pub end_location: SourceLocation, pub except_handler: Option, - pub folded_from_nonliteral_expr: bool, /// Override line number for linetable (e.g., line 0 for module RESUME) pub lineno_override: Option, /// Number of CACHE code units emitted after this instruction pub cache_entries: u32, - /// Preserve a redundant jump until final emission so a zero-width jump - /// materializes as a line-marker NOP, matching CPython's late CFG shape. - pub preserve_redundant_jump_as_nop: bool, - /// Drop this NOP before line propagation if it still has no location. - pub remove_no_location_nop: bool, - /// This no-location NOP was created by CPython-style nop_out() folding. - pub folded_operand_nop: bool, - /// This instruction was emitted as part of a synthetic no-location exit. - pub no_location_exit: bool, - /// Keep this no-location NOP until line propagation when it starts a block. - pub preserve_block_start_no_location_nop: bool, - /// This is the success jump emitted after a non-final match case body. - pub match_success_jump: bool, - /// This is the final jump emitted by codegen_break()/codegen_continue() - /// after unwinding cleanup blocks. - pub break_continue_cleanup_jump: bool, - /// This is the final jump emitted by codegen_break() after unwinding the - /// iterator for a for-loop break. - pub for_loop_break_cleanup_jump: bool, - /// Keep this conditional jump's own location when the preceding TO_BOOL - /// normally propagates its condition location into the jump. - pub preserve_tobool_jump_location: bool, - /// Keep the jump location copied from the second STORE_FAST NOP created - /// by STORE_FAST_STORE_FAST fusion. - pub preserve_store_fast_store_fast_jump_location: bool, } /// Exception handler information for an instruction. @@ -244,25 +217,350 @@ fn set_to_nop(info: &mut InstructionInfo) { info.instr = Instruction::Nop.into(); info.arg = OpArg::new(0); info.target = BlockIdx::NULL; - info.folded_from_nonliteral_expr = false; info.cache_entries = 0; - info.preserve_redundant_jump_as_nop = false; - info.remove_no_location_nop = false; - info.folded_operand_nop = false; - info.no_location_exit = false; - info.preserve_block_start_no_location_nop = false; - info.match_success_jump = false; - info.break_continue_cleanup_jump = false; - info.for_loop_break_cleanup_jump = false; - info.preserve_tobool_jump_location = false; - info.preserve_store_fast_store_fast_jump_location = false; } fn nop_out_no_location(info: &mut InstructionInfo) { set_to_nop(info); info.lineno_override = Some(-1); - info.remove_no_location_nop = true; - info.folded_operand_nop = true; +} + +/// flowgraph.c basicblock_addop +fn basicblock_addop(block: &mut Block, mut info: InstructionInfo) { + if let Some(stale) = block.cpython_spare_instr_slots.first().copied() { + block.cpython_spare_instr_slots.remove(0); + info.except_handler = stale.except_handler; + } + info.target = BlockIdx::NULL; + block.instructions.push(info); +} + +/// flowgraph.c basicblock_add_jump +fn basicblock_add_jump_op( + block: &mut Block, + info: InstructionInfo, + target: BlockIdx, +) -> crate::InternalResult<()> { + if block + .instructions + .last() + .is_some_and(|last| last.instr.has_jump()) + { + return Err(InternalError::MalformedControlFlowGraph); + } + basicblock_addop(block, info); + block.instructions.last_mut().expect("missing jump").target = target; + Ok(()) +} + +/// flowgraph.c basicblock_insert_instruction +fn basicblock_insert_instruction(block: &mut Block, pos: usize, info: InstructionInfo) { + if !block.cpython_spare_instr_slots.is_empty() { + block.cpython_spare_instr_slots.remove(0); + } + block.instructions.insert(pos, info); +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct InstructionSequenceLabel(usize); + +impl InstructionSequenceLabel { + pub(crate) fn idx(self) -> usize { + self.0 + } +} + +#[derive(Clone, Copy)] +struct InstructionSequenceExceptHandlerInfo { + target_label: Option, + target_offset: Option, + stack_depth: u32, + preserve_lasti: bool, +} + +#[derive(Clone, Copy)] +struct InstructionSequenceEntry { + info: InstructionInfo, + target_label: Option, + target_offset: Option, + except_handler: Option, + is_target: bool, +} + +impl InstructionSequenceEntry { + fn new( + info: InstructionInfo, + target_label: Option, + except_handler: Option, + ) -> Self { + Self { + info, + target_label, + target_offset: None, + except_handler, + is_target: false, + } + } +} + +const INSTRUCTION_SEQUENCE_UNSET_LABEL: isize = -111; + +#[derive(Clone)] +enum InstructionSequenceLabelOffsets { + Active(Vec), + Applied, +} + +#[derive(Clone)] +pub(crate) struct InstructionSequence { + instrs: Vec, + label_map: InstructionSequenceLabelOffsets, + annotations_code: Option>, +} + +impl InstructionSequence { + pub(crate) fn new() -> Self { + Self { + instrs: Vec::new(), + label_map: InstructionSequenceLabelOffsets::Active(Vec::new()), + annotations_code: None, + } + } + + /// instruction_sequence.c _PyInstructionSequence_Addop asserts. + fn debug_check_addop(info: &InstructionInfo) { + let opcode = AnyOpcode::from(info.instr); + debug_assert!( + opcode.has_arg() || info.instr.has_target() || u32::from(info.arg) == 0, + "CPython _PyInstructionSequence_Addop requires either OPCODE_HAS_ARG, HAS_TARGET, or oparg == 0" + ); + debug_assert!( + u32::from(info.arg) < (1 << 30), + "CPython _PyInstructionSequence_Addop requires 0 <= oparg < (1 << 30)" + ); + } + + fn set_annotations_code(&mut self, annotations_code: Option>) { + debug_assert!(self.annotations_code.is_none()); + self.annotations_code = annotations_code; + } + + fn use_label(&mut self, label: InstructionSequenceLabel) { + let InstructionSequenceLabelOffsets::Active(label_map) = &mut self.label_map else { + panic!("instruction sequence label map already applied"); + }; + let old_len = label_map.len(); + if label_map.len() <= label.idx() { + label_map.resize(label.idx() + 1, INSTRUCTION_SEQUENCE_UNSET_LABEL); + } + for slot in &mut label_map[old_len..] { + *slot = INSTRUCTION_SEQUENCE_UNSET_LABEL; + } + label_map[label.idx()] = self.instrs.len() as isize; + } + + fn addop( + &mut self, + info: InstructionInfo, + target_label: Option, + except_handler: Option, + ) { + Self::debug_check_addop(&info); + self.instrs.push(InstructionSequenceEntry::new( + info, + target_label, + except_handler, + )); + } + + fn debug_check_pop_preserves_cpython_label_map(&self) { + let Some((popped_index, entry)) = self.instrs.len().checked_sub(1).and_then(|index| { + self.instrs.last().map(|entry| { + ( + isize::try_from(index).expect("too many instructions"), + entry, + ) + }) + }) else { + return; + }; + debug_assert!( + !entry.info.instr.has_target() + && entry.info.target == BlockIdx::NULL + && entry.target_label.is_none() + && entry.target_offset.is_none() + && entry.except_handler.is_none(), + "RustPython-only instruction-sequence pop must not remove CPython label/target state" + ); + let InstructionSequenceLabelOffsets::Active(label_map) = &self.label_map else { + debug_assert!(false, "cannot pop after CPython label map application"); + return; + }; + debug_assert!( + label_map.iter().all(|&target| target < popped_index), + "RustPython-only instruction-sequence pop must not change CPython label offsets" + ); + } + + fn pop(&mut self) -> Option { + self.debug_check_pop_preserves_cpython_label_map(); + self.instrs.pop().map(|entry| entry.info) + } + + fn last_info_mut(&mut self) -> Option<&mut InstructionInfo> { + self.instrs.last_mut().map(|entry| &mut entry.info) + } + + fn insert_instruction( + &mut self, + pos: usize, + info: InstructionInfo, + target_label: Option, + ) { + debug_assert!(pos <= self.instrs.len()); + Self::debug_check_addop(&info); + self.instrs + .insert(pos, InstructionSequenceEntry::new(info, target_label, None)); + if let InstructionSequenceLabelOffsets::Active(label_map) = &mut self.label_map { + for target in label_map.iter_mut() { + if *target >= pos as isize { + *target += 1; + } + } + } + } + + fn apply_label_map(&mut self) -> crate::InternalResult<()> { + let label_map = match core::mem::replace( + &mut self.label_map, + InstructionSequenceLabelOffsets::Applied, + ) { + InstructionSequenceLabelOffsets::Active(label_map) => label_map, + InstructionSequenceLabelOffsets::Applied => return Ok(()), + }; + let resolve_label = |label: InstructionSequenceLabel| -> crate::InternalResult { + label_map + .get(label.idx()) + .copied() + .filter(|target_index| *target_index >= 0) + .and_then(|target_index| usize::try_from(target_index).ok()) + .ok_or(InternalError::MalformedControlFlowGraph) + }; + for entry in &mut self.instrs { + if entry.info.instr.has_target() { + let label_id = entry + .target_label + .take() + .ok_or(InternalError::MalformedControlFlowGraph)?; + let target_index = resolve_label(label_id)?; + entry.info.arg = OpArg::new( + target_index + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); + entry.target_offset = Some(target_index); + } else if entry.target_label.take().is_some() { + return Err(InternalError::MalformedControlFlowGraph); + } + if let Some(handler) = &mut entry.except_handler + && let Some(label_id) = handler.target_label.take() + { + handler.target_offset = Some(resolve_label(label_id)?); + } + } + Ok(()) + } + + fn mark_targets(&mut self) -> crate::InternalResult<()> { + for entry in &mut self.instrs { + entry.is_target = false; + } + let targets: Vec = self + .instrs + .iter() + .filter_map(|entry| entry.target_offset) + .collect(); + for target_offset in targets { + let target = self + .instrs + .get_mut(target_offset) + .ok_or(InternalError::MalformedControlFlowGraph)?; + target.is_target = true; + } + Ok(()) + } +} + +/// flowgraph.c _PyCfg_ToInstructionSequence +fn cfg_to_instruction_sequence( + blocks: &mut [Block], + block_order: &[BlockIdx], +) -> crate::InternalResult { + for block in blocks.iter_mut() { + block.cpython_label_id = None; + } + + for (label_id, block_idx) in block_order.iter().copied().enumerate() { + blocks[block_idx.idx()].cpython_label_id = Some(InstructionSequenceLabel(label_id)); + } + + let mut block_to_label = vec![None; blocks.len()]; + for block_idx in block_order.iter().copied() { + block_to_label[block_idx.idx()] = blocks[block_idx.idx()].cpython_label_id; + } + + let mut instr_sequence = InstructionSequence::new(); + for block_idx in block_order.iter().copied() { + let block_label = blocks[block_idx.idx()] + .cpython_label_id + .ok_or(InternalError::MalformedControlFlowGraph)?; + instr_sequence.use_label(block_label); + + for info in &blocks[block_idx.idx()].instructions { + let mut info = *info; + let target_label = if info.target != BlockIdx::NULL { + if !info.instr.has_target() { + return Err(InternalError::MalformedControlFlowGraph); + } + let label_id = block_to_label + .get(info.target.idx()) + .copied() + .flatten() + .ok_or(InternalError::MalformedControlFlowGraph)?; + info.arg = OpArg::new( + label_id + .idx() + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); + info.target = BlockIdx::NULL; + Some(label_id) + } else { + None + }; + + let except_handler = if let Some(handler) = info.except_handler.take() { + let label_id = block_to_label + .get(handler.handler_block.idx()) + .copied() + .flatten() + .ok_or(InternalError::MalformedControlFlowGraph)?; + Some(InstructionSequenceExceptHandlerInfo { + target_label: Some(label_id), + target_offset: None, + stack_depth: handler.stack_depth, + preserve_lasti: handler.preserve_lasti, + }) + } else { + None + }; + + instr_sequence.addop(info, target_label, except_handler); + } + } + + instr_sequence.apply_label_map()?; + Ok(instr_sequence) } // spell-checker:ignore petgraph @@ -281,20 +579,12 @@ pub struct Block { pub start_depth: Option, /// Whether this block is only reachable via exception table (b_cold) pub cold: bool, - /// Whether LOAD_FAST borrow optimization should be suppressed for this block. - pub disable_load_fast_borrow: bool, - /// Entry block for a try-else orelse suite split after POP_BLOCK. - pub try_else_orelse_entry: bool, - /// Whether this block corresponds to a compiler label (b_label). - pub label: bool, - /// Preserve empty unconditional-jump targets through optimize_load_fast_borrow. - pub load_fast_barrier: bool, - /// Let optimize_load_fast_borrow pass through this empty compiler artifact. - pub load_fast_passthrough: bool, - /// Continuation label that CPython attaches to a preceding empty block. - pub load_fast_label_reuse_passthrough: bool, - /// If-expression orelse label emitted inside another conditional statement. - pub conditional_ifexp_orelse_entry: bool, + /// CPython `basicblock.b_label` used by translate_jump_labels_to_targets. + cpython_label_id: Option, + /// CPython keeps `b_instr` allocated beyond `b_iused`. Instructions removed + /// by compaction remain in those spare slots until `basicblock_next_instr()` + /// reuses them. + cpython_spare_instr_slots: Vec, } impl Default for Block { @@ -306,13 +596,185 @@ impl Default for Block { preserve_lasti: false, start_depth: None, cold: false, - disable_load_fast_borrow: false, - try_else_orelse_entry: false, - label: false, - load_fast_barrier: false, - load_fast_passthrough: false, - load_fast_label_reuse_passthrough: false, - conditional_ifexp_orelse_entry: false, + cpython_label_id: None, + cpython_spare_instr_slots: Vec::new(), + } + } +} + +impl Block { + pub(crate) fn has_cpython_cfg_label(&self) -> bool { + self.cpython_label_id.is_some() + } +} + +#[derive(Clone, Debug)] +pub(crate) struct InstructionSequenceLabelMap { + next_free_label: usize, + block_labels: Vec, + /// Direct-CFG shadow for CPython labels that map to the same instruction + /// offset in `_PyInstructionSequence_UseLabel()`. + direct_block_by_label: Vec>, +} + +impl InstructionSequenceLabelMap { + pub(crate) fn new() -> Self { + Self { + next_free_label: 0, + block_labels: vec![InstructionSequenceLabel(0)], + direct_block_by_label: vec![Some(BlockIdx::new(0))], + } + } + + /// instruction_sequence.c _PyInstructionSequence_NewLabel + fn new_label(&mut self) -> InstructionSequenceLabel { + self.next_free_label += 1; + let label = InstructionSequenceLabel(self.next_free_label); + if self.direct_block_by_label.len() <= label.idx() { + self.direct_block_by_label.resize(label.idx() + 1, None); + } + label + } + + pub(crate) fn push_unmapped_label(&mut self) { + let label = self.new_label(); + let block = BlockIdx( + self.block_labels + .len() + .to_u32() + .expect("too many direct-CFG blocks"), + ); + self.direct_block_by_label[label.idx()] = Some(block); + self.block_labels.push(label); + } + + fn label_for_block(&self, block: BlockIdx) -> InstructionSequenceLabel { + debug_assert_ne!(block, BlockIdx::NULL); + self.block_labels + .get(block.idx()) + .copied() + .expect("basic block must have an instruction-sequence label") + } + + fn block_for_label(&self, label: InstructionSequenceLabel) -> Option { + self.direct_block_by_label + .get(label.idx()) + .copied() + .flatten() + } + + pub(crate) fn resolve_label(&self, block: BlockIdx) -> BlockIdx { + if block == BlockIdx::NULL { + return BlockIdx::NULL; + } + self.block_for_label(self.label_for_block(block)) + .unwrap_or_else(|| { + debug_assert!(false, "CPython label must map to a direct-CFG block"); + BlockIdx::NULL + }) + } + + pub(crate) fn resolve_label_to_block( + &self, + label: Option, + ) -> BlockIdx { + let Some(label) = label else { + return BlockIdx::NULL; + }; + self.block_for_label(label).unwrap_or_else(|| { + debug_assert!(false, "CPython label must map to a direct-CFG block"); + BlockIdx::NULL + }) + } + + pub(crate) fn use_label_at_block(&mut self, from: BlockIdx, to: BlockIdx) { + if from == BlockIdx::NULL || from == to { + return; + } + let from_label = self.label_for_block(from); + let to_block = self.resolve_label(to); + if to_block == BlockIdx::NULL { + debug_assert!(false, "CPython label target must map to a direct-CFG block"); + return; + } + self.direct_block_by_label[from_label.idx()] = Some(to_block); + } + + fn debug_check_for_blocks(&self, blocks_len: usize) { + debug_assert_eq!( + self.block_labels.len(), + blocks_len, + "every direct-CFG block must have a CPython instruction-sequence label" + ); + debug_assert_eq!(self.block_labels[0], InstructionSequenceLabel(0)); + debug_assert!(self.next_free_label + 1 >= self.block_labels.len()); + let mut seen_labels = vec![false; self.next_free_label + 1]; + for &label in &self.block_labels { + debug_assert!( + label.idx() <= self.next_free_label, + "direct-CFG block labels must come from _PyInstructionSequence_NewLabel()" + ); + debug_assert!( + !seen_labels[label.idx()], + "direct-CFG blocks must not share a CPython instruction-sequence label" + ); + seen_labels[label.idx()] = true; + } + for &block in &self.direct_block_by_label { + if let Some(block) = block { + debug_assert!( + block.idx() < blocks_len, + "CPython label must map to an existing direct-CFG block" + ); + } + } + for &label in &self.block_labels { + debug_assert!( + self.block_for_label(label) + .is_some_and(|block| block.idx() < blocks_len), + "direct-CFG block label must map to a direct-CFG block" + ); + } + } + + fn debug_check_label_blocks_match_instruction_sequence( + &self, + instr_sequence: &InstructionSequence, + ) { + let InstructionSequenceLabelOffsets::Active(label_map) = &instr_sequence.label_map else { + debug_assert!( + false, + "direct-CFG label map must be checked before CPython label-map application" + ); + return; + }; + for (label_idx, block) in self.direct_block_by_label.iter().copied().enumerate() { + let Some(block) = block else { + continue; + }; + let Some(&label_offset) = label_map.get(label_idx) else { + continue; + }; + if label_offset < 0 { + continue; + } + let Some(block_label) = self.block_labels.get(block.idx()).copied() else { + debug_assert!( + false, + "CPython label must map to an existing direct-CFG block" + ); + continue; + }; + let Some(&block_offset) = label_map.get(block_label.idx()) else { + continue; + }; + if block_offset < 0 { + continue; + } + debug_assert!( + label_offset == block_offset, + "direct-CFG labels may share a block only when CPython maps them to the same instruction offset" + ); } } } @@ -324,7 +786,9 @@ pub struct CodeInfo { pub blocks: Vec, pub current_block: BlockIdx, - pub annotations_blocks: Option>, + pub(crate) instr_sequence: InstructionSequence, + pub(crate) instr_sequence_label_map: InstructionSequenceLabelMap, + pub(crate) annotations_instr_sequence: Option, pub metadata: CodeUnitMetadata, @@ -344,210 +808,251 @@ pub struct CodeInfo { // u_in_conditional_block pub in_conditional_block: u32, - // Track when compiling the final direct statement in a sync with body. - pub in_final_with_cleanup_statement: u32, - - // Track when compiling the orelse suite of try/except. - pub in_try_else_orelse: u32, - // PEP 649: Next index for conditional annotation tracking // u_next_conditional_annotation_index pub next_conditional_annotation_index: u32, } impl CodeInfo { - pub fn finalize_code( - mut self, - opts: &crate::compile::CompileOpts, - ) -> crate::InternalResult { - self.splice_annotations_blocks(); - if self.flags.contains(CodeFlags::OPTIMIZED) { - // CPython's cfg_builder_maybe_start_new_block() starts a fresh - // basicblock before emitting any instruction after a terminator, - // including conditional jumps. Keep that invariant before - // optimize_cfg-style constant folding so blocks that fold to empty - // remain as b_next barriers for optimize_load_fast(). - split_blocks_at_jumps(&mut self.blocks); - } - // CPython-style per-block forward walk that interleaves tuple, list, - // set, unary, and binop folding. Matches optimize_basic_block() in - // flowgraph.c so co_consts get the same insertion order CPython - // produces (e.g. negated literals like -1 land in co_consts at their - // source position, not before every tuple folded by a separate pass). - self.fold_constants_per_block(); - // Legacy global passes still run after the walker so any cascade the - // single per-block iteration missed gets picked up. - self.fold_binop_constants(); - self.fold_unary_constants(); - self.fold_binop_constants(); // re-run after unary folding: -1 + 2 → 1 - self.fold_unary_constants(); // re-run after binop folding: -(2.0 ** -54) → const - self.fold_tuple_constants(); - self.fold_binop_constants(); // re-run after tuple folding: (1,) * 2 → const - self.fold_list_constants(); - self.fold_set_constants(); - self.optimize_lists_and_sets(); - self.convert_to_load_small_int(); + pub(crate) fn addop_to_instr_sequence( + &mut self, + mut info: InstructionInfo, + ) -> crate::InternalResult<()> { + let target_label = if info.instr.has_target() && info.target != BlockIdx::NULL { + let label = self.instr_sequence_label_map.label_for_block(info.target); + info.arg = OpArg::new( + label + .idx() + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); + info.target = BlockIdx::NULL; + Some(label) + } else { + None + }; + self.instr_sequence.addop(info, target_label, None); + Ok(()) + } - // DCE always runs (removes dead code after terminal instructions) - self.dce(); - // Peephole optimizer handles constant and compare folding. - self.peephole_optimize(); - // Per-block walker first to preserve CPython-style instruction-order - // const registration after peephole transformations exposed new - // foldable patterns. - self.fold_constants_per_block(); - self.fold_tuple_constants(); - self.fold_binop_constants(); - self.fold_list_constants(); - self.fold_set_constants(); - self.optimize_lists_and_sets(); - self.convert_to_load_small_int(); - // CPython's CFG builder starts a new basic block after a terminator. - // Peephole constant-jump folding can create new terminators, so split - // before DCE clears unreachable successor instructions; otherwise the - // empty placeholders CPython leaves in b_next are lost. - split_blocks_at_jumps(&mut self.blocks); - self.dce(); + pub(crate) fn addop_to_instr_sequence_with_target_label( + &mut self, + mut info: InstructionInfo, + target_label: InstructionSequenceLabel, + ) -> crate::InternalResult<()> { + if !info.instr.has_target() { + return Err(InternalError::MalformedControlFlowGraph); + } + info.arg = OpArg::new( + target_label + .idx() + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); + info.target = BlockIdx::NULL; + self.instr_sequence.addop(info, Some(target_label), None); + Ok(()) + } - // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) - // Split blocks so each block has at most one branch as its last instruction - split_blocks_at_jumps(&mut self.blocks); - mark_except_handlers(&mut self.blocks); - label_exception_targets(&mut self.blocks); - // CPython's CFG builder does not leave empty unconditional-jump targets - // in front of small exit blocks. Redirect only unconditional jumps - // here so inline_small_or_no_lineno_blocks() can see direct exit - // targets without erasing conditional target NOP anchors. - redirect_empty_unconditional_jump_targets(&mut self.blocks); - // CPython optimize_cfg starts by inlining tiny exit/no-lineno blocks - // before unreachable elimination and later jump cleanup. - inline_small_or_no_lineno_blocks(&mut self.blocks); - // optimize_cfg: jump threading (before push_cold_blocks_to_end) - jump_threading(&mut self.blocks); - self.eliminate_unreachable_blocks_without_exception_roots(); - // CPython optimize_cfg resolves line numbers before local checks and - // superinstruction insertion, so fusion decisions see propagated - // source locations. - resolve_line_numbers(&mut self.blocks); - // CPython flowgraph.c::optimize_cfg() runs optimize_basic_block() - // after the first resolve_line_numbers(). Keep tuple-unpack SWAP - // creation, duplicate STORE_FAST cleanup, and apply_static_swaps() - // here so synthetic no-location exits inherit the same pre-swap - // source locations as CPython. - self.optimize_build_tuple_unpack(); - self.eliminate_dead_stores(); - self.apply_static_swaps(); - self.remove_nops(); - self.add_checks_for_loads_of_uninitialized_variables(); - // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before - // later jump normalization / block reordering can create adjacencies - // that never exist at this stage in flowgraph.c. - self.insert_superinstructions(); - self.remove_redundant_const_pop_top_pairs(); - inline_single_predecessor_artificial_expr_exit_blocks(&mut self.blocks); - push_cold_blocks_to_end(&mut self.blocks); - // CPython resolves line numbers again after cold-block extraction. - resolve_line_numbers(&mut self.blocks); - reorder_conditional_chain_and_jump_back_blocks(&mut self.blocks); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); + pub(crate) fn pop_instr_sequence(&mut self) -> Option { + self.instr_sequence.pop() + } - // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) - normalize_jumps(&mut self.blocks); - reorder_conditional_exit_and_jump_blocks(&mut self.blocks); - reorder_conditional_jump_and_exit_blocks(&mut self.blocks); - reorder_conditional_break_continue_blocks(&mut self.blocks); - reorder_conditional_explicit_continue_scope_exit_blocks(&mut self.blocks); - reorder_conditional_implicit_continue_scope_exit_blocks(&mut self.blocks); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); - reorder_exception_handler_conditional_continue_scope_exit_blocks(&mut self.blocks); - deduplicate_adjacent_jump_back_blocks(&mut self.blocks); - reorder_conditional_body_and_implicit_continue_blocks(&mut self.blocks); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, false); - self.dce(); // re-run within-block DCE after normalize_jumps creates new instructions - self.eliminate_unreachable_blocks(); - resolve_line_numbers(&mut self.blocks); - // Keep these empty targets until after optimize_load_fast_borrow. - // CPython's optimize_load_fast() visits empty blocks directly; because - // basicblock_last_instr() is NULL, they do not push fallthrough. - materialize_empty_conditional_exit_targets(&mut self.blocks); - retarget_assert_conditional_jumps_to_empty_predecessor(&mut self.blocks); - canonicalize_empty_label_blocks(&mut self.blocks); - inline_small_fast_return_blocks(&mut self.blocks); - duplicate_end_returns(&mut self.blocks, &self.metadata); - retarget_conditional_jumps_to_empty_while_exit_epilogue(&mut self.blocks); - duplicate_fallthrough_jump_back_targets(&mut self.blocks); - duplicate_shared_jump_back_targets(&mut self.blocks); - self.dce(); // truncate after terminal in blocks that got return duplicated - self.eliminate_unreachable_blocks(); // remove now-unreachable last block - remove_redundant_nops_and_jumps(&mut self.blocks); - // Some jump-only blocks only appear after late CFG cleanup. Thread them - // once more so loop backedges stay direct instead of becoming - // JUMP_FORWARD -> JUMP_BACKWARD chains. - jump_threading_unconditional(&mut self.blocks); - self.eliminate_unreachable_blocks(); - remove_redundant_nops_and_jumps(&mut self.blocks); - inline_with_suppress_return_blocks(&mut self.blocks); - inline_pop_except_return_blocks(&mut self.blocks); - duplicate_named_except_cleanup_returns(&mut self.blocks, &self.metadata); - self.eliminate_unreachable_blocks(); - resolve_line_numbers(&mut self.blocks); - reorder_lineful_jump_back_runs_by_descending_line(&mut self.blocks); - let cellfixedoffsets = build_cellfixedoffsets( - &self.metadata.varnames, - &self.metadata.cellvars, - &self.metadata.freevars, - ); - // Late CFG cleanup can create or reshuffle handler entry blocks. - // Refresh exceptional block flags before optimize_load_fast_borrow so - // borrow loads are not introduced into exception-handler paths. - mark_except_handlers(&mut self.blocks); - // CPython's optimize_load_fast runs with block start depths already known. - // Compute them here so the abstract stack simulation can use the real - // CFG entry depth for each block. - let max_stackdepth = self.max_stackdepth()?; - // Match CPython order: pseudo ops are lowered after stackdepth - // calculation, then redundant NOPs from pseudo lowering are removed - // before optimize_load_fast. - convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); - remove_redundant_nops_and_jumps(&mut self.blocks); - inline_named_except_cleanup_jump_blocks(&mut self.blocks, &self.metadata); - deduplicate_adjacent_pop_except_jump_back_blocks(&mut self.blocks); - self.eliminate_unreachable_blocks(); - remove_redundant_nops_and_jumps(&mut self.blocks); - // CPython optimize_load_fast() runs after CFG cleanup, so synthetic - // empty labels that are not intended as load-fast barriers must not - // stop its worklist traversal. - redirect_load_fast_passthrough_targets(&mut self.blocks); - self.compute_load_fast_start_depths(); - // optimize_load_fast: after normalize_jumps - self.optimize_load_fast_borrow(); - // Assembly/late cleanup still wants concrete instruction targets, so - // redirect empty targets after preserving CPython's load-fast barrier. - redirect_empty_block_targets(&mut self.blocks); - self.apply_static_swaps(); - self.optimize_load_global_push_null(); - self.reorder_entry_prefix_cell_setup(); - self.remove_unused_consts(); + pub(crate) fn set_last_instr_sequence_lineno_override(&mut self, lineno_override: i32) { + if let Some(last) = self.instr_sequence.last_info_mut() { + last.lineno_override = Some(lineno_override); + } + } - let Self { - flags, - source_path, - private: _, // private is only used during compilation + pub(crate) fn use_instr_sequence_label(&mut self, block: BlockIdx) { + let label = self.instr_sequence_label_map.label_for_block(block); + self.instr_sequence.use_label(label); + } - mut blocks, - current_block: _, - annotations_blocks: _, - metadata, - static_attributes: _, - in_inlined_comp: _, - fblock: _, - symbol_table_index: _, + pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) { + let label = self.instr_sequence_label_map.label_for_block(block); + self.blocks[block.idx()].cpython_label_id = Some(label); + } + + pub(crate) fn resolve_instr_sequence_label(&self, block: BlockIdx) -> BlockIdx { + self.instr_sequence_label_map.resolve_label(block) + } + + pub(crate) fn block_for_instr_sequence_label( + &self, + label: Option, + ) -> BlockIdx { + self.instr_sequence_label_map.resolve_label_to_block(label) + } + + pub(crate) fn use_instr_sequence_label_at_block(&mut self, from: BlockIdx, to: BlockIdx) { + self.instr_sequence_label_map.use_label_at_block(from, to); + } + + pub(crate) fn instr_sequence_label_for_block( + &self, + block: BlockIdx, + ) -> Option { + if block == BlockIdx::NULL { + None + } else { + Some(self.instr_sequence_label_map.label_for_block(block)) + } + } + + pub(crate) fn insert_start_setup_cleanup(&mut self, handler_block: BlockIdx) { + let handler_label = self.instr_sequence_label_map.label_for_block(handler_block); + self.instr_sequence.insert_instruction( + 0, + InstructionInfo { + instr: PseudoInstruction::SetupCleanup { + delta: Arg::marker(), + } + .into(), + arg: OpArg::new(handler_label.idx().to_u32().expect("too many labels")), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(-1), + cache_entries: 0, + }, + Some(handler_label), + ); + } + + fn take_recorded_instr_sequence(&mut self) -> crate::InternalResult { + let mut instr_sequence = + core::mem::replace(&mut self.instr_sequence, InstructionSequence::new()); + if let Some(mut annotations_instr_sequence) = self.annotations_instr_sequence.take() { + annotations_instr_sequence.apply_label_map()?; + instr_sequence.set_annotations_code(Some(Box::new(annotations_instr_sequence))); + } + Ok(instr_sequence) + } + + /// flowgraph.c cfg_builder_check + fn debug_check_recorded_cfg_builder(&self) { + debug_assert!(!self.blocks.is_empty()); + debug_assert!(!self.blocks[0].instructions.is_empty()); + debug_assert!(!self.instr_sequence.instrs.is_empty()); + debug_assert!(self.current_block.idx() < self.blocks.len()); + self.instr_sequence_label_map + .debug_check_for_blocks(self.blocks.len()); + self.instr_sequence_label_map + .debug_check_label_blocks_match_instruction_sequence(&self.instr_sequence); + for block in &self.blocks { + if block.next != BlockIdx::NULL { + debug_assert!(block.next.idx() < self.blocks.len()); + } + for instr in &block.instructions { + debug_assert!(!instr.instr.is_assembler()); + if instr.target != BlockIdx::NULL { + debug_assert!(instr.target.idx() < self.blocks.len()); + } + } + } + } + + fn prepare_cfg_from_codegen(&mut self) -> crate::InternalResult { + // CPython compile.c optimize_and_assemble_code_unit passes + // u_instr_sequence directly into flowgraph.c _PyCfg_FromInstructionSequence(). + self.debug_check_recorded_cfg_builder(); + self.take_recorded_instr_sequence() + } + + fn optimize_code_unit( + &mut self, + instr_sequence: InstructionSequence, + ) -> crate::InternalResult<()> { + // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) + self.blocks = cfg_from_instruction_sequence(instr_sequence)?; + translate_jump_labels_to_targets(&mut self.blocks)?; + mark_except_handlers(&mut self.blocks)?; + label_exception_targets(&mut self.blocks)?; + // CPython optimize_cfg() starts with check_cfg() and raises + // SystemError if a jump or scope exit is not the last instruction in + // its block. + check_cfg(&self.blocks)?; + inline_small_or_no_lineno_blocks(&mut self.blocks); + // CPython does not re-run instruction-sequence label-map/CFG conversion + // after this point. Unreferenced label blocks left by jump inlining + // remain block boundaries and can preserve line-marker NOPs. + self.remove_unreachable_blocks(); + // CPython optimize_cfg resolves line numbers before local checks and + // superinstruction insertion, so fusion decisions see propagated + // source locations. + resolve_line_numbers(&mut self.blocks); + // CPython optimize_cfg() runs optimize_load_const() and then + // optimize_basic_block() after line numbers are resolved. + self.convert_to_load_small_int(); + self.peephole_optimize(); + self.convert_to_load_small_int(); + self.optimize_basic_blocks()?; + self.remove_redundant_nops_and_pairs(); + // CPython optimize_cfg() removes newly-unreachable blocks and + // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes + // unused constants. + self.remove_unreachable_blocks(); + remove_redundant_nops_and_jumps(&mut self.blocks)?; + debug_assert!(no_redundant_jumps(&self.blocks)); + self.remove_unused_consts(); + self.add_checks_for_loads_of_uninitialized_variables(); + // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before + // later jump normalization / block reordering can create adjacencies + // that never exist at this stage in flowgraph.c. + self.insert_superinstructions(); + push_cold_blocks_to_end(&mut self.blocks)?; + // CPython resolves line numbers again after cold-block extraction. + resolve_line_numbers(&mut self.blocks); + Ok(()) + } + + fn optimized_cfg_to_instruction_sequence(&mut self) -> crate::InternalResult<(u32, usize)> { + // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) + convert_pseudo_conditional_jumps(&mut self.blocks); + let max_stackdepth = self.max_stackdepth()?; + debug_assert!( + !self.flags.intersects( + CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR + ) || max_stackdepth != 0 + ); + let nlocalsplus = self.prepare_localsplus(); + // Match CPython order: pseudo ops are lowered after stackdepth and + // localsplus preparation, before normalize_jumps. + convert_pseudo_ops(&mut self.blocks)?; + normalize_jumps(&mut self.blocks)?; + debug_assert!(no_redundant_jumps(&self.blocks)); + // optimize_load_fast: after normalize_jumps + self.optimize_load_fast_borrow(); + + Ok((max_stackdepth, nlocalsplus)) + } + + pub fn finalize_code( + mut self, + opts: &crate::compile::CompileOpts, + ) -> crate::InternalResult { + let instr_sequence = self.prepare_cfg_from_codegen()?; + self.optimize_code_unit(instr_sequence)?; + let (max_stackdepth, nlocalsplus) = self.optimized_cfg_to_instruction_sequence()?; + + let Self { + flags, + source_path, + private: _, // private is only used during compilation + + mut blocks, + current_block: _, + instr_sequence: _, + instr_sequence_label_map: _, + annotations_instr_sequence: _, + metadata, + static_attributes: _, + in_inlined_comp: _, + fblock: _, + symbol_table_index: _, in_conditional_block: _, - in_final_with_cleanup_statement: _, - in_try_else_orelse: _, next_conditional_annotation_index: _, } = self; @@ -571,106 +1076,12 @@ impl CodeInfo { let mut locations = Vec::new(); let mut linetable_locations: Vec = Vec::new(); - // Build cellfixedoffsets for cell-local merging - let cellfixedoffsets = + // Rebuild and adjust cellfixedoffsets for localsplus metadata. Pseudo + // lowering and deref operand fixups already ran in + // optimized_cfg_to_instruction_sequence(). + let mut cellfixedoffsets = build_cellfixedoffsets(&varname_cache, &cellvar_cache, &freevar_cache); - // Convert pseudo ops (LoadClosure uses cellfixedoffsets) and fixup DEREF opargs - convert_pseudo_ops(&mut blocks, &cellfixedoffsets); - fixup_deref_opargs(&mut blocks, &cellfixedoffsets); - // Remove redundant NOPs, keeping line-marker NOPs only when - // they are needed to preserve tracing. - let mut block_order = Vec::new(); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - block_order.push(current); - current = blocks[current.idx()].next; - } - for block_idx in block_order { - let bi = block_idx.idx(); - let mut src_instructions = core::mem::take(&mut blocks[bi].instructions); - let mut kept = Vec::with_capacity(src_instructions.len()); - let mut prev_lineno = -1i32; - - for src in 0..src_instructions.len() { - let instr = src_instructions[src]; - let lineno = instr - .lineno_override - .unwrap_or_else(|| instr.location.line.get() as i32); - let mut remove = false; - - if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if instr.preserve_redundant_jump_as_nop { - remove = false; - } else if lineno < 0 || prev_lineno == lineno { - remove = true; - } else if src < src_instructions.len() - 1 { - if src_instructions[src + 1].instr.is_block_push() { - remove = false; - } else if src_instructions[src + 1].folded_from_nonliteral_expr { - remove = true; - } else { - let next_lineno = src_instructions[src + 1] - .lineno_override - .unwrap_or_else(|| { - src_instructions[src + 1].location.line.get() as i32 - }); - if next_lineno == lineno { - remove = true; - } else if next_lineno < 0 { - copy_instruction_location(instr, &mut src_instructions[src + 1]); - remove = true; - } - } - } else { - let mut next = blocks[bi].next; - while next != BlockIdx::NULL && blocks[next.idx()].instructions.is_empty() { - next = blocks[next.idx()].next; - } - if next != BlockIdx::NULL { - let mut next_lineno = None; - for next_instr in &blocks[next.idx()].instructions { - let line = next_instr - .lineno_override - .unwrap_or_else(|| next_instr.location.line.get() as i32); - if matches!(next_instr.instr.real(), Some(Instruction::Nop)) - && line < 0 - { - continue; - } - next_lineno = Some(line); - break; - } - if next_lineno.is_some_and(|line| line == lineno) { - remove = true; - } - } - } - } - - if !remove { - kept.push(instr); - prev_lineno = lineno; - } - } - - blocks[bi].instructions = kept; - } - - // Final DCE: truncate instructions after terminal ops in linearized blocks. - // This catches dead code created by normalize_jumps after the initial DCE. - for block in &mut blocks { - if let Some(pos) = block - .instructions - .iter() - .position(|ins| ins.instr.is_scope_exit() || ins.instr.is_unconditional_jump()) - { - block.instructions.truncate(pos + 1); - } - } - - resolve_next_location_overrides(&mut blocks); - propagate_store_fast_store_fast_jump_locations(&mut blocks); - propagate_tobool_conditional_jump_locations(&mut blocks); + let numdropped = fix_cellfixedoffsets(varname_cache.len(), &mut cellfixedoffsets); // Pre-compute cache_entries for real (non-pseudo) instructions for block in &mut blocks { @@ -681,155 +1092,107 @@ impl CodeInfo { } } - let mut block_to_offset = vec![Label::from_u32(0); blocks.len()]; - // block_to_index: maps block idx to instruction index (for exception table) - // This is the index into the final instructions array, including EXTENDED_ARG and CACHE - let mut block_to_index = vec![0u32; blocks.len()]; - // CPython's jump resolver starts from the target instruction index in - // the instruction sequence, before CACHE and EXTENDED_ARG code units. - let mut block_to_instr_index = vec![0u32; blocks.len()]; + let block_order = layout_block_order(&blocks); + let mut instr_sequence = cfg_to_instruction_sequence(&mut blocks, &block_order)?; + let mut instruction_offsets = vec![0u32; instr_sequence.instrs.len()]; + let mut end_offset; // The offset (in code units) of END_SEND from SEND in the yield-from sequence. const END_SEND_OFFSET: u32 = 5; loop { let mut num_instructions = 0; - let mut instr_index = 0; - for (idx, block) in iter_blocks(&blocks) { - block_to_offset[idx.idx()] = Label::from_u32(num_instructions as u32); - // block_to_index uses the same value as block_to_offset but as u32 - // because lasti in frame.rs is the index into instructions array - // and instructions array index == byte offset (each instruction is 1 CodeUnit) - block_to_index[idx.idx()] = num_instructions as u32; - block_to_instr_index[idx.idx()] = instr_index; - instr_index += block.instructions.len() as u32; - for instr in &block.instructions { - num_instructions += instr.arg.instr_size() + instr.cache_entries as usize; - } - } - let mut extended_forward_jumps = Vec::new(); - for (idx, block) in iter_blocks(&blocks) { - let mut current_offset = block_to_offset[idx.idx()].as_u32(); - for info in &block.instructions { - let instr_size = info.arg.instr_size(); - let cache_entries = info.cache_entries; - let offset_after = current_offset + instr_size as u32 + cache_entries; - if info.target != BlockIdx::NULL - && instr_size > 1 - && let Some(op) = info.instr.real() - && !matches!( - op, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - | Instruction::EndAsyncFor - ) - { - let target_offset = block_to_offset[info.target.idx()].as_u32(); - if target_offset >= offset_after { - extended_forward_jumps.push((current_offset, target_offset)); - } - } - current_offset = offset_after; - } + for (idx, instr) in instr_sequence.instrs.iter().enumerate() { + instruction_offsets[idx] = num_instructions as u32; + num_instructions += instr.info.arg.instr_size() + instr.info.cache_entries as usize; } + end_offset = num_instructions as u32; instructions.reserve_exact(num_instructions); locations.reserve_exact(num_instructions); let mut recompile = false; - let mut next_block = BlockIdx(0); - while next_block != BlockIdx::NULL { - let block = &mut blocks[next_block]; - // Track current instruction offset for jump direction resolution - let mut current_offset = block_to_offset[next_block.idx()].as_u32(); - for info in &mut block.instructions { - let target = info.target; - let mut op = info.instr.expect_real(); + for (current_instr_index, entry) in instr_sequence.instrs.iter_mut().enumerate() { + // Track current instruction offset for jump offset resolution. + let current_offset = instruction_offsets[current_instr_index]; + { + let info = &mut entry.info; let old_arg_size = info.arg.instr_size(); let old_cache_entries = info.cache_entries; // Keep offsets fixed within this pass: changes in jump // arg/cache sizes only take effect in the next iteration. let offset_after = current_offset + old_arg_size as u32 + old_cache_entries; - - if target != BlockIdx::NULL { - let target_offset = block_to_offset[target.idx()].as_u32(); - if info.instr.is_unconditional_jump() && target_offset == offset_after { - op = Opcode::Nop.into(); - info.instr = op.into(); - info.target = BlockIdx::NULL; - recompile = true; - let updated_cache = op.cache_entries() as u32; - recompile |= updated_cache != old_cache_entries; - info.cache_entries = updated_cache; - let new_arg = OpArg::NULL; - recompile |= new_arg.instr_size() != old_arg_size; - info.arg = new_arg; - } else { - // Direction must be based on concrete instruction offsets. - // Empty blocks can share offsets, so block-order-based resolution - // may classify some jumps incorrectly. - op = match op.into() { - Opcode::JumpForward if target_offset < offset_after => { - Opcode::JumpBackward.into() + let op = match info.instr { + AnyInstruction::Pseudo( + PseudoInstruction::Jump { .. } + | PseudoInstruction::JumpNoInterrupt { .. }, + ) if entry.target_offset.is_none() => { + return Err(InternalError::MalformedControlFlowGraph); + } + // CPython assemble.c::resolve_unconditional_jumps() + // resolves pseudo JUMP/JUMP_NO_INTERRUPT after label-map + // application, using instruction indexes rather than CFG + // block order or byte offsets. + AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { + if entry.target_offset.expect("missing jump target") + > current_instr_index + { + Instruction::JumpForward { + delta: Arg::marker(), } - Opcode::JumpBackward if target_offset >= offset_after => { - Opcode::JumpForward.into() + } else { + Instruction::JumpBackward { + delta: Arg::marker(), } - Opcode::JumpBackwardNoInterrupt - if target_offset >= offset_after => - { - Opcode::JumpForward.into() + } + } + AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) => { + if entry.target_offset.expect("missing jump target") + > current_instr_index + { + Instruction::JumpForward { + delta: Arg::marker(), } - _ => op, - }; - info.instr = op.into(); - let updated_cache = op.cache_entries() as u32; - recompile |= updated_cache != old_cache_entries; - info.cache_entries = updated_cache; - let mut new_arg = if matches!(op, Instruction::EndAsyncFor) { - let arg = offset_after - .checked_sub(target_offset + END_SEND_OFFSET) - .expect("END_ASYNC_FOR target must be before instruction"); - OpArg::new(arg) - } else if matches!( - op.into(), - Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt - ) { - let arg = offset_after - .checked_sub(target_offset) - .expect("backward jump target must be before instruction"); - OpArg::new(arg) } else { - let arg = target_offset - .checked_sub(offset_after) - .expect("forward jump target must be after instruction"); - OpArg::new(arg) - }; - if matches!( - op.into(), - Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt - ) && u32::from(new_arg) == 0xff - && (block_to_instr_index[target.idx()] > 0xff - || extended_forward_jumps.iter().any( - |&(jump_offset, jump_target_offset)| { - target_offset < jump_offset - && jump_offset < current_offset - && current_offset < jump_target_offset - }, - )) - { - // CPython assemble.c::resolve_jump_offsets() - // bootstraps jump sizing from the unresolved - // target instruction index in i_oparg and loops - // until EXTENDED_ARG sizes stop changing. A - // backward jump exactly on the one-byte boundary - // can therefore remain at 256 if either its - // target index initially needed EXTENDED_ARG, or - // an already-extended forward jump between the - // target and jump crosses past it. - new_arg = OpArg::new(0x100); + Instruction::JumpBackwardNoInterrupt { + delta: Arg::marker(), + } } - recompile |= new_arg.instr_size() != old_arg_size; - info.arg = new_arg; } + _ => info.instr.expect_real(), + }; + + if let Some(target_index) = entry.target_offset { + let target_offset = instruction_offsets[target_index]; + // CPython assemble.c::resolve_jump_offsets() only + // converts label/instruction indexes to bytecode + // offsets here. Direction selection for pseudo + // unconditional jumps has already happened in + // resolve_unconditional_jumps(), and redundant-jump + // removal belongs to flowgraph.c optimization. + info.instr = op.into(); + let updated_cache = op.cache_entries() as u32; + recompile |= updated_cache != old_cache_entries; + info.cache_entries = updated_cache; + let new_arg = if matches!(op, Instruction::EndAsyncFor) { + let arg = offset_after + .checked_sub(target_offset + END_SEND_OFFSET) + .expect("END_ASYNC_FOR target must be before instruction"); + OpArg::new(arg) + } else if matches!( + op.into(), + Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt + ) { + let arg = offset_after + .checked_sub(target_offset) + .expect("backward jump target must be before instruction"); + OpArg::new(arg) + } else { + let arg = target_offset + .checked_sub(offset_after) + .expect("forward jump target must be after instruction"); + OpArg::new(arg) + }; + recompile |= new_arg.instr_size() != old_arg_size; + info.arg = new_arg; } let cache_count = info.cache_entries as usize; @@ -853,6 +1216,12 @@ impl CodeInfo { col: -1, end_col: -1, }, + Some(NEXT_LOCATION_OVERRIDE) => LineTableLocation { + line: NEXT_LOCATION_OVERRIDE, + end_line: NEXT_LOCATION_OVERRIDE, + col: NEXT_LOCATION_OVERRIDE, + end_col: NEXT_LOCATION_OVERRIDE, + }, Some(lineno) => LineTableLocation { line: lineno, end_line: info.end_location.line.get() as i32, @@ -883,9 +1252,7 @@ impl CodeInfo { cache_count, )); } - current_offset = offset_after; } - next_block = block.next; } if !recompile { @@ -897,6 +1264,12 @@ impl CodeInfo { linetable_locations.clear(); } + // CPython assemble.c::assemble_location_info() resolves NEXT_LOCATION + // after final instruction sizing, scanning backward through the + // instruction sequence. Non-terminators inherit the following + // instruction's location; terminators become NO_LOCATION. + resolve_next_locations(&instructions, &mut linetable_locations); + // Generate linetable from linetable_locations (supports line 0 for RESUME) let linetable = generate_linetable( &linetable_locations, @@ -910,7 +1283,8 @@ impl CodeInfo { ); // Generate exception table before moving source_path - let exceptiontable = generate_exception_table(&blocks, &block_to_index); + let exceptiontable = + generate_exception_table(&instr_sequence.instrs, &instruction_offsets, end_offset); // CPython builds u_cellvars in dictbytype() order, but the public // co_cellvars tuple follows localsplus order from assemble.c: @@ -930,11 +1304,11 @@ impl CodeInfo { let nlocals = varname_cache.len(); let ncells = cellvar_cache.len(); let nfrees = freevar_cache.len(); - let numdropped = cellvar_cache - .iter() - .filter(|cv| varname_cache.contains(cv.as_str())) - .count(); - let nlocalsplus = nlocals + ncells - numdropped + nfrees; + debug_assert_eq!( + nlocalsplus, + nlocals + ncells - numdropped + nfrees, + "CPython prepare_localsplus() result must match assemble.c localsplus sizing" + ); let mut localspluskinds = vec![0u8; nlocalsplus]; // Mark locals for kind in localspluskinds.iter_mut().take(nlocals) { @@ -987,148 +1361,143 @@ impl CodeInfo { }) } - fn dce(&mut self) { - // Truncate instructions after terminal instructions within each block - for block in &mut self.blocks { - let mut last_instr = None; - for (i, ins) in block.instructions.iter().enumerate() { - if ins.instr.is_scope_exit() || ins.instr.is_unconditional_jump() { - last_instr = Some(i); - break; - } - } - if let Some(i) = last_instr { - block.instructions.truncate(i + 1); - } - } - } - - fn reorder_entry_prefix_cell_setup(&mut self) { + /// flowgraph.c insert_prefix_instructions + fn insert_prefix_instructions(&mut self, cellfixedoffsets: &[u32]) { let Some(entry) = self.blocks.first_mut() else { return; }; let ncells = self.metadata.cellvars.len(); let nfrees = self.metadata.freevars.len(); - if ncells == 0 && nfrees == 0 { - return; - } + let firstlineno = self.metadata.firstlineno; - let prefix_len = entry - .instructions - .iter() - .take_while(|info| { - matches!( - info.instr.real(), - Some(Instruction::MakeCell { .. } | Instruction::CopyFreeVars { .. }) - ) - }) - .count(); - if prefix_len == 0 { - return; + if self + .flags + .intersects(CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR) + { + let location = SourceLocation { + line: firstlineno, + character_offset: OneIndexed::MIN, + }; + basicblock_insert_instruction( + entry, + 0, + InstructionInfo { + instr: Instruction::ReturnGenerator.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location, + end_location: location, + except_handler: None, + lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), + cache_entries: 0, + }, + ); + basicblock_insert_instruction( + entry, + 1, + InstructionInfo { + instr: Instruction::PopTop.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location, + end_location: location, + except_handler: None, + lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), + cache_entries: 0, + }, + ); } - let original_prefix = entry.instructions[..prefix_len].to_vec(); - let anchor = original_prefix[0]; - let rest = entry.instructions.split_off(prefix_len); - entry.instructions.clear(); + let mut sorted = vec![None; self.metadata.varnames.len() + ncells]; + for (oldindex, fixed) in cellfixedoffsets.iter().copied().take(ncells).enumerate() { + sorted[fixed as usize] = Some(oldindex + 1); + } + for (ncellsused, oldindex) in sorted.into_iter().flatten().enumerate() { + basicblock_insert_instruction( + entry, + ncellsused, + InstructionInfo { + instr: Instruction::MakeCell { i: Arg::marker() }.into(), + arg: OpArg::new((oldindex - 1) as u32), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(-1), + cache_entries: 0, + }, + ); + } if nfrees > 0 { - entry.instructions.push(InstructionInfo { - instr: Instruction::CopyFreeVars { n: Arg::marker() }.into(), - arg: OpArg::new(nfrees as u32), - ..anchor - }); + basicblock_insert_instruction( + entry, + 0, + InstructionInfo { + instr: Instruction::CopyFreeVars { n: Arg::marker() }.into(), + arg: OpArg::new(nfrees as u32), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(-1), + cache_entries: 0, + }, + ); } + } - let cellfixedoffsets = build_cellfixedoffsets( + /// flowgraph.c prepare_localsplus + fn prepare_localsplus(&mut self) -> usize { + let nlocals = self.metadata.varnames.len(); + let ncells = self.metadata.cellvars.len(); + let nfrees = self.metadata.freevars.len(); + let mut nlocalsplus = nlocals + ncells + nfrees; + let mut cellfixedoffsets = build_cellfixedoffsets( &self.metadata.varnames, &self.metadata.cellvars, &self.metadata.freevars, ); - let mut sorted = vec![None; self.metadata.varnames.len() + ncells]; - for (oldindex, fixed) in cellfixedoffsets.iter().copied().take(ncells).enumerate() { - sorted[fixed as usize] = Some(oldindex); - } - for oldindex in sorted.into_iter().flatten() { - entry.instructions.push(InstructionInfo { - instr: Instruction::MakeCell { i: Arg::marker() }.into(), - arg: OpArg::new(oldindex as u32), - ..anchor - }); - } - - entry.instructions.extend(rest); - } - /// Clear blocks that are unreachable (not entry, not a jump target, - /// and only reachable via fall-through from a terminal block). - fn eliminate_unreachable_blocks(&mut self) { - self.eliminate_unreachable_blocks_impl(true); - } + // This must be called before fix_cell_offsets(). + self.insert_prefix_instructions(&cellfixedoffsets); - fn eliminate_unreachable_blocks_without_exception_roots(&mut self) { - self.eliminate_unreachable_blocks_impl(false); + let numdropped = fix_cell_offsets(&mut self.blocks, nlocals, &mut cellfixedoffsets); + nlocalsplus -= numdropped; + nlocalsplus } - fn eliminate_unreachable_blocks_impl(&mut self, root_exception_handlers: bool) { + /// flowgraph.c remove_unreachable + fn remove_unreachable_blocks(&mut self) { let mut reachable = vec![false; self.blocks.len()]; reachable[0] = true; - if root_exception_handlers { - // Late Rust CFG cleanup can run after pseudo SETUP_* opcodes have - // been lowered. At that point exception-handler blocks still need - // to remain roots even when no concrete block-push instruction is - // left to point at them. - for (i, block) in self.blocks.iter().enumerate() { - if block.except_handler { - reachable[i] = true; - } + let mut stack = vec![BlockIdx(0)]; + while let Some(block_idx) = stack.pop() { + let idx = block_idx.idx(); + let block = &self.blocks[idx]; + let next = block.next; + if next != BlockIdx::NULL && block_has_fallthrough(block) && !reachable[next.idx()] { + reachable[next.idx()] = true; + stack.push(next); } - } - // Fixpoint: only mark targets of already-reachable blocks - let mut changed = true; - while changed { - changed = false; - for i in 0..self.blocks.len() { - if !reachable[i] { - continue; - } - // Mark jump targets and exception handlers - for ins in &self.blocks[i].instructions { - if ins.target != BlockIdx::NULL && !reachable[ins.target.idx()] { - reachable[ins.target.idx()] = true; - changed = true; - } - if let Some(eh) = &ins.except_handler - && !reachable[eh.handler_block.idx()] - { - reachable[eh.handler_block.idx()] = true; - changed = true; - } - } - // Mark fall-through - let next = self.blocks[i].next; - if next != BlockIdx::NULL - && !reachable[next.idx()] - && !self.blocks[i].instructions.last().is_some_and(|ins| { - ins.instr.is_scope_exit() || ins.instr.is_unconditional_jump() - }) + for ins in &block.instructions { + if (is_jump_instruction(ins) || ins.instr.is_block_push()) + && ins.target != BlockIdx::NULL + && !reachable[ins.target.idx()] { - reachable[next.idx()] = true; - changed = true; + reachable[ins.target.idx()] = true; + stack.push(ins.target); } } } - for i in 0..self.blocks.len() { - if !reachable[i] - && !preserves_cpython_unreachable_fallthrough_return_epilogue( - &self.blocks, - &reachable, - BlockIdx(i as u32), - ) - { + for block_idx in self.block_next_order() { + let i = block_idx.idx(); + let is_reachable = reachable[i]; + if !is_reachable { let block = &mut self.blocks[i]; block.instructions.clear(); + block.except_handler = false; } } } @@ -1257,25 +1626,9 @@ impl CodeInfo { prev = idx; } Self::instr_make_load_const(metadata, &mut block.instructions[i], folded_const); - block.instructions[i].folded_from_nonliteral_expr = false; true } - /// Fold constant unary operations following CPython fold_const_unaryop(). - fn fold_unary_constants(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - if Self::fold_unary_constant_at(&mut self.metadata, block, i) { - i = i.saturating_sub(1); - } else { - i += 1; - } - } - } - } - fn get_const_loading_instr_indices( block: &Block, mut start: usize, @@ -1313,29 +1666,15 @@ impl CodeInfo { let mut elements = Vec::with_capacity(size); for &j in &operand_indices { - let load_instr = &block.instructions[j]; - if load_instr.folded_from_nonliteral_expr { - return None; - } - elements.push(Self::get_const_value_from(metadata, load_instr)?); + elements.push(Self::get_const_value_from( + metadata, + &block.instructions[j], + )?); } Some((operand_indices, elements)) } - fn get_non_nop_instr_indices(block: &Block, start: usize, count: usize) -> Option> { - let mut indices = Vec::with_capacity(count); - for idx in start..block.instructions.len() { - if !matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { - indices.push(idx); - if indices.len() == count { - return Some(indices); - } - } - } - None - } - fn block_next_order(&self) -> Vec { let mut order = Vec::new(); let mut current = BlockIdx(0); @@ -1376,66 +1715,17 @@ impl CodeInfo { let Some(result_const) = Self::eval_binop(&left_val, &right_val, op) else { return false; }; - let folded_from_nonliteral_expr = operand_indices - .iter() - .any(|&idx| block.instructions[idx].folded_from_nonliteral_expr); for &idx in &operand_indices { nop_out_no_location(&mut block.instructions[idx]); } Self::instr_make_load_const(metadata, &mut block.instructions[i], result_const); - block.instructions[i].folded_from_nonliteral_expr = folded_from_nonliteral_expr; true } - /// Constant folding: fold LOAD_CONST/LOAD_SMALL_INT + LOAD_CONST/LOAD_SMALL_INT + BINARY_OP - /// into a single LOAD_CONST when the result is computable at compile time. - /// = fold_binops_on_constants in CPython flowgraph.c - fn fold_binop_constants(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - if Self::fold_binop_constant_at(&mut self.metadata, block, i) { - i = i.saturating_sub(1); - } else { - i += 1; - } - } - } - } - - /// CPython-style per-block forward walk that interleaves tuple, list, set, - /// unary, and binop constant folding. Mirrors optimize_basic_block() in - /// flowgraph.c so constants are registered in co_consts in instruction - /// order rather than in the order separate global passes would discover - /// them. CPython runs optimize_basic_block() once per basic block, with a - /// single forward scan, so this deliberately does not iterate to a fixed - /// point. - fn fold_constants_per_block(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - let _ = Self::fold_tuple_constant_at(&mut self.metadata, block, i) - || Self::optimize_iterable_or_contains_collection_at( - &mut self.metadata, - block, - i, - ) - || Self::fold_list_constant_at(&mut self.metadata, block, i) - || Self::fold_set_constant_at(&mut self.metadata, block, i) - || Self::fold_constant_intrinsic_list_to_tuple_at(&mut self.metadata, block, i) - || Self::fold_unary_constant_at(&mut self.metadata, block, i) - || Self::fold_binop_constant_at(&mut self.metadata, block, i); - i += 1; - } - } - } - - fn get_const_value_from_dummy(info: &InstructionInfo) -> Option<()> { - match info.instr.real() { - Some(Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }) => Some(()), - _ => None, + fn get_const_value_from_dummy(info: &InstructionInfo) -> Option<()> { + match info.instr.real() { + Some(Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }) => Some(()), + _ => None, } } @@ -2057,7 +2347,6 @@ impl CodeInfo { return true; } - let folded_from_nonliteral_expr = block.instructions[i].folded_from_nonliteral_expr; let Some((operand_indices, elements)) = Self::get_const_sequence(metadata, block, i, tuple_size) else { @@ -2074,7 +2363,6 @@ impl CodeInfo { block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); - block.instructions[i].folded_from_nonliteral_expr = folded_from_nonliteral_expr; true } @@ -2116,7 +2404,6 @@ impl CodeInfo { } let mut elements = Vec::with_capacity(consts_found); - let mut folded_from_nonliteral_expr = false; let mut expect_load = true; for idx in pos + 1..i { let instr = &block.instructions[idx]; @@ -2127,7 +2414,6 @@ impl CodeInfo { let Some(value) = Self::get_const_value_from(metadata, instr) else { return false; }; - folded_from_nonliteral_expr |= instr.folded_from_nonliteral_expr; elements.push(value); } else if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) || u32::from(instr.arg) != 1 @@ -2151,7 +2437,6 @@ impl CodeInfo { } .into(); block.instructions[i].arg = OpArg::new(const_idx as u32); - block.instructions[i].folded_from_nonliteral_expr = folded_from_nonliteral_expr; return true; } @@ -2231,14 +2516,12 @@ impl CodeInfo { true } - /// CPython's optimize_basic_block() calls optimize_lists_and_sets() in - /// place for BUILD_LIST/BUILD_SET. This handles the GET_ITER/CONTAINS_OP - /// subset there so small membership collections are folded before later - /// unary/binop constants in the same block. - fn optimize_iterable_or_contains_collection_at( + /// Port of CPython's flowgraph.c optimize_lists_and_sets(). + fn optimize_lists_and_sets_at( metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize, + nextop: Option, ) -> bool { let Some(instr) = block.instructions[i].instr.real() else { return false; @@ -2249,36 +2532,39 @@ impl CodeInfo { return false; } - let next_is_iter_or_contains = block - .instructions - .get(i + 1) - .and_then(|next| next.instr.real()) - .is_some_and(|next| { - matches!(next, Instruction::GetIter | Instruction::ContainsOp { .. }) - }); - if !next_is_iter_or_contains { - return false; - } - + let contains_or_iter = matches!( + nextop, + Some(Instruction::GetIter | Instruction::ContainsOp { .. }) + ); let seq_size = u32::from(block.instructions[i].arg) as usize; - if seq_size > STACK_USE_GUIDELINE { + if seq_size > STACK_USE_GUIDELINE + || (seq_size < MIN_CONST_SEQUENCE_SIZE && !contains_or_iter) + { return false; } let Some((operand_indices, elements)) = Self::get_const_sequence(metadata, block, i, seq_size) else { - if is_list { + if contains_or_iter && is_list { block.instructions[i].instr = Opcode::BuildTuple.into(); return true; } return false; }; - let const_data = if is_set { - ConstantData::Frozenset { elements } - } else { + if !contains_or_iter { + return if is_list { + Self::fold_list_constant_at(metadata, block, i) + } else { + Self::fold_set_constant_at(metadata, block, i) + }; + } + + let const_data = if is_list { ConstantData::Tuple { elements } + } else { + ConstantData::Frozenset { elements } }; let (const_idx, _) = metadata.consts.insert_full(const_data); let folded_loc = block.instructions[i].location; @@ -2299,275 +2585,6 @@ impl CodeInfo { true } - /// Constant folding: fold LOAD_CONST/LOAD_SMALL_INT + BUILD_TUPLE into LOAD_CONST tuple - /// fold_tuple_of_constants. This also folds constant list/set literals - /// in block order to match CPython's optimize_basic_block() const-table order. - fn fold_tuple_constants(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - if Self::fold_tuple_constant_at(&mut self.metadata, block, i) - || Self::fold_list_constant_at(&mut self.metadata, block, i) - || Self::fold_set_constant_at(&mut self.metadata, block, i) - { - i += 1; - continue; - } - i += 1; - } - } - } - - /// Fold constant list literals: LOAD_CONST* + BUILD_LIST N → - /// BUILD_LIST 0 + LOAD_CONST (tuple) + LIST_EXTEND 1 - fn fold_list_constants(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - let instr = &block.instructions[i]; - let Some(Instruction::BuildList { .. }) = instr.instr.real() else { - i += 1; - continue; - }; - - let list_size = u32::from(instr.arg) as usize; - if list_size == 0 || list_size > STACK_USE_GUIDELINE { - i += 1; - continue; - } - - let Some((operand_indices, elements)) = - Self::get_const_sequence(&self.metadata, block, i, list_size) - else { - i += 1; - continue; - }; - if list_size < MIN_CONST_SEQUENCE_SIZE { - i += 1; - continue; - } - - let tuple_const = ConstantData::Tuple { elements }; - let (const_idx, _) = self.metadata.consts.insert_full(tuple_const); - - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - let eh = block.instructions[i].except_handler; - - let build_idx = operand_indices[0]; - let const_idx_slot = operand_indices[1]; - - block.instructions[build_idx].instr = Instruction::BuildList { - count: Arg::marker(), - } - .into(); - block.instructions[build_idx].arg = OpArg::new(0); - block.instructions[build_idx].location = folded_loc; - block.instructions[build_idx].end_location = end_loc; - block.instructions[build_idx].except_handler = eh; - - block.instructions[const_idx_slot].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); - block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); - block.instructions[const_idx_slot].location = folded_loc; - block.instructions[const_idx_slot].end_location = end_loc; - block.instructions[const_idx_slot].except_handler = eh; - - // NOP the rest - for &j in &operand_indices[2..] { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - } - - // slot[i] (was BUILD_LIST) → LIST_EXTEND 1 - block.instructions[i].instr = Opcode::ListExtend.into(); - block.instructions[i].arg = OpArg::new(1); - - i += 1; - } - } - } - - /// Port of CPython's flowgraph.c optimize_lists_and_sets(). - /// - /// For GET_ITER / CONTAINS_OP users: - /// - Constant BUILD_LIST/BUILD_SET becomes LOAD_CONST tuple/frozenset. - /// - Non-constant BUILD_LIST becomes BUILD_TUPLE. - /// - Previously folded BUILD_LIST 0 + LOAD_CONST + LIST_EXTEND and - /// BUILD_SET 0 + LOAD_CONST + SET_UPDATE collapse back to LOAD_CONST. - fn optimize_lists_and_sets(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i + 1 < block.instructions.len() { - if matches!( - block.instructions[i].instr.real(), - Some(Instruction::CallIntrinsic1 { func }) - if func.get(block.instructions[i].arg) == IntrinsicFunction1::ListToTuple - ) { - if matches!( - block - .instructions - .get(i + 1) - .and_then(|instr| instr.instr.real()), - Some(Instruction::GetIter) - ) { - set_to_nop(&mut block.instructions[i]); - i += 2; - continue; - } - if Self::fold_constant_intrinsic_list_to_tuple_at(&mut self.metadata, block, i) - { - i += 1; - continue; - } - } - - if let Some(non_nop4) = Self::get_non_nop_instr_indices(block, i, 4) { - let is_build_list = non_nop4[0] == i - && matches!( - block.instructions[non_nop4[0]].instr.real(), - Some(Instruction::BuildList { .. }) - ) - && u32::from(block.instructions[non_nop4[0]].arg) == 0; - let is_const = matches!( - block.instructions[non_nop4[1]].instr.real(), - Some(Instruction::LoadConst { .. }) - ); - let is_list_extend = matches!( - block.instructions[non_nop4[2]].instr.real(), - Some(Instruction::ListExtend { .. }) - ) && u32::from(block.instructions[non_nop4[2]].arg) == 1; - let uses_iter_or_contains = matches!( - block.instructions[non_nop4[3]].instr.real(), - Some(Instruction::GetIter | Instruction::ContainsOp { .. }) - ); - - if is_build_list && is_const && is_list_extend && uses_iter_or_contains { - let loc = block.instructions[i].location; - set_to_nop(&mut block.instructions[i]); - block.instructions[i].location = loc; - set_to_nop(&mut block.instructions[non_nop4[2]]); - block.instructions[non_nop4[2]].location = loc; - i += 1; - continue; - } - - let is_build_set = non_nop4[0] == i - && matches!( - block.instructions[non_nop4[0]].instr.real(), - Some(Instruction::BuildSet { .. }) - ) - && u32::from(block.instructions[non_nop4[0]].arg) == 0; - let is_set_update = matches!( - block.instructions[non_nop4[2]].instr.real(), - Some(Instruction::SetUpdate { .. }) - ) && u32::from(block.instructions[non_nop4[2]].arg) == 1; - - if is_build_set && is_const && is_set_update && uses_iter_or_contains { - let loc = block.instructions[i].location; - set_to_nop(&mut block.instructions[i]); - block.instructions[i].location = loc; - set_to_nop(&mut block.instructions[non_nop4[2]]); - block.instructions[non_nop4[2]].location = loc; - i += 1; - continue; - } - } - - let Some(non_nop2) = Self::get_non_nop_instr_indices(block, i, 2) else { - i += 1; - continue; - }; - let uses_iter_or_contains = non_nop2[0] == i - && matches!( - block.instructions[non_nop2[1]].instr.real(), - Some(Instruction::GetIter | Instruction::ContainsOp { .. }) - ); - if !uses_iter_or_contains { - i += 1; - continue; - } - - if matches!( - block.instructions[i].instr.real(), - Some(Instruction::BuildList { .. }) - ) { - let seq_size = u32::from(block.instructions[i].arg) as usize; - if seq_size > STACK_USE_GUIDELINE { - i += 2; - continue; - } - if let Some((operand_indices, elements)) = - Self::get_const_sequence(&self.metadata, block, i, seq_size) - { - let const_data = ConstantData::Tuple { elements }; - let (const_idx, _) = self.metadata.consts.insert_full(const_data); - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - let eh = block.instructions[i].except_handler; - - for &j in &operand_indices { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - block.instructions[j].end_location = end_loc; - } - - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - block.instructions[i].location = folded_loc; - block.instructions[i].end_location = end_loc; - block.instructions[i].except_handler = eh; - i += 2; - continue; - } - - block.instructions[i].instr = Opcode::BuildTuple.into(); - i += 2; - } else if matches!( - block.instructions[i].instr.real(), - Some(Instruction::BuildSet { .. }) - ) { - let seq_size = u32::from(block.instructions[i].arg) as usize; - if seq_size > STACK_USE_GUIDELINE { - i += 2; - continue; - } - let Some((operand_indices, elements)) = - Self::get_const_sequence(&self.metadata, block, i, seq_size) - else { - i += 2; - continue; - }; - let const_data = ConstantData::Frozenset { elements }; - let (const_idx, _) = self.metadata.consts.insert_full(const_data); - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - let eh = block.instructions[i].except_handler; - - for &j in &operand_indices { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - block.instructions[j].end_location = end_loc; - } - - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - block.instructions[i].location = folded_loc; - block.instructions[i].end_location = end_loc; - block.instructions[i].except_handler = eh; - i += 2; - } else { - i += 1; - } - } - } - } - fn fold_set_constant_at(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { let Some(Instruction::BuildSet { .. }) = block.instructions[i].instr.real() else { return false; @@ -2622,112 +2639,6 @@ impl CodeInfo { true } - /// Fold constant set literals: LOAD_CONST* + BUILD_SET N → - /// BUILD_SET 0 + LOAD_CONST (frozenset-as-tuple) + SET_UPDATE 1 - fn fold_set_constants(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - let instr = &block.instructions[i]; - let Some(Instruction::BuildSet { .. }) = instr.instr.real() else { - i += 1; - continue; - }; - - let set_size = u32::from(instr.arg) as usize; - if !(3..=STACK_USE_GUIDELINE).contains(&set_size) { - i += 1; - continue; - } - - let Some((operand_indices, elements)) = - Self::get_const_sequence(&self.metadata, block, i, set_size) - else { - i += 1; - continue; - }; - let const_data = ConstantData::Frozenset { elements }; - let (const_idx, _) = self.metadata.consts.insert_full(const_data); - - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - let eh = block.instructions[i].except_handler; - - let build_idx = operand_indices[0]; - let const_idx_slot = operand_indices[1]; - - block.instructions[build_idx].instr = Instruction::BuildSet { - count: Arg::marker(), - } - .into(); - block.instructions[build_idx].arg = OpArg::new(0); - block.instructions[build_idx].location = folded_loc; - block.instructions[build_idx].end_location = end_loc; - block.instructions[build_idx].except_handler = eh; - - block.instructions[const_idx_slot].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); - block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); - block.instructions[const_idx_slot].location = folded_loc; - block.instructions[const_idx_slot].end_location = end_loc; - block.instructions[const_idx_slot].except_handler = eh; - - for &j in &operand_indices[2..] { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - } - - block.instructions[i].instr = Opcode::SetUpdate.into(); - block.instructions[i].arg = OpArg::new(1); - - i += 1; - } - } - } - - /// BUILD_TUPLE n + UNPACK_SEQUENCE n optimization. - /// - /// Ported from CPython flowgraph.c optimize_basic_block: - /// - n == 1: both become NOP (identity operation) - /// - n == 2 or 3: BUILD_TUPLE → NOP, UNPACK_SEQUENCE → SWAP - fn optimize_build_tuple_unpack(&mut self) { - for block in &mut self.blocks { - let instructions = &mut block.instructions; - let len = instructions.len(); - for i in 0..len.saturating_sub(1) { - let Some(Instruction::BuildTuple { .. }) = instructions[i].instr.real() else { - continue; - }; - let n = u32::from(instructions[i].arg); - let Some(Instruction::UnpackSequence { .. }) = instructions[i + 1].instr.real() - else { - continue; - }; - if u32::from(instructions[i + 1].arg) != n { - continue; - } - match n { - 1 => { - instructions[i].instr = Opcode::Nop.into(); - instructions[i].arg = OpArg::new(0); - instructions[i + 1].instr = Opcode::Nop.into(); - instructions[i + 1].arg = OpArg::new(0); - } - 2 | 3 => { - instructions[i].instr = Opcode::Nop.into(); - instructions[i].arg = OpArg::new(0); - instructions[i + 1].instr = Opcode::Swap.into(); - instructions[i + 1].arg = OpArg::new(n); - } - _ => {} - } - } - } - } - /// apply_static_swaps: eliminate SWAPs by reordering target stores/pops. /// /// Ported from CPython Python/flowgraph.c::apply_static_swaps. @@ -2738,7 +2649,7 @@ impl CodeInfo { /// Safety: abort if the two stores write the same variable, or if any /// intervening swappable stores to one of the same variables. Do not /// cross line-number boundaries (user-visible name bindings). - fn apply_static_swaps(&mut self) { + fn apply_static_swaps_block(block: &mut Block) { const VISITED: i32 = -1; /// Instruction classes that are safe to reorder around SWAP. @@ -2921,89 +2832,14 @@ impl CodeInfo { } } - for block in &mut self.blocks { - optimize_swap_block(&mut block.instructions); - let len = block.instructions.len(); - for i in 0..len { - if matches!( - block.instructions[i].instr.real(), - Some(Instruction::Swap { .. }) - ) { - apply_from(&mut block.instructions, i as isize); - } - } - } - } - - /// Eliminate dead stores in STORE_FAST sequences (apply_static_swaps). - /// - /// In sequences of consecutive STORE_FAST instructions (from tuple unpacking), - /// only collapse directly adjacent duplicate targets. - /// - /// CPython preserves non-adjacent duplicates such as `_, expr, _` so the - /// store layout still reflects the original unpack order. Replacing the - /// first `_` with POP_TOP there changes the emitted superinstructions and - /// bytecode shape even though the final value is the same. - fn eliminate_dead_stores(&mut self) { - for block in &mut self.blocks { - let instructions = &mut block.instructions; - let len = instructions.len(); - let mut i = 0; - while i < len { - // Look for UNPACK_SEQUENCE or UNPACK_EX - let is_unpack = matches!( - instructions[i].instr.into(), - AnyOpcode::Real(Opcode::UnpackSequence | Opcode::UnpackEx) - ); - if !is_unpack { - i += 1; - continue; - } - // Scan the run of STORE_FAST right after the unpack - let run_start = i + 1; - let mut run_end = run_start; - while run_end < len - && matches!( - instructions[run_end].instr.into(), - AnyOpcode::Real(Opcode::StoreFast) - ) - { - run_end += 1; - } - if run_end - run_start >= 2 { - let mut j = run_start; - while j < run_end { - let arg = u32::from(instructions[j].arg); - let mut group_end = j + 1; - while group_end < run_end && u32::from(instructions[group_end].arg) == arg { - group_end += 1; - } - for instr in &mut instructions[j..group_end.saturating_sub(1)] { - instr.instr = Opcode::PopTop.into(); - instr.arg = OpArg::new(0); - } - j = group_end; - } - } - i = run_end.max(i + 1); - } - - // General same-line duplicate STORE_FAST elimination from - // flowgraph.c optimize_basic_block(). This is required for - // apply_static_swaps() patterns such as `a, a = x, y`. - for i in 0..instructions.len().saturating_sub(1) { - let lhs = &instructions[i]; - let rhs = &instructions[i + 1]; - if !matches!(lhs.instr.real(), Some(Instruction::StoreFast { .. })) - || !matches!(rhs.instr.real(), Some(Instruction::StoreFast { .. })) - || u32::from(lhs.arg) != u32::from(rhs.arg) - || instruction_lineno(lhs) != instruction_lineno(rhs) - { - continue; - } - instructions[i].instr = Instruction::PopTop.into(); - instructions[i].arg = OpArg::NULL; - instructions[i].target = BlockIdx::NULL; + optimize_swap_block(&mut block.instructions); + let len = block.instructions.len(); + for i in 0..len { + if matches!( + block.instructions[i].instr.real(), + Some(Instruction::Swap { .. }) + ) { + apply_from(&mut block.instructions, i as isize); } } } @@ -3032,7 +2868,8 @@ impl CodeInfo { Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), _ => None, }; - for (block_idx, block) in self.blocks.iter_mut().enumerate() { + for block_idx in self.block_next_order() { + let block = &mut self.blocks[block_idx]; let mut i = 0; while i + 1 < block.instructions.len() { let curr = &block.instructions[i]; @@ -3040,46 +2877,42 @@ impl CodeInfo { let curr_arg = curr.arg; let next_arg = next.arg; - // Only combine if both are real instructions (not pseudo) - let (Some(curr_instr), Some(next_instr)) = (curr.instr.real(), next.instr.real()) - else { + // Only combine if the source is a real instruction. + let Some(curr_instr) = curr.instr.real() else { i += 1; continue; }; - if matches!(curr_instr, Instruction::ToBool) - && matches!(next_instr, Instruction::UnaryNot) - && let Some(Instruction::ToBool) = block - .instructions - .get(i + 2) - .and_then(|info| info.instr.real()) - && let Some(Instruction::UnaryNot) = block - .instructions - .get(i + 3) - .and_then(|info| info.instr.real()) - { - block.instructions.drain(i + 1..=i + 3); - continue; - } - - if matches!(curr_instr, Instruction::UnaryNot | Instruction::ToBool) - && matches!(next_instr, Instruction::ToBool) - { - block.instructions.remove(i + 1); - continue; + if let Some(is_true) = const_truthiness(curr_instr, curr.arg, &self.metadata) { + let jump_if_true = match next.instr.pseudo() { + Some(PseudoInstruction::JumpIfTrue { .. }) => Some(true), + Some(PseudoInstruction::JumpIfFalse { .. }) => Some(false), + _ => None, + }; + if let Some(jump_if_true) = jump_if_true { + // CPython flowgraph.c::basicblock_optimize_load_const() + // folds LOAD_CONST/LOAD_SMALL_INT followed by + // JUMP_IF_TRUE/FALSE. Unlike POP_JUMP_IF_*, these + // pseudo jumps do not consume the condition, so keep + // the constant for the following POP_TOP pair removal. + if is_true == jump_if_true { + block.instructions[i + 1].instr = PseudoInstruction::Jump { + delta: Arg::marker(), + } + .into(); + } else { + set_to_nop(&mut block.instructions[i + 1]); + } + i += 1; + continue; + } } - if matches!( - curr_instr, - Instruction::ContainsOp { .. } | Instruction::IsOp { .. } - ) && matches!(next_instr, Instruction::UnaryNot) - { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = curr_instr.into(); - block.instructions[i + 1].arg = OpArg::new(u32::from(curr_arg) ^ 1); + // The remaining combinations require both instructions to be real. + let Some(next_instr) = next.instr.real() else { i += 1; continue; - } + }; if let Some(is_true) = const_truthiness(curr_instr, curr.arg, &self.metadata) { let jump_if_true = match next_instr { @@ -3094,22 +2927,6 @@ impl CodeInfo { _ => unreachable!(), }; set_to_nop(&mut block.instructions[i]); - let preserves_pure_self_loop_anchor = i == 0 - && block.instructions[i + 1..].iter().all(|info| { - if info.target == BlockIdx(block_idx as u32) - && info.instr.is_unconditional_jump() - { - return true; - } - matches!(info.instr.real(), Some(Instruction::Nop)) - }) - && block.instructions[i + 1..].iter().any(|info| { - info.target == BlockIdx(block_idx as u32) - && info.instr.is_unconditional_jump() - }); - if preserves_pure_self_loop_anchor { - block.instructions[i].preserve_block_start_no_location_nop = true; - } if is_true == jump_if_true { block.instructions[i + 1].instr = PseudoInstruction::Jump { delta: Arg::marker(), @@ -3191,122 +3008,211 @@ impl CodeInfo { if matches!( curr_instr, Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } - ) && matches!(next_instr, Instruction::PopTop) + ) && matches!(next_instr, Instruction::ToBool) + && let Some(value) = const_truthiness(curr_instr, curr.arg, &self.metadata) { + let (const_idx, _) = self + .metadata + .consts + .insert_full(ConstantData::Boolean { value }); set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); + block.instructions[i + 1].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[i + 1].arg = OpArg::new(const_idx as u32); i += 1; continue; } - if matches!(curr_instr, Instruction::Copy { i } if i.get(curr.arg) == 1) - && matches!(next_instr, Instruction::PopTop) + if let (Instruction::LoadConst { consti }, Instruction::UnaryNot) = + (curr_instr, next_instr) { - set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; - continue; + let constant = &self.metadata.consts[consti.get(curr.arg).as_usize()]; + if let ConstantData::Boolean { value } = constant { + let (const_idx, _) = self + .metadata + .consts + .insert_full(ConstantData::Boolean { value: !value }); + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[i + 1].arg = OpArg::new(const_idx as u32); + i += 1; + continue; + } } - let combined = { - match (curr_instr, next_instr) { - // Note: StoreFast + LoadFast → StoreFastLoadFast is done in a - // later pass aligned with CPython insert_superinstructions(). - ( - Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }, - Instruction::ToBool, - ) => { - if let Some(value) = - const_truthiness(curr_instr, curr.arg, &self.metadata) - { - let (const_idx, _) = self - .metadata - .consts - .insert_full(ConstantData::Boolean { value }); - Some(( - Instruction::LoadConst { - consti: Arg::marker(), - }, - OpArg::new(const_idx as u32), - )) - } else { - None - } - } - (Instruction::CompareOp { .. }, Instruction::ToBool) => Some(( - curr_instr, - OpArg::new(u32::from(curr.arg) | oparg::COMPARE_OP_BOOL_MASK), - )), - ( - Instruction::ContainsOp { .. } | Instruction::IsOp { .. }, - Instruction::ToBool, - ) => Some((curr_instr, curr.arg)), - (Instruction::LoadConst { consti }, Instruction::UnaryNot) => { - let constant = &self.metadata.consts[consti.get(curr.arg).as_usize()]; - match constant { - ConstantData::Boolean { value } => { - let (const_idx, _) = self - .metadata - .consts - .insert_full(ConstantData::Boolean { value: !value }); - Some(((Opcode::LoadConst.into()), OpArg::new(const_idx as u32))) - } - _ => None, - } - } - _ => None, - } - }; - - if let Some((new_instr, new_arg)) = combined { - // Combine: keep first instruction's location, replace with combined instruction - block.instructions[i].instr = new_instr.into(); - block.instructions[i].arg = new_arg; - // Remove the second instruction - block.instructions.remove(i + 1); - // Don't increment i - check if we can combine again with the next instruction - } else { - i += 1; - } + i += 1; } } } - /// LOAD_GLOBAL + PUSH_NULL -> LOAD_GLOBAL , NOP - fn optimize_load_global_push_null(&mut self) { - for block in &mut self.blocks { - let mut i = 0; - while i + 1 < block.instructions.len() { - let curr = &block.instructions[i]; - let next = &block.instructions[i + 1]; - - let (Some(Instruction::LoadGlobal { .. }), Some(Instruction::PushNull)) = - (curr.instr.real(), next.instr.real()) - else { - i += 1; - continue; - }; + /// flowgraph.c optimize_basic_block + fn optimize_basic_blocks(&mut self) -> crate::InternalResult<()> { + for block_idx in self.block_next_order() { + { + let metadata = &mut self.metadata; + let block = &mut self.blocks[block_idx]; + let mut i = 0; + while i < block.instructions.len() { + let inst = block.instructions[i]; + let Some(opcode) = inst.instr.real() else { + i += 1; + continue; + }; + let nextop = block + .instructions + .get(i + 1) + .and_then(|next| next.instr.real()); + + match opcode { + Instruction::BuildTuple { .. } => { + let oparg = u32::from(inst.arg); + if matches!(nextop, Some(Instruction::UnpackSequence { .. })) + && u32::from(block.instructions[i + 1].arg) == oparg + { + match oparg { + 1 => { + set_to_nop(&mut block.instructions[i]); + set_to_nop(&mut block.instructions[i + 1]); + i += 1; + continue; + } + 2 | 3 => { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = + Instruction::Swap { i: Arg::marker() }.into(); + block.instructions[i + 1].arg = OpArg::new(oparg); + i += 1; + continue; + } + _ => {} + } + } + Self::fold_tuple_constant_at(metadata, block, i); + } + Instruction::BuildList { .. } | Instruction::BuildSet { .. } => { + Self::optimize_lists_and_sets_at(metadata, block, i, nextop); + } + Instruction::StoreFast { .. } + if matches!(nextop, Some(Instruction::StoreFast { .. })) + && u32::from(inst.arg) + == u32::from(block.instructions[i + 1].arg) + && instruction_lineno(&block.instructions[i]) + == instruction_lineno(&block.instructions[i + 1]) => + { + block.instructions[i].instr = Instruction::PopTop.into(); + block.instructions[i].arg = OpArg::NULL; + block.instructions[i].target = BlockIdx::NULL; + } + Instruction::Swap { .. } if u32::from(inst.arg) == 1 => { + set_to_nop(&mut block.instructions[i]); + } + Instruction::LoadGlobal { .. } + if matches!(nextop, Some(Instruction::PushNull)) + && (u32::from(inst.arg) & 1) == 0 => + { + block.instructions[i].arg = OpArg::new(u32::from(inst.arg) | 1); + set_to_nop(&mut block.instructions[i + 1]); + } + Instruction::CompareOp { .. } + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = Instruction::CompareOp { + opname: Arg::marker(), + } + .into(); + block.instructions[i + 1].arg = + OpArg::new(u32::from(inst.arg) | oparg::COMPARE_OP_BOOL_MASK); + i += 1; + continue; + } + Instruction::ContainsOp { .. } | Instruction::IsOp { .. } + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = opcode.into(); + block.instructions[i + 1].arg = inst.arg; + i += 1; + continue; + } + Instruction::ContainsOp { .. } | Instruction::IsOp { .. } + if matches!(nextop, Some(Instruction::UnaryNot)) => + { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = opcode.into(); + block.instructions[i + 1].arg = OpArg::new(u32::from(inst.arg) ^ 1); + i += 1; + continue; + } + Instruction::ToBool if matches!(nextop, Some(Instruction::ToBool)) => { + set_to_nop(&mut block.instructions[i]); + i += 1; + continue; + } + Instruction::UnaryNot => { + if matches!(nextop, Some(Instruction::ToBool)) { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = Instruction::UnaryNot.into(); + block.instructions[i + 1].arg = OpArg::new(0); + i += 1; + continue; + } + if matches!(nextop, Some(Instruction::UnaryNot)) { + set_to_nop(&mut block.instructions[i]); + set_to_nop(&mut block.instructions[i + 1]); + i += 1; + continue; + } + Self::fold_unary_constant_at(metadata, block, i); + } + Instruction::UnaryInvert | Instruction::UnaryNegative => { + Self::fold_unary_constant_at(metadata, block, i); + } + Instruction::CallIntrinsic1 { func } => match func.get(inst.arg) { + IntrinsicFunction1::ListToTuple => { + if matches!(nextop, Some(Instruction::GetIter)) { + set_to_nop(&mut block.instructions[i]); + } else { + Self::fold_constant_intrinsic_list_to_tuple_at( + metadata, block, i, + ); + } + } + IntrinsicFunction1::UnaryPositive => { + Self::fold_unary_constant_at(metadata, block, i); + } + _ => {} + }, + Instruction::BinaryOp { .. } => { + Self::fold_binop_constant_at(metadata, block, i); + } + _ => {} + } - let oparg = u32::from(block.instructions[i].arg); - if (oparg & 1) != 0 { i += 1; - continue; } - - block.instructions[i].arg = OpArg::new(oparg | 1); - block.instructions.remove(i + 1); } + jump_threading_block(&mut self.blocks, block_idx)?; + Self::apply_static_swaps_block(&mut self.blocks[block_idx]); } + Ok(()) } - fn remove_redundant_const_pop_top_pairs(&mut self) { + /// flowgraph.c remove_redundant_nops_and_pairs + fn remove_redundant_nops_and_pairs(&mut self) { loop { let mut changed = false; let mut prev: Option<(BlockIdx, usize)> = None; let mut block_idx = BlockIdx::new(0); while block_idx != BlockIdx::NULL { - if self.blocks[block_idx.idx()].label { + basicblock_remove_redundant_nops(&mut self.blocks, block_idx); + if self.blocks[block_idx.idx()].has_cpython_cfg_label() { prev = None; } @@ -3356,14 +3262,6 @@ impl CodeInfo { if !changed { break; } - for block in &mut self.blocks { - block.instructions.retain(|info| { - !matches!(info.instr.real(), Some(Instruction::Nop)) - || instruction_lineno(info) >= 0 - || info.preserve_redundant_jump_as_nop - || info.preserve_block_start_no_location_nop - }); - } } } @@ -3464,99 +3362,12 @@ impl CodeInfo { } } - /// Remove NOP instructions from all blocks, but keep NOPs that introduce - /// a new source line (they serve as line markers for monitoring LINE events). - fn remove_nops(&mut self) { - let layout_predecessors = compute_layout_predecessors(&self.blocks); - let keep_target_start_nops: Vec<_> = (0..self.blocks.len()) - .map(|idx| { - keep_target_start_no_location_nop( - &self.blocks, - BlockIdx(idx as u32), - &layout_predecessors, - ) - }) - .collect(); - let mut conditional_targets = vec![false; self.blocks.len()]; - for block in &self.blocks { - for instr in &block.instructions { - if instr.target != BlockIdx::NULL && is_conditional_jump(&instr.instr) { - let target = next_nonempty_block(&self.blocks, instr.target); - if target != BlockIdx::NULL { - conditional_targets[target.idx()] = true; - } - } - } - } - let preserve_loop_exit_pop_block_nops: Vec<_> = (0..self.blocks.len()) - .map(|idx| { - let block_idx = BlockIdx(idx as u32); - let block = &self.blocks[idx]; - let layout_pred = layout_predecessors[idx]; - block.instructions.first().is_some_and(|instr| { - matches!(instr.instr.real(), Some(Instruction::Nop)) - && instr.remove_no_location_nop - && instruction_lineno(instr) < 0 - && conditional_targets[idx] - && layout_pred != BlockIdx::NULL - && self.blocks[layout_pred.idx()] - .instructions - .last() - .is_some_and(|last| { - matches!( - last.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) && next_nonempty_block(&self.blocks, last.target) != block_idx - }) - }) - }) - .collect(); - - for (block_idx, block) in self.blocks.iter_mut().enumerate() { - let mut prev_line = None; - let mut src = 0usize; - block.instructions.retain(|ins| { - let keep = 'keep: { - if matches!(ins.instr.real(), Some(Instruction::Nop)) { - let keep_loop_exit_pop_block = src == 0 - && preserve_loop_exit_pop_block_nops - .get(block_idx) - .copied() - .unwrap_or(false); - let keep_target_start = src == 0 - && keep_target_start_nops - .get(block_idx) - .copied() - .unwrap_or(false); - if ins.remove_no_location_nop - && instruction_lineno(ins) < 0 - && !keep_loop_exit_pop_block - && (!keep_target_start || ins.folded_operand_nop) - { - break 'keep false; - } - let line = ins.location.line.get() as i32; - if prev_line == Some(line) { - break 'keep false; - } - } - prev_line = Some(instruction_lineno(ins)); - true - }; - src += 1; - keep - }); - } - } - /// insert_superinstructions (flowgraph.c): combine adjacent same-line /// LOAD_FAST / STORE_FAST pairs before later flowgraph passes change /// block layout. fn insert_superinstructions(&mut self) { - for block in &mut self.blocks { + for block_idx in self.block_next_order() { + let block = &mut self.blocks[block_idx]; let mut i = 0; while i + 1 < block.instructions.len() { let curr = &block.instructions[i]; @@ -3608,28 +3419,6 @@ impl CodeInfo { i += 1; continue; } - let first_store_location = curr.location; - let second_store_location = next.location; - let second_store_end_location = next.end_location; - let second_store_lineno_override = next.lineno_override; - let mut after_idx = i + 2; - while after_idx < block.instructions.len() { - let after = &mut block.instructions[after_idx]; - if after.instr.is_unconditional_jump() { - if instruction_lineno(after) < 0 - || after.location == first_store_location - { - after.location = second_store_location; - after.end_location = second_store_end_location; - after.lineno_override = second_store_lineno_override; - } - break; - } - if instruction_lineno(after) >= 0 { - break; - } - after_idx += 1; - } let packed = (idx1 << 4) | idx2; block.instructions[i].instr = Instruction::StoreFastStoreFast { var_nums: Arg::marker(), @@ -3637,15 +3426,14 @@ impl CodeInfo { .into(); block.instructions[i].arg = OpArg::new(packed); set_to_nop(&mut block.instructions[i + 1]); - block.instructions[i + 1].preserve_store_fast_store_fast_jump_location = - true; i += 1; } _ => i += 1, } } } - self.remove_nops(); + remove_redundant_nops(&mut self.blocks); + debug_assert!(no_redundant_nops(&self.blocks)); } fn optimize_load_fast_borrow(&mut self) { @@ -3708,49 +3496,31 @@ impl CodeInfo { target: BlockIdx, start_depth: usize, ) { + debug_assert!(target != BlockIdx::NULL); let expected = blocks[target.idx()].start_depth.map(|depth| depth as usize); - if expected != Some(start_depth) { - debug_assert!( - expected == Some(start_depth), - "optimize_load_fast_borrow start_depth mismatch: source={source:?} target={target:?} expected={expected:?} actual={:?} source_last={:?} target_instrs={:?}", - Some(start_depth), - blocks[source.idx()] - .instructions - .last() - .and_then(|info| info.instr.real()), - blocks[target.idx()] - .instructions - .iter() - .map(|info| info.instr) - .collect::>(), - ); - return; - } + debug_assert!( + expected == Some(start_depth), + "optimize_load_fast_borrow start_depth mismatch: source={source:?} target={target:?} expected={expected:?} actual={:?} source_last={:?} target_instrs={:?}", + Some(start_depth), + blocks[source.idx()] + .instructions + .last() + .and_then(|info| info.instr.real()), + blocks[target.idx()] + .instructions + .iter() + .map(|info| info.instr) + .collect::>(), + ); if !visited[target.idx()] { visited[target.idx()] = true; worklist.push(target); } } - fn folded_load_escapes_block(instructions: &[InstructionInfo], idx: usize) -> bool { - if !instructions[idx].folded_from_nonliteral_expr { - return false; - } - // CPython codegen_boolop() always emits USE_LABEL(end) after the - // selected tail expression. When flowgraph.c folds away a - // constant head such as "False or x", optimize_load_fast() still - // sees a direct local tail load as unconsumed at that basic-block - // boundary, so it must remain a strong LOAD_FAST. - matches!( - instructions[idx].instr.real(), - Some(Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. }) - ) - } - let mut visited = vec![false; self.blocks.len()]; let mut worklist = vec![BlockIdx(0)]; visited[0] = true; - while let Some(block_idx) = worklist.pop() { let block = &self.blocks[block_idx]; @@ -3810,17 +3580,15 @@ impl CodeInfo { } AnyInstruction::Real(Instruction::Copy { i: _ }) => { let depth = arg_u32 as usize; - if depth == 0 || refs.len() < depth { - continue; - } + assert!(depth > 0); + assert!(refs.len() >= depth); let r = at_ref(&refs, refs.len() - depth); push_ref(&mut refs, r.instr, r.local); } AnyInstruction::Real(Instruction::Swap { i: _ }) => { let depth = arg_u32 as usize; - if depth < 2 || refs.len() < depth { - continue; - } + assert!(depth >= 2); + assert!(refs.len() >= depth); swap_top(&mut refs, depth); } AnyInstruction::Real( @@ -3928,7 +3696,7 @@ impl CodeInfo { let num_popped = effect.popped() as usize; let num_pushed = effect.pushed() as usize; let target = info.target; - if target != BlockIdx::NULL { + if instr.has_target() && target != BlockIdx::NULL { let target_depth = refs .len() .saturating_sub(num_popped) @@ -3956,11 +3724,10 @@ impl CodeInfo { if let Some(term) = block.instructions.last() && block.next != BlockIdx::NULL - && block_has_fallthrough(block) - && !block.disable_load_fast_borrow && !term.instr.is_unconditional_jump() && !term.instr.is_scope_exit() { + debug_assert!(block_has_fallthrough(block)); push_block( &mut worklist, &mut visited, @@ -3978,14 +3745,8 @@ impl CodeInfo { } let block = &mut self.blocks[block_idx]; - if block.disable_load_fast_borrow { - continue; - } - let folded_load_escapes: Vec<_> = (0..block.instructions.len()) - .map(|i| folded_load_escapes_block(&block.instructions, i)) - .collect(); for (i, info) in block.instructions.iter_mut().enumerate() { - if instr_flags[i] != 0 || folded_load_escapes[i] { + if instr_flags[i] != 0 { continue; } match info.instr.real() { @@ -4007,87 +3768,14 @@ impl CodeInfo { } } - fn compute_load_fast_start_depths(&mut self) { - fn stackdepth_push( - stack: &mut Vec, - start_depths: &mut [u32], - target: BlockIdx, - depth: u32, - ) { - let idx = target.idx(); - let block_depth = &mut start_depths[idx]; - debug_assert!( - *block_depth == u32::MAX || *block_depth == depth, - "Invalid CFG, inconsistent optimize_load_fast stackdepth for block {:?}: existing={}, new={}", - target, - *block_depth, - depth, - ); - if *block_depth == u32::MAX { - *block_depth = depth; - stack.push(target); - } - } - - let mut stack = Vec::with_capacity(self.blocks.len()); - let mut start_depths = vec![u32::MAX; self.blocks.len()]; - stackdepth_push(&mut stack, &mut start_depths, BlockIdx(0), 0); - - 'process_blocks: while let Some(block_idx) = stack.pop() { - let mut depth = start_depths[block_idx.idx()]; - let block = &self.blocks[block_idx]; - for ins in &block.instructions { - let instr = &ins.instr; - let effect = instr.stack_effect(ins.arg.into()); - let new_depth = depth.saturating_add_signed(effect); - if ins.target != BlockIdx::NULL { - let jump_effect = instr.stack_effect_jump(ins.arg.into()); - let target_depth = depth.saturating_add_signed(jump_effect); - stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth); - } - depth = new_depth; - if instr.is_scope_exit() || instr.is_unconditional_jump() { - continue 'process_blocks; - } - } - if block.next != BlockIdx::NULL { - stackdepth_push(&mut stack, &mut start_depths, block.next, depth); - } - } - - for (block, &start_depth) in self.blocks.iter_mut().zip(&start_depths) { - block.start_depth = (start_depth != u32::MAX).then_some(start_depth); - } - } - - fn fast_scan_many_locals( - &mut self, - nlocals: usize, - nparams: usize, - merged_cell_local: &impl Fn(usize) -> Option, - ) { - const PARAM_INITIALIZED: usize = usize::MAX; - + fn fast_scan_many_locals(&mut self, nlocals: usize) { debug_assert!(nlocals > 64); let mut states = vec![0usize; nlocals - 64]; - let high_params = nparams.saturating_sub(64).min(states.len()); - for state in states.iter_mut().take(high_params) { - *state = PARAM_INITIALIZED; - } - - let is_known = |idx: usize, state: usize, blocknum: usize| { - state == blocknum || (idx < nparams && state == PARAM_INITIALIZED) - }; - let mut blocknum = 0usize; let mut current = BlockIdx(0); while current != BlockIdx::NULL { blocknum += 1; - let old_instructions = self.blocks[current.idx()].instructions.clone(); - let mut new_instructions = Vec::with_capacity(old_instructions.len()); - let mut changed = false; - - for mut info in old_instructions { + for info in &mut self.blocks[current.idx()].instructions { match info.instr.real() { Some( Instruction::DeleteFast { var_num } @@ -4097,7 +3785,6 @@ impl CodeInfo { if idx >= 64 && idx < nlocals { states[idx - 64] = blocknum - 1; } - new_instructions.push(info); } None if matches!( info.instr.pseudo(), @@ -4113,146 +3800,25 @@ impl CodeInfo { if idx >= 64 && idx < nlocals { states[idx - 64] = blocknum - 1; } - new_instructions.push(info); - } - Some(Instruction::DeleteDeref { i }) => { - let cell_relative = usize::from(i.get(info.arg)); - if let Some(idx) = merged_cell_local(cell_relative) - && idx >= 64 - && idx < nlocals - { - states[idx - 64] = blocknum - 1; - } - new_instructions.push(info); } Some(Instruction::StoreFast { var_num }) => { let idx = usize::from(var_num.get(info.arg)); if idx >= 64 && idx < nlocals { states[idx - 64] = blocknum; } - new_instructions.push(info); - } - Some(Instruction::StoreDeref { i }) => { - let cell_relative = usize::from(i.get(info.arg)); - if let Some(idx) = merged_cell_local(cell_relative) - && idx >= 64 - && idx < nlocals - { - states[idx - 64] = blocknum; - } - new_instructions.push(info); - } - Some(Instruction::StoreFastStoreFast { var_nums }) => { - let packed = var_nums.get(info.arg); - let (idx1, idx2) = packed.indexes(); - let idx1 = usize::from(idx1); - let idx2 = usize::from(idx2); - if idx1 >= 64 && idx1 < nlocals { - states[idx1 - 64] = blocknum; - } - if idx2 >= 64 && idx2 < nlocals { - states[idx2 - 64] = blocknum; - } - new_instructions.push(info); - } - Some(Instruction::StoreFastLoadFast { var_nums }) => { - let packed = var_nums.get(info.arg); - let (store_idx, load_idx) = packed.indexes(); - let store_idx = usize::from(store_idx); - let load_idx = usize::from(load_idx); - if store_idx >= 64 && store_idx < nlocals { - states[store_idx - 64] = blocknum; - } - if load_idx >= 64 && load_idx < nlocals { - if !is_known(load_idx, states[load_idx - 64], blocknum) { - let mut first = info; - first.instr = Instruction::StoreFast { - var_num: Arg::marker(), - } - .into(); - first.arg = OpArg::new(store_idx as u32); - - let mut second = info; - second.instr = Opcode::LoadFastCheck.into(); - second.arg = OpArg::new(load_idx as u32); - - new_instructions.push(first); - new_instructions.push(second); - changed = true; - } else { - new_instructions.push(info); - } - } else { - new_instructions.push(info); - } } Some(Instruction::LoadFast { var_num }) => { let idx = usize::from(var_num.get(info.arg)); - if idx >= 64 && idx < nlocals && !is_known(idx, states[idx - 64], blocknum) - { + if idx >= 64 && idx < nlocals && states[idx - 64] != blocknum { info.instr = Opcode::LoadFastCheck.into(); - states[idx - 64] = blocknum; - changed = true; - } - new_instructions.push(info); - } - Some(Instruction::LoadFastLoadFast { var_nums }) => { - let packed = var_nums.get(info.arg); - let (idx1, idx2) = packed.indexes(); - let idx1 = usize::from(idx1); - let idx2 = usize::from(idx2); - let needs_check_1 = idx1 >= 64 - && idx1 < nlocals - && !is_known(idx1, states[idx1 - 64], blocknum); - if needs_check_1 { - states[idx1 - 64] = blocknum; - } - let needs_check_2 = idx2 >= 64 - && idx2 < nlocals - && !is_known(idx2, states[idx2 - 64], blocknum); - - if needs_check_1 || needs_check_2 { - let mut first = info; - first.instr = if needs_check_1 { - Opcode::LoadFastCheck - } else { - Opcode::LoadFast - } - .into(); - first.arg = OpArg::new(idx1 as u32); - - let mut second = info; - second.instr = if needs_check_2 { - Opcode::LoadFastCheck.into() - } else { - Opcode::LoadFast.into() - }; - second.arg = OpArg::new(idx2 as u32); - - new_instructions.push(first); - new_instructions.push(second); - changed = true; - if needs_check_2 { - states[idx2 - 64] = blocknum; - } - } else { - new_instructions.push(info); } - } - Some(Instruction::LoadFastCheck { var_num }) => { - let idx = usize::from(var_num.get(info.arg)); if idx >= 64 && idx < nlocals { states[idx - 64] = blocknum; } - new_instructions.push(info); } - _ => new_instructions.push(info), + _ => {} } } - - if changed { - self.blocks[current.idx()].instructions = new_instructions; - } current = self.blocks[current.idx()].next; } } @@ -4263,15 +3829,6 @@ impl CodeInfo { return; } - let cell_to_local: Vec<_> = self - .metadata - .cellvars - .iter() - .map(|name| self.metadata.varnames.get_index_of(name.as_str())) - .collect(); - let merged_cell_local = - |cell_relative: usize| cell_to_local.get(cell_relative).copied().flatten(); - let mut nparams = self.metadata.argcount as usize + self.metadata.kwonlyargcount as usize; if self.flags.contains(CodeFlags::VARARGS) { nparams += 1; @@ -4282,244 +3839,89 @@ impl CodeInfo { nparams = nparams.min(nlocals); if nlocals > 64 { - self.fast_scan_many_locals(nlocals, nparams, &merged_cell_local); + self.fast_scan_many_locals(nlocals); nlocals = 64; } - let mut in_masks: Vec>> = vec![None; self.blocks.len()]; - let mut start_mask = vec![false; nlocals]; - for slot in start_mask.iter_mut().skip(nparams) { - *slot = true; + let mut unsafe_masks = vec![0u64; self.blocks.len()]; + let mut on_stack = vec![false; self.blocks.len()]; + let mut worklist = Vec::with_capacity(self.blocks.len()); + let mut start_mask = 0u64; + for i in nparams..nlocals { + start_mask |= 1u64 << i; } - in_masks[0] = Some(start_mask); + maybe_push_local_block( + &mut worklist, + &mut on_stack, + &mut unsafe_masks, + BlockIdx(0), + start_mask, + ); - let mut worklist = vec![BlockIdx(0)]; - while let Some(block_idx) = worklist.pop() { - let idx = block_idx.idx(); - let Some(mut unsafe_mask) = in_masks[idx].clone() else { - continue; - }; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + scan_block_for_locals( + &mut self.blocks, + current, + &mut worklist, + &mut on_stack, + &mut unsafe_masks, + ); + current = self.blocks[current.idx()].next; + } - let old_instructions = self.blocks[idx].instructions.clone(); - let mut new_instructions = Vec::with_capacity(old_instructions.len()); - let mut changed = false; + while let Some(block_idx) = worklist.pop() { + on_stack[block_idx.idx()] = false; + scan_block_for_locals( + &mut self.blocks, + block_idx, + &mut worklist, + &mut on_stack, + &mut unsafe_masks, + ); + } + } - for info in old_instructions { - let mut info = info; - if let Some(eh) = info.except_handler { - let target = next_nonempty_block(&self.blocks, eh.handler_block); - if target != BlockIdx::NULL - && merge_unsafe_mask(&mut in_masks[target.idx()], &unsafe_mask) - { - worklist.push(target); + fn max_stackdepth(&mut self) -> crate::InternalResult { + let mut maxdepth = 0u32; + let mut stack = Vec::with_capacity(self.blocks.len()); + let mut start_depths = vec![u32::MAX; self.blocks.len()]; + stackdepth_push(&mut stack, &mut start_depths, BlockIdx(0), 0)?; + const DEBUG: bool = false; + 'process_blocks: while let Some(block_idx) = stack.pop() { + let idx = block_idx.idx(); + let mut depth = start_depths[idx]; + if DEBUG { + eprintln!("===BLOCK {}===", block_idx.0); + } + let block = &self.blocks[block_idx]; + for ins in &block.instructions { + let instr = &ins.instr; + let effect = instr.stack_effect(ins.arg.into()); + if DEBUG { + let display_arg = if ins.target == BlockIdx::NULL { + ins.arg + } else { + OpArg::new(ins.target.0) + }; + eprint!("{display_arg:?}: {depth} {effect:+} => "); + } + let new_depth = depth.checked_add_signed(effect).ok_or({ + if effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow } + })?; + if DEBUG { + eprintln!("{new_depth}"); } - if matches!(info.instr.real(), Some(Instruction::ForIter { .. })) - && info.target != BlockIdx::NULL - && merge_unsafe_mask(&mut in_masks[info.target.idx()], &unsafe_mask) + maxdepth = maxdepth.max(depth); + // Process target blocks for branching instructions + if instr.has_target() + && ins.target != BlockIdx::NULL + && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) { - worklist.push(info.target); - } - match info.instr.real() { - None if matches!( - info.instr.pseudo(), - Some(PseudoInstruction::StoreFastMaybeNull { .. }) - ) => - { - let Some(PseudoInstruction::StoreFastMaybeNull { var_num }) = - info.instr.pseudo() - else { - unreachable!(); - }; - let var_idx = var_num.get(info.arg) as usize; - if var_idx < nlocals { - unsafe_mask[var_idx] = true; - } - new_instructions.push(info); - } - Some(Instruction::DeleteFast { var_num }) => { - let var_idx = usize::from(var_num.get(info.arg)); - if var_idx < nlocals { - unsafe_mask[var_idx] = true; - } - new_instructions.push(info); - } - Some(Instruction::LoadFastAndClear { var_num }) => { - let var_idx = usize::from(var_num.get(info.arg)); - if var_idx < nlocals { - unsafe_mask[var_idx] = true; - } - new_instructions.push(info); - } - Some(Instruction::StoreFast { var_num }) => { - let var_idx = usize::from(var_num.get(info.arg)); - if var_idx < nlocals { - unsafe_mask[var_idx] = false; - } - new_instructions.push(info); - } - Some(Instruction::StoreDeref { i }) => { - let cell_relative = usize::from(i.get(info.arg)); - if let Some(var_idx) = merged_cell_local(cell_relative) - && var_idx < nlocals - { - unsafe_mask[var_idx] = false; - } - new_instructions.push(info); - } - Some(Instruction::StoreFastStoreFast { var_nums }) => { - let packed = var_nums.get(info.arg); - let (idx1, idx2) = packed.indexes(); - let idx1 = usize::from(idx1); - let idx2 = usize::from(idx2); - if idx1 < nlocals { - unsafe_mask[idx1] = false; - } - if idx2 < nlocals { - unsafe_mask[idx2] = false; - } - new_instructions.push(info); - } - Some(Instruction::LoadFastCheck { var_num }) => { - let var_idx = usize::from(var_num.get(info.arg)); - if var_idx < nlocals { - unsafe_mask[var_idx] = false; - } - new_instructions.push(info); - } - Some(Instruction::DeleteDeref { i }) => { - let cell_relative = usize::from(i.get(info.arg)); - if let Some(var_idx) = merged_cell_local(cell_relative) - && var_idx < nlocals - { - unsafe_mask[var_idx] = true; - } - new_instructions.push(info); - } - Some( - Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num }, - ) => { - let var_idx = usize::from(var_num.get(info.arg)); - if var_idx < nlocals && unsafe_mask[var_idx] { - info.instr = Opcode::LoadFastCheck.into(); - changed = true; - } - if var_idx < nlocals { - unsafe_mask[var_idx] = false; - } - new_instructions.push(info); - } - Some( - Instruction::LoadFastLoadFast { var_nums } - | Instruction::LoadFastBorrowLoadFastBorrow { var_nums }, - ) => { - let packed = var_nums.get(info.arg); - let (idx1, idx2) = packed.indexes(); - let idx1 = usize::from(idx1); - let idx2 = usize::from(idx2); - let needs_check_1 = idx1 < nlocals && unsafe_mask[idx1]; - let needs_check_2 = idx2 < nlocals && unsafe_mask[idx2]; - if needs_check_1 || needs_check_2 { - let mut first = info; - first.instr = if needs_check_1 { - Opcode::LoadFastCheck - } else { - Opcode::LoadFast - } - .into(); - first.arg = OpArg::new(idx1 as u32); - - let mut second = info; - second.instr = if needs_check_2 { - Opcode::LoadFastCheck.into() - } else { - Opcode::LoadFast.into() - }; - second.arg = OpArg::new(idx2 as u32); - - new_instructions.push(first); - new_instructions.push(second); - changed = true; - } else { - new_instructions.push(info); - } - if idx1 < nlocals { - unsafe_mask[idx1] = false; - } - if idx2 < nlocals { - unsafe_mask[idx2] = false; - } - } - _ => new_instructions.push(info), - } - } - - if changed { - self.blocks[idx].instructions = new_instructions; - } - - let block = &self.blocks[idx]; - if block_has_fallthrough(block) { - let next = next_nonempty_block(&self.blocks, block.next); - if next != BlockIdx::NULL - && merge_unsafe_mask(&mut in_masks[next.idx()], &unsafe_mask) - { - worklist.push(next); - } - } - - if let Some(last) = block.instructions.last() - && is_jump_instruction(last) - { - let target = next_nonempty_block(&self.blocks, last.target); - if target != BlockIdx::NULL - && merge_unsafe_mask(&mut in_masks[target.idx()], &unsafe_mask) - { - worklist.push(target); - } - } - } - } - - fn max_stackdepth(&mut self) -> crate::InternalResult { - let mut maxdepth = 0u32; - let mut stack = Vec::with_capacity(self.blocks.len()); - let mut start_depths = vec![u32::MAX; self.blocks.len()]; - stackdepth_push(&mut stack, &mut start_depths, BlockIdx(0), 0); - const DEBUG: bool = false; - 'process_blocks: while let Some(block_idx) = stack.pop() { - let idx = block_idx.idx(); - let mut depth = start_depths[idx]; - if DEBUG { - eprintln!("===BLOCK {}===", block_idx.0); - } - let block = &self.blocks[block_idx]; - for ins in &block.instructions { - let instr = &ins.instr; - let effect = instr.stack_effect(ins.arg.into()); - if DEBUG { - let display_arg = if ins.target == BlockIdx::NULL { - ins.arg - } else { - OpArg::new(ins.target.0) - }; - eprint!("{display_arg:?}: {depth} {effect:+} => "); - } - let new_depth = depth.checked_add_signed(effect).ok_or({ - if effect < 0 { - InternalError::StackUnderflow - } else { - InternalError::StackOverflow - } - })?; - if DEBUG { - eprintln!("{new_depth}"); - } - if new_depth > maxdepth { - maxdepth = new_depth - } - // Process target blocks for branching instructions - if ins.target != BlockIdx::NULL { let jump_effect = instr.stack_effect_jump(ins.arg.into()); let target_depth = depth.checked_add_signed(jump_effect).ok_or({ if jump_effect < 0 { @@ -4528,32 +3930,32 @@ impl CodeInfo { InternalError::StackOverflow } })?; - if target_depth > maxdepth { - maxdepth = target_depth; - } - stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth); + maxdepth = maxdepth.max(depth); + stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth)?; } depth = new_depth; - if instr.is_scope_exit() || instr.is_unconditional_jump() { + if instr.is_no_fallthrough() { continue 'process_blocks; } } // Only push next block if it's not NULL if block.next != BlockIdx::NULL { - stackdepth_push(&mut stack, &mut start_depths, block.next, depth); + stackdepth_push(&mut stack, &mut start_depths, block.next, depth)?; } } if DEBUG { eprintln!("DONE: {maxdepth}"); } - for (block, &start_depth) in self.blocks.iter_mut().zip(&start_depths) { - block.start_depth = (start_depth != u32::MAX).then_some(start_depth); + for block_idx in self.block_next_order() { + let start_depth = start_depths[block_idx.idx()]; + self.blocks[block_idx].start_depth = (start_depth != u32::MAX).then_some(start_depth); } // Fix up handler stack_depth in ExceptHandlerInfo using start_depths // computed above: depth = start_depth - 1 - preserve_lasti - for block in &mut self.blocks { + for block_idx in self.block_next_order() { + let block = &mut self.blocks[block_idx]; for ins in &mut block.instructions { if let Some(ref mut handler) = ins.except_handler { let h_start = start_depths[handler.handler_block.idx()]; @@ -4581,7 +3983,7 @@ impl CodeInfo { use core::fmt::Write; let _ = writeln!( out, - "block {} next={} cold={} except={} preserve_lasti={} disable_borrow={} start_depth={}", + "block {} next={} cold={} except={} preserve_lasti={} start_depth={}", u32::from(block_idx), if block.next == BlockIdx::NULL { String::from("NULL") @@ -4591,7 +3993,6 @@ impl CodeInfo { block.cold, block.except_handler, block.preserve_lasti, - block.disable_load_fast_borrow, block .start_depth .map_or_else(|| String::from("None"), |depth| depth.to_string()), @@ -4600,9 +4001,13 @@ impl CodeInfo { let lineno = instruction_lineno(info); let _ = writeln!( out, - " [disp={} raw={} override={:?}] {:?} arg={} target={}", + " [disp={}:{} raw={}:{}-{}:{} override={:?}] {:?} arg={} target={}", lineno, + info.location.character_offset.get(), info.location.line.get(), + info.location.character_offset.get(), + info.end_location.line.get(), + info.end_location.character_offset.get(), info.lineno_override, info.instr, u32::from(info.arg), @@ -4621,77 +4026,47 @@ impl CodeInfo { let mut trace = Vec::new(); trace.push(("initial".to_owned(), self.debug_block_dump())); - self.splice_annotations_blocks(); - if self.flags.contains(CodeFlags::OPTIMIZED) { - split_blocks_at_jumps(&mut self.blocks); - trace.push(( - "after_initial_split_blocks_at_jumps".to_owned(), - self.debug_block_dump(), - )); - } - self.fold_constants_per_block(); - self.fold_binop_constants(); - self.fold_unary_constants(); - self.fold_binop_constants(); - self.fold_unary_constants(); - self.fold_tuple_constants(); - self.fold_binop_constants(); - self.fold_list_constants(); - self.fold_set_constants(); - self.optimize_lists_and_sets(); - self.convert_to_load_small_int(); - self.dce(); - self.peephole_optimize(); - trace.push(( - "after_peephole_optimize".to_owned(), - self.debug_block_dump(), - )); - self.fold_constants_per_block(); - self.fold_tuple_constants(); - self.fold_binop_constants(); - self.fold_list_constants(); - self.fold_set_constants(); - self.optimize_lists_and_sets(); - self.convert_to_load_small_int(); - split_blocks_at_jumps(&mut self.blocks); - trace.push(( - "after_split_blocks_at_jumps".to_owned(), - self.debug_block_dump(), - )); - self.dce(); + let instr_sequence = self.prepare_cfg_from_codegen()?; + self.blocks = cfg_from_instruction_sequence(instr_sequence)?; trace.push(( - "after_split_dce_unreachable".to_owned(), + "after_cfg_from_instruction_sequence".to_owned(), self.debug_block_dump(), )); - mark_except_handlers(&mut self.blocks); - label_exception_targets(&mut self.blocks); - redirect_empty_unconditional_jump_targets(&mut self.blocks); + translate_jump_labels_to_targets(&mut self.blocks)?; + mark_except_handlers(&mut self.blocks)?; + label_exception_targets(&mut self.blocks)?; + check_cfg(&self.blocks)?; inline_small_or_no_lineno_blocks(&mut self.blocks); trace.push(( "after_inline_small_or_no_lineno_blocks".to_owned(), self.debug_block_dump(), )); - jump_threading(&mut self.blocks); - trace.push(("after_jump_threading".to_owned(), self.debug_block_dump())); - self.eliminate_unreachable_blocks(); + self.remove_unreachable_blocks(); resolve_line_numbers(&mut self.blocks); - self.optimize_build_tuple_unpack(); - self.eliminate_dead_stores(); - self.apply_static_swaps(); + self.convert_to_load_small_int(); + self.peephole_optimize(); + trace.push(( + "after_peephole_optimize".to_owned(), + self.debug_block_dump(), + )); + self.convert_to_load_small_int(); + self.optimize_basic_blocks()?; trace.push(( - "after_first_resolve_line_numbers".to_owned(), + "after_optimize_basic_block".to_owned(), self.debug_block_dump(), )); - self.remove_nops(); + self.remove_redundant_nops_and_pairs(); + self.remove_unreachable_blocks(); + remove_redundant_nops_and_jumps(&mut self.blocks)?; + debug_assert!(no_redundant_jumps(&self.blocks)); + self.remove_unused_consts(); trace.push(( - "after_early_remove_nops".to_owned(), + "after_optimize_cfg_cleanup".to_owned(), self.debug_block_dump(), )); self.add_checks_for_loads_of_uninitialized_variables(); self.insert_superinstructions(); - self.remove_redundant_const_pop_top_pairs(); - inline_single_predecessor_artificial_expr_exit_blocks(&mut self.blocks); - push_cold_blocks_to_end(&mut self.blocks); + push_cold_blocks_to_end(&mut self.blocks)?; trace.push(( "after_push_cold_before_chain_reorder".to_owned(), self.debug_block_dump(), @@ -4701,230 +4076,39 @@ impl CodeInfo { "after_push_cold_resolve_line_numbers".to_owned(), self.debug_block_dump(), )); - reorder_conditional_chain_and_jump_back_blocks(&mut self.blocks); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); trace.push(( "after_push_cold_blocks_to_end".to_owned(), self.debug_block_dump(), )); - normalize_jumps(&mut self.blocks); - trace.push(("after_normalize_jumps".to_owned(), self.debug_block_dump())); - reorder_conditional_exit_and_jump_blocks(&mut self.blocks); - reorder_conditional_jump_and_exit_blocks(&mut self.blocks); - reorder_conditional_break_continue_blocks(&mut self.blocks); - reorder_conditional_explicit_continue_scope_exit_blocks(&mut self.blocks); - reorder_conditional_implicit_continue_scope_exit_blocks(&mut self.blocks); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); - reorder_exception_handler_conditional_continue_scope_exit_blocks(&mut self.blocks); - deduplicate_adjacent_jump_back_blocks(&mut self.blocks); - reorder_conditional_body_and_implicit_continue_blocks(&mut self.blocks); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, true, true); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, true); - reorder_conditional_scope_exit_and_jump_back_blocks(&mut self.blocks, false, false); - trace.push(("after_reorder".to_owned(), self.debug_block_dump())); - - self.dce(); - self.eliminate_unreachable_blocks(); - trace.push(("after_dce_unreachable".to_owned(), self.debug_block_dump())); - - resolve_line_numbers(&mut self.blocks); - trace.push(( - "after_resolve_line_numbers".to_owned(), - self.debug_block_dump(), - )); - - materialize_empty_conditional_exit_targets(&mut self.blocks); - trace.push(( - "after_materialize_empty_conditional_exit_targets".to_owned(), - self.debug_block_dump(), - )); - retarget_assert_conditional_jumps_to_empty_predecessor(&mut self.blocks); - trace.push(( - "after_retarget_assert_conditional_jumps_to_empty_predecessor".to_owned(), - self.debug_block_dump(), - )); - canonicalize_empty_label_blocks(&mut self.blocks); - trace.push(( - "after_canonicalize_empty_label_blocks".to_owned(), - self.debug_block_dump(), - )); - inline_small_fast_return_blocks(&mut self.blocks); - trace.push(( - "after_inline_small_fast_return_blocks".to_owned(), - self.debug_block_dump(), - )); - - duplicate_end_returns(&mut self.blocks, &self.metadata); - retarget_conditional_jumps_to_empty_while_exit_epilogue(&mut self.blocks); - duplicate_fallthrough_jump_back_targets(&mut self.blocks); - duplicate_shared_jump_back_targets(&mut self.blocks); - trace.push(( - "after_duplicate_jump_back_targets".to_owned(), - self.debug_block_dump(), - )); - - self.dce(); - self.eliminate_unreachable_blocks(); - trace.push(( - "after_second_dce_unreachable".to_owned(), - self.debug_block_dump(), - )); - - resolve_line_numbers(&mut self.blocks); - trace.push(( - "after_final_resolve_line_numbers".to_owned(), - self.debug_block_dump(), - )); - - remove_redundant_nops_and_jumps(&mut self.blocks); - trace.push(( - "after_remove_redundant_nops_and_jumps".to_owned(), - self.debug_block_dump(), - )); - - jump_threading_unconditional(&mut self.blocks); - self.eliminate_unreachable_blocks(); - remove_redundant_nops_and_jumps(&mut self.blocks); - inline_with_suppress_return_blocks(&mut self.blocks); - inline_pop_except_return_blocks(&mut self.blocks); - duplicate_named_except_cleanup_returns(&mut self.blocks, &self.metadata); - self.eliminate_unreachable_blocks(); - trace.push(( - "after_final_cfg_cleanup".to_owned(), - self.debug_block_dump(), - )); - - resolve_line_numbers(&mut self.blocks); - reorder_lineful_jump_back_runs_by_descending_line(&mut self.blocks); + convert_pseudo_conditional_jumps(&mut self.blocks); trace.push(( - "after_post_cleanup_resolve_line_numbers".to_owned(), + "after_convert_pseudo_conditional_jumps".to_owned(), self.debug_block_dump(), )); - let cellfixedoffsets = build_cellfixedoffsets( - &self.metadata.varnames, - &self.metadata.cellvars, - &self.metadata.freevars, - ); - mark_except_handlers(&mut self.blocks); - let _ = self.max_stackdepth()?; - convert_pseudo_ops(&mut self.blocks, &cellfixedoffsets); - remove_redundant_nops_and_jumps(&mut self.blocks); - inline_named_except_cleanup_jump_blocks(&mut self.blocks, &self.metadata); - deduplicate_adjacent_pop_except_jump_back_blocks(&mut self.blocks); - self.eliminate_unreachable_blocks(); - remove_redundant_nops_and_jumps(&mut self.blocks); - // Keep debug_late_cfg_trace in the same order as finalize(). - redirect_load_fast_passthrough_targets(&mut self.blocks); + let _max_stackdepth = self.max_stackdepth()?; + let _nlocalsplus = self.prepare_localsplus(); + convert_pseudo_ops(&mut self.blocks)?; trace.push(( "after_convert_pseudo_ops".to_owned(), self.debug_block_dump(), )); - self.compute_load_fast_start_depths(); - trace.push(( - "after_compute_load_fast_start_depths".to_owned(), - self.debug_block_dump(), - )); + + normalize_jumps(&mut self.blocks)?; + debug_assert!(no_redundant_jumps(&self.blocks)); + trace.push(("after_normalize_jumps".to_owned(), self.debug_block_dump())); self.optimize_load_fast_borrow(); - redirect_empty_block_targets(&mut self.blocks); trace.push(( "after_raw_optimize_load_fast_borrow".to_owned(), self.debug_block_dump(), )); - self.apply_static_swaps(); - self.optimize_load_global_push_null(); - self.reorder_entry_prefix_cell_setup(); - self.remove_unused_consts(); Ok(trace) } } -impl CodeInfo { - fn remap_block_idx(idx: BlockIdx, base: u32) -> BlockIdx { - if idx == BlockIdx::NULL { - idx - } else { - BlockIdx::new(u32::from(idx) + base) - } - } - - fn splice_annotations_blocks(&mut self) { - let mut placeholder = None; - for (block_idx, block) in self.blocks.iter().enumerate() { - if let Some(instr_idx) = block.instructions.iter().position(|info| { - matches!( - info.instr.pseudo(), - Some(PseudoInstruction::AnnotationsPlaceholder) - ) - }) { - placeholder = Some((block_idx, instr_idx)); - break; - } - } - - let Some((block_idx, instr_idx)) = placeholder else { - return; - }; - - let Some(mut annotations_blocks) = self.annotations_blocks.take() else { - self.blocks[block_idx].instructions.remove(instr_idx); - return; - }; - if annotations_blocks.is_empty() { - self.blocks[block_idx].instructions.remove(instr_idx); - return; - } - - let base = self.blocks.len() as u32; - for block in &mut annotations_blocks { - block.next = Self::remap_block_idx(block.next, base); - for info in &mut block.instructions { - info.target = Self::remap_block_idx(info.target, base); - if let Some(handler) = &mut info.except_handler { - handler.handler_block = Self::remap_block_idx(handler.handler_block, base); - } - } - } - - let ann_entry = BlockIdx::new(base); - let ann_tail = { - let mut cursor = ann_entry; - while annotations_blocks[(u32::from(cursor) - base) as usize].next != BlockIdx::NULL { - cursor = annotations_blocks[(u32::from(cursor) - base) as usize].next; - } - cursor - }; - - let old_next = self.blocks[block_idx].next; - let suffix = self.blocks[block_idx].instructions.split_off(instr_idx + 1); - self.blocks[block_idx].instructions.pop(); - - let suffix_block = if suffix.is_empty() { - old_next - } else { - let suffix_idx = BlockIdx::new(base + annotations_blocks.len() as u32); - let disable_load_fast_borrow = self.blocks[block_idx].disable_load_fast_borrow; - let block = Block { - instructions: suffix, - next: old_next, - disable_load_fast_borrow, - ..Default::default() - }; - annotations_blocks.push(block); - suffix_idx - }; - - self.blocks[block_idx].next = ann_entry; - let ann_tail_local = (u32::from(ann_tail) - base) as usize; - annotations_blocks[ann_tail_local].next = suffix_block; - self.blocks.extend(annotations_blocks); - } -} - impl InstrDisplayContext for CodeInfo { type Constant = ConstantData; @@ -4961,13 +4145,17 @@ fn stackdepth_push( start_depths: &mut [u32], target: BlockIdx, depth: u32, -) { +) -> crate::InternalResult<()> { let idx = target.idx(); let block_depth = &mut start_depths[idx]; - if depth > *block_depth || *block_depth == u32::MAX { + if *block_depth != u32::MAX && *block_depth != depth { + return Err(InternalError::InconsistentStackDepth); + } + if *block_depth == u32::MAX { *block_depth = depth; stack.push(target); } + Ok(()) } fn iter_blocks(blocks: &[Block]) -> impl Iterator + '_ { @@ -5129,117 +4317,59 @@ fn generate_linetable( linetable.into_boxed_slice() } -/// Generate Python 3.11+ exception table from instruction handler info -fn generate_exception_table(blocks: &[Block], block_to_index: &[u32]) -> Box<[u8]> { - let mut entries: Vec = Vec::new(); - let mut current_entry: Option<(ExceptHandlerInfo, u32)> = None; // (handler_info, start_index) - let mut instr_index = 0u32; - let instructions: Vec<(BlockIdx, usize, &InstructionInfo)> = iter_blocks(blocks) - .flat_map(|(idx, block)| { - block - .instructions - .iter() - .enumerate() - .map(move |(instr_idx, instr)| (idx, instr_idx, instr)) - }) - .collect(); - let mut jump_targets = vec![false; blocks.len()]; - for (_, block) in iter_blocks(blocks) { - for instr in &block.instructions { - if instr.target != BlockIdx::NULL { - jump_targets[instr.target.idx()] = true; - } - } +fn no_linetable_location() -> LineTableLocation { + LineTableLocation { + line: -1, + end_line: -1, + col: -1, + end_col: -1, } - let same_handler = |left: ExceptHandlerInfo, right: ExceptHandlerInfo| { - block_to_index[left.handler_block.idx()] == block_to_index[right.handler_block.idx()] - && left.stack_depth == right.stack_depth - && left.preserve_lasti == right.preserve_lasti - }; - let mut conditional_jumps_since_exit = 0usize; - - // Iterate through all instructions in block order - // instr_index is the index into the final instructions array (including EXTENDED_ARG) - // This matches how frame.rs uses lasti - for (pos, &(block_idx, instr_idx, instr)) in instructions.iter().enumerate() { - // CPython's final exception table is keyed by bytecode offsets after - // empty cleanup labels have been resolved. RustPython can still have - // distinct block ids for those labels here, so compare handler offsets. - let next = instructions.get(pos + 1).copied(); - let next_is_jump_target_block = next.is_some_and(|(next_block, _, _)| { - next_block != block_idx - && instr_idx + 1 == blocks[block_idx.idx()].instructions.len() - && jump_targets[next_block.idx()] - }); - let next_is_normalized_backward_jump = next.is_some_and(|(next_block, _, _)| { - next_block != block_idx - && instr_idx + 1 == blocks[block_idx.idx()].instructions.len() - && matches!( - blocks[next_block.idx()].instructions.as_slice(), - [not_taken, jump] - if matches!(not_taken.instr.real(), Some(Instruction::NotTaken)) - && jump.instr.is_unconditional_jump() - && jump.target != BlockIdx::NULL - && comes_before(blocks, jump.target, next_block) - ) - }); - let previous_is_conditional_ifexp_jump = pos.checked_sub(1).is_some_and(|prev_pos| { - let (_, _, previous) = instructions[prev_pos]; - previous.target != BlockIdx::NULL - && is_conditional_jump(&previous.instr) - && blocks[previous.target.idx()].conditional_ifexp_orelse_entry - }); - let previous_is_general_bool_conditional_jump = - pos.checked_sub(1).is_some_and(|prev_pos| { - let (_, _, previous) = instructions[prev_pos]; - matches!( - previous.instr.real(), - Some(Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. }) - ) - }); - let previous_jump_uses_to_bool = pos.checked_sub(1).is_some_and(|prev_pos| { - let (_, _, previous) = instructions[prev_pos]; - matches!( - previous.instr.real(), - Some(Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. }) - ) && instructions[..prev_pos] - .iter() - .rev() - .find(|(_, _, info)| !matches!(info.instr.real(), Some(Instruction::Cache))) - .is_some_and(|(_, _, info)| matches!(info.instr.real(), Some(Instruction::ToBool))) - }); - let effective_except_handler = - if is_conditional_jump(&instr.instr) && next_is_normalized_backward_jump { - None - } else if instr.except_handler.is_none() - && matches!(instr.instr.real(), Some(Instruction::NotTaken)) - && let Some((current_handler, _)) = current_entry - && let Some((_, _, next)) = next - && let Some(next_handler) = next.except_handler - && same_handler(current_handler, next_handler) - && !next.instr.is_scope_exit() - && !(next_is_jump_target_block && previous_jump_uses_to_bool) - && !previous_is_conditional_ifexp_jump - && !(conditional_jumps_since_exit > 1 - && previous_jump_uses_to_bool - && previous_is_general_bool_conditional_jump) - { - Some(current_handler) +} + +fn instruction_is_terminator(op: Instruction) -> bool { + op.is_terminator() +} + +fn resolve_next_locations(instructions: &[CodeUnit], locations: &mut [LineTableLocation]) { + debug_assert_eq!(instructions.len(), locations.len()); + let mut next_location = no_linetable_location(); + for (instruction, location) in instructions.iter().zip(locations.iter_mut()).rev() { + if location.line == NEXT_LOCATION_OVERRIDE { + *location = if instruction_is_terminator(instruction.op) { + no_linetable_location() } else { - instr.except_handler + next_location }; + } + next_location = *location; + } +} - // instr_size includes EXTENDED_ARG and CACHE entries - let instr_size = instr.arg.instr_size() as u32 + instr.cache_entries; +/// assemble.c assemble_exception_table +fn generate_exception_table( + instrs: &[InstructionSequenceEntry], + instruction_offsets: &[u32], + end_offset: u32, +) -> Box<[u8]> { + let mut entries: Vec = Vec::new(); + let mut current_entry: Option<(InstructionSequenceExceptHandlerInfo, u32)> = None; + let same_handler = |left: InstructionSequenceExceptHandlerInfo, + right: InstructionSequenceExceptHandlerInfo| { + // CPython assemble_exception_table() starts a new table entry only + // when h_label changes. h_startdepth and h_preserve_lasti come from + // the active handler for that label. + left.target_offset == right.target_offset + }; - match (¤t_entry, effective_except_handler) { + for (idx, instr) in instrs.iter().enumerate() { + let instr_offset = instruction_offsets[idx]; + match (¤t_entry, instr.except_handler) { // No current entry, no handler - nothing to do (None, None) => {} // No current entry, handler starts - begin new entry (None, Some(handler)) => { - current_entry = Some((handler, instr_index)); - conditional_jumps_since_exit = 0; + current_entry = Some((handler, instr_offset)); } // Current entry exists, same handler - continue @@ -5247,49 +4377,42 @@ fn generate_exception_table(blocks: &[Block], block_to_index: &[u32]) -> Box<[u8 // Current entry exists, different handler - finish current, start new (Some((curr_handler, start)), Some(handler)) => { - let target_index = block_to_index[curr_handler.handler_block.idx()]; + let target_offset = instruction_offsets + [curr_handler.target_offset.expect("missing handler target")]; entries.push(ExceptionTableEntry::new( *start, - instr_index, - target_index, + instr_offset, + target_offset, curr_handler.stack_depth as u16, curr_handler.preserve_lasti, )); - current_entry = Some((handler, instr_index)); - conditional_jumps_since_exit = 0; + current_entry = Some((handler, instr_offset)); } // Current entry exists, no handler - finish current entry (Some((curr_handler, start)), None) => { - let target_index = block_to_index[curr_handler.handler_block.idx()]; + let target_offset = instruction_offsets + [curr_handler.target_offset.expect("missing handler target")]; entries.push(ExceptionTableEntry::new( *start, - instr_index, - target_index, + instr_offset, + target_offset, curr_handler.stack_depth as u16, curr_handler.preserve_lasti, )); current_entry = None; } } - - if effective_except_handler.is_some() && is_conditional_jump(&instr.instr) { - conditional_jumps_since_exit += 1; - } - if instr.instr.is_scope_exit() { - conditional_jumps_since_exit = 0; - } - - instr_index += instr_size; // Account for EXTENDED_ARG instructions } // Finish any remaining entry if let Some((curr_handler, start)) = current_entry { - let target_index = block_to_index[curr_handler.handler_block.idx()]; + let target_offset = + instruction_offsets[curr_handler.target_offset.expect("missing handler target")]; entries.push(ExceptionTableEntry::new( start, - instr_index, - target_index, + end_offset, + target_offset, curr_handler.stack_depth as u16, curr_handler.preserve_lasti, )); @@ -5300,29 +4423,35 @@ fn generate_exception_table(blocks: &[Block], block_to_index: &[u32]) -> Box<[u8 /// Mark exception handler target blocks. /// flowgraph.c mark_except_handlers -pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { - // Reset handler flags - for block in blocks.iter_mut() { - block.except_handler = false; - block.preserve_lasti = false; +pub(crate) fn mark_except_handlers(blocks: &mut [Block]) -> crate::InternalResult<()> { + let block_order = layout_block_order(blocks); + for block_idx in block_order.iter().copied() { + debug_assert!(!blocks[block_idx.idx()].except_handler); } - // Mark target blocks of SETUP_* as except handlers - let targets: Vec = blocks - .iter() - .flat_map(|b| b.instructions.iter()) - .filter(|i| i.instr.is_block_push() && i.target != BlockIdx::NULL) - .map(|i| i.target.idx()) - .collect(); + + let mut targets = Vec::new(); + for block_idx in block_order.iter().copied() { + for instr in &blocks[block_idx.idx()].instructions { + if instr.instr.is_block_push() { + if instr.target == BlockIdx::NULL { + return Err(InternalError::MalformedControlFlowGraph); + } + targets.push(instr.target.idx()); + } + } + } + for idx in targets { blocks[idx].except_handler = true; } + Ok(()) } /// flowgraph.c mark_cold (two-pass to match CPython). /// -/// Phase 1 (mark_warm): propagate "warm" from entry via forward edges -/// (fall-through and jump targets). Skip except_handler targets, matching -/// the practical effect of CPython's assertion in mark_warm. +/// Phase 1 (mark_warm): propagate "warm" from entry via fall-through and +/// jump targets. CPython asserts while visiting warm blocks that they are not +/// exception handlers. /// /// Phase 2 (mark_cold): propagate "cold" from except_handler blocks via /// forward edges. Blocks reached only via runtime exception dispatch are @@ -5334,35 +4463,37 @@ pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { /// continuation for a nested try/except whose inner_end was emptied by /// optimize_cfg). This matches CPython's behavior and is necessary for /// optimize_load_fast_borrow to terminate fall-through at those placeholders. -fn mark_cold(blocks: &mut [Block]) { +fn mark_cold(blocks: &mut [Block]) -> Vec { let n = blocks.len(); + let block_order = layout_block_order(blocks); + for block_idx in block_order.iter().copied() { + let block = &blocks[block_idx.idx()]; + debug_assert!(!block.cold); + } let mut warm = vec![false; n]; - let mut queue = VecDeque::new(); + let mut stack = Vec::new(); warm[0] = true; - queue.push_back(BlockIdx(0)); + stack.push(BlockIdx(0)); - while let Some(block_idx) = queue.pop_front() { + while let Some(block_idx) = stack.pop() { let block = &blocks[block_idx.idx()]; + debug_assert!(!block.except_handler); - let has_fallthrough = block - .instructions - .last() - .is_none_or(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()); - if has_fallthrough && block.next != BlockIdx::NULL { + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { let next_idx = block.next.idx(); - if !blocks[next_idx].except_handler && !warm[next_idx] { + if !warm[next_idx] { warm[next_idx] = true; - queue.push_back(block.next); + stack.push(block.next); } } for instr in &block.instructions { - if instr.target != BlockIdx::NULL && !instr.instr.is_block_push() { + if is_jump_instruction(instr) && instr.target != BlockIdx::NULL { let target_idx = instr.target.idx(); - if !blocks[target_idx].except_handler && !warm[target_idx] { + if !warm[target_idx] { warm[target_idx] = true; - queue.push_back(instr.target); + stack.push(instr.target); } } } @@ -5370,61 +4501,62 @@ fn mark_cold(blocks: &mut [Block]) { let mut cold = vec![false; n]; let mut cold_visited = vec![false; n]; - let mut cold_queue: VecDeque = VecDeque::new(); - for (i, block) in blocks.iter().enumerate() { + let mut cold_stack = Vec::new(); + for block_idx in block_order.iter().copied() { + let i = block_idx.idx(); + let block = &blocks[i]; if block.except_handler { - cold_queue.push_back(BlockIdx::new(i as u32)); + debug_assert!(!warm[i]); + cold_stack.push(block_idx); cold_visited[i] = true; } } - while let Some(block_idx) = cold_queue.pop_front() { + while let Some(block_idx) = cold_stack.pop() { let idx = block_idx.idx(); cold[idx] = true; let block = &blocks[idx]; - let has_fallthrough = block - .instructions - .last() - .is_none_or(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()); - if has_fallthrough && block.next != BlockIdx::NULL { + if block_has_fallthrough(block) && block.next != BlockIdx::NULL { let next_idx = block.next.idx(); if !warm[next_idx] && !cold_visited[next_idx] { cold_visited[next_idx] = true; - cold_queue.push_back(block.next); + cold_stack.push(block.next); } } - for instr in &block.instructions { - if instr.target != BlockIdx::NULL && !instr.instr.is_block_push() { + let instr_count = block.instructions.len(); + for (i, instr) in block.instructions.iter().enumerate() { + if is_jump_instruction(instr) && instr.target != BlockIdx::NULL { + debug_assert_eq!(i, instr_count - 1); let target_idx = instr.target.idx(); if !warm[target_idx] && !cold_visited[target_idx] { cold_visited[target_idx] = true; - cold_queue.push_back(instr.target); + cold_stack.push(instr.target); } } } } - for (i, block) in blocks.iter_mut().enumerate() { - block.cold = cold[i]; + for block_idx in block_order { + let i = block_idx.idx(); + blocks[i].cold = cold[i]; } + warm } /// flowgraph.c push_cold_blocks_to_end -fn push_cold_blocks_to_end(blocks: &mut Vec) { +fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> { if blocks.len() <= 1 { - return; + return Ok(()); } - mark_cold(blocks); + let warm = mark_cold(blocks); // If a cold block falls through to a warm block, add an explicit jump let fixups: Vec<(BlockIdx, BlockIdx)> = iter_blocks(blocks) .filter(|(_, block)| { block.cold && block.next != BlockIdx::NULL - && !blocks[block.next.idx()].cold - && block.instructions.last().is_none_or(|ins| { - !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump() - }) + && warm[block.next.idx()] + && block_has_fallthrough(block) }) .map(|(idx, block)| (idx, block.next)) .collect(); @@ -5435,27 +4567,20 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) { cold: true, ..Block::default() }; - jump_block.instructions.push(InstructionInfo { - instr: PseudoOpcode::JumpNoInterrupt.into(), - arg: OpArg::new(0), - target: warm_next, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: Some(-1), - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, - }); + basicblock_add_jump_op( + &mut jump_block, + InstructionInfo { + instr: PseudoOpcode::JumpNoInterrupt.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(-1), + cache_entries: 0, + }, + warm_next, + )?; jump_block.next = blocks[cold_idx.idx()].next; blocks[cold_idx.idx()].next = jump_block_idx; blocks.push(jump_block); @@ -5503,6740 +4628,1208 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) { last = blocks[last.idx()].next; } blocks[last.idx()].next = cold_head; - remove_redundant_nops_and_jumps(blocks); + remove_redundant_nops_and_jumps(blocks)?; } + Ok(()) } -/// Split blocks at branch points so each block has at most one branch -/// (conditional/unconditional jump) as its last instruction. -/// This matches CPython's CFG structure where each basic block has one exit. -fn split_blocks_at_jumps(blocks: &mut Vec) { - let mut bi = 0; - while bi < blocks.len() { - // Find the first jump/branch instruction in the block - let split_at = { - let block = &blocks[bi]; - let mut found = None; - for (i, ins) in block.instructions.iter().enumerate() { - if is_conditional_jump(&ins.instr) - || ins.instr.is_unconditional_jump() - || ins.instr.is_scope_exit() - { - if i + 1 < block.instructions.len() { - found = Some(i + 1); - } - break; - } +/// flowgraph.c check_cfg +fn check_cfg(blocks: &[Block]) -> crate::InternalResult<()> { + for (_, block) in iter_blocks(blocks) { + for (i, ins) in block.instructions.iter().enumerate() { + debug_assert!(!ins.instr.is_assembler()); + if ins.instr.is_terminator() && i + 1 != block.instructions.len() { + return Err(InternalError::MalformedControlFlowGraph); } - found - }; - if let Some(pos) = split_at { - let new_block_idx = BlockIdx(blocks.len() as u32); - let tail: Vec = blocks[bi].instructions.drain(pos..).collect(); - let old_next = blocks[bi].next; - let cold = blocks[bi].cold; - let disable_load_fast_borrow = blocks[bi].disable_load_fast_borrow; - blocks[bi].next = new_block_idx; - blocks.push(Block { - instructions: tail, - next: old_next, - cold, - disable_load_fast_borrow, - ..Block::default() - }); - // Don't increment bi - re-check current block (it might still have issues) - } else { - bi += 1; } } + Ok(()) } -fn retarget_assert_conditional_jumps_to_empty_predecessor(blocks: &mut [Block]) { - fn is_assertion_error_load(info: &InstructionInfo) -> bool { - matches!( - info.instr.real(), - Some(Instruction::LoadCommonConstant { idx }) - if idx.get(info.arg) == oparg::CommonConstant::AssertionError - ) - } +#[derive(Clone, Copy, PartialEq, Eq)] +enum JumpThreadKind { + Plain, + NoInterrupt, +} - fn is_plain_empty_label(block: &Block) -> bool { - block.instructions.is_empty() - && block.next != BlockIdx::NULL - && !block.except_handler - && !block.preserve_lasti - && block.start_depth.is_none() - && !block.cold - && !block.disable_load_fast_borrow - && !block.try_else_orelse_entry - && !block.label - } +fn jump_thread_kind(instr: AnyInstruction) -> Option { + Some(match instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::Jump) + | AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward) => JumpThreadKind::Plain, + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) + | AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => JumpThreadKind::NoInterrupt, + _ => return None, + }) +} - fn assertion_failure_start_line(block: &Block) -> Option { - block - .instructions - .iter() - .find(|info| is_assertion_error_load(info)) - .map(instruction_lineno) +fn threaded_jump_instr( + source: AnyInstruction, + target: AnyInstruction, + conditional: bool, +) -> Option { + let target_kind = jump_thread_kind(target)?; + if conditional { + return (target_kind == JumpThreadKind::Plain).then_some(source); } - let empty_predecessors: Vec> = (0..blocks.len()) - .map(|target| { - let target = BlockIdx::new(target as u32); - blocks - .iter() - .enumerate() - .find(|(_, block)| is_plain_empty_label(block) && block.next == target) - .map(|(idx, _)| BlockIdx::new(idx as u32)) - }) - .collect(); - let assertion_lines: Vec> = - blocks.iter().map(assertion_failure_start_line).collect(); + let source_kind = jump_thread_kind(source)?; + let result_kind = if source_kind == JumpThreadKind::NoInterrupt + && target_kind == JumpThreadKind::NoInterrupt + { + JumpThreadKind::NoInterrupt + } else { + JumpThreadKind::Plain + }; - for block in &mut *blocks { - for instr in &mut block.instructions { - if instr.target == BlockIdx::NULL || !is_conditional_jump(&instr.instr) { - continue; + Some(match (source.into(), result_kind) { + (AnyOpcode::Pseudo(_), JumpThreadKind::Plain) => PseudoOpcode::Jump.into(), + (AnyOpcode::Pseudo(_), JumpThreadKind::NoInterrupt) => PseudoOpcode::JumpNoInterrupt.into(), + (AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt), JumpThreadKind::Plain) => { + Opcode::JumpBackward.into() + } + (AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt), JumpThreadKind::NoInterrupt) => source, + (AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward), JumpThreadKind::Plain) => { + source + } + ( + AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward), + JumpThreadKind::NoInterrupt, + ) => PseudoOpcode::JumpNoInterrupt.into(), + _ => return None, + }) +} + +/// flowgraph.c optimize_basic_block + jump_thread +fn jump_threading_block(blocks: &mut [Block], block_idx: BlockIdx) -> crate::InternalResult<()> { + let bi = block_idx.idx(); + while let Some(last_idx) = blocks[bi].instructions.len().checked_sub(1) { + let ins = blocks[bi].instructions[last_idx]; + let target = ins.target; + if target == BlockIdx::NULL { + return Ok(()); + } + if !(ins.instr.is_unconditional_jump() || is_conditional_jump(&ins.instr)) { + return Ok(()); + } + if blocks[target.idx()].instructions.is_empty() { + return Ok(()); + } + let target_ins = blocks[target.idx()].instructions[0]; + match ( + ins.instr.pseudo().map(Into::into), + target_ins.instr.pseudo().map(Into::into), + ) { + ( + Some(source @ (PseudoOpcode::JumpIfFalse | PseudoOpcode::JumpIfTrue)), + Some(PseudoOpcode::Jump), + ) + | (Some(source @ PseudoOpcode::JumpIfFalse), Some(PseudoOpcode::JumpIfFalse)) + | (Some(source @ PseudoOpcode::JumpIfTrue), Some(PseudoOpcode::JumpIfTrue)) => { + let final_target = target_ins.target; + if final_target == BlockIdx::NULL || final_target == ins.target { + return Ok(()); + } + set_to_nop(&mut blocks[bi].instructions[last_idx]); + basicblock_add_jump(blocks, block_idx, source.into(), final_target, target_ins)?; + return Ok(()); } - let target = instr.target; - let Some(empty) = empty_predecessors[target.idx()] else { - continue; - }; - let Some(assert_line) = assertion_lines[target.idx()] else { + (Some(PseudoOpcode::JumpIfFalse), Some(PseudoOpcode::JumpIfTrue)) + | (Some(PseudoOpcode::JumpIfTrue), Some(PseudoOpcode::JumpIfFalse)) => { + let next = blocks[target.idx()].next; + if next == BlockIdx::NULL || next == target { + return Ok(()); + } + blocks[bi].instructions[last_idx].target = next; continue; - }; - if instruction_lineno(instr) != assert_line { - instr.target = empty; } + _ => {} + } + if !target_ins.instr.is_unconditional_jump() + || target_ins.target == BlockIdx::NULL + || target_ins.target == target + { + return Ok(()); + } + let conditional = is_conditional_jump(&ins.instr); + let final_target = target_ins.target; + let Some(threaded_instr) = (if conditional { + match jump_thread_kind(target_ins.instr) { + Some(JumpThreadKind::Plain) => Some(ins.instr), + _ => None, + } + } else { + threaded_jump_instr(ins.instr, target_ins.instr, false) + }) else { + return Ok(()); + }; + if ins.target == final_target { + return Ok(()); } + set_to_nop(&mut blocks[bi].instructions[last_idx]); + basicblock_add_jump(blocks, block_idx, threaded_instr, final_target, target_ins)?; + return Ok(()); } + Ok(()) } -fn canonicalize_empty_label_blocks(blocks: &mut [Block]) { - fn is_assertion_error_load(info: &InstructionInfo) -> bool { - matches!( - info.instr.real(), - Some(Instruction::LoadCommonConstant { idx }) - if idx.get(info.arg) == oparg::CommonConstant::AssertionError - ) - } - - fn block_has_explicit_reference(blocks: &[Block], idx: BlockIdx) -> bool { - blocks.iter().any(|block| { - block.instructions.iter().any(|info| { - info.target == idx - || info - .except_handler - .is_some_and(|handler| handler.handler_block == idx) - }) - }) - } +/// flowgraph.c basicblock_add_jump +fn basicblock_add_jump( + blocks: &mut [Block], + block_idx: BlockIdx, + instr: AnyInstruction, + target: BlockIdx, + loc_source: InstructionInfo, +) -> crate::InternalResult<()> { + let bi = block_idx.idx(); + basicblock_add_jump_op( + &mut blocks[bi], + InstructionInfo { + instr, + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: loc_source.location, + end_location: loc_source.end_location, + except_handler: None, + lineno_override: loc_source.lineno_override, + cache_entries: 0, + }, + target, + ) +} - fn is_plain_empty_label(blocks: &[Block], idx: BlockIdx) -> bool { - let block = &blocks[idx.idx()]; - block.instructions.is_empty() - && block.next != BlockIdx::NULL - && !block.except_handler - && !block.preserve_lasti - && block.start_depth.is_none() - && !block.cold - && !block.disable_load_fast_borrow - && !block.try_else_orelse_entry - && !block.load_fast_barrier - && (!block.label - || block.load_fast_passthrough - || !block_has_explicit_reference(blocks, idx)) - } - - fn block_falls_through(block: &Block) -> bool { - block - .instructions - .last() - .is_some_and(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()) - } +pub(crate) fn is_conditional_jump(instr: &AnyInstruction) -> bool { + matches!( + instr.real().map(Into::into), + Some( + Opcode::PopJumpIfFalse + | Opcode::PopJumpIfTrue + | Opcode::PopJumpIfNone + | Opcode::PopJumpIfNotNone + ) + ) || matches!( + instr.pseudo(), + Some(PseudoInstruction::JumpIfFalse { .. } | PseudoInstruction::JumpIfTrue { .. }) + ) +} - fn has_fallthrough_predecessor_matching( - blocks: &[Block], - idx: BlockIdx, - mut matches_predecessor: F, - ) -> bool - where - F: FnMut(&Block) -> bool, - { - let mut seen = vec![false; blocks.len()]; - let mut stack = vec![idx]; - while let Some(target) = stack.pop() { - if target == BlockIdx::NULL { +/// flowgraph.c convert_pseudo_conditional_jumps +fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { + let block_order = layout_block_order(blocks); + for block_idx in block_order { + let block = &mut blocks[block_idx.idx()]; + let mut i = 0; + while i < block.instructions.len() { + let Some(pseudo) = block.instructions[i].instr.pseudo() else { + i += 1; continue; - } - for (block_idx, block) in blocks.iter().enumerate() { - if block.next != target { - continue; + }; + let jump = match pseudo { + PseudoInstruction::JumpIfFalse { .. } => { + debug_assert_eq!(i, block.instructions.len() - 1); + Instruction::PopJumpIfFalse { + delta: Arg::marker(), + } } - if block_falls_through(block) { - if matches_predecessor(block) { - return true; + PseudoInstruction::JumpIfTrue { .. } => { + debug_assert_eq!(i, block.instructions.len() - 1); + Instruction::PopJumpIfTrue { + delta: Arg::marker(), } - } else if block.instructions.is_empty() - && block.load_fast_passthrough - && !seen[block_idx] - { - seen[block_idx] = true; - stack.push(BlockIdx::new(block_idx as u32)); } - } + _ => { + i += 1; + continue; + } + }; + + let jump_info = InstructionInfo { + instr: jump.into(), + ..block.instructions[i] + }; + block.instructions[i].instr = Instruction::Copy { i: Arg::marker() }.into(); + block.instructions[i].arg = OpArg::new(1); + block.instructions[i].target = BlockIdx::NULL; + + let mut to_bool = block.instructions[i]; + to_bool.instr = Instruction::ToBool.into(); + to_bool.arg = OpArg::new(0); + to_bool.target = BlockIdx::NULL; + + basicblock_insert_instruction(block, i + 1, to_bool); + basicblock_insert_instruction(block, i + 2, jump_info); + i += 3; } - false } +} - fn block_has_exception_setup_or_handler(block: &Block) -> bool { - block_is_protected(block) - || block.instructions.iter().any(|info| { - matches!( - info.instr, - AnyInstruction::Pseudo( - PseudoInstruction::SetupFinally { .. } - | PseudoInstruction::SetupCleanup { .. } - ) - ) - }) - } +/// Invert a conditional jump opcode. +fn reversed_conditional(instr: &AnyInstruction) -> Option { + Some(match AnyOpcode::from(*instr).real()? { + Opcode::PopJumpIfFalse => Opcode::PopJumpIfTrue.into(), + Opcode::PopJumpIfTrue => Opcode::PopJumpIfFalse.into(), + Opcode::PopJumpIfNone => Opcode::PopJumpIfNotNone.into(), + Opcode::PopJumpIfNotNone => Opcode::PopJumpIfNone.into(), + _ => return None, + }) +} - fn block_has_fast_load(block: &Block) -> bool { - block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some( - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - ) - }) +/// flowgraph.c normalize_jumps_in_block +fn normalize_jumps_in_block( + blocks: &mut Vec, + block_idx: BlockIdx, + visited: &mut Vec, +) -> crate::InternalResult<()> { + let idx = block_idx.idx(); + let Some(last_ins) = blocks[idx].instructions.last().copied() else { + return Ok(()); + }; + if !is_conditional_jump(&last_ins.instr) || last_ins.target == BlockIdx::NULL { + return Ok(()); } - fn block_starts_with_make_closure_from_fast_loads(block: &Block) -> bool { - let mut iter = block.instructions.iter().map(|info| info.instr.real()); - let mut fast_loads = 0; - while matches!( - iter.clone().next(), - Some(Some( - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - )) - ) { - fast_loads += 1; - iter.next(); - } - fast_loads > 0 - && matches!(iter.next(), Some(Some(Instruction::BuildTuple { .. }))) - && matches!(iter.next(), Some(Some(Instruction::LoadConst { .. }))) - && matches!(iter.next(), Some(Some(Instruction::MakeFunction))) - && matches!( - iter.next(), - Some(Some(Instruction::SetFunctionAttribute { .. })) - ) + let target = last_ins.target; + let is_forward = !visited[target.idx()]; + + if is_forward { + // Insert NOT_TAKEN after forward conditional jump. + let not_taken = InstructionInfo { + instr: Opcode::NotTaken.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: last_ins.location, + end_location: last_ins.end_location, + except_handler: None, + lineno_override: last_ins.lineno_override, + cache_entries: 0, + }; + basicblock_addop(&mut blocks[idx], not_taken); + return Ok(()); } - fn block_is_for_cleanup(block: &Block) -> bool { - matches!( - block.instructions.as_slice(), - [end_for, pop_iter] - if matches!(end_for.instr.real(), Some(Instruction::EndFor)) - && matches!(pop_iter.instr.real(), Some(Instruction::PopIter)) - ) + // Backward conditional jump: invert and create new block + // Transform: `cond_jump T` (backward) + // Into: `reversed_cond_jump b_next` + new block [NOT_TAKEN, JUMP T] + let loc = last_ins.location; + let end_loc = last_ins.end_location; + + if let Some(reversed) = reversed_conditional(&last_ins.instr) { + let old_next = blocks[idx].next; + let is_cold = blocks[idx].cold; + + // Create new block with NOT_TAKEN + JUMP to original backward target + let new_block_idx = BlockIdx(blocks.len() as u32); + let mut new_block = Block { + cold: is_cold, + start_depth: blocks[target.idx()].start_depth, + ..Block::default() + }; + basicblock_addop( + &mut new_block, + InstructionInfo { + instr: Opcode::NotTaken.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: loc, + end_location: end_loc, + except_handler: None, + lineno_override: last_ins.lineno_override, + cache_entries: 0, + }, + ); + basicblock_add_jump_op( + &mut new_block, + InstructionInfo { + instr: PseudoOpcode::Jump.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: loc, + end_location: end_loc, + except_handler: None, + lineno_override: last_ins.lineno_override, + cache_entries: 0, + }, + target, + )?; + new_block.next = old_next; + + // Update the conditional jump: invert opcode, target = old next block + let last_mut = blocks[idx].instructions.last_mut().unwrap(); + last_mut.instr = reversed; + last_mut.target = old_next; + + // Splice new block between current and old next + blocks[idx].next = new_block_idx; + blocks.push(new_block); + + // Extend visited array and update visit order + visited.push(true); } + Ok(()) +} - fn block_returns_call_with_fast_load(block: &Block) -> bool { - let mut seen_fast_load = false; - let mut previous_was_call_after_fast_load = false; - for info in &block.instructions { - match info.instr.real() { - Some( - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. }, - ) => { - seen_fast_load = true; - previous_was_call_after_fast_load = false; - } - Some(Instruction::Call { .. } | Instruction::CallKw { .. }) if seen_fast_load => { - previous_was_call_after_fast_load = true; - } - Some(Instruction::ReturnValue) if previous_was_call_after_fast_load => { - return true; - } - Some(_) => previous_was_call_after_fast_load = false, - None => {} - } - } - false +/// flowgraph.c normalize_jumps +fn normalize_jumps(blocks: &mut Vec) -> crate::InternalResult<()> { + let mut visited = vec![false; blocks.len()]; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let idx = current.idx(); + visited[idx] = true; + normalize_jumps_in_block(blocks, current, &mut visited)?; + current = blocks[idx].next; } + Ok(()) +} - fn block_has_exception_match_handler(blocks: &[Block], block: &Block) -> bool { - let mut seen = vec![false; blocks.len()]; - let mut stack: Vec<_> = block - .instructions - .iter() - .filter_map(|info| info.except_handler.map(|handler| handler.handler_block)) - .collect(); - while let Some(idx) = stack.pop() { - if idx == BlockIdx::NULL || seen[idx.idx()] { - continue; - } - seen[idx.idx()] = true; - let handler = &blocks[idx.idx()]; - if handler.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) - ) - }) { - return true; - } - for info in &handler.instructions { - if info.target != BlockIdx::NULL { - stack.push(info.target); - } - } - if block_has_fallthrough(handler) && handler.next != BlockIdx::NULL { - stack.push(handler.next); - } - } - false +/// flowgraph.c basicblock_inline_small_or_no_lineno_blocks +fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: BlockIdx) -> bool { + const MAX_COPY_SIZE: usize = 4; + + let Some(last) = blocks[block_idx.idx()].instructions.last().copied() else { + return false; + }; + if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { + return false; } - fn block_is_calling_finally_cleanup(block: &Block) -> bool { - let has_call = block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some(Instruction::Call { .. } | Instruction::CallKw { .. }) - ) - }); - has_call - && block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::Reraise { .. }))) - && !block.instructions.iter().any(|info| { - matches!( - info.instr.real(), - Some(Instruction::CheckExcMatch | Instruction::CheckEgMatch) - ) - }) + let target = last.target; + let small_exit_block = is_scope_exit_block(&blocks[target.idx()]) + && blocks[target.idx()].instructions.len() <= MAX_COPY_SIZE; + let no_lineno_no_fallthrough = + block_has_no_lineno(&blocks[target.idx()]) && !block_has_fallthrough(&blocks[target.idx()]); + if !small_exit_block && !no_lineno_no_fallthrough { + return false; } - fn handler_chain_calls_finally_cleanup(blocks: &[Block], handler: BlockIdx) -> bool { - let mut seen = vec![false; blocks.len()]; - let mut stack = vec![handler]; - while let Some(idx) = stack.pop() { - if idx == BlockIdx::NULL || seen[idx.idx()] { - continue; - } - seen[idx.idx()] = true; - let block = &blocks[idx.idx()]; - if block_is_calling_finally_cleanup(block) { - return true; - } - for info in &block.instructions { - if info.target != BlockIdx::NULL { - stack.push(info.target); - } - } - if block_has_fallthrough(block) && block.next != BlockIdx::NULL { - stack.push(block.next); - } - } - false - } - - fn block_has_finally_cleanup_handler(blocks: &[Block], block: &Block) -> bool { - block.instructions.iter().any(|info| { - let setup_finally_handler = matches!( - info.instr, - AnyInstruction::Pseudo(PseudoInstruction::SetupFinally { .. }) - ) - .then_some(info.target); - setup_finally_handler - .into_iter() - .chain(info.except_handler.map(|handler| handler.handler_block)) - .any(|handler| handler_chain_calls_finally_cleanup(blocks, handler)) - }) - } - - fn has_cpython_empty_load_fast_barrier(blocks: &[Block], idx: BlockIdx) -> bool { - if !is_plain_empty_label(blocks, idx) { - return false; - } - let target = next_nonempty_block(blocks, blocks[idx.idx()].next); - if target == BlockIdx::NULL { - return false; - } - let Some(assertion) = blocks[target.idx()] - .instructions - .iter() - .find(|info| is_assertion_error_load(info)) - else { - return false; - }; - let assert_line = instruction_lineno(assertion); - let mut has_jump_predecessor = false; - for block in blocks { - if block.instructions.iter().any(|info| { - info.target == idx - && instruction_lineno(info) != assert_line - && is_conditional_jump(&info.instr) - }) { - has_jump_predecessor = true; - } - } - has_jump_predecessor - } - - fn has_cpython_empty_join_barrier(blocks: &[Block], idx: BlockIdx) -> bool { - if !is_plain_empty_label(blocks, idx) { - return false; - } - let target = next_nonempty_block(blocks, blocks[idx.idx()].next); - if target != BlockIdx::NULL - && blocks.iter().any(|block| { - !block.cold - && !block.except_handler - && block.instructions.iter().any(|info| { - info.target == target - && matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) - }) - }) - { - return false; - } - let has_conditional_target_predecessor = blocks.iter().any(|block| { - block - .instructions - .iter() - .any(|info| info.target == idx && is_conditional_jump(&info.instr)) - }); - let has_fallthrough_predecessor = blocks - .iter() - .any(|block| block.next == idx && block_falls_through(block)); - has_conditional_target_predecessor && has_fallthrough_predecessor - } - - fn has_cpython_empty_try_end_barrier(blocks: &[Block], idx: BlockIdx) -> bool { - if !is_plain_empty_label(blocks, idx) { - return false; - } - let target = next_nonempty_block(blocks, blocks[idx.idx()].next); - if target == BlockIdx::NULL { - return false; - } - let falls_through_from_finally_cleanup = - has_fallthrough_predecessor_matching(blocks, idx, |block| { - block_has_finally_cleanup_handler(blocks, block) - }); - if blocks[idx.idx()].load_fast_passthrough && falls_through_from_finally_cleanup { - return false; - } - if block_starts_with_make_closure_from_fast_loads(&blocks[target.idx()]) { - // CPython codegen_make_closure() emits LOAD_CLOSURE/BUILD_TUPLE, - // LOAD_CONST, MAKE_FUNCTION, SET_FUNCTION_ATTRIBUTE in the - // continuation block. A Rust-only empty try-end label before that - // sequence must not stop optimize_load_fast() from visiting it. - return false; - } - let handler_resumes_to_target = blocks.iter().any(|block| { - (block.cold || block.except_handler) - && block.instructions.iter().any(|info| { - (info.target == target || next_nonempty_block(blocks, info.target) == target) - && matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - | Instruction::JumpForward { .. } - ) - ) - }) - }); - let normal_backedge_to_target = blocks.iter().any(|block| { - !block.cold - && !block.except_handler - && block.instructions.iter().any(|info| { - info.target == target - && matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) - }) - }); - if handler_resumes_to_target && normal_backedge_to_target { - return false; - } - let falls_through_from_for_cleanup = blocks.iter().any(|block| { - block.next == idx && block_falls_through(block) && block_is_for_cleanup(block) - }); - if handler_resumes_to_target && falls_through_from_for_cleanup { - return false; - } - if handler_resumes_to_target - && block_has_fast_load(&blocks[target.idx()]) - && has_fallthrough_predecessor_matching(blocks, idx, |block| { - !block.cold && !block.except_handler - }) - { - // CPython's codegen_try_except() can leave an empty USE_LABEL(end) - // between the normal protected path and the following statement, - // while a cold handler resumes directly to that following block. - // flowgraph.c::optimize_load_fast() visits the empty end block and - // stops because basicblock_last_instr() is NULL, so the following - // block's LOAD_FAST instructions are not borrowed. - return true; - } - if !handler_resumes_to_target - && block_has_fast_load(&blocks[target.idx()]) - && has_fallthrough_predecessor_matching(blocks, idx, |block| { - block_has_exception_setup_or_handler(block) - && block_has_exception_match_handler(blocks, block) - && !(handler_resumes_to_target - && block_returns_call_with_fast_load(&blocks[target.idx()])) - }) - { - return true; - } - if block_has_fast_load(&blocks[target.idx()]) - && has_fallthrough_predecessor_matching(blocks, idx, |block| { - block_has_exception_setup_or_handler(block) - }) - { - // CPython optimize_load_fast() only pushes b_next when - // basicblock_last_instr(block) is not NULL. codegen_try_except() - // can leave the normal try path falling through an empty end label, - // including bare except handlers that have no CHECK_EXC_MATCH. - return true; - } - let reaches_backedge = normal_region_reaches_backedge(blocks, target); - has_fallthrough_predecessor_matching(blocks, idx, |block| { - if reaches_backedge { - block_has_exception_setup_or_handler(block) - || block_has_exception_match_handler(blocks, block) - } else { - block_has_finally_cleanup_handler(blocks, block) - && block_has_exception_match_handler(blocks, block) - && block_has_fast_load(&blocks[target.idx()]) - } - }) - } - - fn block_tail_calls_with_fast_pair(block: &Block) -> bool { - let mut seen_fast_pair = false; - for info in &block.instructions { - if matches!( - info.instr.real(), - Some( - Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) - ) { - seen_fast_pair = true; - } else if seen_fast_pair - && matches!( - info.instr.real(), - Some(Instruction::Call { .. } | Instruction::CallKw { .. }) - ) - { - return true; - } - } - false - } - - fn has_cpython_empty_while_break_call_barrier(blocks: &[Block], idx: BlockIdx) -> bool { - if !is_plain_empty_label(blocks, idx) { - return false; - } - let target = next_nonempty_block(blocks, blocks[idx.idx()].next); - if target == BlockIdx::NULL || !block_tail_calls_with_fast_pair(&blocks[target.idx()]) { - return false; - } - blocks.iter().any(|block| { - if block.next != idx { - return false; - } - if matches!( - block.instructions.as_slice(), - [ - InstructionInfo { - instr: AnyInstruction::Real(Instruction::Nop), - .. - }, - jump - ] if jump.instr.is_unconditional_jump() - && jump.target != BlockIdx::NULL - && next_nonempty_block(blocks, jump.target) == target - ) { - return true; - } - block_falls_through(block) - && block - .instructions - .last() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - }) - } - - fn has_cpython_empty_loop_entry_barrier(blocks: &[Block], idx: BlockIdx) -> bool { - if !is_plain_empty_label(blocks, idx) { - return false; - } - let target = next_nonempty_block(blocks, blocks[idx.idx()].next); - if target == BlockIdx::NULL { - return false; - } - let Some(target_first) = blocks[target.idx()].instructions.first() else { - return false; - }; - if !matches!(target_first.instr.real(), Some(Instruction::Nop)) { - return false; - } - let target_line = instruction_lineno(target_first); - if target_line <= 0 { - return false; - } - let has_loop_backedge = blocks.iter().any(|block| { - block.instructions.iter().any(|info| { - info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - && matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) - }) - }); - if !has_loop_backedge { - return false; - } - blocks.iter().any(|block| { - if block.next != idx { - return false; - } - if matches!( - block.instructions.as_slice(), - [nop, jump] if matches!(nop.instr.real(), Some(Instruction::Nop)) - && { - let line = instruction_lineno(nop); - line > 0 && line < target_line - } - && jump.instr.is_unconditional_jump() - && jump.target != BlockIdx::NULL - && next_nonempty_block(blocks, jump.target) == target - ) { - return true; - } - block_falls_through(block) - && block.instructions.last().is_some_and(|info| { - matches!(info.instr.real(), Some(Instruction::Nop)) && { - let line = instruction_lineno(info); - line > 0 && line < target_line - } - }) - }) - } - - fn has_cpython_empty_unreachable_fallthrough_barrier(blocks: &[Block], idx: BlockIdx) -> bool { - if !is_plain_empty_label(blocks, idx) { - return false; - } - let target = next_nonempty_block(blocks, blocks[idx.idx()].next); - if target == BlockIdx::NULL || !block_has_fast_load(&blocks[target.idx()]) { - return false; - } - blocks.iter().any(|block| { - if block.next != idx { - return false; - } - if block.instructions.last().is_some_and(|info| { - info.instr.is_unconditional_jump() - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) { - return true; - } - blocks[target.idx()].disable_load_fast_borrow - && block_falls_through(block) - && block - .instructions - .last() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - }) - } - - fn has_cpython_empty_assert_success_barrier(blocks: &[Block], idx: BlockIdx) -> bool { - if !is_plain_empty_label(blocks, idx) { - return false; - } - let target = next_nonempty_block(blocks, blocks[idx.idx()].next); - if target == BlockIdx::NULL || !block_has_fast_load(&blocks[target.idx()]) { - return false; - } - blocks.iter().any(|block| { - let jumps_to_empty = block - .instructions - .last() - .is_some_and(|info| info.target == idx && is_conditional_jump(&info.instr)); - if !jumps_to_empty || block.next == BlockIdx::NULL { - return false; - } - blocks[block.next.idx()] - .instructions - .iter() - .any(is_assertion_error_load) - }) - } - - fn normal_region_reaches_backedge(blocks: &[Block], target: BlockIdx) -> bool { - let mut stack = vec![target]; - let mut seen = vec![false; blocks.len()]; - while let Some(idx) = stack.pop() { - if idx == BlockIdx::NULL || seen[idx.idx()] { - continue; - } - seen[idx.idx()] = true; - let block = &blocks[idx.idx()]; - if block.cold || block.except_handler { - continue; - } - for info in &block.instructions { - if matches!( - info.instr, - AnyInstruction::Pseudo(PseudoInstruction::SetupWith { .. }) - ) { - return false; - } - if matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - ) - ) { - return true; - } - if info.target != BlockIdx::NULL && is_conditional_jump(&info.instr) { - stack.push(info.target); - } - } - let last_real = block - .instructions - .iter() - .rev() - .find(|info| info.instr.real().is_some()); - if last_real.is_some_and(|info| { - info.target != BlockIdx::NULL && info.instr.is_unconditional_jump() - }) { - stack.push(last_real.unwrap().target); - } else if !last_real.is_some_and(|info| info.instr.is_scope_exit()) { - stack.push(block.next); - } - } - false - } - - fn canonical_target(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { - while idx != BlockIdx::NULL - && is_plain_empty_label(blocks, idx) - && ((blocks[idx.idx()].load_fast_passthrough - && !has_cpython_empty_try_end_barrier(blocks, idx)) - || (!has_cpython_empty_load_fast_barrier(blocks, idx) - && !has_cpython_empty_join_barrier(blocks, idx) - && !has_cpython_empty_try_end_barrier(blocks, idx) - && !has_cpython_empty_while_break_call_barrier(blocks, idx) - && !has_cpython_empty_loop_entry_barrier(blocks, idx) - && !has_cpython_empty_unreachable_fallthrough_barrier(blocks, idx) - && !has_cpython_empty_assert_success_barrier(blocks, idx))) - { - idx = blocks[idx.idx()].next; - } - idx - } - - let replacements: Vec = (0..blocks.len()) - .map(|idx| canonical_target(blocks, BlockIdx(idx as u32))) - .collect(); - - for block in blocks.iter_mut() { - if block.next != BlockIdx::NULL { - block.next = replacements[block.next.idx()]; - } - for instr in &mut block.instructions { - if instr.target != BlockIdx::NULL { - instr.target = replacements[instr.target.idx()]; - } - if let Some(handler) = &mut instr.except_handler - && handler.handler_block != BlockIdx::NULL - { - handler.handler_block = replacements[handler.handler_block.idx()]; - } - } - } - - for idx in 1..blocks.len() { - if replacements[idx] != BlockIdx(idx as u32) { - blocks[idx].next = BlockIdx::NULL; - } - } -} - -/// Jump threading: when a block's last jump targets a block whose first -/// instruction is an unconditional jump, redirect to the final target. -/// flowgraph.c optimize_basic_block + jump_thread -fn jump_threading(blocks: &mut [Block]) { - jump_threading_impl(blocks, true); -} - -fn jump_threading_unconditional(blocks: &mut [Block]) { - jump_threading_impl(blocks, false); -} - -fn short_circuit_stub_conditional(block: &Block) -> Option { - let cond_idx = trailing_conditional_jump_index(block)?; - if cond_idx < 2 { - return None; - } - let [first, second, ..] = block.instructions.as_slice() else { - return None; - }; - if !matches!(first.instr.real(), Some(Instruction::Copy { i }) if i.get(first.arg) == 1) - || !matches!(second.instr.real(), Some(Instruction::ToBool)) - { - return None; - } - - let only_markers_between = block.instructions[2..cond_idx].iter().all(|info| { - matches!( - info.instr.real(), - None | Some(Instruction::Nop | Instruction::NotTaken) - ) - }); - if !only_markers_between { - return None; - } - - block.instructions[cond_idx].instr.real() -} - -fn opposite_short_circuit_target(block: &Block, source: AnyInstruction) -> bool { - let Some(conditional) = short_circuit_stub_conditional(block) else { - return false; - }; - matches!( - (source.real(), Some(conditional)), - ( - Some(Instruction::PopJumpIfFalse { .. }), - Some(Instruction::PopJumpIfTrue { .. }) - ) | ( - Some(Instruction::PopJumpIfTrue { .. }), - Some(Instruction::PopJumpIfFalse { .. }) - ) - ) -} - -fn same_short_circuit_target(block: &Block, source: AnyInstruction) -> Option { - let conditional = short_circuit_stub_conditional(block)?; - matches!( - (source.real(), Some(conditional)), - ( - Some(Instruction::PopJumpIfFalse { .. }), - Some(Instruction::PopJumpIfFalse { .. }) - ) | ( - Some(Instruction::PopJumpIfTrue { .. }), - Some(Instruction::PopJumpIfTrue { .. }) - ) - ) - .then_some(block.instructions[trailing_conditional_jump_index(block)?].target) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum JumpThreadKind { - Plain, - NoInterrupt, -} - -fn jump_thread_kind(instr: AnyInstruction) -> Option { - Some(match instr.into() { - AnyOpcode::Pseudo(PseudoOpcode::Jump) - | AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward) => JumpThreadKind::Plain, - AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) - | AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => JumpThreadKind::NoInterrupt, - _ => return None, - }) -} - -fn threaded_jump_instr( - source: AnyInstruction, - target: AnyInstruction, - conditional: bool, -) -> Option { - let target_kind = jump_thread_kind(target)?; - if conditional { - return (target_kind == JumpThreadKind::Plain).then_some(source); - } - - let source_kind = jump_thread_kind(source)?; - let result_kind = if source_kind == JumpThreadKind::NoInterrupt - && target_kind == JumpThreadKind::NoInterrupt - { - JumpThreadKind::NoInterrupt - } else { - JumpThreadKind::Plain - }; - - Some(match (source.into(), result_kind) { - (AnyOpcode::Pseudo(_), JumpThreadKind::Plain) => PseudoOpcode::Jump.into(), - (AnyOpcode::Pseudo(_), JumpThreadKind::NoInterrupt) => PseudoOpcode::JumpNoInterrupt.into(), - (AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt), JumpThreadKind::Plain) => { - Opcode::JumpBackward.into() - } - (AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt), JumpThreadKind::NoInterrupt) => source, - (AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward), JumpThreadKind::Plain) => { - source - } - ( - AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward), - JumpThreadKind::NoInterrupt, - ) => PseudoOpcode::JumpNoInterrupt.into(), - _ => return None, - }) -} - -fn can_thread_conditional_through_forward_nointerrupt( - source: AnyInstruction, - target_pos: u32, - final_target_pos: u32, -) -> bool { - matches!( - source.real(), - Some(Instruction::PopJumpIfNone { .. } | Instruction::PopJumpIfNotNone { .. }) - ) && final_target_pos > target_pos -} - -fn block_has_with_suppress_prefix(block: &Block, jump_idx: usize) -> bool { - let tail: Vec<_> = block.instructions[..jump_idx] - .iter() - .filter_map(|info| info.instr.real()) - .rev() - .take(5) - .collect(); - matches!( - tail.as_slice(), - [ - Instruction::PopTop, - Instruction::PopTop, - Instruction::PopTop, - Instruction::PopExcept, - Instruction::PopTop, - ] - ) -} - -fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { - let mut changed = true; - while changed { - changed = false; - let mut block_order = vec![u32::MAX; blocks.len()]; - let mut cursor = BlockIdx(0); - let mut pos = 0u32; - while cursor != BlockIdx::NULL { - block_order[cursor.idx()] = pos; - pos += 1; - cursor = blocks[cursor.idx()].next; - } - for bi in 0..blocks.len() { - let last_idx = match blocks[bi].instructions.len().checked_sub(1) { - Some(i) => i, - None => continue, - }; - let ins = blocks[bi].instructions[last_idx]; - let mut target = ins.target; - if target == BlockIdx::NULL { - continue; - } - if !(ins.instr.is_unconditional_jump() - || include_conditional && is_conditional_jump(&ins.instr)) - { - continue; - } - target = next_nonempty_block(blocks, target); - if target == BlockIdx::NULL { - continue; - } - if include_conditional - && is_conditional_jump(&ins.instr) - && opposite_short_circuit_target(&blocks[target.idx()], ins.instr) - { - let final_target = next_nonempty_block(blocks, blocks[target.idx()].next); - if final_target != BlockIdx::NULL && ins.target != final_target { - blocks[bi].instructions[last_idx].target = final_target; - changed = true; - continue; - } - } - if include_conditional - && is_conditional_jump(&ins.instr) - && let Some(final_target) = - same_short_circuit_target(&blocks[target.idx()], ins.instr) - && final_target != BlockIdx::NULL - && ins.target != final_target - { - blocks[bi].instructions[last_idx].target = final_target; - changed = true; - continue; - } - if include_conditional && is_conditional_jump(&ins.instr) { - let source_pos = block_order[bi]; - let target_pos = block_order.get(target.idx()).copied().unwrap_or(u32::MAX); - if target_pos <= source_pos { - continue; - } - } - // Match CPython's early flowgraph jump threading: inspect the - // target block's first instruction only. A later unconditional-only - // cleanup pass may thread through line-anchor NOPs introduced after - // jump normalization. - let target_jump = if include_conditional { - blocks[target.idx()].instructions.first().copied() - } else { - blocks[target.idx()] - .instructions - .iter() - .find(|info| !matches!(info.instr.real(), Some(Instruction::Nop))) - .copied() - }; - if let Some(target_ins) = target_jump - && target_ins.instr.is_unconditional_jump() - && target_ins.target != BlockIdx::NULL - && target_ins.target != target - { - let conditional = is_conditional_jump(&ins.instr); - if !include_conditional - && blocks[target.idx()] - .instructions - .iter() - .take_while(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - .any(instruction_has_lineno) - { - continue; - } - let threads_with_suppress_exit = !include_conditional - && block_has_with_suppress_prefix(&blocks[bi], last_idx) - && blocks[target.idx()].instructions.len() == 1; - let source_has_break_cleanup_pop = !include_conditional - && ins.for_loop_break_cleanup_jump - && blocks[bi].instructions[..last_idx] - .iter() - .rev() - .find(|info| !matches!(info.instr.real(), Some(Instruction::Nop))) - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::PopTop))); - let target_is_marker_prefixed_jump_back = !include_conditional - && matches!( - target_ins.instr.real(), - Some(Instruction::JumpBackward { .. }) - ) - && blocks[target.idx()].instructions - [..blocks[target.idx()].instructions.len() - 1] - .iter() - .all(|info| matches!(info.instr.real(), Some(Instruction::Nop))); - let threads_break_cleanup = - source_has_break_cleanup_pop && target_is_marker_prefixed_jump_back; - if !include_conditional - && instruction_has_lineno(&target_ins) - && !threads_with_suppress_exit - && !threads_break_cleanup - { - continue; - } - let source_pos = block_order[bi]; - let target_pos = block_order.get(target.idx()).copied().unwrap_or(u32::MAX); - let final_target = target_ins.target; - let final_target_pos = block_order - .get(final_target.idx()) - .copied() - .unwrap_or(u32::MAX); - if !include_conditional - && source_pos < target_pos - && final_target_pos < target_pos - && !threads_break_cleanup - { - // Keep the forward hop when threading would turn it into a - // backward edge. CPython preserves this shape for chained - // compare loop exits to avoid wraparound-style jumps, but - // codegen_break() for a for-loop emits POP_TOP before an - // empty end label and does thread that label to the - // surrounding loop backedge. - continue; - } - if !include_conditional - && matches!( - jump_thread_kind(ins.instr), - Some(JumpThreadKind::NoInterrupt) - ) - && matches!( - jump_thread_kind(target_ins.instr), - Some(JumpThreadKind::Plain) - ) - { - // CPython does not late-thread WITH suppress exits through - // the line-anchored continue/break jump that follows. - continue; - } - let Some(threaded_instr) = (if conditional { - match jump_thread_kind(target_ins.instr) { - Some(JumpThreadKind::Plain) => Some(ins.instr), - Some(JumpThreadKind::NoInterrupt) - if target_ins.match_success_jump - && can_thread_conditional_through_forward_nointerrupt( - ins.instr, - target_pos, - final_target_pos, - ) => - { - // A forward JUMP_NO_INTERRUPT assembles to the same - // JUMP_FORWARD opcode as CPython's plain match - // success jump. Limit this to None-check match - // success tests; boolean finally-cleanup jumps need - // the stronger no-interrupt shape for later CFG - // cleanup decisions. - Some(ins.instr) - } - _ => None, - } - } else { - threaded_jump_instr(ins.instr, target_ins.instr, false) - }) else { - continue; - }; - if ins.target == final_target { - continue; - } - set_to_nop(&mut blocks[bi].instructions[last_idx]); - let mut threaded = ins; - threaded.instr = threaded_instr; - threaded.arg = OpArg::new(0); - threaded.target = final_target; - threaded.location = target_ins.location; - threaded.end_location = target_ins.end_location; - threaded.lineno_override = target_ins.lineno_override; - threaded.cache_entries = 0; - blocks[bi].instructions.push(threaded); - changed = true; - } - } - if include_conditional { - break; - } - } -} - -pub(crate) fn is_conditional_jump(instr: &AnyInstruction) -> bool { - matches!( - instr.real().map(Into::into), - Some( - Opcode::PopJumpIfFalse - | Opcode::PopJumpIfTrue - | Opcode::PopJumpIfNone - | Opcode::PopJumpIfNotNone - ) - ) -} - -fn is_false_path_conditional_jump(instr: &AnyInstruction) -> bool { - matches!( - instr.real().map(Into::into), - Some(Opcode::PopJumpIfFalse | Opcode::PopJumpIfNone | Opcode::PopJumpIfNotNone) - ) -} - -/// Invert a conditional jump opcode. -fn reversed_conditional(instr: &AnyInstruction) -> Option { - Some(match AnyOpcode::from(*instr).real()? { - Opcode::PopJumpIfFalse => Opcode::PopJumpIfTrue.into(), - Opcode::PopJumpIfTrue => Opcode::PopJumpIfFalse.into(), - Opcode::PopJumpIfNone => Opcode::PopJumpIfNotNone.into(), - Opcode::PopJumpIfNotNone => Opcode::PopJumpIfNone.into(), - _ => return None, - }) -} - -/// flowgraph.c normalize_jumps -fn normalize_jumps(blocks: &mut Vec) { - let mut visit_order = Vec::new(); - let mut visited = vec![false; blocks.len()]; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - visit_order.push(current); - visited[current.idx()] = true; - current = blocks[current.idx()].next; - } - - visited.fill(false); - - for &block_idx in &visit_order { - let idx = block_idx.idx(); - visited[idx] = true; - - // Normalize conditional jumps: forward gets NOT_TAKEN, backward gets inverted - let last = blocks[idx].instructions.last(); - if let Some(last_ins) = last - && is_conditional_jump(&last_ins.instr) - && last_ins.target != BlockIdx::NULL - { - let target = last_ins.target; - let is_forward = !visited[target.idx()]; - - if is_forward { - // Insert NOT_TAKEN after forward conditional jump - let not_taken = InstructionInfo { - instr: Opcode::NotTaken.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location: last_ins.location, - end_location: last_ins.end_location, - // CPython adds NOT_TAKEN in normalize_jumps(), after - // label_exception_targets(), so the synthetic instruction - // has no i_except edge. - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: last_ins.lineno_override, - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, - }; - blocks[idx].instructions.push(not_taken); - } else { - // Backward conditional jump: invert and create new block - // Transform: `cond_jump T` (backward) - // Into: `reversed_cond_jump b_next` + new block [NOT_TAKEN, JUMP T] - let loc = last_ins.location; - let end_loc = last_ins.end_location; - - if let Some(reversed) = reversed_conditional(&last_ins.instr) { - let old_next = blocks[idx].next; - let is_cold = blocks[idx].cold; - let disable_load_fast_borrow = blocks[idx].disable_load_fast_borrow; - - // Create new block with NOT_TAKEN + JUMP to original backward target - let new_block_idx = BlockIdx(blocks.len() as u32); - let mut new_block = Block { - cold: is_cold, - disable_load_fast_borrow, - start_depth: blocks[target.idx()].start_depth, - ..Block::default() - }; - new_block.instructions.push(InstructionInfo { - instr: Opcode::NotTaken.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location: loc, - end_location: end_loc, - // CPython creates this block in normalize_jumps(), - // after exception targets were labelled. - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: last_ins.lineno_override, - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, - }); - new_block.instructions.push(InstructionInfo { - instr: PseudoOpcode::Jump.into(), - arg: OpArg::new(0), - target, - location: loc, - end_location: end_loc, - // CPython's synthetic NOT_TAKEN/JUMP pair is not in - // an exception-table range. - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: last_ins.lineno_override, - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, - }); - new_block.next = old_next; - - // Update the conditional jump: invert opcode, target = old next block - let last_mut = blocks[idx].instructions.last_mut().unwrap(); - last_mut.instr = reversed; - last_mut.target = old_next; - - // Splice new block between current and old next - blocks[idx].next = new_block_idx; - blocks.push(new_block); - - // Extend visited array and update visit order - visited.push(true); - } - } - } - } - - // Rebuild visit_order since backward normalization may have added new blocks - let mut visit_order = Vec::new(); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - visit_order.push(current); - current = blocks[current.idx()].next; - } - - // Resolve JUMP/JUMP_NO_INTERRUPT pseudo instructions before offset fixpoint. - let mut block_order = vec![0u32; blocks.len()]; - for (pos, &block_idx) in visit_order.iter().enumerate() { - block_order[block_idx.idx()] = pos as u32; - } - - for &block_idx in &visit_order { - let source_pos = block_order[block_idx.idx()]; - for info in &mut blocks[block_idx.idx()].instructions { - let target = info.target; - if target == BlockIdx::NULL { - continue; - } - let target_pos = block_order[target.idx()]; - info.instr = match info.instr.into() { - AnyOpcode::Pseudo(PseudoOpcode::Jump) => { - if target_pos > source_pos { - Opcode::JumpForward.into() - } else { - Opcode::JumpBackward.into() - } - } - AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { - if target_pos > source_pos { - Opcode::JumpForward.into() - } else { - Opcode::JumpBackwardNoInterrupt.into() - } - } - _ => info.instr, - }; - } - } -} - -/// flowgraph.c inline_small_or_no_lineno_blocks -fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { - const MAX_COPY_SIZE: usize = 4; - - let block_exits_scope = |block: &Block| { - block - .instructions - .last() - .is_some_and(|ins| ins.instr.is_scope_exit()) - }; - let block_has_no_lineno = |block: &Block| { - block - .instructions - .iter() - .all(|ins| !instruction_has_lineno(ins)) - }; - loop { - let mut changes = false; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let next = blocks[current.idx()].next; - let Some(last) = blocks[current.idx()].instructions.last().copied() else { - current = next; - continue; - }; - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { - current = next; - continue; - } - - let direct_target = last.target; - let nonempty_target = next_nonempty_block(blocks, direct_target); - let target = if blocks[direct_target.idx()].instructions.is_empty() - && nonempty_target != BlockIdx::NULL - { - // CPython's cfg_builder_maybe_start_new_block() attaches a - // label to the current empty block instead of leaving a - // separate empty block in front of a small exit target. Treat - // Rust-only empty labels the same way for this CPython - // flowgraph.c::basicblock_inline_small_or_no_lineno_blocks() - // transform. - nonempty_target - } else { - direct_target - }; - let small_exit_block = block_exits_scope(&blocks[target.idx()]) - && blocks[target.idx()].instructions.len() <= MAX_COPY_SIZE; - let no_lineno_no_fallthrough = block_has_no_lineno(&blocks[target.idx()]) - && !block_has_fallthrough(&blocks[target.idx()]); - if small_exit_block || no_lineno_no_fallthrough { - let removed_jump_kind = jump_thread_kind(last.instr); - if let Some(last_instr) = blocks[current.idx()].instructions.last_mut() { - set_to_nop(last_instr); - } - blocks[current.idx()] - .instructions - .extend(blocks[target.idx()].instructions.clone()); - if no_lineno_no_fallthrough - && removed_jump_kind == Some(JumpThreadKind::Plain) - && let Some(last) = blocks[current.idx()].instructions.last_mut() - && jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) - { - last.instr = match last.instr.into() { - AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { - PseudoOpcode::Jump.into() - } - AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => { - Opcode::JumpBackward.into() - } - _ => last.instr, - }; - } - changes = true; - } - - current = next; - } - - if !changes { - break; - } - } -} - -fn is_artificial_expr_stmt_exit_block(block: &Block) -> bool { - matches!( - block.instructions.as_slice(), - [ - InstructionInfo { - instr: AnyInstruction::Real(Instruction::PopTop), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::LoadConst { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::ReturnValue), - .. - } - ] - ) -} - -fn inline_single_predecessor_artificial_expr_exit_blocks(blocks: &mut [Block]) { - let predecessors = compute_predecessors(blocks); - - for idx in 0..blocks.len() { - let Some(last) = blocks[idx].instructions.last().copied() else { - continue; - }; - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { - continue; - } - - let target = next_nonempty_block(blocks, last.target); - if target == BlockIdx::NULL - || predecessors[target.idx()] != 1 - || !is_artificial_expr_stmt_exit_block(&blocks[target.idx()]) - { - continue; - } - - let is_jump_wrapper = blocks[idx] - .instructions - .split_last() - .is_some_and(|(_, prefix)| { - prefix - .iter() - .all(|ins| matches!(ins.instr.real(), Some(Instruction::Nop))) - }); - if is_jump_wrapper { - continue; - } - - if blocks[idx] - .instructions - .last() - .is_some_and(instruction_has_lineno) - { - if let Some(last_instr) = blocks[idx].instructions.last_mut() { - set_to_nop(last_instr); - } - } else { - let _ = blocks[idx].instructions.pop(); - } - blocks[idx] - .instructions - .extend(blocks[target.idx()].instructions.clone()); - } -} - -struct TargetPredecessorFlags { - plain_jump: Vec, -} - -fn compute_target_predecessor_flags(blocks: &[Block]) -> TargetPredecessorFlags { - let mut plain_jump = vec![false; blocks.len()]; - for block in blocks { - for instr in &block.instructions { - if instr.target == BlockIdx::NULL { - continue; - } - let target = next_nonempty_block(blocks, instr.target); - if target == BlockIdx::NULL { - continue; - } - let idx = target.idx(); - if matches!(jump_thread_kind(instr.instr), Some(JumpThreadKind::Plain)) { - plain_jump[idx] = true; - } - } - } - TargetPredecessorFlags { plain_jump } -} - -fn compute_break_continue_cleanup_jump_target_lines(blocks: &[Block]) -> Vec> { - let mut lines = vec![Vec::new(); blocks.len()]; - for block in blocks { - for instr in &block.instructions { - if instr.target == BlockIdx::NULL - || !instr.break_continue_cleanup_jump - || !matches!(jump_thread_kind(instr.instr), Some(JumpThreadKind::Plain)) - { - continue; - } - let target = next_nonempty_block(blocks, instr.target); - if target == BlockIdx::NULL { - continue; - } - let lineno = if instr.lineno_override.is_some_and(|line| line < 0) { - instr.location.line.get() as i32 - } else { - instruction_lineno(instr) - }; - if lineno > 0 { - lines[target.idx()].push(lineno); - } - } - } - lines -} - -fn remove_redundant_nops_in_blocks(blocks: &mut [Block]) -> usize { - let mut changes = 0; - let plain_jump_targets = compute_target_predecessor_flags(blocks).plain_jump; - let break_continue_cleanup_jump_target_lines = - compute_break_continue_cleanup_jump_target_lines(blocks); - let layout_predecessors = compute_layout_predecessors(blocks); - let mut block_order = Vec::new(); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - block_order.push(current); - current = blocks[current.idx()].next; - } - for block_idx in block_order { - let bi = block_idx.idx(); - let keep_target_start_nop = - keep_target_start_no_location_nop(blocks, block_idx, &layout_predecessors); - let follows_same_line_pop_iter = - layout_predecessor_ends_with_pop_iter_on_line(blocks, block_idx, &layout_predecessors); - let mut src_instructions = core::mem::take(&mut blocks[bi].instructions); - let mut kept = Vec::with_capacity(src_instructions.len()); - let mut prev_lineno = -1i32; - - for src in 0..src_instructions.len() { - let instr = src_instructions[src]; - let lineno = instruction_lineno(&instr); - let mut remove = false; - - if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if instr.remove_no_location_nop && instr.folded_operand_nop { - remove = true; - } else if instr.no_location_exit && instr.preserve_redundant_jump_as_nop { - remove = false; - } else if src == 0 - && lineno > 0 - && ((!keep_target_start_nop && follows_same_line_pop_iter == Some(lineno)) - || (instr.preserve_block_start_no_location_nop - && block_tail_starts_with_async_with_normal_exit( - &src_instructions[src + 1..], - ))) - { - remove = true; - } else if src == 0 - && lineno > 0 - && plain_jump_targets[block_idx.idx()] - && !instr.preserve_redundant_jump_as_nop - && !instr.preserve_block_start_no_location_nop - && break_continue_cleanup_jump_target_lines[block_idx.idx()].contains(&lineno) - { - // CPython basicblock_remove_redundant_nops() removes NOPs that - // only carry the same source line as neighboring control-flow. - // RustPython may hold the target block's instructions out of - // the block array while filtering, so keep incoming jump lines - // from the unmodified CFG. - let next_lineno = src_instructions[src + 1..].iter().find_map(|next_instr| { - let line = instruction_lineno(next_instr); - if matches!(next_instr.instr.real(), Some(Instruction::Nop)) { - None - } else { - Some(line) - } - }); - if next_lineno.is_some_and(|next_lineno| next_lineno > lineno) { - remove = true; - } - } else if instr.preserve_redundant_jump_as_nop - || instr.preserve_block_start_no_location_nop - { - remove = false; - } else if lineno < 0 { - remove = true; - } else if instr.remove_no_location_nop - && src == 0 - && plain_jump_targets[block_idx.idx()] - && instr.lineno_override.is_some() - && !keep_target_start_nop - { - let next_lineno = src_instructions[src + 1..].iter().find_map(|next_instr| { - let line = instruction_lineno(next_instr); - if matches!(next_instr.instr.real(), Some(Instruction::Nop)) && line < 0 { - None - } else { - Some(line) - } - }); - if next_lineno.is_some_and(|next_lineno| lineno < next_lineno) { - remove = true; - } - } else if prev_lineno == lineno { - remove = true; - } else if src < src_instructions.len() - 1 { - if src_instructions[src + 1].instr.is_block_push() { - remove = false; - } else if src_instructions[src + 1].instr.is_unconditional_jump() - && src_instructions[src + 1].target != block_idx - { - let next_lineno = instruction_lineno(&src_instructions[src + 1]); - if next_lineno < 0 { - copy_instruction_location(instr, &mut src_instructions[src + 1]); - remove = true; - } else if next_lineno == lineno { - remove = true; - } - } else if src_instructions[src + 1].folded_from_nonliteral_expr { - remove = true; - } else { - let next_lineno = instruction_lineno(&src_instructions[src + 1]); - if next_lineno == lineno { - remove = true; - } else if next_lineno < 0 { - copy_instruction_location(instr, &mut src_instructions[src + 1]); - remove = true; - } - } - } else { - let next = next_nonempty_block(blocks, blocks[bi].next); - if next != BlockIdx::NULL { - let mut next_info = None; - for next_instr in &blocks[next.idx()].instructions { - let line = instruction_lineno(next_instr); - if matches!(next_instr.instr.real(), Some(Instruction::Nop)) && line < 0 - { - continue; - } - next_info = Some(line); - break; - } - if let Some(next_lineno) = next_info { - // CPython basicblock_remove_redundant_nops() - // does not copy a block-ending NOP's line onto a - // no-location instruction in the next block. - if next_lineno == lineno { - remove = true; - } - } - } - } - } - - if remove { - changes += 1; - } else { - kept.push(instr); - prev_lineno = lineno; - } - } - - blocks[bi].instructions = kept; - } - - changes -} - -fn remove_redundant_jumps_in_blocks(blocks: &mut [Block]) -> usize { - let mut changes = 0; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = next_nonempty_block(blocks, blocks[idx].next); - if next != BlockIdx::NULL { - let Some(last_instr) = blocks[idx].instructions.last().copied() else { - current = blocks[idx].next; - continue; - }; - if last_instr.instr.is_unconditional_jump() - && last_instr.target != BlockIdx::NULL - && next_nonempty_block(blocks, last_instr.target) == next - { - let preserve_redundant_jump_nop = if last_instr.preserve_redundant_jump_as_nop { - let line = instruction_lineno(&last_instr); - let next_line = blocks[next.idx()].instructions.iter().find_map(|instr| { - let line = instruction_lineno(instr); - (!matches!(instr.instr.real(), Some(Instruction::Nop)) || line >= 0) - .then_some(line) - }); - line < 0 - || line > 0 - && !block_jump_follows_async_send_pop(&blocks[idx]) - && !(block_jump_follows_with_normal_exit(&blocks[idx]) - && block_tail_starts_with_async_with_normal_exit( - &blocks[next.idx()].instructions, - )) - && next_line.is_some_and(|next_line| next_line < line) - } else { - false - }; - let last_instr = blocks[idx].instructions.last_mut().unwrap(); - let remove_no_location_nop = last_instr.remove_no_location_nop; - let folded_operand_nop = last_instr.folded_operand_nop; - let preserve_block_start_no_location_nop = - last_instr.preserve_block_start_no_location_nop; - set_to_nop(last_instr); - last_instr.preserve_redundant_jump_as_nop = preserve_redundant_jump_nop; - last_instr.remove_no_location_nop = remove_no_location_nop; - last_instr.folded_operand_nop = folded_operand_nop; - last_instr.preserve_block_start_no_location_nop = - preserve_block_start_no_location_nop || preserve_redundant_jump_nop; - changes += 1; - current = blocks[idx].next; - continue; - } - } - current = blocks[idx].next; - } - changes -} - -fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) { - loop { - let removed_nops = remove_redundant_nops_in_blocks(blocks); - let removed_jumps = remove_redundant_jumps_in_blocks(blocks); - if removed_nops + removed_jumps == 0 { - break; - } - } -} - -fn redirect_empty_block_targets(blocks: &mut [Block]) { - let redirected_targets: Vec> = blocks - .iter() - .map(|block| { - block - .instructions - .iter() - .map(|instr| { - if instr.target == BlockIdx::NULL { - BlockIdx::NULL - } else { - next_nonempty_block(blocks, instr.target) - } - }) - .collect() - }) - .collect(); - - for (block, block_targets) in blocks.iter_mut().zip(redirected_targets) { - for (instr, target) in block.instructions.iter_mut().zip(block_targets) { - if target != BlockIdx::NULL { - instr.target = target; - } - } - } -} - -fn preserves_cpython_unreachable_fallthrough_return_epilogue( - blocks: &[Block], - reachable: &[bool], - idx: BlockIdx, -) -> bool { - let block = &blocks[idx.idx()]; - if !matches!( - block.instructions.as_slice(), - [load, ret] - if load.no_location_exit - && ret.no_location_exit - && instruction_lineno(load) < 0 - && instruction_lineno(ret) < 0 - && matches!(load.instr.real(), Some(Instruction::LoadConst { .. })) - && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) - ) { - return false; - } - - let mut seen = vec![false; blocks.len()]; - let mut stack = vec![idx]; - while let Some(target) = stack.pop() { - if target == BlockIdx::NULL { - continue; - } - for (source_idx, source) in blocks.iter().enumerate() { - if source.next != target || seen[source_idx] { - continue; - } - seen[source_idx] = true; - if reachable[source_idx] { - let [.., load, ret] = source.instructions.as_slice() else { - continue; - }; - if load.no_location_exit - && ret.no_location_exit - && matches!(load.instr.real(), Some(Instruction::LoadConst { .. })) - && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) - && instruction_lineno(ret) > 0 - { - return true; - } - } else if source.instructions.is_empty() { - stack.push(BlockIdx(source_idx as u32)); - } - } - } - false -} - -fn redirect_load_fast_passthrough_targets(blocks: &mut [Block]) { - fn is_assertion_error_load(info: &InstructionInfo) -> bool { - matches!( - info.instr.real(), - Some(Instruction::LoadCommonConstant { idx }) - if idx.get(info.arg) == oparg::CommonConstant::AssertionError - ) - } - - fn block_returns_call_with_fast_load(block: &Block) -> bool { - let mut seen_fast_load = false; - let mut previous_was_call_after_fast_load = false; - for info in &block.instructions { - match info.instr.real() { - Some( - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. }, - ) => { - seen_fast_load = true; - previous_was_call_after_fast_load = false; - } - Some(Instruction::Call { .. } | Instruction::CallKw { .. }) if seen_fast_load => { - previous_was_call_after_fast_load = true; - } - Some(Instruction::ReturnValue) if previous_was_call_after_fast_load => { - return true; - } - Some(_) => previous_was_call_after_fast_load = false, - None => {} - } - } - false - } - - fn handler_resumes_to_target(blocks: &[Block], target: BlockIdx) -> bool { - blocks.iter().any(|block| { - (block.cold || block.except_handler) - && block.instructions.iter().any(|info| { - info.target != BlockIdx::NULL - && (info.target == target - || next_nonempty_block(blocks, info.target) == target) - && matches!( - info.instr.real(), - Some( - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } - | Instruction::JumpForward { .. } - ) - ) - }) - }) - } - - fn has_warm_fallthrough_predecessor(blocks: &[Block], target: BlockIdx) -> bool { - let mut seen = vec![false; blocks.len()]; - let mut stack = vec![target]; - while let Some(target) = stack.pop() { - if target == BlockIdx::NULL { - continue; - } - for (block_idx, block) in blocks.iter().enumerate() { - if block.next != target || block.cold || block.except_handler { - continue; - } - if block.instructions.is_empty() - && (block.load_fast_passthrough || block.load_fast_label_reuse_passthrough) - { - if !seen[block_idx] { - seen[block_idx] = true; - stack.push(BlockIdx::new(block_idx as u32)); - } - continue; - } - if block_has_fallthrough(block) { - return true; - } - } - } - false - } - - fn has_protected_warm_fallthrough_predecessor(blocks: &[Block], target: BlockIdx) -> bool { - blocks.iter().any(|block| { - block.next == target - && !block.cold - && !block.except_handler - && block_has_fallthrough(block) - && block - .instructions - .iter() - .any(|info| info.except_handler.is_some()) - }) - } - - fn assertion_success_nop_passthrough(blocks: &[Block], target: BlockIdx) -> bool { - let block = &blocks[target.idx()]; - if !block.instructions.is_empty() - || block.next == BlockIdx::NULL - || block.except_handler - || block.preserve_lasti - || block.cold - || block.disable_load_fast_borrow - { - return false; - } - // CPython's assertion success jump targets the following statement's - // NOP block directly. RustPython can synthesize an empty label before - // that NOP while preserving assertion failure layout, so do not let - // that synthetic label stop flowgraph.c::optimize_load_fast() parity. - if !blocks[block.next.idx()] - .instructions - .first() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - { - return false; - } - let has_conditional_target = blocks.iter().any(|block| { - block - .instructions - .iter() - .any(|info| info.target == target && is_conditional_jump(&info.instr)) - }); - let has_assertion_failure_predecessor = blocks.iter().any(|block| { - block.next == target - && block.instructions.iter().any(is_assertion_error_load) - && block - .instructions - .last() - .is_some_and(|info| info.instr.is_scope_exit()) - }); - has_conditional_target && has_assertion_failure_predecessor - } - - fn assertion_failure_passthrough(blocks: &[Block], target: BlockIdx) -> bool { - let block = &blocks[target.idx()]; - if !block.instructions.is_empty() - || block.next == BlockIdx::NULL - || block.except_handler - || block.preserve_lasti - || block.cold - || block.disable_load_fast_borrow - || blocks.iter().any(|predecessor| { - predecessor.instructions.iter().any(|info| { - (info.target == target || info.target == block.next) - && is_conditional_jump(&info.instr) - }) - }) - { - return false; - } - if !blocks.iter().any(|predecessor| { - predecessor.next == target - && predecessor.instructions.last().is_some_and(|info| { - matches!( - info.instr.real(), - Some(Instruction::EndFor | Instruction::PopIter) - ) - }) - }) { - return false; - } - // CPython codegen_assert() emits LOAD_COMMON_CONSTANT AssertionError - // immediately after codegen_jump_if() for a failing assertion. If - // RustPython leaves an empty label before that failure block, do not - // let it stop flowgraph.c::optimize_load_fast() from visiting the - // AssertionError construction. - blocks[block.next.idx()] - .instructions - .first() - .is_some_and(is_assertion_error_load) - } - - fn try_else_orelse_entry_passthrough(blocks: &[Block], target: BlockIdx) -> bool { - let block = &blocks[target.idx()]; - if !block.instructions.is_empty() - || block.next == BlockIdx::NULL - || block.except_handler - || block.preserve_lasti - || block.cold - || block.disable_load_fast_borrow - { - return false; - } - - // CPython codegen_try_except() emits Try.orelse with VISIT_SEQ() - // immediately after the normal try-body path. RustPython can leave - // an empty join block immediately before the marked orelse entry, - // often from a conditional at the end of the try body. - block.try_else_orelse_entry || blocks[block.next.idx()].try_else_orelse_entry - } - - fn labeled_passthrough_successor(blocks: &[Block], target: BlockIdx) -> bool { - let block = &blocks[target.idx()]; - if !block.instructions.is_empty() - || block.next == BlockIdx::NULL - || block.label - || block.load_fast_barrier - || block.except_handler - || block.preserve_lasti - || block.cold - || block.disable_load_fast_borrow - { - return false; - } - - // CPython flowgraph.c::cfg_builder_current_block_is_terminated() - // attaches a pending label to the current empty block instead of - // creating an intervening b_next block. RustPython can leave that - // unlabeled empty block immediately before the labeled continuation. - blocks[block.next.idx()].load_fast_label_reuse_passthrough - } - - fn passthrough_target(blocks: &[Block], mut target: BlockIdx) -> BlockIdx { - while target != BlockIdx::NULL { - let block = &blocks[target.idx()]; - let next = next_nonempty_block(blocks, block.next); - let handler_resume_end = block.instructions.is_empty() - && next != BlockIdx::NULL - && handler_resumes_to_target(blocks, next) - && block_returns_call_with_fast_load(&blocks[next.idx()]); - let handler_resume_end = - handler_resume_end && !has_warm_fallthrough_predecessor(blocks, target); - // CPython codegen_try_except() emits USE_LABEL(end), then the - // following statement directly into that end block. After - // RustPython pushes cold handlers to the end, a protected normal - // path can still fall through an empty synthetic block before the - // handler and normal path rejoin at a return-call block. Treat - // that split as a label-reuse passthrough for optimize_load_fast() - // parity. - let try_except_return_end = block.instructions.is_empty() - && next != BlockIdx::NULL - && !block.label - && !block.load_fast_barrier - && !block.except_handler - && !block.preserve_lasti - && !block.cold - && !block.disable_load_fast_borrow - && handler_resumes_to_target(blocks, next) - && block_returns_call_with_fast_load(&blocks[next.idx()]) - && has_protected_warm_fallthrough_predecessor(blocks, target); - let assertion_success_nop = assertion_success_nop_passthrough(blocks, target); - let assertion_failure = assertion_failure_passthrough(blocks, target); - let try_else_orelse_entry = try_else_orelse_entry_passthrough(blocks, target); - let labeled_passthrough_successor = labeled_passthrough_successor(blocks, target); - if !(block.load_fast_passthrough - || handler_resume_end - || try_except_return_end - || assertion_success_nop - || assertion_failure - || try_else_orelse_entry - || labeled_passthrough_successor) - || !block.instructions.is_empty() - { - break; - } - target = block.next; - } - target - } - - let redirected_next: Vec<_> = blocks - .iter() - .map(|block| passthrough_target(blocks, block.next)) - .collect(); - let redirected_targets: Vec> = blocks - .iter() - .map(|block| { - block - .instructions - .iter() - .map(|instr| passthrough_target(blocks, instr.target)) - .collect() - }) - .collect(); - - for ((block, next), block_targets) in blocks - .iter_mut() - .zip(redirected_next) - .zip(redirected_targets) - { - block.next = next; - for (instr, target) in block.instructions.iter_mut().zip(block_targets) { - instr.target = target; - } - } -} - -fn redirect_empty_unconditional_jump_targets(blocks: &mut [Block]) { - const MAX_COPY_SIZE: usize = 4; - - let block_exits_to_large_reraise = |block_idx: BlockIdx| { - let block = &blocks[block_idx.idx()]; - let Some(last) = block.instructions.last() else { - return false; - }; - let reraise_block = if matches!(last.instr.real(), Some(Instruction::Reraise { .. })) { - block_idx - } else if last.instr.is_unconditional_jump() && last.target != BlockIdx::NULL { - next_nonempty_block(blocks, last.target) - } else { - BlockIdx::NULL - }; - reraise_block != BlockIdx::NULL - && blocks[reraise_block.idx()].instructions.len() > MAX_COPY_SIZE - && blocks[reraise_block.idx()] - .instructions - .last() - .is_some_and(|instr| { - matches!(instr.instr.real(), Some(Instruction::Reraise { .. })) - }) - }; - - let mut raw_predecessors = vec![0u32; blocks.len()]; - for block in blocks.iter() { - if block_has_fallthrough(block) && block.next != BlockIdx::NULL { - raw_predecessors[block.next.idx()] += 1; - } - for instr in &block.instructions { - if instr.target != BlockIdx::NULL { - raw_predecessors[instr.target.idx()] += 1; - } - } - } - - let redirected_targets: Vec> = blocks - .iter() - .map(|block| { - block - .instructions - .iter() - .map(|instr| { - if instr.target == BlockIdx::NULL - || !instr.instr.is_unconditional_jump() - || blocks[instr.target.idx()].load_fast_barrier - { - instr.target - } else { - if blocks[instr.target.idx()].instructions.is_empty() - && raw_predecessors[instr.target.idx()] > 1 - && { - let target = next_nonempty_block(blocks, instr.target); - target != BlockIdx::NULL && block_exits_to_large_reraise(target) - } - { - return instr.target; - } - let target = next_nonempty_block(blocks, instr.target); - if matches!( - jump_thread_kind(instr.instr), - Some(JumpThreadKind::NoInterrupt) - ) && (target == BlockIdx::NULL - || is_jump_back_only_block(blocks, target)) - { - instr.target - } else { - target - } - } - }) - .collect() - }) - .collect(); - - for (block, block_targets) in blocks.iter_mut().zip(redirected_targets) { - for (instr, target) in block.instructions.iter_mut().zip(block_targets) { - if target != BlockIdx::NULL { - instr.target = target; - } - } - } -} - -fn materialize_empty_conditional_exit_targets(blocks: &mut [Block]) { - fn block_starts_with_with_normal_exit(block: &Block) -> bool { - matches!( - block.instructions.as_slice(), - [ - InstructionInfo { - instr: AnyInstruction::Real(Instruction::LoadConst { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::LoadConst { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::LoadConst { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::Call { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::PopTop), - .. - }, - .. - ] - ) - } - - fn with_normal_exit_is_followed_by_try(blocks: &[Block], block_idx: BlockIdx) -> bool { - if block_idx == BlockIdx::NULL - || !block_starts_with_with_normal_exit(&blocks[block_idx.idx()]) - { - return false; - } - let next = next_nonempty_block(blocks, blocks[block_idx.idx()].next); - next != BlockIdx::NULL - && blocks[next.idx()].instructions.first().is_some_and(|info| { - matches!( - info.instr, - AnyInstruction::Pseudo(PseudoInstruction::SetupFinally { .. }) - | AnyInstruction::Real(Instruction::Nop) - ) - }) - } - - fn has_loop_backedge_to(blocks: &[Block], target: BlockIdx) -> bool { - blocks.iter().enumerate().any(|(source_idx, block)| { - let source = BlockIdx(source_idx as u32); - comes_before(blocks, target, source) - && block.instructions.iter().any(|info| { - info.instr.is_unconditional_jump() - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) - }) - } - - let mut jump_back_inserts = Vec::new(); - let mut inserts = Vec::new(); - let mut target_start_inserts = Vec::new(); - let mut jump_back_target_locations = Vec::new(); - for (block_idx, block) in blocks.iter().enumerate() { - let source = BlockIdx(block_idx as u32); - let (last, allow_scope_exit_target) = if let Some(last) = block - .instructions - .last() - .filter(|info| is_conditional_jump(&info.instr)) - { - (last, true) - } else if let Some(cond_idx) = trailing_conditional_jump_index(block) { - (&block.instructions[cond_idx], false) - } else { - continue; - }; - if last.target == BlockIdx::NULL { - continue; - } - let target = last.target; - if !blocks[target.idx()].instructions.is_empty() { - if is_jump_back_only_block(blocks, target) && block_has_no_lineno(&blocks[target.idx()]) - { - jump_back_target_locations.push((*last, target)); - } - if with_normal_exit_is_followed_by_try(blocks, target) - && has_loop_backedge_to(blocks, source) - && !matches!( - blocks[target.idx()] - .instructions - .first() - .and_then(|info| info.instr.real()), - Some(Instruction::Nop) - ) - { - target_start_inserts.push((*last, target)); - } - continue; - } - let next = next_nonempty_block(blocks, blocks[target.idx()].next); - if next != BlockIdx::NULL - && is_jump_only_block(&blocks[next.idx()]) - && block_has_no_lineno(&blocks[next.idx()]) - && comes_before( - blocks, - next_nonempty_block(blocks, blocks[next.idx()].instructions[0].target), - next, - ) - { - jump_back_inserts.push((BlockIdx(block_idx as u32), target, next)); - continue; - } - if next == BlockIdx::NULL - || !((allow_scope_exit_target && is_scope_exit_block(&blocks[next.idx()])) - || (with_normal_exit_is_followed_by_try(blocks, next) - && has_loop_backedge_to(blocks, source))) - { - continue; - } - inserts.push((*last, target)); - } - - for (source, target) in jump_back_target_locations { - if !is_jump_back_only_block(blocks, target) || !block_has_no_lineno(&blocks[target.idx()]) { - continue; - } - if let Some(first) = blocks[target.idx()].instructions.first_mut() { - overwrite_location( - first, - source.location, - source.end_location, - source.lineno_override, - ); - } - } - - for (source, target, next) in jump_back_inserts { - if !blocks[target.idx()].instructions.is_empty() { - continue; - } - let Some(last) = blocks[source.idx()].instructions.last().copied() else { - continue; - }; - let mut cloned = blocks[next.idx()].instructions[0]; - overwrite_location( - &mut cloned, - last.location, - last.end_location, - last.lineno_override, - ); - blocks[target.idx()].instructions.push(cloned); - } - - for (source, target) in inserts { - if !blocks[target.idx()].instructions.is_empty() { - continue; - } - blocks[target.idx()].instructions.push(InstructionInfo { - instr: Instruction::Nop.into(), - arg: OpArg::NULL, - target: BlockIdx::NULL, - location: source.location, - end_location: source.end_location, - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: source.lineno_override, - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, - }); - } - - for (source, target) in target_start_inserts.into_iter().rev() { - if !with_normal_exit_is_followed_by_try(blocks, target) - || matches!( - blocks[target.idx()] - .instructions - .first() - .and_then(|info| info.instr.real()), - Some(Instruction::Nop) - ) - { - continue; - } - blocks[target.idx()].instructions.insert( - 0, - InstructionInfo { - instr: Instruction::Nop.into(), - arg: OpArg::NULL, - target: BlockIdx::NULL, - location: source.location, - end_location: source.end_location, - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: source.lineno_override, - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, - }, - ); - } -} - -fn merge_unsafe_mask(slot: &mut Option>, incoming: &[bool]) -> bool { - match slot { - Some(existing) => { - let mut changed = false; - for (dst, src) in existing.iter_mut().zip(incoming.iter().copied()) { - if src && !*dst { - *dst = true; - changed = true; - } - } - changed - } - None => { - *slot = Some(incoming.to_vec()); - true - } - } -} - -/// Follow chain of empty blocks to find first non-empty block. -fn next_nonempty_block(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { - while idx != BlockIdx::NULL && blocks[idx.idx()].instructions.is_empty() { - idx = blocks[idx.idx()].next; - } - idx -} - -fn is_load_const_none(instr: &InstructionInfo, metadata: &CodeUnitMetadata) -> bool { - matches!(instr.instr.real(), Some(Instruction::LoadConst { .. })) - && matches!( - metadata.consts.get_index(u32::from(instr.arg) as usize), - Some(ConstantData::None) - ) -} - -fn block_tail_starts_with_async_with_normal_exit(instructions: &[InstructionInfo]) -> bool { - matches!( - instructions, - [ - InstructionInfo { - instr: AnyInstruction::Real(Instruction::LoadConst { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::LoadConst { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::LoadConst { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::Call { .. }), - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::GetAwaitable { .. }), - .. - }, - .. - ] - ) -} - -fn instruction_lineno(instr: &InstructionInfo) -> i32 { - match instr.lineno_override { - Some(LINE_ONLY_LOCATION_OVERRIDE) | None => instr.location.line.get() as i32, - Some(lineno) => lineno, - } -} - -fn instruction_has_lineno(instr: &InstructionInfo) -> bool { - instruction_lineno(instr) >= 0 -} - -fn copy_instruction_location(source: InstructionInfo, target: &mut InstructionInfo) { - target.location = source.location; - target.end_location = source.end_location; - target.lineno_override = source.lineno_override; - target.preserve_store_fast_store_fast_jump_location = - source.preserve_store_fast_store_fast_jump_location; -} - -fn propagation_location( - instr: &InstructionInfo, -) -> Option<(SourceLocation, SourceLocation, Option)> { - instruction_has_lineno(instr).then_some(( - instr.location, - instr.end_location, - instr.lineno_override, - )) -} - -fn block_has_fallthrough(block: &Block) -> bool { - block - .instructions - .last() - .is_none_or(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()) -} - -fn is_jump_instruction(instr: &InstructionInfo) -> bool { - instr.instr.is_unconditional_jump() || is_conditional_jump(&instr.instr) -} - -fn last_jump_for_line_propagation(block: &Block) -> Option { - let last = block.instructions.last().copied()?; - if matches!(last.instr.real(), Some(Instruction::NotTaken)) { - block - .instructions - .iter() - .rev() - .copied() - .find(|instr| !matches!(instr.instr.real(), Some(Instruction::NotTaken))) - .filter(is_jump_instruction) - } else { - is_jump_instruction(&last).then_some(last) - } -} - -fn is_exit_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { - let block = &blocks[block_idx.idx()]; - let Some(first) = block.instructions.first() else { - return false; - }; - if instruction_has_lineno(first) || !block_has_no_lineno(block) { - return false; - } - - if block - .instructions - .last() - .is_some_and(|last| last.instr.is_scope_exit()) - { - return true; - } - - // CPython duplicates no-lineno exit blocks before propagating locations. - // RustPython's late CFG can inline the following synthetic jump-back block - // into that exit block first, collapsing `POP_EXCEPT; JUMP_BACKWARD` into a - // single block. Treat that merged tail as exit-like only for real loop - // backedges so resolve_line_numbers() can recover CPython's loop cleanup - // structure without duplicating cold exception-handler jumps to normal - // forward continuation code. - let Some((last, prefix)) = block.instructions.split_last() else { - return false; - }; - last.instr.is_unconditional_jump() - && prefix.iter().all(|info| { - matches!( - info.instr.real(), - Some(Instruction::PopExcept | Instruction::Nop) - ) - }) - && prefix - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) - && has_non_exception_loop_backedge_to(blocks, block_idx, last.target) -} - -fn is_eval_break_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { - let block = &blocks[block_idx.idx()]; - let Some(first) = block.instructions.first() else { - return false; - }; - !instruction_has_lineno(first) && block_has_no_lineno(block) && block_has_eval_break(block) -} - -fn block_has_eval_break(block: &Block) -> bool { - block.instructions.iter().any(|info| { - matches!( - info.instr, - AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) - | AnyInstruction::Real( - Instruction::Call { .. } - | Instruction::CallFunctionEx - | Instruction::CallKw { .. } - | Instruction::JumpBackward { .. } - | Instruction::Resume { .. } - ) - ) - }) -} - -fn block_has_no_lineno(block: &Block) -> bool { - block - .instructions - .iter() - .all(|ins| !instruction_has_lineno(ins)) -} - -fn shared_jump_back_target(block: &Block) -> Option { - if !block_has_no_lineno(block) { - return None; - } - - let (last, prefix) = block.instructions.split_last()?; - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { - return None; - } - - if !prefix.iter().all(|info| { - matches!( - info.instr.real(), - Some(Instruction::PopExcept | Instruction::Nop) - ) - }) { - return None; - } - - Some(last.target) -} - -fn block_has_break_continue_cleanup_jump(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| info.break_continue_cleanup_jump) -} - -fn block_has_non_exception_loop_backedge_to( - blocks: &[Block], - source: BlockIdx, - target: BlockIdx, -) -> bool { - let target = next_nonempty_block(blocks, target); - source != BlockIdx::NULL - && target != BlockIdx::NULL - && !block_is_exceptional(&blocks[source.idx()]) - && comes_before(blocks, target, source) - && blocks[source.idx()].instructions.iter().any(|info| { - info.instr.is_unconditional_jump() - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) -} - -fn has_non_exception_loop_backedge_to( - blocks: &[Block], - cleanup_block: BlockIdx, - target: BlockIdx, -) -> bool { - blocks.iter().enumerate().any(|(source_idx, block)| { - let source = BlockIdx(source_idx as u32); - source != cleanup_block - && !block_is_exceptional(block) - && block_has_non_exception_loop_backedge_to(blocks, source, target) - }) -} - -fn is_jump_only_block(block: &Block) -> bool { - let [instr] = block.instructions.as_slice() else { - return false; - }; - instr.instr.is_unconditional_jump() && instr.target != BlockIdx::NULL -} - -fn is_jump_back_only_block(blocks: &[Block], block_idx: BlockIdx) -> bool { - if block_idx == BlockIdx::NULL || !is_jump_only_block(&blocks[block_idx.idx()]) { - return false; - } - comes_before( - blocks, - next_nonempty_block(blocks, blocks[block_idx.idx()].instructions[0].target), - block_idx, - ) -} - -fn lineful_shared_jump_back_target(blocks: &[Block], block_idx: BlockIdx) -> Option { - if block_idx == BlockIdx::NULL { - return None; - } - let block = &blocks[block_idx.idx()]; - let (last, prefix) = block.instructions.split_last()?; - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { - return None; - } - if !prefix - .iter() - .all(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - { - return None; - } - let target = next_nonempty_block(blocks, last.target); - (target != BlockIdx::NULL && comes_before(blocks, target, block_idx)).then_some(target) -} - -fn is_pop_top_jump_block(block: &Block) -> bool { - let mut real_instrs = block - .instructions - .iter() - .filter(|info| !matches!(info.instr.real(), Some(Instruction::Nop))); - let Some(first) = real_instrs.next() else { - return false; - }; - let Some(second) = real_instrs.next() else { - return false; - }; - real_instrs.next().is_none() - && matches!(first.instr.real(), Some(Instruction::PopTop)) - && second.instr.is_unconditional_jump() - && second.target != BlockIdx::NULL -} - -fn is_for_break_cleanup_block(blocks: &[Block], block_idx: BlockIdx) -> bool { - if block_idx == BlockIdx::NULL { - return false; - } - let mut real_instrs = blocks[block_idx.idx()] - .instructions - .iter() - .filter(|info| !matches!(info.instr.real(), Some(Instruction::Nop))); - let Some(first) = real_instrs.next() else { - return false; - }; - let Some(second) = real_instrs.next() else { - return false; - }; - real_instrs.next().is_none() - && matches!(first.instr.real(), Some(Instruction::PopTop)) - && second.instr.is_unconditional_jump() - && second.target != BlockIdx::NULL - && !comes_before(blocks, second.target, block_idx) -} - -fn jump_targets_exception_region_entry(blocks: &[Block], jump_block: BlockIdx) -> bool { - if jump_block == BlockIdx::NULL { - return false; - } - let Some(info) = blocks[jump_block.idx()].instructions.first() else { - return false; - }; - let mut target = info.target; - let mut seen = 0usize; - while target != BlockIdx::NULL && seen < blocks.len() { - seen += 1; - let block = &blocks[target.idx()]; - if block_is_protected(block) - || block.instructions.iter().any(|info| { - info.instr.is_block_push() - || matches!( - info.instr, - AnyInstruction::Pseudo( - PseudoInstruction::SetupFinally { .. } - | PseudoInstruction::SetupCleanup { .. } - ) - ) - }) - { - return true; - } - if block - .instructions - .iter() - .all(|info| matches!(info.instr.real(), Some(Instruction::Nop))) - { - target = block.next; - continue; - } - return false; - } - false -} - -fn is_scope_exit_block(block: &Block) -> bool { - block - .instructions - .last() - .is_some_and(|instr| instr.instr.is_scope_exit()) -} - -fn is_pop_top_scope_exit_block(block: &Block) -> bool { - is_scope_exit_block(block) - && matches!( - block - .instructions - .first() - .and_then(|info| info.instr.real()), - Some(Instruction::PopTop) - ) -} - -fn is_pop_top_exit_like_block(block: &Block) -> bool { - is_pop_top_scope_exit_block(block) || is_pop_top_jump_block(block) -} - -fn is_loop_cleanup_block(block: &Block) -> bool { - block - .instructions - .iter() - .find_map(|info| info.instr.real()) - .is_some_and(|instr| { - matches!( - instr, - Instruction::EndFor | Instruction::EndAsyncFor | Instruction::PopIter - ) - }) -} - -fn is_async_loop_cleanup_block(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::EndAsyncFor))) -} - -fn is_exception_cleanup_block(block: &Block) -> bool { - block - .instructions - .iter() - .any(|instr| matches!(instr.instr.real(), Some(Instruction::PopExcept))) - && block - .instructions - .last() - .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::Reraise { .. }))) -} - -fn is_reraise_scope_exit_block(block: &Block) -> bool { - block - .instructions - .last() - .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::Reraise { .. }))) -} - -fn block_starts_with_with_exit_none_call(block: &Block) -> bool { - let real_instrs: Vec<_> = block - .instructions - .iter() - .filter_map(|info| { - let instr = info.instr.real()?; - (!matches!(instr, Instruction::Nop)).then_some(instr) - }) - .take(4) - .collect(); - matches!( - real_instrs.as_slice(), - [ - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::LoadConst { .. }, - Instruction::Call { .. }, - ] - ) -} - -fn keep_target_start_no_location_nop( - blocks: &[Block], - target: BlockIdx, - layout_predecessors: &[BlockIdx], -) -> bool { - if target == BlockIdx::NULL { - return false; - } - let Some(first) = blocks[target.idx()].instructions.first() else { - return false; - }; - if !matches!(first.instr.real(), Some(Instruction::Nop)) { - return false; - } - let layout_pred = layout_predecessors[target.idx()]; - if layout_pred == BlockIdx::NULL { - return false; - } - if is_async_loop_cleanup_block(&blocks[layout_pred.idx()]) { - return true; - } - is_exception_cleanup_block(&blocks[layout_pred.idx()]) - && !block_starts_with_with_exit_none_call(&blocks[target.idx()]) -} - -fn layout_predecessor_ends_with_pop_iter_on_line( - blocks: &[Block], - target: BlockIdx, - layout_predecessors: &[BlockIdx], -) -> Option { - let layout_pred = layout_predecessors[target.idx()]; - if layout_pred == BlockIdx::NULL - || !block_has_fallthrough(&blocks[layout_pred.idx()]) - || next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target - { - return None; - } - let last = blocks[layout_pred.idx()].instructions.last()?; - matches!(last.instr.real(), Some(Instruction::PopIter)).then_some(instruction_lineno(last)) -} - -fn is_with_suppress_exit_block(block: &Block) -> bool { - let real_instrs: Vec<_> = block - .instructions - .iter() - .filter_map(|info| info.instr.real()) - .collect(); - matches!( - real_instrs.as_slice(), - [ - Instruction::PopTop, - Instruction::PopExcept, - Instruction::PopTop, - Instruction::PopTop, - Instruction::PopTop, - last, - ] if last.is_unconditional_jump() - ) -} - -fn block_is_protected(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| info.except_handler.is_some()) -} - -fn block_contains_suspension_point(block: &Block) -> bool { - block - .instructions - .iter() - .filter_map(|info| info.instr.real()) - .any(|instr| { - matches!( - instr, - Instruction::YieldValue { .. } - | Instruction::GetAwaitable { .. } - | Instruction::GetAnext - | Instruction::EndAsyncFor - ) - }) -} - -fn block_jump_follows_async_send_pop(block: &Block) -> bool { - let mut before_jump = - block - .instructions - .iter() - .rev() - .skip(1) - .filter_map(|info| match info.instr.real() { - Some(Instruction::Nop) => None, - instr => instr, - }); - matches!( - (before_jump.next(), before_jump.next()), - (Some(Instruction::PopTop), Some(Instruction::EndSend)) - ) -} - -fn block_jump_follows_with_normal_exit(block: &Block) -> bool { - let mut before_jump = - block - .instructions - .iter() - .rev() - .skip(1) - .filter_map(|info| match info.instr.real() { - Some(Instruction::Nop) => None, - instr => instr, - }); - matches!( - ( - before_jump.next(), - before_jump.next(), - before_jump.next(), - before_jump.next(), - before_jump.next(), - ), - ( - Some(Instruction::PopTop), - Some(Instruction::Call { .. }), - Some(Instruction::LoadConst { .. }), - Some(Instruction::LoadConst { .. }), - Some(Instruction::LoadConst { .. }), - ) - ) -} - -fn is_stop_iteration_error_handler_block(block: &Block) -> bool { - matches!( - block.instructions.as_slice(), - [ - InstructionInfo { - instr: AnyInstruction::Real(Instruction::CallIntrinsic1 { func }), - arg, - .. - }, - InstructionInfo { - instr: AnyInstruction::Real(Instruction::Reraise { .. }), - .. - } - ] if matches!(func.get(*arg), oparg::IntrinsicFunction1::StopIterationError) - ) -} - -fn block_has_only_stop_iteration_error_handlers(block: &Block, blocks: &[Block]) -> bool { - let mut saw_handler = false; - for info in &block.instructions { - let Some(handler) = info.except_handler else { - continue; - }; - saw_handler = true; - let target = next_nonempty_block(blocks, handler.handler_block); - if target == BlockIdx::NULL || !is_stop_iteration_error_handler_block(&blocks[target.idx()]) - { - return false; - } - } - saw_handler -} - -fn block_is_exceptional(block: &Block) -> bool { - block.except_handler || block.preserve_lasti || is_exception_cleanup_block(block) -} - -fn has_exceptional_duplicate_lineno(blocks: &[Block], source: BlockIdx, lineno: i32) -> bool { - blocks.iter().enumerate().any(|(idx, block)| { - BlockIdx(idx as u32) != source - && (block.cold || block_is_exceptional(block) || block_is_protected(block)) - && block - .instructions - .iter() - .any(|info| instruction_lineno(info) == lineno) - }) -} - -fn trailing_conditional_jump_index(block: &Block) -> Option { - let last_idx = block.instructions.len().checked_sub(1)?; - if is_conditional_jump(&block.instructions[last_idx].instr) - && block.instructions[last_idx].target != BlockIdx::NULL - { - return Some(last_idx); - } - - let cond_idx = last_idx.checked_sub(1)?; - if matches!( - block.instructions[last_idx].instr.real(), - Some(Instruction::NotTaken) - ) && is_conditional_jump(&block.instructions[cond_idx].instr) - && block.instructions[cond_idx].target != BlockIdx::NULL - { - Some(cond_idx) - } else { - None - } -} - -fn block_is_pure_conditional_test(block: &Block) -> bool { - let Some(cond_idx) = trailing_conditional_jump_index(block) else { - return false; - }; - block.instructions[..cond_idx].iter().all(|info| { - matches!( - info.instr.real(), - Some( - Instruction::Nop - | Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::LoadDeref { .. } - | Instruction::LoadGlobal { .. } - | Instruction::LoadConst { .. } - | Instruction::LoadSmallInt { .. } - | Instruction::LoadAttr { .. } - | Instruction::BinaryOp { .. } - | Instruction::ContainsOp { .. } - | Instruction::IsOp { .. } - | Instruction::CompareOp { .. } - | Instruction::ToBool - ) - ) - }) -} - -fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let last = blocks[idx].instructions[cond_idx]; - let Some(reversed) = reversed_conditional(&last.instr) else { - current = next; - continue; - }; - - let exit_start = next; - let jump_start = last.target; - if exit_start == BlockIdx::NULL || jump_start == BlockIdx::NULL || exit_start == jump_start - { - current = next; - continue; - } - - let mut exit_end = BlockIdx::NULL; - let mut exit_block = BlockIdx::NULL; - let mut cursor = exit_start; - let mut exit_segment_valid = true; - while cursor != BlockIdx::NULL && cursor != jump_start { - if block_is_exceptional(&blocks[cursor.idx()]) { - exit_segment_valid = false; - break; - } - if !blocks[cursor.idx()].instructions.is_empty() { - if exit_block != BlockIdx::NULL { - exit_segment_valid = false; - break; - } - exit_block = cursor; - } - exit_end = cursor; - cursor = blocks[cursor.idx()].next; - } - if !exit_segment_valid - || cursor != jump_start - || exit_end == BlockIdx::NULL - || exit_block == BlockIdx::NULL - || !is_scope_exit_block(&blocks[exit_block.idx()]) - { - current = next; - continue; - } - - let mut jump_end = BlockIdx::NULL; - let mut jump_block = BlockIdx::NULL; - cursor = jump_start; - while cursor != BlockIdx::NULL { - if block_is_exceptional(&blocks[cursor.idx()]) - || block_is_protected(&blocks[cursor.idx()]) - { - jump_block = BlockIdx::NULL; - break; - } - jump_end = cursor; - if blocks[cursor.idx()].instructions.is_empty() { - cursor = blocks[cursor.idx()].next; - continue; - } - if is_jump_only_block(&blocks[cursor.idx()]) { - jump_block = cursor; - } - break; - } - if jump_block == BlockIdx::NULL { - current = next; - continue; - } - let jump_instr = blocks[jump_block.idx()].instructions[0]; - if jump_instr.match_success_jump { - // CPython codegen_pattern_or() emits an explicit success - // JUMP after every alternative, including the last one. - current = next; - continue; - } - let jump_is_forward = matches!( - jump_instr.instr.real(), - Some(Instruction::JumpForward { .. }) - ); - let jump_is_backward = matches!( - jump_instr.instr.real(), - Some(Instruction::JumpBackward { .. }) - ); - let exit_is_reorderable = - blocks[exit_block.idx()] - .instructions - .last() - .is_some_and(|info| { - matches!( - info.instr.real(), - Some(Instruction::ReturnValue | Instruction::RaiseVarargs { .. }) - ) - }); - if !(jump_is_forward || jump_is_backward && exit_is_reorderable) { - current = next; - continue; - } - if jump_is_backward { - if block_is_protected(&blocks[idx]) - || block_is_protected(&blocks[exit_block.idx()]) - || block_is_protected(&blocks[jump_block.idx()]) - || jump_targets_exception_region_entry(blocks, jump_block) - { - current = next; - continue; - } - let after_jump_start = blocks[jump_end.idx()].next; - let after_jump = next_nonempty_block(blocks, after_jump_start); - let after_jump_is_end_async_for = after_jump != BlockIdx::NULL - && blocks[after_jump.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::EndAsyncFor))); - let after_jump_is_adjacent_scope_exit = after_jump != BlockIdx::NULL - && is_pop_top_exit_like_block(&blocks[after_jump.idx()]); - if after_jump_is_end_async_for || after_jump_is_adjacent_scope_exit { - current = next; - continue; - } - // CPython flowgraph.c::normalize_jumps_in_block() only checks - // whether the conditional target was already visited. When the - // jump block is a backward edge to a loop header, the following - // outer block does not prevent rewriting the condition so the - // backedge remains the fallthrough path. - } - - let after_jump = blocks[jump_end.idx()].next; - blocks[idx].next = jump_start; - blocks[jump_end.idx()].next = exit_start; - blocks[exit_end.idx()].next = after_jump; - - let cond_mut = &mut blocks[idx].instructions[cond_idx]; - cond_mut.instr = reversed; - cond_mut.target = exit_start; - - current = after_jump; - } -} - -fn reorder_conditional_jump_and_exit_blocks(blocks: &mut [Block]) { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let last = blocks[idx].instructions[cond_idx]; - - let Some(reversed) = reversed_conditional(&last.instr) else { - current = next; - continue; - }; - - let jump_start = next; - let exit_start = last.target; - if jump_start == BlockIdx::NULL || exit_start == BlockIdx::NULL || jump_start == exit_start - { - current = next; - continue; - } - - let mut jump_end = BlockIdx::NULL; - let mut jump_block = BlockIdx::NULL; - let mut cursor = jump_start; - let mut jump_segment_valid = true; - while cursor != BlockIdx::NULL && cursor != exit_start { - if block_is_exceptional(&blocks[cursor.idx()]) { - jump_segment_valid = false; - break; - } - if !blocks[cursor.idx()].instructions.is_empty() { - if jump_block != BlockIdx::NULL || !is_jump_only_block(&blocks[cursor.idx()]) { - jump_segment_valid = false; - break; - } - jump_block = cursor; - } - jump_end = cursor; - cursor = blocks[cursor.idx()].next; - } - if !jump_segment_valid || cursor != exit_start || jump_block == BlockIdx::NULL { - current = next; - continue; - } - let jump_instr = blocks[jump_block.idx()].instructions[0]; - if jump_instr.match_success_jump { - // CPython codegen_pattern_or() emits an explicit success - // JUMP after every alternative, including the last one. - current = next; - continue; - } - if !matches!( - jump_instr.instr.real(), - Some(Instruction::JumpForward { .. }) - ) { - current = next; - continue; - } - if jump_instr.lineno_override.is_some_and(|line| line >= 0) - && instruction_lineno(&jump_instr) != instruction_lineno(&last) - { - current = next; - continue; - } - - let mut exit_end = BlockIdx::NULL; - let mut exit_block = BlockIdx::NULL; - let after_exit = loop { - if cursor == BlockIdx::NULL { - break BlockIdx::NULL; - } - if block_is_exceptional(&blocks[cursor.idx()]) { - if exit_block != BlockIdx::NULL { - break cursor; - } - exit_block = BlockIdx::NULL; - break BlockIdx::NULL; - } - if !blocks[cursor.idx()].instructions.is_empty() { - if exit_block != BlockIdx::NULL { - break cursor; - } - if !is_scope_exit_block(&blocks[cursor.idx()]) { - exit_block = BlockIdx::NULL; - break BlockIdx::NULL; - } - exit_block = cursor; - } - exit_end = cursor; - cursor = blocks[cursor.idx()].next; - }; - if exit_block == BlockIdx::NULL || exit_end == BlockIdx::NULL { - current = next; - continue; - } - - blocks[idx].next = exit_start; - blocks[exit_end.idx()].next = jump_start; - blocks[jump_end.idx()].next = after_exit; - - let cond_mut = &mut blocks[idx].instructions[cond_idx]; - cond_mut.instr = reversed; - cond_mut.target = jump_start; - - current = after_exit; - } -} - -fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { - let target_comes_before = |target: BlockIdx, block: BlockIdx, blocks: &[Block]| -> bool { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if current == target { - return true; - } - if current == block { - return false; - } - current = blocks[current.idx()].next; - } - false - }; - - fn is_single_delete_subscr_body(block: &Block) -> bool { - let real: Vec<_> = block - .instructions - .iter() - .filter_map(|info| info.instr.real()) - .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) - .collect(); - real.iter() - .filter(|instr| matches!(instr, Instruction::DeleteSubscr)) - .count() - == 1 - && matches!(real.last(), Some(Instruction::DeleteSubscr)) - && real.iter().all(|instr| { - matches!( - instr, - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::LoadSmallInt { .. } - | Instruction::LoadConst { .. } - | Instruction::Copy { .. } - | Instruction::Swap { .. } - | Instruction::BinaryOp { .. } - | Instruction::BinarySlice - | Instruction::BuildSlice { .. } - | Instruction::StoreSubscr - | Instruction::DeleteSubscr - ) - }) - } - - fn chain_is_conditional_single_delete_body( - blocks: &[Block], - chain_start: BlockIdx, - jump_start: BlockIdx, - ) -> bool { - if chain_start == BlockIdx::NULL || jump_start == BlockIdx::NULL { - return false; - } - let Some(chain_cond_idx) = trailing_conditional_jump_index(&blocks[chain_start.idx()]) - else { - return false; - }; - let chain_cond = blocks[chain_start.idx()].instructions[chain_cond_idx]; - if !is_false_path_conditional_jump(&chain_cond.instr) || chain_cond.target != jump_start { - return false; - } - let body = next_nonempty_block(blocks, blocks[chain_start.idx()].next); - body != BlockIdx::NULL - && !block_is_exceptional(&blocks[body.idx()]) - && !block_is_protected(&blocks[body.idx()]) - && next_nonempty_block(blocks, blocks[body.idx()].next) == jump_start - && is_single_delete_subscr_body(&blocks[body.idx()]) - } - - fn jump_targets_for_iter(blocks: &[Block], jump_block: BlockIdx) -> bool { - if jump_block == BlockIdx::NULL { - return false; - } - let Some(info) = blocks[jump_block.idx()].instructions.first() else { - return false; - }; - let target = next_nonempty_block(blocks, info.target); - target != BlockIdx::NULL - && blocks[target.idx()] - .instructions - .first() - .is_some_and(|target_info| { - matches!(target_info.instr.real(), Some(Instruction::ForIter { .. })) - }) - } - - fn true_body_false_backedge_is_already_normalized( - blocks: &[Block], - conditional: InstructionInfo, - false_backedge: BlockIdx, - true_body: BlockIdx, - ) -> bool { - fn comes_before(blocks: &[Block], target: BlockIdx, block: BlockIdx) -> bool { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if current == target { - return true; - } - if current == block { - return false; - } - current = blocks[current.idx()].next; - } - false - } - - if !matches!( - conditional.instr.real(), - Some(Instruction::PopJumpIfTrue { .. }) - ) || false_backedge == BlockIdx::NULL - || true_body == BlockIdx::NULL - || !is_jump_only_block(&blocks[false_backedge.idx()]) - { - return false; - } - let false_target = blocks[false_backedge.idx()].instructions[0].target; - if false_target == BlockIdx::NULL || !comes_before(blocks, false_target, false_backedge) { - return false; - } - let true_tail = next_nonempty_block(blocks, blocks[true_body.idx()].next); - true_tail != BlockIdx::NULL - && is_jump_only_block(&blocks[true_tail.idx()]) - && blocks[true_tail.idx()].instructions[0].target == false_target - } - - fn has_other_conditional_predecessor_to( - blocks: &[Block], - conditional_target_counts: &[usize], - target: BlockIdx, - current: BlockIdx, - ) -> bool { - let current_targets = blocks[current.idx()] - .instructions - .iter() - .filter(|info| info.target == target && is_conditional_jump(&info.instr)) - .count(); - conditional_target_counts[target.idx()] > current_targets - } - - let has_exceptional_lineno_sources = blocks - .iter() - .any(|block| block.cold || block_is_exceptional(block) || block_is_protected(block)); - let mut conditional_target_counts = vec![0usize; blocks.len()]; - for block in blocks.iter() { - for info in &block.instructions { - if info.target != BlockIdx::NULL && is_conditional_jump(&info.instr) { - conditional_target_counts[info.target.idx()] += 1; - } - } - } - - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let last = blocks[idx].instructions[cond_idx]; - - let Some(reversed) = reversed_conditional(&last.instr) else { - current = next; - continue; - }; - - let chain_start = next; - let jump_start = last.target; - if chain_start == BlockIdx::NULL - || jump_start == BlockIdx::NULL - || chain_start == jump_start - { - current = next; - continue; - } - if true_body_false_backedge_is_already_normalized(blocks, last, chain_start, jump_start) { - current = next; - continue; - } - let mut chain_has_suspension_point = false; - let mut scan = chain_start; - while scan != BlockIdx::NULL && scan != jump_start { - if block_contains_suspension_point(&blocks[scan.idx()]) { - chain_has_suspension_point = true; - break; - } - scan = blocks[scan.idx()].next; - } - let chain_starts_with_false_path_jump = trailing_conditional_jump_index( - &blocks[chain_start.idx()], - ) - .is_some_and(|chain_cond_idx| { - is_false_path_conditional_jump( - &blocks[chain_start.idx()].instructions[chain_cond_idx].instr, - ) - }); - let chain_is_single_exit_block = is_scope_exit_block(&blocks[chain_start.idx()]) - && next_nonempty_block(blocks, blocks[chain_start.idx()].next) == jump_start; - let chain_is_jump_only_exit_block = is_jump_only_block(&blocks[chain_start.idx()]) - && !target_comes_before( - blocks[chain_start.idx()].instructions[0].target, - chain_start, - blocks, - ); - let chain_is_jump_back_exit_block = is_jump_only_block(&blocks[chain_start.idx()]) - && target_comes_before( - blocks[chain_start.idx()].instructions[0].target, - chain_start, - blocks, - ); - let allow_true_path_jump_back_reorder = - matches!(last.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) - && (chain_has_suspension_point - || chain_starts_with_false_path_jump - || chain_is_single_exit_block - || chain_is_jump_only_exit_block - || chain_is_jump_back_exit_block); - let is_generic_false_path_reorder = !allow_true_path_jump_back_reorder; - if !is_false_path_conditional_jump(&last.instr) && !allow_true_path_jump_back_reorder { - current = next; - continue; - } - if is_generic_false_path_reorder && is_scope_exit_block(&blocks[chain_start.idx()]) { - current = next; - continue; - } - if has_exceptional_lineno_sources - && is_generic_false_path_reorder - && has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&last)) - { - current = next; - continue; - } - if is_generic_false_path_reorder - && has_other_conditional_predecessor_to( - blocks, - &conditional_target_counts, - chain_start, - current, - ) - { - current = next; - continue; - } - if block_is_protected(&blocks[idx]) && block_contains_suspension_point(&blocks[idx]) { - current = next; - continue; - } - if let Some(chain_cond_idx) = trailing_conditional_jump_index(&blocks[chain_start.idx()]) { - let chain_cond = blocks[chain_start.idx()].instructions[chain_cond_idx]; - if matches!( - chain_cond.instr.real().map(Into::into), - Some(Opcode::PopJumpIfTrue) - ) { - let chain_true_target = next_nonempty_block(blocks, chain_cond.target); - if chain_true_target != BlockIdx::NULL - && !is_scope_exit_block(&blocks[chain_true_target.idx()]) - && !is_jump_only_block(&blocks[chain_true_target.idx()]) - && !is_pop_top_jump_block(&blocks[chain_true_target.idx()]) - { - current = next; - continue; - } - } - } - - let mut chain_end = BlockIdx::NULL; - let mut saw_nonempty = false; - let mut nonempty_blocks = 0usize; - let mut real_instr_count = 0usize; - let mut chain_has_block_push = false; - let mut cursor = chain_start; - let mut chain_valid = true; - while cursor != BlockIdx::NULL && cursor != jump_start { - if block_is_exceptional(&blocks[cursor.idx()]) - || (block_is_protected(&blocks[cursor.idx()]) - && block_contains_suspension_point(&blocks[cursor.idx()]) - && !block_has_only_stop_iteration_error_handlers(&blocks[cursor.idx()], blocks)) - { - chain_valid = false; - break; - } - if !blocks[cursor.idx()].instructions.is_empty() { - saw_nonempty = true; - nonempty_blocks += 1; - real_instr_count += blocks[cursor.idx()] - .instructions - .iter() - .filter(|info| info.instr.real().is_some()) - .count(); - chain_has_block_push |= blocks[cursor.idx()] - .instructions - .iter() - .any(|info| info.instr.is_block_push()); - } - chain_end = cursor; - cursor = blocks[cursor.idx()].next; - } - if !chain_valid || !saw_nonempty || chain_end == BlockIdx::NULL || cursor != jump_start { - current = next; - continue; - } - if is_generic_false_path_reorder && chain_has_block_push { - current = next; - continue; - } - let chain_is_conditional_single_delete_body = - chain_is_conditional_single_delete_body(blocks, chain_start, jump_start); - if is_generic_false_path_reorder - && nonempty_blocks > 1 - && !chain_is_conditional_single_delete_body - { - current = next; - continue; - } - if !is_generic_false_path_reorder && (nonempty_blocks > 8 || real_instr_count > 80) { - current = next; - continue; - } - - let mut jump_end = BlockIdx::NULL; - let mut jump_block = BlockIdx::NULL; - cursor = jump_start; - while cursor != BlockIdx::NULL { - if block_is_exceptional(&blocks[cursor.idx()]) { - jump_block = BlockIdx::NULL; - break; - } - jump_end = cursor; - if blocks[cursor.idx()].instructions.is_empty() { - cursor = blocks[cursor.idx()].next; - continue; - } - if !is_jump_only_block(&blocks[cursor.idx()]) - || !target_comes_before(blocks[cursor.idx()].instructions[0].target, cursor, blocks) - { - jump_block = BlockIdx::NULL; - } else { - jump_block = cursor; - } - break; - } - if jump_block == BlockIdx::NULL || jump_end == BlockIdx::NULL { - current = next; - continue; - } - let after_jump = next_nonempty_block(blocks, blocks[jump_block.idx()].next); - let jump_is_artificial = blocks[jump_block.idx()] - .instructions - .first() - .is_some_and(|info| matches!(info.lineno_override, Some(line) if line < 0)); - if is_generic_false_path_reorder - && jump_targets_for_iter(blocks, jump_block) - && is_for_break_cleanup_block(blocks, next_nonempty_block(blocks, chain_start)) - { - current = next; - continue; - } - if jump_targets_exception_region_entry(blocks, jump_block) { - current = next; - continue; - } - let after_jump_is_adjacent_scope_exit = - after_jump != BlockIdx::NULL && is_pop_top_exit_like_block(&blocks[after_jump.idx()]); - if !is_generic_false_path_reorder - && chain_is_single_exit_block - && after_jump_is_adjacent_scope_exit - { - current = next; - continue; - } - if is_generic_false_path_reorder - && jump_is_artificial - && after_jump != BlockIdx::NULL - && is_loop_cleanup_block(&blocks[after_jump.idx()]) - && !chain_is_conditional_single_delete_body - { - current = next; - continue; - } - if !is_generic_false_path_reorder - && jump_is_artificial - && jump_targets_for_iter(blocks, jump_block) - { - current = next; - continue; - } - if nonempty_blocks == 1 - && !is_jump_only_block(&blocks[chain_start.idx()]) - && after_jump != BlockIdx::NULL - && !blocks[after_jump.idx()].cold - && !block_is_exceptional(&blocks[after_jump.idx()]) - && !is_scope_exit_block(&blocks[after_jump.idx()]) - && !is_loop_cleanup_block(&blocks[after_jump.idx()]) - { - current = next; - continue; - } - - let mut cloned_jump = blocks[jump_block.idx()].clone(); - cloned_jump.next = chain_start; - cloned_jump.start_depth = None; - let cloned_idx = BlockIdx::new(blocks.len() as u32); - blocks.push(cloned_jump); - conditional_target_counts.push(0); - blocks[idx].next = cloned_idx; - let cond_mut = &mut blocks[idx].instructions[cond_idx]; - if cond_mut.target != BlockIdx::NULL { - conditional_target_counts[cond_mut.target.idx()] -= 1; - } - conditional_target_counts[chain_start.idx()] += 1; - cond_mut.instr = reversed; - cond_mut.target = chain_start; - - current = next; - } -} - -fn reorder_conditional_scope_exit_and_jump_back_blocks( - blocks: &mut [Block], - allow_for_iter_jump_targets: bool, - allow_true_scope_exit_reorder: bool, -) { - fn retarget_empty_chain_targets( - blocks: &mut [Block], - chain_start: BlockIdx, - chain_end: BlockIdx, - replacement: BlockIdx, - ) { - if chain_start == BlockIdx::NULL - || chain_end == BlockIdx::NULL - || replacement == BlockIdx::NULL - { - return; - } - - let mut in_chain = vec![false; blocks.len()]; - let mut cursor = chain_start; - while cursor != BlockIdx::NULL && cursor != chain_end { - let block = &blocks[cursor.idx()]; - if !block.instructions.is_empty() { - return; - } - in_chain[cursor.idx()] = true; - cursor = block.next; - } - if cursor != chain_end { - return; - } - - for block in blocks { - for instr in &mut block.instructions { - if instr.target != BlockIdx::NULL && in_chain[instr.target.idx()] { - instr.target = replacement; - } - } - } - } - - fn jump_targets_for_iter(blocks: &[Block], jump_block: BlockIdx) -> bool { - if jump_block == BlockIdx::NULL { - return false; - } - let Some(info) = blocks[jump_block.idx()].instructions.first() else { - return false; - }; - let target = next_nonempty_block(blocks, info.target); - target != BlockIdx::NULL - && blocks[target.idx()] - .instructions - .first() - .is_some_and(|target_info| { - matches!(target_info.instr.real(), Some(Instruction::ForIter { .. })) - }) - } - - fn is_explicit_continue_to_for_iter(blocks: &[Block], jump_block: BlockIdx) -> bool { - if jump_block == BlockIdx::NULL { - return false; - } - let Some(info) = blocks[jump_block.idx()].instructions.iter().find(|info| { - !matches!( - info.instr.real(), - Some(Instruction::Nop | Instruction::NotTaken) - ) - }) else { - return false; - }; - matches!( - info.instr.real(), - Some(Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. }) - ) && !matches!(info.lineno_override, Some(line) if line < 0) - && jump_targets_for_iter(blocks, jump_block) - } - - fn is_explicit_continue_after_conditional( - blocks: &[Block], - jump_block: BlockIdx, - cond: InstructionInfo, - ) -> bool { - if !is_explicit_continue_to_for_iter(blocks, jump_block) { - return false; - } - let Some(info) = blocks[jump_block.idx()].instructions.iter().find(|info| { - !matches!( - info.instr.real(), - Some(Instruction::Nop | Instruction::NotTaken) - ) - }) else { - return false; - }; - instruction_lineno(info) > instruction_lineno(&cond) - } - - fn is_explicit_non_for_jump_back(blocks: &[Block], jump_block: BlockIdx) -> bool { - if jump_block == BlockIdx::NULL || jump_targets_for_iter(blocks, jump_block) { - return false; - } - let Some(info) = blocks[jump_block.idx()].instructions.first() else { - return false; - }; - matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) - && info.lineno_override.is_some_and(|line| line >= 0) - } - - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let cond = blocks[idx].instructions[cond_idx]; - if matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })) { - let exit_start = next; - let exit_block = next_nonempty_block(blocks, exit_start); - let jump_start = cond.target; - let jump_block = next_nonempty_block(blocks, jump_start); - let after_jump = if jump_block != BlockIdx::NULL { - next_nonempty_block(blocks, blocks[jump_block.idx()].next) - } else { - BlockIdx::NULL - }; - let after_jump_continues_conditional_chain = after_jump != BlockIdx::NULL - && block_is_pure_conditional_test(&blocks[after_jump.idx()]); - // Allow reorder when all three blocks share the same except handler - // (or none have one). Reordering within a shared protection region - // matches CPython, which reorders regardless of try/except context. - let cond_handler = blocks[idx] - .instructions - .first() - .and_then(|info| info.except_handler); - let jump_handler = blocks[jump_block.idx()] - .instructions - .first() - .and_then(|info| info.except_handler); - let exit_handler = blocks[exit_block.idx()] - .instructions - .first() - .and_then(|info| info.except_handler); - let mismatched_protection = - !(cond_handler == jump_handler && jump_handler == exit_handler); - if exit_start == BlockIdx::NULL - || exit_block == BlockIdx::NULL - || jump_start == BlockIdx::NULL - || jump_block == BlockIdx::NULL - || jump_block == exit_block - || after_jump_continues_conditional_chain - || block_is_exceptional(&blocks[idx]) - || block_is_exceptional(&blocks[jump_block.idx()]) - || block_is_exceptional(&blocks[exit_block.idx()]) - || mismatched_protection - || !is_scope_exit_block(&blocks[exit_block.idx()]) - || !is_jump_back_only_block(blocks, jump_block) - || jump_targets_exception_region_entry(blocks, jump_block) - || (jump_targets_for_iter(blocks, jump_block) - && is_for_break_cleanup_block(blocks, exit_block)) - || (!allow_for_iter_jump_targets - && is_explicit_continue_to_for_iter(blocks, jump_block)) - && blocks[exit_block.idx()].instructions.iter().any(|info| { - matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. })) - }) - || (!allow_for_iter_jump_targets - && is_explicit_non_for_jump_back(blocks, jump_block)) - || (after_jump != BlockIdx::NULL - && is_pop_top_exit_like_block(&blocks[after_jump.idx()])) - || (after_jump != BlockIdx::NULL - && !blocks[after_jump.idx()].cold - && !is_scope_exit_block(&blocks[after_jump.idx()]) - && !is_loop_cleanup_block(&blocks[after_jump.idx()]) - && jump_targets_for_iter(blocks, jump_block)) - || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block - { - current = next; - continue; - } - - let after_jump = blocks[jump_block.idx()].next; - blocks[idx].instructions[cond_idx].instr = reversed_conditional(&cond.instr) - .expect("PopJumpIfFalse has a reversed conditional"); - blocks[idx].instructions[cond_idx].target = exit_start; - blocks[idx].next = jump_start; - blocks[jump_block.idx()].next = exit_start; - blocks[exit_block.idx()].next = after_jump; - current = next; - continue; - } - if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { - current = next; - continue; - } - if !allow_true_scope_exit_reorder { - current = next; - continue; - } - let exit_block = next_nonempty_block(blocks, cond.target); - let jump_block = next_nonempty_block(blocks, next); - if exit_block == BlockIdx::NULL - || jump_block == BlockIdx::NULL - || !is_scope_exit_block(&blocks[exit_block.idx()]) - || !is_jump_only_block(&blocks[jump_block.idx()]) - // CPython flowgraph.c::normalize_jumps_in_block() leaves a - // backward conditional as reversed_conditional-to-exit followed by - // a fallthrough backward jump. This Rust layout pass must not - // undo that normalized shape. - || (is_jump_back_only_block(blocks, jump_block) - && next_nonempty_block(blocks, blocks[jump_block.idx()].next) == exit_block - && !(jump_targets_for_iter(blocks, jump_block) - && !is_explicit_continue_after_conditional(blocks, jump_block, cond) - && !block_is_protected(&blocks[idx]))) - || next_nonempty_block(blocks, blocks[jump_block.idx()].next) != exit_block - || !comes_before( - blocks, - next_nonempty_block(blocks, blocks[jump_block.idx()].instructions[0].target), - jump_block, - ) - { - current = next; - continue; - } - let after_exit = blocks[exit_block.idx()].next; - retarget_empty_chain_targets(blocks, next, jump_block, jump_block); - blocks[idx].instructions[cond_idx].instr = - reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); - blocks[idx].instructions[cond_idx].target = jump_block; - blocks[idx].next = exit_block; - blocks[exit_block.idx()].next = jump_block; - blocks[jump_block.idx()].next = after_exit; - current = next; - } -} - -fn reorder_conditional_break_continue_blocks(blocks: &mut [Block]) { - fn is_break_exit_like(blocks: &[Block], block_idx: BlockIdx) -> bool { - let block = &blocks[block_idx.idx()]; - if !is_jump_only_block(block) { - return false; - } - matches!( - block.instructions[0].instr.real(), - Some(Instruction::JumpForward { .. }) - ) && !comes_before( - blocks, - next_nonempty_block(blocks, block.instructions[0].target), - block_idx, - ) - } - - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let cond = blocks[idx].instructions[cond_idx]; - if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { - current = next; - continue; - } - - let jump_start = next; - let jump_block = next_nonempty_block(blocks, jump_start); - let exit_start = cond.target; - let exit_block = next_nonempty_block(blocks, exit_start); - if jump_start == BlockIdx::NULL - || jump_block == BlockIdx::NULL - || exit_start == BlockIdx::NULL - || exit_block == BlockIdx::NULL - || jump_block == exit_block - || block_is_exceptional(&blocks[idx]) - || block_is_exceptional(&blocks[jump_block.idx()]) - || block_is_exceptional(&blocks[exit_block.idx()]) - || block_is_protected(&blocks[idx]) - || block_is_protected(&blocks[jump_block.idx()]) - || block_is_protected(&blocks[exit_block.idx()]) - || !is_jump_back_only_block(blocks, jump_block) - || !is_break_exit_like(blocks, exit_block) - || next_nonempty_block(blocks, blocks[jump_block.idx()].next) != exit_block - { - current = next; - continue; - } - - let after_exit = blocks[exit_block.idx()].next; - blocks[idx].instructions[cond_idx].instr = - reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); - blocks[idx].instructions[cond_idx].target = jump_start; - blocks[idx].next = exit_start; - blocks[exit_block.idx()].next = jump_start; - blocks[jump_block.idx()].next = after_exit; - current = next; - } -} - -fn reorder_conditional_explicit_continue_scope_exit_blocks(blocks: &mut [Block]) { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let cond = blocks[idx].instructions[cond_idx]; - if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { - current = next; - continue; - } - - let jump_start = next; - let jump_block = next_nonempty_block(blocks, jump_start); - let exit_start = cond.target; - let exit_block = next_nonempty_block(blocks, exit_start); - if jump_start == BlockIdx::NULL - || jump_block == BlockIdx::NULL - || exit_start == BlockIdx::NULL - || exit_block == BlockIdx::NULL - || jump_block == exit_block - || block_is_exceptional(&blocks[idx]) - || block_is_exceptional(&blocks[jump_block.idx()]) - || block_is_exceptional(&blocks[exit_block.idx()]) - || block_is_protected(&blocks[idx]) - || block_is_protected(&blocks[jump_block.idx()]) - || block_is_protected(&blocks[exit_block.idx()]) - || !is_scope_exit_block(&blocks[exit_block.idx()]) - || !is_jump_back_only_block(blocks, jump_block) - || next_nonempty_block(blocks, blocks[jump_block.idx()].next) != exit_block - || blocks[jump_block.idx()] - .instructions - .first() - .is_none_or(|info| info.lineno_override.is_none_or(|lineno| lineno < 0)) - { - current = next; - continue; - } - - let after_exit = blocks[exit_block.idx()].next; - blocks[idx].instructions[cond_idx].instr = - reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); - blocks[idx].instructions[cond_idx].target = jump_start; - blocks[idx].next = exit_start; - blocks[exit_block.idx()].next = jump_start; - blocks[jump_block.idx()].next = after_exit; - current = next; - } -} - -fn reorder_conditional_implicit_continue_scope_exit_blocks(blocks: &mut [Block]) { - fn jump_back_target(block: &Block) -> Option { - let [info] = block.instructions.as_slice() else { - return None; - }; - if matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) - && info.target != BlockIdx::NULL - { - Some(info.target) - } else { - None - } - } - - fn scope_exit_segment_tail_before_jump( - blocks: &[Block], - start: BlockIdx, - jump_start: BlockIdx, - ) -> Option { - let jump_block = next_nonempty_block(blocks, jump_start); - if jump_block == BlockIdx::NULL { - return None; - } - - let mut segment = Vec::new(); - let mut cursor = next_nonempty_block(blocks, start); - while cursor != BlockIdx::NULL && cursor != jump_block { - if segment.len() >= blocks.len() { - return None; - } - let block = &blocks[cursor.idx()]; - if block_is_exceptional(block) || block_is_protected(block) { - return None; - } - segment.push(cursor); - cursor = next_nonempty_block(blocks, block.next); - } - if cursor != jump_block || segment.is_empty() { - return None; - } - - let mut in_segment = vec![false; blocks.len()]; - for block_idx in &segment { - in_segment[block_idx.idx()] = true; - } - - let mut has_scope_exit = false; - for block_idx in &segment { - let block = &blocks[block_idx.idx()]; - if is_scope_exit_block(block) { - has_scope_exit = true; - continue; - } - - if block_has_fallthrough(block) { - let next = next_nonempty_block(blocks, block.next); - if next == BlockIdx::NULL || !in_segment[next.idx()] { - return None; - } - } - - for info in &block.instructions { - if info.target == BlockIdx::NULL { - continue; - } - let target = next_nonempty_block(blocks, info.target); - if target == BlockIdx::NULL || !in_segment[target.idx()] { - return None; - } - } - } - - has_scope_exit.then_some(*segment.last().expect("non-empty segment")) - } - - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let cond = blocks[idx].instructions[cond_idx]; - if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })) { - current = next; - continue; - } - - let exit_start = next; - let exit_block = next_nonempty_block(blocks, exit_start); - let jump_start = cond.target; - let jump_block = next_nonempty_block(blocks, jump_start); - let jump_target = if jump_block != BlockIdx::NULL { - jump_back_target(&blocks[jump_block.idx()]) - } else { - None - }; - let jumps_to_for_iter = jump_target.is_some_and(|target| { - blocks[target.idx()] - .instructions - .first() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) - }); - let after_jump_start = if jump_block != BlockIdx::NULL { - blocks[jump_block.idx()].next - } else { - BlockIdx::NULL - }; - let after_jump = next_nonempty_block(blocks, after_jump_start); - let jump_exits_to_loop_exit = jump_target.is_some_and(|target| { - trailing_conditional_jump_index(&blocks[target.idx()]).is_some_and(|cond_idx| { - let loop_exit_start = blocks[target.idx()].instructions[cond_idx].target; - loop_exit_start == after_jump_start - || next_nonempty_block(blocks, loop_exit_start) == after_jump - }) - }); - let jump_has_lineno = blocks[jump_block.idx()] - .instructions - .first() - .is_some_and(|info| instruction_lineno(info) >= 0); - let exit_segment_tail = scope_exit_segment_tail_before_jump(blocks, exit_start, jump_start); - if exit_start == BlockIdx::NULL - || exit_block == BlockIdx::NULL - || jump_start == BlockIdx::NULL - || jump_block == BlockIdx::NULL - || exit_block == jump_block - || block_is_exceptional(&blocks[idx]) - || block_is_exceptional(&blocks[exit_block.idx()]) - || block_is_exceptional(&blocks[jump_block.idx()]) - || block_is_protected(&blocks[idx]) - || block_is_protected(&blocks[exit_block.idx()]) - || block_is_protected(&blocks[jump_block.idx()]) - || exit_segment_tail.is_none() - || !is_jump_back_only_block(blocks, jump_block) - || jump_targets_exception_region_entry(blocks, jump_block) - || jumps_to_for_iter - || (after_jump != BlockIdx::NULL - && !blocks[after_jump.idx()].cold - && !jump_exits_to_loop_exit - && !jump_has_lineno) - { - current = next; - continue; - } - - let exit_segment_tail = exit_segment_tail.expect("checked above"); - let after_jump = blocks[jump_block.idx()].next; - blocks[idx].instructions[cond_idx].instr = - reversed_conditional(&cond.instr).expect("PopJumpIfFalse has a reversed conditional"); - blocks[idx].instructions[cond_idx].target = exit_start; - blocks[idx].next = jump_start; - blocks[jump_block.idx()].next = exit_start; - blocks[exit_segment_tail.idx()].next = after_jump; - current = next; - } -} - -fn reorder_exception_handler_conditional_continue_scope_exit_blocks(blocks: &mut [Block]) { - fn handler_scope_exit_returns(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) - && block - .instructions - .last() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ReturnValue))) - } - - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let cond = blocks[idx].instructions[cond_idx]; - if !matches!( - cond.instr.real(), - Some(Instruction::PopJumpIfNotNone { .. } | Instruction::PopJumpIfFalse { .. }) - ) || !(block_is_exceptional(&blocks[idx]) || blocks[idx].cold) - { - current = next; - continue; - } - - let exit_start = next; - let exit_block = next_nonempty_block(blocks, exit_start); - let jump_start = cond.target; - let jump_block = next_nonempty_block(blocks, jump_start); - let jump_target = if jump_block != BlockIdx::NULL { - next_nonempty_block(blocks, blocks[jump_block.idx()].instructions[0].target) - } else { - BlockIdx::NULL - }; - let exit_raises = exit_block != BlockIdx::NULL - && blocks[exit_block.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))); - let exit_returns = - exit_block != BlockIdx::NULL && handler_scope_exit_returns(&blocks[exit_block.idx()]); - if exit_start == BlockIdx::NULL - || exit_block == BlockIdx::NULL - || jump_start == BlockIdx::NULL - || jump_block == BlockIdx::NULL - || jump_target == BlockIdx::NULL - || exit_block == jump_block - || !is_scope_exit_block(&blocks[exit_block.idx()]) - || !is_jump_back_only_block(blocks, jump_block) - || !(exit_raises || exit_returns) - || !blocks[jump_target.idx()] - .instructions - .first() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) - || next_nonempty_block(blocks, blocks[exit_block.idx()].next) != jump_block - { - current = next; - continue; - } - - let after_jump = blocks[jump_block.idx()].next; - blocks[idx].instructions[cond_idx].instr = reversed_conditional(&cond.instr) - .expect("handler conditional has a reversed conditional"); - blocks[idx].instructions[cond_idx].target = exit_start; - blocks[idx].next = jump_start; - blocks[jump_block.idx()].next = exit_start; - blocks[exit_block.idx()].next = after_jump; - current = next; - } -} - -fn deduplicate_adjacent_jump_back_blocks(blocks: &mut [Block]) { - fn jump_back_target(block: &Block) -> Option { - let [info] = block.instructions.as_slice() else { - return None; - }; - if matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) - && info.target != BlockIdx::NULL - { - Some(info.target) - } else { - None - } - } - - fn jump_back_lineno(block: &Block) -> Option { - let [info] = block.instructions.as_slice() else { - return None; - }; - matches!(info.instr.real(), Some(Instruction::JumpBackward { .. })) - .then(|| instruction_lineno(info)) - } - - fn has_protected_conditional_predecessor( - blocks: &[Block], - incoming_origins: &[Vec], - target: BlockIdx, - ) -> bool { - incoming_origins[target.idx()] - .iter() - .copied() - .any(|origin| { - blocks[origin.idx()].instructions.iter().any(|info| { - info.except_handler.is_some() - && is_conditional_jump(&info.instr) - && next_nonempty_block(blocks, info.target) == target - }) - }) - } - - let reachable = compute_reachable_blocks(blocks); - let incoming_origins = compute_incoming_origins(blocks, &reachable); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let Some(target) = jump_back_target(&blocks[current.idx()]) else { - current = blocks[current.idx()].next; - continue; - }; - if block_is_exceptional(&blocks[current.idx()]) - || block_is_protected(&blocks[current.idx()]) - { - current = blocks[current.idx()].next; - continue; - } - - let duplicate = next_nonempty_block(blocks, blocks[current.idx()].next); - if duplicate == BlockIdx::NULL - || duplicate == current - || block_is_exceptional(&blocks[duplicate.idx()]) - || block_is_protected(&blocks[duplicate.idx()]) - || jump_back_target(&blocks[duplicate.idx()]) != Some(target) - || jump_back_lineno(&blocks[duplicate.idx()]) - != jump_back_lineno(&blocks[current.idx()]) - || has_protected_conditional_predecessor(blocks, &incoming_origins, current) - || has_protected_conditional_predecessor(blocks, &incoming_origins, duplicate) - { - current = blocks[current.idx()].next; - continue; - } - - for block in blocks.iter_mut() { - for info in &mut block.instructions { - if info.target == duplicate { - info.target = current; - } - } - } - current = blocks[current.idx()].next; - } -} - -fn reorder_conditional_body_and_implicit_continue_blocks(blocks: &mut Vec) { - fn jump_back_target(blocks: &[Block], block_idx: BlockIdx) -> Option { - if block_idx == BlockIdx::NULL { - return None; - } - let instructions = blocks[block_idx.idx()].instructions.as_slice(); - let jump = match instructions { - [jump] if jump.instr.is_unconditional_jump() => jump, - [not_taken, jump] - if matches!(not_taken.instr.real(), Some(Instruction::NotTaken)) - && jump.instr.is_unconditional_jump() => - { - jump - } - _ => return None, - }; - if jump.target == BlockIdx::NULL - || !comes_before(blocks, next_nonempty_block(blocks, jump.target), block_idx) - { - return None; - } - Some(jump.target) - } - - fn find_body_tail( - blocks: &[Block], - body_start: BlockIdx, - false_jump: BlockIdx, - target: BlockIdx, - ) -> Option { - let mut cursor = body_start; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL && cursor != false_jump { - if visited[cursor.idx()] { - return None; - } - visited[cursor.idx()] = true; - if block_is_exceptional(&blocks[cursor.idx()]) { - return None; - } - if jump_back_target(blocks, cursor) == Some(target) { - return Some(cursor); - } - cursor = blocks[cursor.idx()].next; - } - None - } - - fn find_body_tail_before_jump( - blocks: &[Block], - body_start: BlockIdx, - jump_start: BlockIdx, - ) -> Option { - let mut cursor = body_start; - let mut tail = BlockIdx::NULL; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL && cursor != jump_start { - if visited[cursor.idx()] { - return None; - } - visited[cursor.idx()] = true; - if block_is_exceptional(&blocks[cursor.idx()]) { - return None; - } - tail = cursor; - cursor = blocks[cursor.idx()].next; - } - (tail != BlockIdx::NULL && cursor == jump_start).then_some(tail) - } - - fn body_segment_contains_for_iter( - blocks: &[Block], - body_start: BlockIdx, - body_tail: BlockIdx, - ) -> bool { - let mut cursor = body_start; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL { - if visited[cursor.idx()] { - return false; - } - visited[cursor.idx()] = true; - if blocks[cursor.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::ForIter { .. }))) - { - return true; - } - if cursor == body_tail { - return false; - } - cursor = blocks[cursor.idx()].next; - } - false - } - - fn body_segment_contains_protected_block( - blocks: &[Block], - body_start: BlockIdx, - body_tail: BlockIdx, - ) -> bool { - let mut cursor = body_start; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL { - if visited[cursor.idx()] { - return false; - } - visited[cursor.idx()] = true; - if block_is_protected(&blocks[cursor.idx()]) { - return true; - } - if cursor == body_tail { - return false; - } - cursor = blocks[cursor.idx()].next; - } - false - } - - fn body_segment_contains_scope_exit( - blocks: &[Block], - body_start: BlockIdx, - body_tail: BlockIdx, - ) -> bool { - let mut cursor = body_start; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL { - if visited[cursor.idx()] { - return false; - } - visited[cursor.idx()] = true; - if blocks[cursor.idx()] - .instructions - .iter() - .any(|info| info.instr.is_scope_exit()) - { - return true; - } - if cursor == body_tail { - return false; - } - cursor = blocks[cursor.idx()].next; - } - false - } - - fn body_segment_contains_jump_back_to( - blocks: &[Block], - body_start: BlockIdx, - body_tail: BlockIdx, - target: BlockIdx, - ) -> bool { - let mut cursor = body_start; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL { - if visited[cursor.idx()] { - return false; - } - visited[cursor.idx()] = true; - if jump_back_target(blocks, cursor) == Some(target) { - return true; - } - if cursor == body_tail { - return false; - } - cursor = blocks[cursor.idx()].next; - } - false - } - - fn body_segment_contains_any_jump_back( - blocks: &[Block], - body_start: BlockIdx, - body_tail: BlockIdx, - ) -> bool { - fn jump_back_or_self_target(blocks: &[Block], block_idx: BlockIdx) -> Option { - if block_idx == BlockIdx::NULL { - return None; - } - let jump = blocks[block_idx.idx()].instructions.last()?; - if !jump.instr.is_unconditional_jump() { - return None; - } - if jump.target == BlockIdx::NULL { - return None; - } - let target = next_nonempty_block(blocks, jump.target); - if target == block_idx || comes_before(blocks, target, block_idx) { - Some(jump.target) - } else { - None - } - } - - let mut cursor = body_start; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL { - if visited[cursor.idx()] { - return false; - } - visited[cursor.idx()] = true; - if jump_back_or_self_target(blocks, cursor).is_some() { - return true; - } - if cursor == body_tail { - return false; - } - cursor = blocks[cursor.idx()].next; - } - false - } - - fn block_ends_with_jump_back_to( - blocks: &[Block], - block_idx: BlockIdx, - target: BlockIdx, - ) -> bool { - let Some(last) = blocks[block_idx.idx()].instructions.last() else { - return false; - }; - last.instr.is_unconditional_jump() - && last.target != BlockIdx::NULL - && next_nonempty_block(blocks, last.target) == next_nonempty_block(blocks, target) - && comes_before(blocks, next_nonempty_block(blocks, target), block_idx) - } - - fn conditional_jump_target_count(blocks: &[Block], target: BlockIdx) -> usize { - let target = next_nonempty_block(blocks, target); - blocks - .iter() - .flat_map(|block| &block.instructions) - .filter(|info| { - is_conditional_jump(&info.instr) - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) - .count() - } - - fn empty_chain_reaches(blocks: &[Block], start: BlockIdx, target: BlockIdx) -> bool { - let mut cursor = start; - let mut visited = vec![false; blocks.len()]; - while cursor != BlockIdx::NULL && cursor != target { - if visited[cursor.idx()] - || block_is_exceptional(&blocks[cursor.idx()]) - || block_is_protected(&blocks[cursor.idx()]) - || !blocks[cursor.idx()].instructions.is_empty() - { - return false; - } - visited[cursor.idx()] = true; - cursor = blocks[cursor.idx()].next; - } - cursor == target - } - - fn block_starts_loop_cleanup(blocks: &[Block], block_idx: BlockIdx) -> bool { - block_idx != BlockIdx::NULL - && matches!( - blocks[block_idx.idx()] - .instructions - .first() - .and_then(|info| info.instr.real()), - Some(Instruction::EndFor | Instruction::PopIter) - ) - } - - fn is_single_delete_subscr_body(block: &Block) -> bool { - let real: Vec<_> = block - .instructions - .iter() - .filter_map(|info| info.instr.real()) - .filter(|instr| !matches!(instr, Instruction::Nop | Instruction::NotTaken)) - .collect(); - real.iter() - .filter(|instr| matches!(instr, Instruction::DeleteSubscr)) - .count() - == 1 - && matches!(real.last(), Some(Instruction::DeleteSubscr)) - && real.iter().all(|instr| { - matches!( - instr, - Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastLoadFast { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - | Instruction::LoadSmallInt { .. } - | Instruction::LoadConst { .. } - | Instruction::Copy { .. } - | Instruction::Swap { .. } - | Instruction::BinaryOp { .. } - | Instruction::BinarySlice - | Instruction::BuildSlice { .. } - | Instruction::StoreSubscr - | Instruction::DeleteSubscr - ) - }) - } - - fn block_has_call(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::Call { .. }))) - } - - fn protected_region_has_prior_scope_exit( - blocks: &[Block], - loop_target: BlockIdx, - current: BlockIdx, - ) -> bool { - let mut cursor = next_nonempty_block(blocks, loop_target); - let mut visited = vec![false; blocks.len()]; - let mut saw_protected = false; - while cursor != BlockIdx::NULL && cursor != current { - if visited[cursor.idx()] { - return false; - } - visited[cursor.idx()] = true; - let block = &blocks[cursor.idx()]; - saw_protected |= block_is_protected(block); - if saw_protected - && block - .instructions - .iter() - .any(|info| info.instr.is_scope_exit()) - { - return true; - } - cursor = block.next; - } - false - } - - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - let Some(cond_idx) = trailing_conditional_jump_index(&blocks[idx]) else { - current = next; - continue; - }; - let cond = blocks[idx].instructions[cond_idx]; - let Some(reversed_cond) = reversed_conditional(&cond.instr) else { - current = next; - continue; - }; - - let true_jump_start = cond.target; - let true_jump = next_nonempty_block(blocks, true_jump_start); - let body_start = next; - let body = next_nonempty_block(blocks, body_start); - let true_jump_loop_target = jump_back_target(blocks, true_jump); - let normalized_forward_conditional = blocks[idx] - .instructions - .get(cond_idx + 1) - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::NotTaken))); - if true_jump_loop_target.is_some() - && (true_jump_start == true_jump - || empty_chain_reaches(blocks, true_jump_start, true_jump)) - && body_start != BlockIdx::NULL - && body != BlockIdx::NULL - && true_jump != BlockIdx::NULL - && !block_is_exceptional(&blocks[idx]) - && !block_is_exceptional(&blocks[true_jump.idx()]) - && !block_is_protected(&blocks[idx]) - && !block_is_protected(&blocks[true_jump.idx()]) - && next_nonempty_block(blocks, blocks[true_jump.idx()].next) != body - && let Some(body_tail) = find_body_tail_before_jump(blocks, body, true_jump_start) - { - let loop_target = true_jump_loop_target.expect("checked above"); - let after_jump = blocks[true_jump.idx()].next; - let body_tail_is_conditional = - trailing_conditional_jump_index(&blocks[body_tail.idx()]).is_some(); - let body_is_single_block = body == body_tail; - let body_has_scope_exit = body_segment_contains_scope_exit(blocks, body, body_tail); - let body_tail_has_scope_exit = is_scope_exit_block(&blocks[body_tail.idx()]); - let body_starts_conditional_chain = block_is_pure_conditional_test(&blocks[body.idx()]); - let body_tail_ends_with_loop_backedge = - block_ends_with_jump_back_to(blocks, body_tail, loop_target); - let body_has_loop_backedge = body_tail_ends_with_loop_backedge - || body_segment_contains_jump_back_to(blocks, body, body_tail, loop_target); - if body_has_scope_exit && !body_tail_has_scope_exit { - current = next; - continue; - } - let body_has_inner_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); - let body_has_any_loop_backedge = body_has_inner_for_iter - && body_segment_contains_any_jump_back(blocks, body, body_tail); - let normalized_single_block_can_reorder = - !normalized_forward_conditional || !block_has_call(&blocks[body.idx()]); - let has_exceptional_duplicate_condition_line = - has_exceptional_duplicate_lineno(blocks, current, instruction_lineno(&cond)); - let has_prior_protected_scope_exit = - protected_region_has_prior_scope_exit(blocks, loop_target, current); - let after_jump_target = next_nonempty_block(blocks, after_jump); - let after_jump_starts_loop_cleanup = - block_starts_loop_cleanup(blocks, after_jump_target); - let after_jump_continues_conditional_chain = after_jump_target != BlockIdx::NULL - && block_is_pure_conditional_test(&blocks[after_jump_target.idx()]); - let body_is_pop_top_exit_like = - body_is_single_block && is_pop_top_exit_like_block(&blocks[body.idx()]); - if (body_has_scope_exit || body_is_pop_top_exit_like) - && after_jump_target != BlockIdx::NULL - && is_pop_top_exit_like_block(&blocks[after_jump_target.idx()]) - { - current = next; - continue; - } - if after_jump_continues_conditional_chain { - current = next; - continue; - } - let jump_target_has_multiple_conditional_predecessors = - conditional_jump_target_count(blocks, true_jump_start) > 1; - let simple_single_block_can_reorder = body_is_single_block - && !body_tail_is_conditional - && !has_exceptional_duplicate_condition_line - && after_jump_starts_loop_cleanup - && !body_has_scope_exit - && (!blocks[body.idx()] - .instructions - .iter() - .any(|info| info.instr.is_unconditional_jump()) - || body_tail_ends_with_loop_backedge) - && !after_jump_continues_conditional_chain - && matches!(cond.instr.real(), Some(Instruction::PopJumpIfFalse { .. })); - let trailing_implicit_continue_can_reorder = after_jump != BlockIdx::NULL - && next_nonempty_block(blocks, after_jump) != body - && (!after_jump_starts_loop_cleanup - || body_has_loop_backedge - || body_has_any_loop_backedge) - && !(body_has_scope_exit - && after_jump_target != BlockIdx::NULL - && is_scope_exit_block(&blocks[after_jump_target.idx()]) - && !body_starts_conditional_chain) - && !is_scope_exit_block(&blocks[body.idx()]); - let can_reorder = !has_exceptional_duplicate_condition_line - && !has_prior_protected_scope_exit - && (!jump_target_has_multiple_conditional_predecessors || body_tail_has_scope_exit) - && ((!body_tail_is_conditional - && ((normalized_single_block_can_reorder - && body_is_single_block - && matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. }))) - || (body == body_tail - && is_single_delete_subscr_body(&blocks[body.idx()])) - || simple_single_block_can_reorder)) - || (trailing_implicit_continue_can_reorder - && ((body_has_scope_exit && body_tail_has_scope_exit) - || body_has_loop_backedge - || (after_jump_starts_loop_cleanup && body_has_any_loop_backedge)) - && !after_jump_continues_conditional_chain)); - if can_reorder && after_jump != BlockIdx::NULL && after_jump != body_start { - blocks[idx].instructions[cond_idx].instr = reversed_cond; - blocks[idx].instructions[cond_idx].target = body_start; - if body_tail_ends_with_loop_backedge && true_jump_start == true_jump { - blocks[idx].next = true_jump_start; - blocks[true_jump.idx()].next = body_start; - blocks[body_tail.idx()].next = after_jump; - } else { - let cloned_jump_idx = BlockIdx(blocks.len() as u32); - let mut cloned_jump = blocks[true_jump.idx()].clone(); - cloned_jump.next = body_start; - blocks.push(cloned_jump); - - blocks[idx].next = cloned_jump_idx; - blocks[body_tail.idx()].next = true_jump_start; - } - current = blocks[idx].next; - continue; - } - } - - if !matches!(cond.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) { - current = next; - continue; - } - - let false_jump_start = next; - let false_jump = next_nonempty_block(blocks, false_jump_start); - let body_start = cond.target; - let body = next_nonempty_block(blocks, body_start); - let Some(loop_target) = jump_back_target(blocks, false_jump) else { - current = next; - continue; - }; - if false_jump_start == BlockIdx::NULL - || false_jump == BlockIdx::NULL - || body_start == BlockIdx::NULL - || body == BlockIdx::NULL - || false_jump == body - || block_is_exceptional(&blocks[idx]) - || block_is_exceptional(&blocks[false_jump.idx()]) - || block_is_exceptional(&blocks[body.idx()]) - || block_is_protected(&blocks[idx]) - || block_is_protected(&blocks[false_jump.idx()]) - || block_is_protected(&blocks[body.idx()]) - || next_nonempty_block(blocks, blocks[false_jump.idx()].next) != body - { - current = next; - continue; - } - - let Some(body_tail) = find_body_tail(blocks, body, false_jump, loop_target) else { - current = next; - continue; - }; - let after_body = blocks[body_tail.idx()].next; - if after_body == BlockIdx::NULL || after_body == false_jump_start { - current = next; - continue; - } - let body_contains_for_iter = body_segment_contains_for_iter(blocks, body, body_tail); - let simple_body_can_reorder = !block_has_call(&blocks[body.idx()]) - && jump_back_target(blocks, body_tail) == Some(loop_target); - let false_jump_starts_with_not_taken = blocks[false_jump.idx()] - .instructions - .first() - .is_some_and(|info| matches!(info.instr.real(), Some(Instruction::NotTaken))); - if simple_body_can_reorder && false_jump_starts_with_not_taken && !body_contains_for_iter { - current = next; - continue; - } - if (!body_contains_for_iter && !simple_body_can_reorder) - || body_segment_contains_protected_block(blocks, body, body_tail) - || trailing_conditional_jump_index(&blocks[body.idx()]).is_some() - || block_starts_loop_cleanup(blocks, next_nonempty_block(blocks, after_body)) - { - current = next; - continue; - } - - if blocks[idx] - .instructions - .get(cond_idx + 1) - .is_none_or(|info| !matches!(info.instr.real(), Some(Instruction::NotTaken))) - && matches!( - blocks[false_jump.idx()] - .instructions - .first() - .and_then(|info| info.instr.real()), - Some(Instruction::NotTaken) - ) - { - let not_taken = blocks[false_jump.idx()].instructions.remove(0); - blocks[idx].instructions.insert(cond_idx + 1, not_taken); - } - blocks[idx].instructions[cond_idx].instr = - reversed_conditional(&cond.instr).expect("PopJumpIfTrue has a reversed conditional"); - blocks[idx].instructions[cond_idx].target = false_jump_start; - blocks[idx].next = body_start; - blocks[body_tail.idx()].next = false_jump_start; - blocks[false_jump.idx()].next = after_body; - current = blocks[idx].next; - } -} - -#[allow(dead_code)] -fn reorder_jump_over_exception_cleanup_blocks(blocks: &mut [Block]) { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let next = blocks[idx].next; - if blocks[idx].cold && is_with_suppress_exit_block(&blocks[idx]) { - current = next; - continue; - } - let Some(last) = blocks[idx].instructions.last().copied() else { - current = next; - continue; - }; - if !matches!(last.instr.real(), Some(Instruction::JumpForward { .. })) - || last.target == BlockIdx::NULL - { - current = next; - continue; - } - - let cleanup_start = next; - let target_start = last.target; - let target = next_nonempty_block(blocks, target_start); - if cleanup_start == BlockIdx::NULL || target == BlockIdx::NULL || cleanup_start == target { - current = next; - continue; - } - // Keep the target anchored to the first target block. If we have to - // skip leading empty blocks here, reordering can leave the jump shape - // inconsistent in nested cleanup chains such as poplib.POP3.close(). - if target_start != target { - current = next; - continue; - } - - let mut cleanup_end = BlockIdx::NULL; - let mut saw_exceptional = false; - let mut cursor = cleanup_start; - while cursor != BlockIdx::NULL && cursor != target { - if blocks[cursor.idx()].instructions.is_empty() { - cleanup_end = cursor; - cursor = blocks[cursor.idx()].next; - continue; - } - if !(block_is_exceptional(&blocks[cursor.idx()]) - || is_exception_cleanup_block(&blocks[cursor.idx()]) - || blocks[cursor.idx()].cold && is_reraise_scope_exit_block(&blocks[cursor.idx()])) - { - cleanup_end = BlockIdx::NULL; - break; - } - saw_exceptional = true; - cleanup_end = cursor; - cursor = blocks[cursor.idx()].next; - } - if !saw_exceptional || cleanup_end == BlockIdx::NULL || cursor != target { - current = next; - continue; - } - - let mut target_end = BlockIdx::NULL; - let mut target_exit = BlockIdx::NULL; - let mut nonempty_target_blocks = 0usize; - cursor = target; - while cursor != BlockIdx::NULL { - if block_is_exceptional(&blocks[cursor.idx()]) { - break; - } - target_end = cursor; - if !blocks[cursor.idx()].instructions.is_empty() { - nonempty_target_blocks += 1; - target_exit = cursor; - } - cursor = blocks[cursor.idx()].next; - } - - const MAX_REORDERED_EXIT_BLOCK_SIZE: usize = 4; - - if target_end == BlockIdx::NULL - || target_exit == BlockIdx::NULL - || nonempty_target_blocks != 1 - || target_exit != target_end - || !is_scope_exit_block(&blocks[target_exit.idx()]) - || blocks[target_exit.idx()].instructions.len() > MAX_REORDERED_EXIT_BLOCK_SIZE - || blocks[target_exit.idx()] - .instructions - .iter() - .any(|info| info.instr.is_block_push()) - { - current = next; - continue; - } - - let after_target = blocks[target_end.idx()].next; - blocks[idx].next = target_start; - blocks[target_end.idx()].next = cleanup_start; - blocks[cleanup_end.idx()].next = after_target; - current = after_target; - } -} - -fn maybe_propagate_location( - instr: &mut InstructionInfo, - location: SourceLocation, - end_location: SourceLocation, - lineno_override: Option, -) { - if instr.lineno_override != Some(-2) && !instruction_has_lineno(instr) { - instr.location = location; - instr.end_location = end_location; - instr.lineno_override = lineno_override; - } -} - -fn overwrite_location( - instr: &mut InstructionInfo, - location: SourceLocation, - end_location: SourceLocation, - lineno_override: Option, -) { - instr.location = location; - instr.end_location = end_location; - instr.lineno_override = lineno_override; -} - -fn compute_reachable_blocks(blocks: &[Block]) -> Vec { - let mut reachable = vec![false; blocks.len()]; - if blocks.is_empty() { - return reachable; - } - - reachable[0] = true; - let mut changed = true; - while changed { - changed = false; - for i in 0..blocks.len() { - if !reachable[i] { - continue; - } - for ins in &blocks[i].instructions { - if ins.target != BlockIdx::NULL && !reachable[ins.target.idx()] { - reachable[ins.target.idx()] = true; - changed = true; - } - if let Some(eh) = &ins.except_handler - && !reachable[eh.handler_block.idx()] - { - reachable[eh.handler_block.idx()] = true; - changed = true; - } - } - let next = blocks[i].next; - if next != BlockIdx::NULL - && !reachable[next.idx()] - && !blocks[i].instructions.last().is_some_and(|ins| { - ins.instr.is_scope_exit() || ins.instr.is_unconditional_jump() - }) - { - reachable[next.idx()] = true; - changed = true; - } - } - } - - reachable -} - -fn compute_predecessors(blocks: &[Block]) -> Vec { - let mut predecessors = vec![0u32; blocks.len()]; - if blocks.is_empty() { - return predecessors; - } - - let reachable = compute_reachable_blocks(blocks); - predecessors[0] = 1; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if !reachable[current.idx()] { - current = blocks[current.idx()].next; - continue; - } - - let block = &blocks[current.idx()]; - if block_has_fallthrough(block) { - let next = next_nonempty_block(blocks, block.next); - if next != BlockIdx::NULL && reachable[next.idx()] { - predecessors[next.idx()] += 1; - } - } - for ins in &block.instructions { - if ins.target != BlockIdx::NULL { - let target = next_nonempty_block(blocks, ins.target); - if target != BlockIdx::NULL && reachable[target.idx()] { - predecessors[target.idx()] += 1; - } - } - } - current = block.next; - } - predecessors -} - -fn record_incoming_origin(origins: &mut [Vec], target: BlockIdx, source: BlockIdx) { - let incoming = &mut origins[target.idx()]; - if !incoming.contains(&source) { - incoming.push(source); - } -} - -fn compute_incoming_origins(blocks: &[Block], reachable: &[bool]) -> Vec> { - let mut origins = vec![Vec::new(); blocks.len()]; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if !reachable[current.idx()] { - current = blocks[current.idx()].next; - continue; - } - - let block = &blocks[current.idx()]; - if block_has_fallthrough(block) { - let next = next_nonempty_block(blocks, block.next); - if next != BlockIdx::NULL && reachable[next.idx()] { - record_incoming_origin(&mut origins, next, current); - } - } - for ins in &block.instructions { - if ins.target != BlockIdx::NULL { - let target = next_nonempty_block(blocks, ins.target); - if target != BlockIdx::NULL && reachable[target.idx()] { - record_incoming_origin(&mut origins, target, current); - } - } - } - current = block.next; - } - origins -} - -fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Vec) { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let block = &blocks[current.idx()]; - let last = match block.instructions.last() { - Some(ins) if ins.target != BlockIdx::NULL && is_jump_instruction(ins) => ins, - _ => { - current = blocks[current.idx()].next; - continue; - } - }; - - let target = next_nonempty_block(blocks, last.target); - if target == BlockIdx::NULL { - current = blocks[current.idx()].next; - continue; - } - let target_is_exit_without_lineno = is_exit_without_lineno(blocks, target); - let target_is_protected_eval_break = - is_eval_break_without_lineno(blocks, target) && last.except_handler.is_some(); - if !target_is_exit_without_lineno && !target_is_protected_eval_break { - current = blocks[current.idx()].next; - continue; - } - if predecessors[target.idx()] <= 1 { - current = blocks[current.idx()].next; - continue; - } - - // Copy the exit block and splice it into the linked list after the - // original target block, matching CPython's copy_basicblock() layout. - let new_idx = BlockIdx(blocks.len() as u32); - let mut new_block = blocks[target.idx()].clone(); - if let Some(first) = new_block.instructions.first_mut() - && let Some((location, end_location, lineno_override)) = propagation_location(last) - { - overwrite_location(first, location, end_location, lineno_override); - } - let old_next = blocks[target.idx()].next; - new_block.next = old_next; - blocks.push(new_block); - blocks[target.idx()].next = new_idx; - - // Update the jump target - let last_mut = blocks[current.idx()].instructions.last_mut().unwrap(); - last_mut.target = new_idx; - predecessors[target.idx()] -= 1; - predecessors.push(1); - current = blocks[current.idx()].next; - } - - let reachable = compute_reachable_blocks(blocks); - let incoming_origins = compute_incoming_origins(blocks, &reachable); - current = BlockIdx(0); - while current != BlockIdx::NULL { - let block = &blocks[current.idx()]; - if let Some(last) = block.instructions.last() - && block_has_fallthrough(block) - { - let target = next_nonempty_block(blocks, block.next); - if target != BlockIdx::NULL - && (predecessors[target.idx()] == 1 - || has_unique_fallthrough_origin( - blocks, - &reachable, - &incoming_origins, - current, - target, - )) - && (is_exit_without_lineno(blocks, target) - || is_eval_break_without_lineno(blocks, target)) - && let Some((location, end_location, lineno_override)) = propagation_location(last) - && let Some(first) = blocks[target.idx()].instructions.first_mut() - { - maybe_propagate_location(first, location, end_location, lineno_override); - } - } - current = blocks[current.idx()].next; - } -} - -fn propagate_line_numbers(blocks: &mut [Block], predecessors: &[u32]) { - let reachable = compute_reachable_blocks(blocks); - let incoming_origins = compute_incoming_origins(blocks, &reachable); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if !blocks[current.idx()].instructions.is_empty() { - let (next_block, has_fallthrough) = { - let block = &blocks[current.idx()]; - (block.next, block_has_fallthrough(block)) - }; - - let prev_location = { - let block = &mut blocks[current.idx()]; - let mut prev_location = None; - for instr in &mut block.instructions { - if let Some((location, end_location, lineno_override)) = prev_location { - maybe_propagate_location(instr, location, end_location, lineno_override); - } - prev_location = propagation_location(instr); - } - prev_location - }; - let last_jump = last_jump_for_line_propagation(&blocks[current.idx()]); - - if has_fallthrough { - let target = next_nonempty_block(blocks, next_block); - if target != BlockIdx::NULL - && (predecessors[target.idx()] == 1 - || has_unique_fallthrough_origin( - blocks, - &reachable, - &incoming_origins, - current, - target, - )) - && let Some((location, end_location, lineno_override)) = prev_location - && let Some(first) = blocks[target.idx()].instructions.first_mut() - { - maybe_propagate_location(first, location, end_location, lineno_override); - } - } - - if let Some(last_jump) = last_jump { - let mut target = next_nonempty_block(blocks, last_jump.target); - while target != BlockIdx::NULL - && blocks[target.idx()].instructions.is_empty() - && predecessors[target.idx()] == 1 - { - target = blocks[target.idx()].next; - } - if target != BlockIdx::NULL - && (predecessors[target.idx()] == 1 - || has_unique_jump_origin( - blocks, - &reachable, - &incoming_origins, - current, - target, - )) - && let Some((location, end_location, lineno_override)) = prev_location - && let Some(first) = blocks[target.idx()].instructions.first_mut() - { - maybe_propagate_location(first, location, end_location, lineno_override); - } - } - } - current = blocks[current.idx()].next; - } -} - -fn resolve_line_numbers(blocks: &mut Vec) { - let mut predecessors = compute_predecessors(blocks); - duplicate_exits_without_lineno(blocks, &mut predecessors); - propagate_line_numbers(blocks, &predecessors); -} - -fn resolve_next_location_overrides(blocks: &mut [Block]) { - let mut order = Vec::new(); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - for instr_idx in 0..blocks[current.idx()].instructions.len() { - order.push((current, instr_idx)); - } - current = blocks[current.idx()].next; - } - - for pos in (0..order.len()).rev() { - let (block_idx, instr_idx) = order[pos]; - if blocks[block_idx.idx()].instructions[instr_idx].lineno_override != Some(-2) { - continue; - } - if blocks[block_idx.idx()].instructions[instr_idx] - .instr - .is_scope_exit() - || blocks[block_idx.idx()].instructions[instr_idx] - .instr - .is_unconditional_jump() - || is_conditional_jump(&blocks[block_idx.idx()].instructions[instr_idx].instr) - { - blocks[block_idx.idx()].instructions[instr_idx].lineno_override = Some(-1); - continue; - } - if let Some(&(next_block, next_instr)) = order.get(pos + 1) { - let next = blocks[next_block.idx()].instructions[next_instr]; - blocks[block_idx.idx()].instructions[instr_idx].location = next.location; - blocks[block_idx.idx()].instructions[instr_idx].end_location = next.end_location; - blocks[block_idx.idx()].instructions[instr_idx].lineno_override = next.lineno_override; - } else { - blocks[block_idx.idx()].instructions[instr_idx].lineno_override = Some(-1); - } - } -} - -fn propagate_store_fast_store_fast_jump_locations(blocks: &mut [Block]) { - for block in blocks.iter_mut() { - for i in 1..block.instructions.len() { - let previous = block.instructions[i - 1]; - let follows_copy = i >= 2 - && matches!( - block.instructions[i - 2].instr.real(), - Some(Instruction::Copy { .. }) - ); - if !matches!( - previous.instr.real(), - Some(Instruction::StoreFastStoreFast { .. }) - ) || !block.instructions[i].instr.is_unconditional_jump() - || block.instructions[i].preserve_store_fast_store_fast_jump_location - || (follows_copy - && instruction_lineno(&block.instructions[i]) == instruction_lineno(&previous) - && block.instructions[i].location != previous.location) - { - continue; - } - let follows_unpack = i >= 2 - && matches!( - block.instructions[i - 2].instr.real(), - Some(Instruction::UnpackSequence { .. } | Instruction::UnpackEx { .. }) - ); - if follows_unpack && instruction_lineno(&block.instructions[i]) >= 0 { - continue; - } - block.instructions[i].location = previous.location; - block.instructions[i].end_location = previous.end_location; - block.instructions[i].lineno_override = previous.lineno_override; - } - } -} - -fn propagate_tobool_conditional_jump_locations(blocks: &mut [Block]) { - for block in blocks.iter_mut() { - let mut i = 1; - while i < block.instructions.len() { - if !matches!( - block.instructions[i - 1].instr.real(), - Some(Instruction::ToBool) - ) || !is_conditional_jump(&block.instructions[i].instr) - { - i += 1; - continue; - } - - let (location, end_location, lineno_override) = - if block.instructions[i].preserve_tobool_jump_location { - ( - block.instructions[i].location, - block.instructions[i].end_location, - block.instructions[i].lineno_override, - ) - } else { - ( - block.instructions[i - 1].location, - block.instructions[i - 1].end_location, - block.instructions[i - 1].lineno_override, - ) - }; - block.instructions[i].location = location; - block.instructions[i].end_location = end_location; - block.instructions[i].lineno_override = lineno_override; - - let mut j = i + 1; - if j < block.instructions.len() - && matches!( - block.instructions[j].instr.real(), - Some(Instruction::NotTaken) - ) - { - block.instructions[j].location = location; - block.instructions[j].end_location = end_location; - block.instructions[j].lineno_override = lineno_override; - j += 1; - } - if j < block.instructions.len() && block.instructions[j].instr.is_unconditional_jump() { - block.instructions[j].location = location; - block.instructions[j].end_location = end_location; - block.instructions[j].lineno_override = lineno_override; - } - - i = j; - } - } - - for idx in 0..blocks.len() { - let Some(last) = blocks[idx].instructions.last().copied() else { - continue; - }; - if !is_conditional_jump(&last.instr) { - continue; - } - let next = blocks[idx].next; - if next == BlockIdx::NULL { - continue; - } - let next_block = &mut blocks[next.idx()]; - if !next_block - .instructions - .first() - .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::NotTaken))) - { - continue; - } - for instr in next_block.instructions.iter_mut().take(2) { - if matches!(instr.instr.real(), Some(Instruction::NotTaken)) - || instr.instr.is_unconditional_jump() - { - instr.location = last.location; - instr.end_location = last.end_location; - instr.lineno_override = last.lineno_override; - } - } - } -} - -fn find_layout_predecessor(blocks: &[Block], target: BlockIdx) -> BlockIdx { - if target == BlockIdx::NULL { - return BlockIdx::NULL; - } - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if blocks[current.idx()].next == target { - return current; - } - current = blocks[current.idx()].next; - } - BlockIdx::NULL -} - -fn compute_layout_predecessors(blocks: &[Block]) -> Vec { - let mut predecessors = vec![BlockIdx::NULL; blocks.len()]; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let next = blocks[current.idx()].next; - if next != BlockIdx::NULL { - predecessors[next.idx()] = current; - } - current = next; + let removed_jump_kind = jump_thread_kind(last.instr); + let target_instructions = blocks[target.idx()].instructions.clone(); + if let Some(last_instr) = blocks[block_idx.idx()].instructions.last_mut() { + set_to_nop(last_instr); } - predecessors -} - -fn has_unique_fallthrough_origin( - blocks: &[Block], - reachable: &[bool], - incoming_origins: &[Vec], - source: BlockIdx, - target: BlockIdx, -) -> bool { - if source == BlockIdx::NULL - || target == BlockIdx::NULL - || !reachable[source.idx()] - || !block_has_fallthrough(&blocks[source.idx()]) - || next_nonempty_block(blocks, blocks[source.idx()].next) != target + blocks[block_idx.idx()] + .instructions + .extend(target_instructions); + if no_lineno_no_fallthrough + && removed_jump_kind == Some(JumpThreadKind::Plain) + && let Some(last) = blocks[block_idx.idx()].instructions.last_mut() + && jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) { - return false; - } - - let chain_start = blocks[source.idx()].next; - let mut current = chain_start; - while current != target { - if current == BlockIdx::NULL { - return false; - } - if !blocks[current.idx()].instructions.is_empty() { - return false; - } - current = blocks[current.idx()].next; - } - - fn empty_chain_contains( - blocks: &[Block], - mut current: BlockIdx, - target: BlockIdx, - needle: BlockIdx, - ) -> bool { - while current != target { - if current == needle { - return true; - } - current = blocks[current.idx()].next; - } - false + last.instr = match last.instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => PseudoOpcode::Jump.into(), + AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => Opcode::JumpBackward.into(), + _ => last.instr, + }; } - - incoming_origins[target.idx()].iter().all(|&origin| { - origin == source || empty_chain_contains(blocks, chain_start, target, origin) - }) + true } -fn has_unique_jump_origin( - blocks: &[Block], - reachable: &[bool], - incoming_origins: &[Vec], - source: BlockIdx, - target: BlockIdx, -) -> bool { - if source == BlockIdx::NULL - || target == BlockIdx::NULL - || !reachable[source.idx()] - || !blocks[source.idx()] - .instructions - .last() - .is_some_and(|instr| is_jump_instruction(instr) && instr.target != BlockIdx::NULL) - { - return false; - } - - incoming_origins[target.idx()].iter().all(|&origin| { - origin == source - || (blocks[origin.idx()].instructions.is_empty() - && next_nonempty_block(blocks, blocks[origin.idx()].next) == target) - }) -} +/// flowgraph.c inline_small_or_no_lineno_blocks +fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { + loop { + let mut changes = false; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let next = blocks[current.idx()].next; + changes |= basicblock_inline_small_or_no_lineno_blocks(blocks, current); -fn comes_before(blocks: &[Block], first: BlockIdx, second: BlockIdx) -> bool { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if current == first { - return true; + current = next; } - if current == second { - return false; + + if !changes { + break; } - current = blocks[current.idx()].next; } - false } -fn duplicate_shared_jump_back_targets(blocks: &mut Vec) { - let predecessors = compute_predecessors(blocks); - let mut clones = Vec::new(); - let mut lineful_clones_before_target = Vec::new(); - let mut block_order = vec![usize::MAX; blocks.len()]; - let mut current = BlockIdx(0); - let mut pos = 0usize; - while current != BlockIdx::NULL { - block_order[current.idx()] = pos; - pos += 1; - current = blocks[current.idx()].next; - } - - for target in 0..blocks.len() { - let target = BlockIdx(target as u32); - if let Some(jump_target) = lineful_shared_jump_back_target(blocks, target) - && instruction_lineno(&blocks[target.idx()].instructions[0]) >= 0 - && !block_has_break_continue_cleanup_jump(&blocks[target.idx()]) - { - let layout_pred = find_layout_predecessor(blocks, target); - let has_break_continue_jump_predecessor = blocks.iter().any(|block| { - block.instructions.iter().any(|info| { - info.break_continue_cleanup_jump - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) - }); - if jump_target != BlockIdx::NULL - && comes_before(blocks, jump_target, target) - && layout_pred != BlockIdx::NULL - && !block_has_fallthrough(&blocks[layout_pred.idx()]) - && predecessors[target.idx()] >= 2 - && !has_break_continue_jump_predecessor - { - let target_location = blocks[target.idx()].instructions[0].location; - let target_end_location = blocks[target.idx()].instructions[0].end_location; - let target_follows_forward_jump = blocks[layout_pred.idx()] - .instructions - .last() - .is_some_and(|info| { - matches!(info.instr.real(), Some(Instruction::JumpForward { .. })) - }); - for block_idx in 0..blocks.len() { - let block_idx = BlockIdx(block_idx as u32); - for (instr_idx, info) in blocks[block_idx.idx()].instructions.iter().enumerate() - { - if !is_conditional_jump(&info.instr) - || info.target == BlockIdx::NULL - || next_nonempty_block(blocks, info.target) != target - || block_order[block_idx.idx()] >= block_order[target.idx()] - { - continue; - } - let jump_lineno = instruction_lineno(info); - // CPython codegen_if() gives each nested if/elif arm - // its own end/next labels. When those labels collapse - // to the same loop backedge, keep distinct lineful - // jump-back blocks unless the incoming conditional and - // the existing target came from the same source span. - if target_follows_forward_jump && target_location.line == info.location.line +/// flowgraph.c basicblock_remove_redundant_nops +fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) -> usize { + let mut changes = 0; + let bi = block_idx.idx(); + let mut instructions = core::mem::take(&mut blocks[bi].instructions); + let spare_instr_slots = core::mem::take(&mut blocks[bi].cpython_spare_instr_slots); + let mut dest = 0; + let mut prev_lineno = -1i32; + + for src in 0..instructions.len() { + let instr = instructions[src]; + let lineno = instruction_lineno(&instr); + let mut remove = false; + + if matches!(instr.instr.real(), Some(Instruction::Nop)) { + if lineno < 0 || prev_lineno == lineno { + remove = true; + } else if src < instructions.len() - 1 { + let next_lineno = instruction_lineno(&instructions[src + 1]); + if next_lineno == lineno { + remove = true; + } else if next_lineno < 0 { + copy_instruction_location(instr, &mut instructions[src + 1]); + remove = true; + } + } else { + let next = next_nonempty_block(blocks, blocks[bi].next); + if next != BlockIdx::NULL { + let mut first_next = None; + for (next_i, next_instr) in blocks[next.idx()].instructions.iter().enumerate() { + let next_lineno = instruction_lineno(next_instr); + if matches!(next_instr.instr.real(), Some(Instruction::Nop)) + && next_lineno < 0 { continue; } - if jump_lineno >= 0 - && (info.location != target_location - || info.end_location != target_end_location) - { - lineful_clones_before_target.push((target, block_idx, instr_idx)); - } + first_next = Some((next_i, next_lineno)); + break; } - } - } - } - - let Some(jump_target) = shared_jump_back_target(&blocks[target.idx()]) else { - continue; - }; - if blocks[target.idx()] - .instructions - .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::PopExcept))) - { - continue; - } - - let jump_target = next_nonempty_block(blocks, jump_target); - if jump_target == BlockIdx::NULL || !comes_before(blocks, jump_target, target) { - continue; - } - if !has_non_exception_loop_backedge_to(blocks, target, jump_target) { - continue; - } - - let layout_pred = find_layout_predecessor(blocks, target); - if layout_pred != BlockIdx::NULL - && (!block_has_fallthrough(&blocks[layout_pred.idx()]) - || next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target) - && predecessors[target.idx()] >= 2 - { - let mut jump_predecessors = Vec::new(); - for block_idx in 0..blocks.len() { - let block_idx = BlockIdx(block_idx as u32); - for (instr_idx, info) in blocks[block_idx.idx()].instructions.iter().enumerate() { - if !is_jump_instruction(info) || info.target == BlockIdx::NULL { - continue; - } - if next_nonempty_block(blocks, info.target) == target { - jump_predecessors.push((block_idx, instr_idx)); - } - } - } - if jump_predecessors.len() >= 2 - && jump_predecessors.iter().all(|(block_idx, instr_idx)| { - is_conditional_jump(&blocks[block_idx.idx()].instructions[*instr_idx].instr) - && block_order[block_idx.idx()] < block_order[target.idx()] - }) - && let Some((keep_block, keep_instr)) = jump_predecessors - .iter() - .max_by_key(|(block_idx, _)| block_order[block_idx.idx()]) - .copied() - { - for (block_idx, instr_idx) in jump_predecessors { - if block_idx == keep_block && instr_idx == keep_instr { - continue; + if let Some((_next_i, next_lineno)) = first_next + && next_lineno == lineno + { + remove = true; } - clones.push((target, block_idx, instr_idx)); } - continue; } } - if layout_pred == BlockIdx::NULL - || !block_has_fallthrough(&blocks[layout_pred.idx()]) - || next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target - || predecessors[target.idx()] < 2 - { - continue; - } - - for block_idx in 0..blocks.len() { - let block_idx = BlockIdx(block_idx as u32); - if block_idx == target || block_idx == layout_pred { - continue; - } - - for (instr_idx, info) in blocks[block_idx.idx()].instructions.iter().enumerate() { - if !is_jump_instruction(info) || info.target == BlockIdx::NULL { - continue; - } - if next_nonempty_block(blocks, info.target) != target { - continue; - } - clones.push((target, block_idx, instr_idx)); + if remove { + changes += 1; + } else { + if dest != src { + instructions[dest] = instructions[src]; } + dest += 1; + prev_lineno = lineno; } } - lineful_clones_before_target.sort_by_key(|(target, block_idx, instr_idx)| { - let jump_lineno = instruction_lineno(&blocks[block_idx.idx()].instructions[*instr_idx]); - ( - block_order[target.idx()], - Reverse(jump_lineno), - usize::MAX - block_order[block_idx.idx()], - ) - }); - for (target, block_idx, instr_idx) in lineful_clones_before_target { - if next_nonempty_block( - blocks, - blocks[block_idx.idx()].instructions[instr_idx].target, - ) != target - { - continue; - } - let jump = blocks[block_idx.idx()].instructions[instr_idx]; - let layout_pred = find_layout_predecessor(blocks, target); - if layout_pred == BlockIdx::NULL { - continue; - } - - let mut cloned = blocks[target.idx()].clone(); - if let Some(first) = cloned.instructions.first_mut() { - overwrite_location( - first, - jump.location, - jump.end_location, - jump.lineno_override, - ); - } - let new_idx = BlockIdx(blocks.len() as u32); - cloned.next = target; - blocks.push(cloned); - blocks[layout_pred.idx()].next = new_idx; - blocks[block_idx.idx()].instructions[instr_idx].target = new_idx; - } - - for (target, block_idx, instr_idx) in clones.into_iter().rev() { - let jump = blocks[block_idx.idx()].instructions[instr_idx]; - let mut cloned = blocks[target.idx()].clone(); - if let Some(first) = cloned.instructions.first_mut() { - overwrite_location( - first, - jump.location, - jump.end_location, - jump.lineno_override, - ); - } + blocks[bi] + .cpython_spare_instr_slots + .extend_from_slice(&instructions[dest..]); + blocks[bi] + .cpython_spare_instr_slots + .extend_from_slice(&spare_instr_slots); + instructions.truncate(dest); + blocks[bi].instructions = instructions; + changes +} - let new_idx = BlockIdx(blocks.len() as u32); - let old_next = blocks[target.idx()].next; - cloned.next = old_next; - blocks.push(cloned); - blocks[target.idx()].next = new_idx; - blocks[block_idx.idx()].instructions[instr_idx].target = new_idx; +/// flowgraph.c remove_redundant_nops +fn remove_redundant_nops(blocks: &mut [Block]) -> usize { + let mut block_order = Vec::new(); + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + block_order.push(current); + current = blocks[current.idx()].next; } + block_order + .into_iter() + .map(|block_idx| basicblock_remove_redundant_nops(blocks, block_idx)) + .sum() } -fn reorder_lineful_jump_back_runs_by_descending_line(blocks: &mut [Block]) { - fn follows_while_body_anchor_order(blocks: &[Block], body: BlockIdx, anchor: BlockIdx) -> bool { - blocks.iter().any(|block| { - next_nonempty_block(blocks, block.next) == body - && block.instructions.iter().any(|info| { - is_conditional_jump(&info.instr) - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == anchor - }) - }) - } +/// flowgraph.c no_redundant_nops +fn no_redundant_nops(blocks: &[Block]) -> bool { + let mut blocks = blocks.to_vec(); + remove_redundant_nops(&mut blocks) == 0 +} - let mut prev = BlockIdx::NULL; +/// flowgraph.c remove_redundant_jumps +fn remove_redundant_jumps(blocks: &mut [Block]) -> crate::InternalResult { + let mut changes = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { - let Some(first_target) = blocks[current.idx()] - .instructions - .first() - .filter(|_| is_jump_back_only_block(blocks, current)) - .map(|info| next_nonempty_block(blocks, info.target)) - else { - prev = current; - current = blocks[current.idx()].next; + let idx = current.idx(); + let Some(last_instr) = blocks[idx].instructions.last().copied() else { + current = blocks[idx].next; continue; }; - if first_target == BlockIdx::NULL || !comes_before(blocks, first_target, current) { - prev = current; - current = blocks[current.idx()].next; - continue; - } - - let mut run = Vec::new(); - let mut scan = current; - while scan != BlockIdx::NULL - && is_jump_back_only_block(blocks, scan) - && blocks[scan.idx()] - .instructions - .first() - .is_some_and(|info| next_nonempty_block(blocks, info.target) == first_target) - && instruction_lineno(&blocks[scan.idx()].instructions[0]) >= 0 - { - run.push(scan); - scan = blocks[scan.idx()].next; - } - - if run.len() < 2 { - prev = current; - current = blocks[current.idx()].next; - continue; - } - if follows_while_body_anchor_order(blocks, run[0], run[1]) { - prev = *run.last().expect("non-empty jump-back run"); - current = scan; - continue; - } - - run.sort_by_key(|block_idx| { - Reverse(instruction_lineno(&blocks[block_idx.idx()].instructions[0])) - }); - if prev != BlockIdx::NULL { - blocks[prev.idx()].next = run[0]; - } - for pair in run.windows(2) { - blocks[pair[0].idx()].next = pair[1]; + debug_assert!(!last_instr.instr.is_assembler()); + if last_instr.instr.is_unconditional_jump() { + let jump_target = next_nonempty_block(blocks, last_instr.target); + if jump_target == BlockIdx::NULL { + return Err(InternalError::MalformedControlFlowGraph); + } + let next = next_nonempty_block(blocks, blocks[idx].next); + if jump_target == next { + let last_instr = blocks[idx].instructions.last_mut().unwrap(); + set_to_nop(last_instr); + changes += 1; + current = blocks[idx].next; + continue; + } } - let last = *run.last().expect("non-empty jump-back run"); - blocks[last.idx()].next = scan; - prev = last; - current = scan; + current = blocks[idx].next; } + Ok(changes) } -fn duplicate_fallthrough_jump_back_targets(blocks: &mut Vec) { - fn block_has_real_fallthrough_body(block: &Block) -> bool { - block.instructions.iter().any(|info| { - !matches!( - info.instr.real(), - Some(Instruction::Nop | Instruction::NotTaken | Instruction::PopTop) - ) - }) - } - - let predecessors = compute_predecessors(blocks); - let mut clones = Vec::new(); - - let mut layout_pred = BlockIdx(0); - while layout_pred != BlockIdx::NULL { - if !block_has_fallthrough(&blocks[layout_pred.idx()]) - || !block_has_real_fallthrough_body(&blocks[layout_pred.idx()]) - { - layout_pred = blocks[layout_pred.idx()].next; - continue; - } - - let target = next_nonempty_block(blocks, blocks[layout_pred.idx()].next); - let Some(jump_target) = (target != BlockIdx::NULL) - .then(|| lineful_shared_jump_back_target(blocks, target)) - .flatten() - else { - layout_pred = blocks[layout_pred.idx()].next; +/// flowgraph.c no_redundant_jumps +fn no_redundant_jumps(blocks: &[Block]) -> bool { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let block = &blocks[current.idx()]; + let Some(last) = block.instructions.last() else { + current = block.next; continue; }; - if target == BlockIdx::NULL - || predecessors[target.idx()] < 2 - || block_has_break_continue_cleanup_jump(&blocks[target.idx()]) - || !comes_before(blocks, next_nonempty_block(blocks, jump_target), target) - { - layout_pred = blocks[layout_pred.idx()].next; - continue; - } - let jump = blocks[target.idx()] - .instructions - .last() - .expect("shared_jump_back_target requires a jump"); - if jump.lineno_override.is_some_and(|lineno| lineno >= 0) { - layout_pred = blocks[layout_pred.idx()].next; - continue; - } - if !block_has_no_lineno(&blocks[target.idx()]) - && trailing_conditional_jump_index(&blocks[layout_pred.idx()]).is_some() - { - layout_pred = blocks[layout_pred.idx()].next; - continue; - } - let jump_target = next_nonempty_block(blocks, jump_target); - if jump_target == BlockIdx::NULL - || (!has_non_exception_loop_backedge_to(blocks, target, jump_target) - && !block_has_non_exception_loop_backedge_to(blocks, target, jump_target)) - { - layout_pred = blocks[layout_pred.idx()].next; - continue; - } - - let has_non_layout_jump_predecessor = blocks.iter().enumerate().any(|(idx, block)| { - let block_idx = BlockIdx(idx as u32); - block_idx != layout_pred - && block_idx != target - && block.instructions.iter().any(|info| { - is_jump_instruction(info) - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) - }); - let has_break_continue_jump_predecessor = blocks.iter().enumerate().any(|(idx, block)| { - let block_idx = BlockIdx(idx as u32); - block_idx != layout_pred - && block_idx != target - && block.instructions.iter().any(|info| { - info.break_continue_cleanup_jump - && info.target != BlockIdx::NULL - && next_nonempty_block(blocks, info.target) == target - }) - }); - if has_non_layout_jump_predecessor && !has_break_continue_jump_predecessor { - clones.push((layout_pred, target)); + if last.instr.is_unconditional_jump() { + let next = next_nonempty_block(blocks, block.next); + let jump_target = next_nonempty_block(blocks, last.target); + if jump_target == next { + if next == BlockIdx::NULL { + return false; + } + if let Some(first_next) = blocks[next.idx()].instructions.first() + && instruction_lineno(last) == instruction_lineno(first_next) + { + return false; + } + } } - - layout_pred = blocks[layout_pred.idx()].next; + current = block.next; } + true +} - for (layout_pred, target) in clones.into_iter().rev() { - if next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target { - continue; - } - let Some(last) = blocks[layout_pred.idx()].instructions.last().copied() else { - continue; - }; - - let new_idx = BlockIdx(blocks.len() as u32); - let mut cloned = blocks[target.idx()].clone(); - if let Some(first) = cloned.instructions.first_mut() { - overwrite_location( - first, - last.location, - last.end_location, - last.lineno_override, - ); +fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResult<()> { + loop { + let removed_nops = remove_redundant_nops(blocks); + let removed_jumps = remove_redundant_jumps(blocks)?; + if removed_nops + removed_jumps == 0 { + break; } - cloned.next = blocks[layout_pred.idx()].next; - blocks.push(cloned); - blocks[layout_pred.idx()].next = new_idx; } + Ok(()) } -fn retarget_conditional_jumps_to_empty_while_exit_epilogue(blocks: &mut [Block]) { - fn is_no_location_return_epilogue(block: &Block) -> bool { - matches!( - block.instructions.as_slice(), - [load, ret] - if load.no_location_exit - && ret.no_location_exit - && instruction_lineno(load) < 0 - && instruction_lineno(ret) < 0 - && matches!(load.instr.real(), Some(Instruction::LoadConst { .. })) - && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) - ) +fn layout_block_order(blocks: &[Block]) -> Vec { + let mut order = Vec::new(); + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + order.push(current); + current = blocks[current.idx()].next; } + order +} - fn ends_with_line_marker_implicit_return(block: &Block) -> bool { - matches!( - block.instructions.as_slice(), - [marker, load, ret] - if matches!(marker.instr.real(), Some(Instruction::Nop)) - && instruction_lineno(marker) > 0 - && load.no_location_exit - && ret.no_location_exit - && instruction_lineno(ret) > 0 - && matches!(load.instr.real(), Some(Instruction::LoadConst { .. })) - && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) - ) - } +/// flowgraph.c struct _PyCfgBuilder +struct CfgBuilder { + blocks: Vec, + current: BlockIdx, + current_label: Option, +} - let jump_targets: Vec> = blocks - .iter() - .map(|block| { - block - .instructions - .iter() - .map(|instr| { - if instr.target == BlockIdx::NULL || !is_conditional_jump(&instr.instr) { - return instr.target; - } - let target = instr.target; - if !ends_with_line_marker_implicit_return(&blocks[target.idx()]) { - return target; - } - let next = next_nonempty_block(blocks, blocks[target.idx()].next); - if next != BlockIdx::NULL && is_no_location_return_epilogue(&blocks[next.idx()]) - { - next - } else { - target - } - }) - .collect() - }) - .collect(); +impl CfgBuilder { + /// flowgraph.c init_cfg_builder + fn new_with_capacity(capacity: usize) -> Self { + let mut blocks = Vec::with_capacity(capacity.max(1)); + blocks.push(Block::default()); + Self { + blocks, + current: BlockIdx(0), + current_label: None, + } + } - for (block, block_jump_targets) in blocks.iter_mut().zip(jump_targets) { - for (instr, target) in block.instructions.iter_mut().zip(block_jump_targets) { - if target != BlockIdx::NULL { - instr.target = target; + /// flowgraph.c cfg_builder_current_block_is_terminated + fn current_block_is_terminated(&mut self) -> bool { + let block = &mut self.blocks[self.current.idx()]; + if block + .instructions + .last() + .is_some_and(|instr| instr.instr.is_terminator()) + { + return true; + } + if let Some(label_id) = self.current_label { + if !block.instructions.is_empty() || block.has_cpython_cfg_label() { + return true; } + block.cpython_label_id = Some(label_id); + self.current_label = None; } + false } -} -/// Duplicate `LOAD_CONST None + RETURN_VALUE` for blocks that fall through -/// to the final return block. -fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { - // Walk the block chain and keep the last non-cold non-empty block. - // After cold exception handlers are pushed to the end, the mainline - // return epilogue can sit before trailing cold blocks. - let mut last_block = BlockIdx::NULL; - let mut last_nonempty_block = BlockIdx::NULL; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - if !blocks[current.idx()].instructions.is_empty() { - last_nonempty_block = current; - if !blocks[current.idx()].cold { - last_block = current; - } + /// flowgraph.c cfg_builder_maybe_start_new_block + fn maybe_start_new_block(&mut self) { + if self.current_block_is_terminated() { + let next = BlockIdx(self.blocks.len() as u32); + let block = Block { + cpython_label_id: self.current_label.take(), + ..Default::default() + }; + self.blocks[self.current.idx()].next = next; + self.blocks.push(block); + self.current = next; } - current = blocks[current.idx()].next; } - if last_block == BlockIdx::NULL { - last_block = last_nonempty_block; + + /// flowgraph.c _PyCfgBuilder_UseLabel + fn use_label(&mut self, label_id: InstructionSequenceLabel) { + self.current_label = Some(label_id); + self.maybe_start_new_block(); } - if last_block == BlockIdx::NULL { - return; + + /// flowgraph.c _PyCfgBuilder_Addop + fn addop(&mut self, info: InstructionInfo) { + self.maybe_start_new_block(); + basicblock_addop(&mut self.blocks[self.current.idx()], info); } - let last_insts = &blocks[last_block.idx()].instructions; - // Only apply when the last block is EXACTLY a return-None epilogue. - let is_return_block = last_insts.len() == 2 - && matches!( - last_insts[0].instr, - AnyInstruction::Real(Instruction::LoadConst { .. }) - ) - && is_load_const_none(&last_insts[0], metadata) - && last_insts[0].no_location_exit - && last_insts[1].no_location_exit - && matches!( - last_insts[1].instr, - AnyInstruction::Real(Instruction::ReturnValue) - ); - if !is_return_block { - return; + fn into_blocks(self) -> Vec { + self.blocks } +} - // Get the return instructions to clone - let return_insts: Vec = last_insts[last_insts.len() - 2..].to_vec(); - let predecessors = compute_predecessors(blocks); - let has_exception_nointerrupt_jump_to_last_block = blocks.iter().any(|block| { - (block.cold || block.except_handler) - && block.instructions.iter().any(|instr| { - matches!( - jump_thread_kind(instr.instr), - Some(JumpThreadKind::NoInterrupt) - ) && instr.target != BlockIdx::NULL - && next_nonempty_block(blocks, instr.target) == last_block - }) +/// flowgraph.c translate_jump_labels_to_targets +fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResult<()> { + let max_label = iter_blocks(blocks) + .filter_map(|(_, block)| block.cpython_label_id) + .map(InstructionSequenceLabel::idx) + .max(); + let mut label_to_block = max_label.map_or_else(Vec::new, |max_label| { + vec![BlockIdx::NULL; max_label.saturating_add(1)] }); + for (block_idx, block) in iter_blocks(blocks) { + if let Some(label_id) = block.cpython_label_id { + let slot = label_to_block + .get_mut(label_id.idx()) + .ok_or(InternalError::MalformedControlFlowGraph)?; + *slot = block_idx; + } + } - // Find non-cold blocks that reach the last return block either by - // fallthrough or as an unconditional jump target that should get its own - // cloned epilogue. - let mut fallthrough_blocks_to_fix = Vec::new(); - let mut jump_targets_to_fix = Vec::new(); - current = BlockIdx(0); - while current != BlockIdx::NULL { - let block = &blocks[current.idx()]; - let next = next_nonempty_block(blocks, block.next); - if current != last_block && !block.cold { - let last_ins = block.instructions.last(); - let has_fallthrough = last_ins - .is_none_or(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()); - let has_nointerrupt_jump_to_last_block = block.instructions.iter().any(|instr| { - matches!( - jump_thread_kind(instr.instr), - Some(JumpThreadKind::NoInterrupt) - ) && instr.target != BlockIdx::NULL - && next_nonempty_block(blocks, instr.target) == last_block - }); - // Don't duplicate if block already ends with the same return pattern - let already_has_return = block.instructions.len() >= 2 && { - let n = block.instructions.len(); - matches!( - block.instructions[n - 2].instr, - AnyInstruction::Real(Instruction::LoadConst { .. }) - ) && matches!( - block.instructions[n - 1].instr, - AnyInstruction::Real(Instruction::ReturnValue) - ) - }; - if !block.except_handler - && next == last_block - && has_fallthrough - && trailing_conditional_jump_index(block).is_none() - && !has_nointerrupt_jump_to_last_block - && !has_exception_nointerrupt_jump_to_last_block - && !already_has_return - { - fallthrough_blocks_to_fix.push(current); - } - let jump_idx = trailing_conditional_jump_index(block).or_else(|| { - block.instructions.last().and_then(|last| { - (last.instr.is_unconditional_jump() && last.target != BlockIdx::NULL) - .then_some(block.instructions.len() - 1) - }) - }); - if let Some(jump_idx) = jump_idx { - let jump = &block.instructions[jump_idx]; - let has_lineful_return_fallthrough_to_last_block = block.next != BlockIdx::NULL - && next_nonempty_block(blocks, block.next) == last_block - && block.instructions.last().is_some_and(|instr| { - matches!(instr.instr.real(), Some(Instruction::ReturnValue)) - && instruction_lineno(instr) > 0 - }); - if jump.target != BlockIdx::NULL - && !matches!( - jump_thread_kind(jump.instr), - Some(JumpThreadKind::NoInterrupt) - ) - && !(has_lineful_return_fallthrough_to_last_block - && is_conditional_jump(&jump.instr)) - && next_nonempty_block(blocks, jump.target) == last_block - && (is_conditional_jump(&jump.instr) || predecessors[last_block.idx()] > 1) - { - jump_targets_to_fix.push((current, jump_idx)); + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next = blocks[block_idx.idx()].next; + for info in &mut blocks[block_idx.idx()].instructions { + if info.instr.has_target() { + let label_id = InstructionSequenceLabel(u32::from(info.arg) as usize); + let target = label_to_block + .get(label_id.idx()) + .copied() + .filter(|target| *target != BlockIdx::NULL) + .ok_or(InternalError::MalformedControlFlowGraph)?; + info.target = target; + } + } + block_idx = next; + } + Ok(()) +} + +/// flowgraph.c _PyCfg_FromInstructionSequence +fn cfg_from_instruction_sequence( + mut instr_sequence: InstructionSequence, +) -> crate::InternalResult> { + instr_sequence.apply_label_map()?; + instr_sequence.mark_targets()?; + let InstructionSequence { + instrs, + label_map, + annotations_code, + } = instr_sequence; + debug_assert!(matches!( + label_map, + InstructionSequenceLabelOffsets::Applied + )); + + let final_capacity = instrs.len() + annotations_code.as_ref().map_or(0, |seq| seq.instrs.len()); + let mut sequence = Vec::with_capacity(final_capacity); + let mut builder = CfgBuilder::new_with_capacity(final_capacity); + let mut offset_delta = 0isize; + + for mut entry in instrs { + if matches!( + entry.info.instr.pseudo(), + Some(PseudoInstruction::AnnotationsPlaceholder) + ) { + if let Some(annotations_code) = &annotations_code { + debug_assert!(matches!( + annotations_code.label_map, + InstructionSequenceLabelOffsets::Applied + )); + debug_assert!(annotations_code.annotations_code.is_none()); + for ann_entry in annotations_code.instrs.iter().copied() { + if ann_entry.info.instr.has_target() { + return Err(InternalError::MalformedControlFlowGraph); + } + let mut info = ann_entry.info; + info.target = BlockIdx::NULL; + builder.addop(info); + sequence.push(ann_entry); } + offset_delta += annotations_code.instrs.len() as isize - 1; + } else { + offset_delta -= 1; } + continue; } - current = blocks[current.idx()].next; - } - // Duplicate the return instructions at the end of fall-through blocks - for block_idx in fallthrough_blocks_to_fix { - let propagated_location = blocks[block_idx.idx()] - .instructions - .last() - .map(|instr| (instr.location, instr.end_location, instr.lineno_override)); - let mut cloned_return = return_insts.clone(); - if !instruction_has_lineno(&cloned_return[0]) - && let Some((location, end_location, lineno_override)) = propagated_location - { - for instr in &mut cloned_return { - overwrite_location(instr, location, end_location, lineno_override); - } + if entry.is_target { + let label = InstructionSequenceLabel(sequence.len()); + builder.use_label(label); } - blocks[block_idx.idx()].instructions.extend(cloned_return); - } - // Clone the final return block for jump predecessors so their target layout - // matches CPython's duplicated exit blocks. - for (block_idx, instr_idx) in jump_targets_to_fix.into_iter().rev() { - let jump = blocks[block_idx.idx()].instructions[instr_idx]; - let mut cloned_return = return_insts.clone(); - if let Some(first) = cloned_return.first_mut() { - overwrite_location( - first, - jump.location, - jump.end_location, - jump.lineno_override, + if let Some(target_offset) = entry.target_offset { + entry.target_offset = Some( + target_offset + .checked_add_signed(offset_delta) + .ok_or(InternalError::MalformedControlFlowGraph)?, ); } - let new_idx = BlockIdx(blocks.len() as u32); - let is_conditional = is_conditional_jump(&jump.instr); - let new_block = Block { - cold: blocks[last_block.idx()].cold, - except_handler: blocks[last_block.idx()].except_handler, - disable_load_fast_borrow: blocks[last_block.idx()].disable_load_fast_borrow, - instructions: cloned_return, - next: if is_conditional { - last_block - } else { - blocks[block_idx.idx()].next - }, - ..Block::default() - }; - blocks.push(new_block); - if is_conditional { - let layout_pred = find_layout_predecessor(blocks, last_block); - if layout_pred != BlockIdx::NULL { - blocks[layout_pred.idx()].next = new_idx; - } - } else { - blocks[block_idx.idx()].next = new_idx; + let mut info = entry.info; + if let Some(target_offset) = entry.target_offset { + info.arg = OpArg::new( + target_offset + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); } - blocks[block_idx.idx()].instructions[instr_idx].target = new_idx; + info.target = BlockIdx::NULL; + builder.addop(info); + sequence.push(entry); } -} -fn inline_small_fast_return_blocks(blocks: &mut [Block]) { - fn block_is_small_fast_return(block: &Block) -> bool { - if block.instructions.len() > 3 { - return false; + for entry in &sequence { + if entry + .target_offset + .is_some_and(|target_offset| target_offset >= sequence.len()) + { + return Err(InternalError::MalformedControlFlowGraph); } - let real: Vec<_> = block - .instructions - .iter() - .filter(|info| !matches!(info.instr.real(), Some(Instruction::Nop))) - .collect(); - matches!( - real.as_slice(), - [load, ret] - if matches!( - load.instr.real(), - Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) - ) && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) - ) } - loop { - let mut changed = false; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let next = blocks[current.idx()].next; - let Some(last) = blocks[current.idx()].instructions.last().copied() else { - current = next; - continue; - }; - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { - current = next; - continue; - } - let target = next_nonempty_block(blocks, last.target); - if target == BlockIdx::NULL || !block_is_small_fast_return(&blocks[target.idx()]) { - current = next; - continue; - } + Ok(builder.into_blocks()) +} - if jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) - || instruction_has_lineno(&last) - { - let last_instr = blocks[current.idx()].instructions.last_mut().unwrap(); - let lineno_override = last_instr.lineno_override; - set_to_nop(last_instr); - last_instr.lineno_override = lineno_override; - } else { - blocks[current.idx()].instructions.pop(); - } - let cloned = blocks[target.idx()].instructions.clone(); - blocks[current.idx()].instructions.extend(cloned); - changed = true; - current = next; - } - if !changed { - break; +fn maybe_push_local_block( + worklist: &mut Vec, + on_stack: &mut [bool], + unsafe_masks: &mut [u64], + block: BlockIdx, + unsafe_mask: u64, +) { + if block == BlockIdx::NULL { + return; + } + + let idx = block.idx(); + let both = unsafe_masks[idx] | unsafe_mask; + if unsafe_masks[idx] != both { + unsafe_masks[idx] = both; + if !on_stack[idx] { + worklist.push(block); + on_stack[idx] = true; } } } -fn inline_with_suppress_return_blocks(blocks: &mut [Block]) { - for block_idx in 0..blocks.len() { - let Some(jump_idx) = blocks[block_idx].instructions.len().checked_sub(1) else { - continue; +fn scan_block_for_locals( + blocks: &mut [Block], + block_idx: BlockIdx, + worklist: &mut Vec, + on_stack: &mut [bool], + unsafe_masks: &mut [u64], +) { + let idx = block_idx.idx(); + let mut unsafe_mask = unsafe_masks[idx]; + let instr_count = blocks[idx].instructions.len(); + + for i in 0..instr_count { + let (instr, arg, except_handler) = { + let info = &blocks[idx].instructions[i]; + ( + info.instr, + info.arg, + info.except_handler.map(|eh| eh.handler_block), + ) }; - let jump = blocks[block_idx].instructions[jump_idx]; - if !jump.instr.is_unconditional_jump() || jump.target == BlockIdx::NULL { - continue; - } - if !block_has_with_suppress_prefix(&blocks[block_idx], jump_idx) { - continue; + + if let Some(handler_block) = except_handler { + maybe_push_local_block(worklist, on_stack, unsafe_masks, handler_block, unsafe_mask); } - let target = next_nonempty_block(blocks, jump.target); - if target == BlockIdx::NULL || !is_const_return_block(&blocks[target.idx()]) { + let (local_idx, action) = match instr { + AnyInstruction::Pseudo(PseudoInstruction::StoreFastMaybeNull { var_num }) => { + (var_num.get(arg) as usize, LocalScanAction::SetUnsafe) + } + AnyInstruction::Real( + Instruction::DeleteFast { var_num } | Instruction::LoadFastAndClear { var_num }, + ) => (usize::from(var_num.get(arg)), LocalScanAction::SetUnsafe), + AnyInstruction::Real(Instruction::StoreFast { var_num }) => { + (usize::from(var_num.get(arg)), LocalScanAction::ClearUnsafe) + } + AnyInstruction::Real(Instruction::LoadFastCheck { var_num }) => { + (usize::from(var_num.get(arg)), LocalScanAction::ClearUnsafe) + } + AnyInstruction::Real(Instruction::LoadFast { var_num }) => { + (usize::from(var_num.get(arg)), LocalScanAction::CheckLoad) + } + _ => continue, + }; + if local_idx >= 64 { continue; } - let mut cloned_return = blocks[target.idx()].instructions.clone(); - for instr in &mut cloned_return { - overwrite_location( - instr, - jump.location, - jump.end_location, - jump.lineno_override, - ); + let bit = 1u64 << local_idx; + match action { + LocalScanAction::SetUnsafe => unsafe_mask |= bit, + LocalScanAction::ClearUnsafe => unsafe_mask &= !bit, + LocalScanAction::CheckLoad => { + if unsafe_mask & bit != 0 { + blocks[idx].instructions[i].instr = Opcode::LoadFastCheck.into(); + } + unsafe_mask &= !bit; + } } - blocks[block_idx].instructions.pop(); - blocks[block_idx].instructions.extend(cloned_return); + } + + let block = &blocks[idx]; + if block.next != BlockIdx::NULL && block_has_fallthrough(block) { + maybe_push_local_block(worklist, on_stack, unsafe_masks, block.next, unsafe_mask); + } + + if let Some(last) = block.instructions.last() + && is_jump_instruction(last) + && last.target != BlockIdx::NULL + { + maybe_push_local_block(worklist, on_stack, unsafe_masks, last.target, unsafe_mask); } } -fn is_named_except_cleanup_return_block(block: &Block, metadata: &CodeUnitMetadata) -> bool { - matches!( - block.instructions.as_slice(), - [pop_except, load_none1, store, delete, load_none2, ret] - if matches!(pop_except.instr.real(), Some(Instruction::PopExcept)) - && is_load_const_none(load_none1, metadata) - && matches!( - store.instr.real(), - Some(Instruction::StoreFast { .. } | Instruction::StoreName { .. }) - ) - && matches!( - delete.instr.real(), - Some(Instruction::DeleteFast { .. } | Instruction::DeleteName { .. }) - ) - && is_load_const_none(load_none2, metadata) - && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) - ) +enum LocalScanAction { + SetUnsafe, + ClearUnsafe, + CheckLoad, } -fn duplicate_named_except_cleanup_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { - let predecessors = compute_predecessors(blocks); - let mut clones = Vec::new(); +/// Follow chain of empty blocks to find first non-empty block. +fn next_nonempty_block(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { + while idx != BlockIdx::NULL && blocks[idx.idx()].instructions.is_empty() { + idx = blocks[idx.idx()].next; + } + idx +} - for target in 0..blocks.len() { - let target = BlockIdx(target as u32); - if !is_named_except_cleanup_return_block(&blocks[target.idx()], metadata) { - continue; - } +fn instruction_lineno(instr: &InstructionInfo) -> i32 { + match instr.lineno_override { + Some(LINE_ONLY_LOCATION_OVERRIDE) | None => instr.location.line.get() as i32, + Some(lineno) => lineno, + } +} - let layout_pred = find_layout_predecessor(blocks, target); - if layout_pred == BlockIdx::NULL - || next_nonempty_block(blocks, blocks[layout_pred.idx()].next) != target - { - continue; - } +fn instruction_has_lineno(instr: &InstructionInfo) -> bool { + instruction_lineno(instr) >= 0 +} - let fallthroughs_into_target = blocks[layout_pred.idx()] - .instructions - .last() - .is_none_or(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()); - if !fallthroughs_into_target || predecessors[target.idx()] < 2 { - continue; - } +fn copy_instruction_location(source: InstructionInfo, target: &mut InstructionInfo) { + target.location = source.location; + target.end_location = source.end_location; + target.lineno_override = source.lineno_override; +} - let target_lineno = blocks[target.idx()] - .instructions - .first() - .map_or(-1, instruction_lineno); - let layout_pred_lineno = blocks[layout_pred.idx()] - .instructions - .iter() - .rev() - .find(|info| info.instr.real().is_some()) - .map_or(-1, instruction_lineno); - if target_lineno > 0 && layout_pred_lineno > 0 && target_lineno != layout_pred_lineno { - continue; - } +fn propagation_location( + instr: &InstructionInfo, +) -> Option<(SourceLocation, SourceLocation, Option)> { + instruction_has_lineno(instr).then_some(( + instr.location, + instr.end_location, + instr.lineno_override, + )) +} - let mut conditional_sources = Vec::new(); - for block_idx in 0..blocks.len() { - if block_idx == target.idx() { - continue; - } - let Some(instr_idx) = trailing_conditional_jump_index(&blocks[block_idx]) else { - continue; - }; - if next_nonempty_block(blocks, blocks[block_idx].instructions[instr_idx].target) - != target - { - continue; - } - conditional_sources.push((BlockIdx(block_idx as u32), instr_idx)); - } - if let [(block_idx, instr_idx)] = conditional_sources.as_slice() { - clones.push((*block_idx, *instr_idx, target)); - } +/// flowgraph.c basicblock_nofallthrough / BB_NO_FALLTHROUGH +fn basicblock_nofallthrough(block: &Block) -> bool { + block + .instructions + .last() + .is_some_and(|ins| ins.instr.is_no_fallthrough()) +} + +/// flowgraph.c BB_HAS_FALLTHROUGH +fn block_has_fallthrough(block: &Block) -> bool { + !basicblock_nofallthrough(block) +} + +fn is_jump_instruction(instr: &InstructionInfo) -> bool { + instr.instr.has_jump() +} + +fn last_jump_for_line_propagation(block: &Block) -> Option { + let last = block.instructions.last().copied()?; + is_jump_instruction(&last).then_some(last) +} + +/// flowgraph.c basicblock_exits_scope +fn basicblock_exits_scope(block: &Block) -> bool { + block + .instructions + .last() + .is_some_and(|instr| instr.instr.is_scope_exit()) +} + +/// flowgraph.c is_exit_or_eval_check_without_lineno +fn is_exit_or_eval_check_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { + let block = &blocks[block_idx.idx()]; + if basicblock_exits_scope(block) || basicblock_has_eval_break(block) { + block_has_no_lineno(block) + } else { + false + } +} + +/// flowgraph.c basicblock_has_eval_break +fn basicblock_has_eval_break(block: &Block) -> bool { + block + .instructions + .iter() + .any(|info| info.instr.has_eval_break()) +} + +fn block_has_no_lineno(block: &Block) -> bool { + block + .instructions + .iter() + .all(|ins| !instruction_has_lineno(ins)) +} + +fn is_scope_exit_block(block: &Block) -> bool { + basicblock_exits_scope(block) +} + +fn maybe_propagate_location( + instr: &mut InstructionInfo, + location: SourceLocation, + end_location: SourceLocation, + lineno_override: Option, +) { + if instr.lineno_override != Some(NEXT_LOCATION_OVERRIDE) && !instruction_has_lineno(instr) { + instr.location = location; + instr.end_location = end_location; + instr.lineno_override = lineno_override; } +} - for (block_idx, instr_idx, target) in clones.into_iter().rev() { - let jump = blocks[block_idx.idx()].instructions[instr_idx]; - let mut cloned = blocks[target.idx()].instructions.clone(); - if let Some(first) = cloned.first_mut() { - overwrite_location( - first, - jump.location, - jump.end_location, - jump.lineno_override, - ); +fn propagate_line_numbers_in_block( + block: &mut Block, +) -> Option<(SourceLocation, SourceLocation, Option)> { + let mut prev_location = None; + for instr in &mut block.instructions { + if let Some((location, end_location, lineno_override)) = prev_location { + maybe_propagate_location(instr, location, end_location, lineno_override); } - - let new_idx = BlockIdx(blocks.len() as u32); - let next = blocks[target.idx()].next; - blocks.push(Block { - cold: blocks[target.idx()].cold, - except_handler: blocks[target.idx()].except_handler, - disable_load_fast_borrow: blocks[target.idx()].disable_load_fast_borrow, - instructions: cloned, - next, - ..Block::default() - }); - blocks[target.idx()].next = new_idx; - blocks[block_idx.idx()].instructions[instr_idx].target = new_idx; + prev_location = propagation_location(instr); } + prev_location } -fn is_const_return_block(block: &Block) -> bool { - block.instructions.len() == 2 - && matches!( - block.instructions[0].instr.real(), - Some(Instruction::LoadConst { .. }) - ) - && matches!( - block.instructions[1].instr.real(), - Some(Instruction::ReturnValue) - ) +fn overwrite_location( + instr: &mut InstructionInfo, + location: SourceLocation, + end_location: SourceLocation, + lineno_override: Option, +) { + instr.location = location; + instr.end_location = end_location; + instr.lineno_override = lineno_override; } -fn inline_pop_except_return_blocks(blocks: &mut [Block]) { - for block_idx in 0..blocks.len() { - let Some(jump_idx) = blocks[block_idx].instructions.len().checked_sub(1) else { - continue; - }; - let jump = blocks[block_idx].instructions[jump_idx]; - if !jump.instr.is_unconditional_jump() || jump.target == BlockIdx::NULL { - continue; +fn compute_predecessors(blocks: &[Block]) -> Vec { + let mut predecessors = vec![0u32; blocks.len()]; + if blocks.is_empty() { + return predecessors; + } + + predecessors[0] = 1; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let block = &blocks[current.idx()]; + if block_has_fallthrough(block) { + let next = block.next; + if next != BlockIdx::NULL { + predecessors[next.idx()] += 1; + } } - if matches!( - jump_thread_kind(jump.instr), - Some(JumpThreadKind::NoInterrupt) - ) { - continue; + for ins in &block.instructions { + if ins.instr.has_target() && ins.target != BlockIdx::NULL { + predecessors[ins.target.idx()] += 1; + } } + current = block.next; + } + predecessors +} - let Some(last_real_before_jump) = blocks[block_idx].instructions[..jump_idx] - .iter() - .rev() - .find_map(|info| info.instr.real()) - else { - continue; +/// flowgraph.c copy_basicblock +fn copy_basicblock(blocks: &[Block], block_idx: BlockIdx) -> Block { + let block = &blocks[block_idx.idx()]; + debug_assert!(!block_has_fallthrough(block)); + Block { + instructions: block.instructions.clone(), + ..Block::default() + } +} + +fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Vec) { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let block = &blocks[current.idx()]; + let last = match block.instructions.last() { + Some(ins) if ins.target != BlockIdx::NULL && is_jump_instruction(ins) => ins, + _ => { + current = blocks[current.idx()].next; + continue; + } }; - if !matches!(last_real_before_jump, Instruction::PopExcept) { - continue; - } - let target = next_nonempty_block(blocks, jump.target); - if target == BlockIdx::NULL || !is_const_return_block(&blocks[target.idx()]) { + let target = next_nonempty_block(blocks, last.target); + if target == BlockIdx::NULL + || !is_exit_or_eval_check_without_lineno(blocks, target) + || predecessors[target.idx()] <= 1 + { + current = blocks[current.idx()].next; continue; } - let mut cloned_return = blocks[target.idx()].instructions.clone(); - for instr in &mut cloned_return { + let new_idx = BlockIdx(blocks.len() as u32); + let mut new_block = copy_basicblock(blocks, target); + if let Some(first) = new_block.instructions.first_mut() { overwrite_location( - instr, - jump.location, - jump.end_location, - jump.lineno_override, + first, + last.location, + last.end_location, + last.lineno_override, ); } - blocks[block_idx].instructions.pop(); - blocks[block_idx].instructions.extend(cloned_return); + let old_next = blocks[target.idx()].next; + new_block.next = old_next; + blocks.push(new_block); + blocks[target.idx()].next = new_idx; + + let last_mut = blocks[current.idx()].instructions.last_mut().unwrap(); + last_mut.target = new_idx; + predecessors[target.idx()] -= 1; + predecessors.push(1); + current = blocks[current.idx()].next; } -} -fn is_named_except_cleanup_jump_block(block: &Block, metadata: &CodeUnitMetadata) -> bool { - matches!( - block.instructions.as_slice(), - [pop_except, load_none, store, delete, jump] - if matches!(pop_except.instr.real(), Some(Instruction::PopExcept)) - && is_load_const_none(load_none, metadata) - && matches!( - store.instr.real(), - Some(Instruction::StoreFast { .. } | Instruction::StoreName { .. }) - ) - && matches!( - delete.instr.real(), - Some(Instruction::DeleteFast { .. } | Instruction::DeleteName { .. }) - ) - && jump.instr.is_unconditional_jump() - && jump.target != BlockIdx::NULL - ) + current = BlockIdx(0); + while current != BlockIdx::NULL { + let (next_block, last_location) = { + let block = &blocks[current.idx()]; + ( + block_has_fallthrough(block).then_some(block.next), + block + .instructions + .last() + .map(|last| (last.location, last.end_location, last.lineno_override)), + ) + }; + if let (Some(target), Some((location, end_location, lineno_override))) = + (next_block, last_location) + && target != BlockIdx::NULL + && is_exit_or_eval_check_without_lineno(blocks, target) + && let Some(first) = blocks[target.idx()].instructions.first_mut() + { + overwrite_location(first, location, end_location, lineno_override); + } + current = blocks[current.idx()].next; + } } -fn inline_named_except_cleanup_jump_blocks(blocks: &mut [Block], metadata: &CodeUnitMetadata) { - loop { - let mut replacements = Vec::new(); - for block_idx in 0..blocks.len() { - let Some(jump_idx) = blocks[block_idx].instructions.len().checked_sub(1) else { - continue; +fn propagate_line_numbers(blocks: &mut [Block], predecessors: &[u32]) { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + if !blocks[current.idx()].instructions.is_empty() { + let (next_block, has_fallthrough) = { + let block = &blocks[current.idx()]; + (block.next, block_has_fallthrough(block)) }; - let jump = blocks[block_idx].instructions[jump_idx]; - if !matches!(jump.instr.real(), Some(Instruction::JumpForward { .. })) - || jump.target == BlockIdx::NULL - { - continue; - } - let target = next_nonempty_block(blocks, jump.target); - if target == BlockIdx::NULL - || !is_named_except_cleanup_jump_block(&blocks[target.idx()], metadata) + + let prev_location = propagate_line_numbers_in_block(&mut blocks[current.idx()]); + let last_jump = last_jump_for_line_propagation(&blocks[current.idx()]); + + if has_fallthrough + && next_block != BlockIdx::NULL + && predecessors[next_block.idx()] == 1 + && let Some((location, end_location, lineno_override)) = prev_location + && let Some(first) = blocks[next_block.idx()].instructions.first_mut() { - continue; + maybe_propagate_location(first, location, end_location, lineno_override); } - replacements.push((BlockIdx(block_idx as u32), jump_idx, target)); - } - if replacements.is_empty() { - break; - } - for (block_idx, jump_idx, target) in replacements { - if next_nonempty_block( - blocks, - blocks[block_idx.idx()].instructions[jump_idx].target, - ) != target - { - continue; + if let Some(last_jump) = last_jump { + let target = last_jump.target; + if target != BlockIdx::NULL + && predecessors[target.idx()] == 1 + && let Some((location, end_location, lineno_override)) = prev_location + && let Some(first) = blocks[target.idx()].instructions.first_mut() + { + maybe_propagate_location(first, location, end_location, lineno_override); + } } - let mut cloned = blocks[target.idx()].instructions.clone(); - blocks[block_idx.idx()].instructions.truncate(jump_idx); - blocks[block_idx.idx()].instructions.append(&mut cloned); } + current = blocks[current.idx()].next; } } -fn deduplicate_adjacent_pop_except_jump_back_blocks(blocks: &mut [Block]) { - fn pop_except_jump_back_target(block: &Block) -> Option { - let [pop_except, jump] = block.instructions.as_slice() else { +fn resolve_line_numbers(blocks: &mut Vec) { + let mut predecessors = compute_predecessors(blocks); + duplicate_exits_without_lineno(blocks, &mut predecessors); + propagate_line_numbers(blocks, &predecessors); +} + +pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalResult<()> { + fn except_stack_top(stack: &[BlockIdx], blocks: &[Block]) -> Option { + let handler_block = stack.last().copied()?; + if handler_block == BlockIdx::NULL { return None; - }; - if matches!(pop_except.instr.real(), Some(Instruction::PopExcept)) - && matches!( - jump.instr.real(), - Some(Instruction::JumpBackwardNoInterrupt { .. }) - ) - && jump.target != BlockIdx::NULL - { - Some(jump.target) - } else { - None } + Some(ExceptHandlerInfo { + handler_block, + stack_depth: 0, + preserve_lasti: blocks[handler_block.idx()].preserve_lasti, + }) } - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let next = blocks[current.idx()].next; - let Some(target) = pop_except_jump_back_target(&blocks[current.idx()]) else { - current = next; - continue; - }; - let layout_pred = find_layout_predecessor(blocks, current); - if layout_pred != BlockIdx::NULL - && block_has_fallthrough(&blocks[layout_pred.idx()]) - && next_nonempty_block(blocks, blocks[layout_pred.idx()].next) == current - && blocks[layout_pred.idx()] - .instructions - .iter() - .rev() - .find_map(|info| info.instr.real()) - .is_some_and(|instr| { - matches!( - instr, - Instruction::StoreFast { .. } - | Instruction::StoreName { .. } - | Instruction::StoreGlobal { .. } - | Instruction::StoreDeref { .. } - | Instruction::PopExcept - ) - }) - { - current = next; - continue; - } - let duplicate = next_nonempty_block(blocks, next); - if duplicate == BlockIdx::NULL - || duplicate == current - || pop_except_jump_back_target(&blocks[duplicate.idx()]) != Some(target) - { - current = next; - continue; + fn push_except_block( + stack: &mut Vec, + instr: AnyInstruction, + target: BlockIdx, + blocks: &mut [Block], + ) -> crate::InternalResult> { + debug_assert!(instr.is_block_push()); + if target == BlockIdx::NULL { + return Err(InternalError::MalformedControlFlowGraph); } - - for block in blocks.iter_mut() { - for info in &mut block.instructions { - if info.target == duplicate { - info.target = current; - } - } + if matches!( + instr.pseudo(), + Some(PseudoInstruction::SetupWith { .. } | PseudoInstruction::SetupCleanup { .. }) + ) { + blocks[target.idx()].preserve_lasti = true; } - blocks[current.idx()].next = blocks[duplicate.idx()].next; + stack.push(target); + Ok(except_stack_top(stack, blocks)) } -} -/// Label exception targets: walk CFG with except stack, set per-instruction -/// handler info and block preserve_lasti flag. Converts POP_BLOCK to NOP. -/// flowgraph.c label_exception_targets + push_except_block -pub(crate) fn label_exception_targets(blocks: &mut [Block]) { - #[derive(Clone)] - struct ExceptEntry { - handler_block: BlockIdx, - preserve_lasti: bool, + fn pop_except_block( + stack: &mut Vec, + blocks: &[Block], + ) -> crate::InternalResult> { + debug_assert!(!stack.is_empty()); + if stack.is_empty() { + return Err(InternalError::MalformedControlFlowGraph); + } + stack.pop(); + Ok(except_stack_top(stack, blocks)) } let num_blocks = blocks.len(); if num_blocks == 0 { - return; + return Ok(()); } let mut visited = vec![false; num_blocks]; - let mut block_stacks: Vec>> = vec![None; num_blocks]; + let mut block_stacks: Vec>> = vec![None; num_blocks]; // Entry block visited[0] = true; @@ -12247,127 +5840,83 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { while let Some(block_idx) = todo.pop() { let bi = block_idx.idx(); let mut stack = block_stacks[bi].take().unwrap_or_default(); + let mut handler = except_stack_top(&stack, blocks); let mut last_yield_except_depth: i32 = -1; let instr_count = blocks[bi].instructions.len(); for i in 0..instr_count { - // Read all needed fields (each temporary borrow ends immediately) + let instr = blocks[bi].instructions[i].instr; let target = blocks[bi].instructions[i].target; let arg = blocks[bi].instructions[i].arg; - let is_push = blocks[bi].instructions[i].instr.is_block_push(); - let is_pop = blocks[bi].instructions[i].instr.is_pop_block(); - - if is_push { - // Determine preserve_lasti from instruction type (push_except_block) - let preserve_lasti = matches!( - blocks[bi].instructions[i].instr.pseudo(), - Some( - PseudoInstruction::SetupWith { .. } - | PseudoInstruction::SetupCleanup { .. } - ) - ); - // Set preserve_lasti on handler block - if preserve_lasti && target != BlockIdx::NULL { - blocks[target.idx()].preserve_lasti = true; + if instr.is_block_push() { + if target == BlockIdx::NULL { + return Err(InternalError::MalformedControlFlowGraph); } - - // Propagate except stack to handler block if not visited - if target != BlockIdx::NULL && !visited[target.idx()] { + if !visited[target.idx()] { visited[target.idx()] = true; block_stacks[target.idx()] = Some(stack.clone()); todo.push(target); } - - // Push handler onto except stack - stack.push(ExceptEntry { - handler_block: target, - preserve_lasti, - }); - } else if is_pop { - debug_assert!( - !stack.is_empty(), - "POP_BLOCK with empty except stack at block {bi} instruction {i}" - ); - stack.pop(); - // POP_BLOCK → NOP - let remove_no_location_nop = blocks[bi].instructions[i].remove_no_location_nop; - let folded_operand_nop = blocks[bi].instructions[i].folded_operand_nop; - let preserve_block_start_no_location_nop = - blocks[bi].instructions[i].preserve_block_start_no_location_nop; + handler = push_except_block(&mut stack, instr, target, blocks)?; + } else if instr.is_pop_block() { + handler = pop_except_block(&mut stack, blocks)?; set_to_nop(&mut blocks[bi].instructions[i]); - blocks[bi].instructions[i].remove_no_location_nop = remove_no_location_nop; - blocks[bi].instructions[i].folded_operand_nop = folded_operand_nop; - blocks[bi].instructions[i].preserve_block_start_no_location_nop = - preserve_block_start_no_location_nop; - } else { - // Set except_handler for this instruction from except stack top - // stack_depth placeholder: filled by fixup_handler_depths - let handler_info = stack.last().map(|e| ExceptHandlerInfo { - handler_block: e.handler_block, - stack_depth: 0, - preserve_lasti: e.preserve_lasti, - }); - blocks[bi].instructions[i].except_handler = handler_info; - - // Track YIELD_VALUE except stack depth - // Record the except stack depth at the point of yield. - // With the StopIteration wrapper, depth is naturally correct: - // - plain yield outside try: depth=1 → DEPTH1 set - // - yield inside try: depth=2+ → no DEPTH1 - // - yield-from/await: has internal SETUP_FINALLY → depth=2+ → no DEPTH1 - if let Some(Instruction::YieldValue { .. }) = - blocks[bi].instructions[i].instr.real() - { - last_yield_except_depth = stack.len() as i32; - } + } else if is_jump_instruction(&blocks[bi].instructions[i]) { + blocks[bi].instructions[i].except_handler = handler; + debug_assert_eq!(i, instr_count - 1); - // Set RESUME DEPTH1 flag based on last yield's except depth - if let Some(Instruction::Resume { context }) = - blocks[bi].instructions[i].instr.real() - { - let location = context.get(arg).location(); - match location { - oparg::ResumeLocation::AtFuncStart => {} - _ => { - if last_yield_except_depth == 1 { - blocks[bi].instructions[i].arg = - OpArg::new(oparg::ResumeContext::new(location, true).as_u32()); - } - last_yield_except_depth = -1; - } - } + // CPython label_exception_targets(): copy the except stack + // when this block can also fall through, otherwise transfer it + // to the jump target. + if target == BlockIdx::NULL { + return Err(InternalError::MalformedControlFlowGraph); } - - // For jump instructions, propagate except stack to target - if target != BlockIdx::NULL && !visited[target.idx()] { + if !visited[target.idx()] { visited[target.idx()] = true; - block_stacks[target.idx()] = Some(stack.clone()); + block_stacks[target.idx()] = Some(if block_has_fallthrough(&blocks[bi]) { + stack.clone() + } else { + core::mem::take(&mut stack) + }); todo.push(target); } + } else if matches!(instr.real(), Some(Instruction::YieldValue { .. })) { + blocks[bi].instructions[i].except_handler = handler; + last_yield_except_depth = stack.len() as i32; + } else if let Some(Instruction::Resume { context }) = instr.real() { + blocks[bi].instructions[i].except_handler = handler; + let location = context.get(arg).location(); + if !matches!(location, oparg::ResumeLocation::AtFuncStart) { + debug_assert!(last_yield_except_depth >= 0); + if last_yield_except_depth == 1 { + blocks[bi].instructions[i].arg = + OpArg::new(oparg::ResumeContext::new(location, true).as_u32()); + } + last_yield_except_depth = -1; + } + } else { + blocks[bi].instructions[i].except_handler = handler; } } - // Propagate to fallthrough block (block.next) let next = blocks[bi].next; - if next != BlockIdx::NULL && !visited[next.idx()] { - let has_fallthrough = blocks[bi] - .instructions - .last() - .is_none_or(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()); // Empty block falls through - if has_fallthrough { - visited[next.idx()] = true; - block_stacks[next.idx()] = Some(stack); - todo.push(next); - } + if block_has_fallthrough(&blocks[bi]) && next != BlockIdx::NULL && !visited[next.idx()] { + visited[next.idx()] = true; + block_stacks[next.idx()] = Some(stack); + todo.push(next); } } + debug_assert!(block_stacks.iter().all(Option::is_none)); + Ok(()) } /// Convert remaining pseudo ops to real instructions or NOP. /// flowgraph.c convert_pseudo_ops -pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) { - for block in blocks.iter_mut() { +pub(crate) fn convert_pseudo_ops(blocks: &mut [Block]) -> crate::InternalResult<()> { + let block_order = layout_block_order(blocks); + for block_idx in block_order { + let block = &mut blocks[block_idx.idx()]; for info in &mut block.instructions { let Some(pseudo) = info.instr.pseudo() else { continue; @@ -12377,30 +5926,9 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) PseudoInstruction::SetupCleanup { .. } | PseudoInstruction::SetupFinally { .. } | PseudoInstruction::SetupWith { .. } => { - let preserve_block_start_no_location_nop = - info.preserve_block_start_no_location_nop; - set_to_nop(info); - info.preserve_block_start_no_location_nop = - preserve_block_start_no_location_nop; - } - // PopBlock in reachable blocks is converted to NOP by - // label_exception_targets. Dead blocks may still have them. - PseudoInstruction::PopBlock => { - let remove_no_location_nop = info.remove_no_location_nop; - let folded_operand_nop = info.folded_operand_nop; - let preserve_block_start_no_location_nop = - info.preserve_block_start_no_location_nop; set_to_nop(info); - info.remove_no_location_nop = remove_no_location_nop; - info.folded_operand_nop = folded_operand_nop; - info.preserve_block_start_no_location_nop = - preserve_block_start_no_location_nop; } - // LOAD_CLOSURE → LOAD_FAST (using cellfixedoffsets for merged layout) - PseudoInstruction::LoadClosure { i } => { - let cell_relative = i.get(info.arg) as usize; - let new_idx = cellfixedoffsets[cell_relative]; - info.arg = OpArg::new(new_idx); + PseudoInstruction::LoadClosure { .. } => { info.instr = Opcode::LoadFast.into(); } // Jump pseudo ops are resolved during block linearization @@ -12412,7 +5940,8 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) .into(); } // These should have been resolved earlier - PseudoInstruction::AnnotationsPlaceholder + PseudoInstruction::PopBlock + | PseudoInstruction::AnnotationsPlaceholder | PseudoInstruction::JumpIfFalse { .. } | PseudoInstruction::JumpIfTrue { .. } => { unreachable!("Unexpected pseudo instruction in convert_pseudo_ops: {pseudo:?}") @@ -12420,11 +5949,12 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) } } } + // CPython flowgraph.c::convert_pseudo_ops() finishes by calling + // remove_redundant_nops_and_jumps(). + remove_redundant_nops_and_jumps(blocks) } -/// Build cellfixedoffsets mapping: cell/free index -> localsplus index. -/// Merged cells (cellvar also in varnames) get the local slot index. -/// Non-merged cells get slots after nlocals. Free vars follow. +/// flowgraph.c build_cellfixedoffsets pub(crate) fn build_cellfixedoffsets( varnames: &IndexSet, cellvars: &IndexSet, @@ -12433,140 +5963,61 @@ pub(crate) fn build_cellfixedoffsets( let nlocals = varnames.len(); let ncells = cellvars.len(); let nfrees = freevars.len(); - let mut fixed = Vec::with_capacity(ncells + nfrees); - let mut numdropped = 0usize; - for (i, cellvar) in cellvars.iter().enumerate() { + let mut fixed = (0..ncells + nfrees) + .map(|i| (nlocals + i).to_u32().expect("too many localsplus slots")) + .collect::>(); + for (oldindex, cellvar) in cellvars.iter().enumerate() { if let Some(local_idx) = varnames.get_index_of(cellvar) { - fixed.push(local_idx as u32); - numdropped += 1; - } else { - fixed.push((nlocals + i - numdropped) as u32); + fixed[oldindex] = local_idx.to_u32().expect("too many localsplus slots"); } } - for i in 0..nfrees { - fixed.push((nlocals + ncells - numdropped + i) as u32); - } fixed } -/// Convert DEREF instruction opargs from cell-relative indices to localsplus indices -/// using the cellfixedoffsets mapping. -pub(crate) fn fixup_deref_opargs(blocks: &mut [Block], cellfixedoffsets: &[u32]) { - for block in blocks.iter_mut() { +/// First half of flowgraph.c fix_cell_offsets. +fn fix_cellfixedoffsets(nlocals: usize, fixedmap: &mut [u32]) -> usize { + let mut numdropped = 0usize; + for (i, fixed) in fixedmap.iter_mut().enumerate() { + if usize::try_from(*fixed).expect("localsplus index overflow") == i + nlocals { + *fixed -= numdropped.to_u32().expect("too many dropped cell vars"); + } else { + numdropped += 1; + } + } + numdropped +} + +/// flowgraph.c fix_cell_offsets +pub(crate) fn fix_cell_offsets( + blocks: &mut [Block], + nlocals: usize, + cellfixedoffsets: &mut [u32], +) -> usize { + let numdropped = fix_cellfixedoffsets(nlocals, cellfixedoffsets); + let block_order = layout_block_order(blocks); + for block_idx in block_order { + let block = &mut blocks[block_idx.idx()]; for info in &mut block.instructions { - let Some(instr) = info.instr.real() else { - continue; - }; + debug_assert!( + !matches!(info.instr.real(), Some(Instruction::ExtendedArg)), + "fix_cell_offsets is called before extended args are generated" + ); let needs_fixup = matches!( - instr.into(), - Opcode::LoadDeref - | Opcode::StoreDeref - | Opcode::DeleteDeref - | Opcode::LoadFromDictOrDeref - | Opcode::MakeCell + info.instr, + AnyInstruction::Real( + Instruction::LoadDeref { .. } + | Instruction::StoreDeref { .. } + | Instruction::DeleteDeref { .. } + | Instruction::LoadFromDictOrDeref { .. } + | Instruction::MakeCell { .. } + ) | AnyInstruction::Pseudo(PseudoInstruction::LoadClosure { .. }) ); if needs_fixup { let cell_relative = u32::from(info.arg) as usize; + debug_assert!(cell_relative < cellfixedoffsets.len()); info.arg = OpArg::new(cellfixedoffsets[cell_relative]); } } } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn instruction_info(instr: Instruction, arg: u32, target: BlockIdx) -> InstructionInfo { - InstructionInfo { - instr: instr.into(), - arg: OpArg::new(arg), - target, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - folded_from_nonliteral_expr: false, - lineno_override: None, - cache_entries: 0, - preserve_redundant_jump_as_nop: false, - remove_no_location_nop: false, - folded_operand_nop: false, - no_location_exit: false, - preserve_block_start_no_location_nop: false, - match_success_jump: false, - break_continue_cleanup_jump: false, - for_loop_break_cleanup_jump: false, - preserve_tobool_jump_location: false, - preserve_store_fast_store_fast_jump_location: false, - } - } - - #[test] - fn short_circuit_stub_allows_only_marker_instructions_before_jump() { - let final_target = BlockIdx(7); - let block = Block { - instructions: vec![ - instruction_info(Instruction::Copy { i: Arg::marker() }, 1, BlockIdx::NULL), - instruction_info(Instruction::ToBool, 0, BlockIdx::NULL), - instruction_info(Instruction::Nop, 0, BlockIdx::NULL), - instruction_info(Instruction::NotTaken, 0, BlockIdx::NULL), - instruction_info( - Instruction::PopJumpIfFalse { - delta: Arg::marker(), - }, - 0, - final_target, - ), - ], - ..Block::default() - }; - - assert_eq!( - same_short_circuit_target( - &block, - Instruction::PopJumpIfFalse { - delta: Arg::marker(), - } - .into(), - ), - Some(final_target) - ); - } - - #[test] - fn short_circuit_stub_rejects_real_instruction_before_jump() { - let block = Block { - instructions: vec![ - instruction_info(Instruction::Copy { i: Arg::marker() }, 1, BlockIdx::NULL), - instruction_info(Instruction::ToBool, 0, BlockIdx::NULL), - instruction_info(Instruction::PopTop, 0, BlockIdx::NULL), - instruction_info( - Instruction::PopJumpIfFalse { - delta: Arg::marker(), - }, - 0, - BlockIdx(7), - ), - ], - ..Block::default() - }; - - assert_eq!( - same_short_circuit_target( - &block, - Instruction::PopJumpIfFalse { - delta: Arg::marker(), - } - .into(), - ), - None - ); - assert!(!opposite_short_circuit_target( - &block, - Instruction::PopJumpIfTrue { - delta: Arg::marker(), - } - .into() - )); - } + numdropped } diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 04389306521..3cc1cc98d08 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -156,6 +156,36 @@ macro_rules! define_opcodes { self.as_opcode().is_scope_exit() } + #[must_use] + $instr_vis const fn is_terminator(&self) -> bool { + self.as_opcode().is_terminator() + } + + #[must_use] + $instr_vis const fn is_no_fallthrough(&self) -> bool { + self.as_opcode().is_no_fallthrough() + } + + #[must_use] + $instr_vis const fn has_target(&self) -> bool { + self.as_opcode().has_target() + } + + #[must_use] + $instr_vis const fn has_jump(&self) -> bool { + self.as_opcode().has_jump() + } + + #[must_use] + $instr_vis const fn has_eval_break(&self) -> bool { + self.as_opcode().has_eval_break() + } + + #[must_use] + $instr_vis const fn is_assembler(&self) -> bool { + self.as_opcode().is_assembler() + } + #[must_use] $instr_vis const fn cache_entries(&self) -> usize{ self.as_opcode().cache_entries() @@ -673,11 +703,68 @@ impl Opcode { ) } + /// CPython's `IS_ASSEMBLER_OPCODE`. + #[must_use] + pub const fn is_assembler(&self) -> bool { + matches!( + self, + Self::JumpForward | Self::JumpBackward | Self::JumpBackwardNoInterrupt + ) + } + #[must_use] pub const fn is_scope_exit(&self) -> bool { matches!(self, Self::ReturnValue | Self::RaiseVarargs | Self::Reraise) } + /// CPython's `IS_TERMINATOR_OPCODE`. + #[must_use] + pub const fn is_terminator(&self) -> bool { + self.has_jump() || self.is_scope_exit() + } + + /// CPython's `IS_SCOPE_EXIT_OPCODE || IS_UNCONDITIONAL_JUMP_OPCODE`. + #[must_use] + pub const fn is_no_fallthrough(&self) -> bool { + self.is_scope_exit() || self.is_unconditional_jump() + } + + /// CPython's `HAS_TARGET`. + #[must_use] + pub const fn has_target(&self) -> bool { + self.has_jump() || self.is_block_push() + } + + /// Does this opcode have CPython's `HAS_EVAL_BREAK_FLAG` set. + #[must_use] + pub const fn has_eval_break(&self) -> bool { + matches!( + self, + Self::Call + | Self::CallBuiltinClass + | Self::CallBuiltinFast + | Self::CallBuiltinFastWithKeywords + | Self::CallBuiltinO + | Self::CallFunctionEx + | Self::CallKwNonPy + | Self::CallMethodDescriptorFast + | Self::CallMethodDescriptorFastWithKeywords + | Self::CallMethodDescriptorNoargs + | Self::CallMethodDescriptorO + | Self::CallNonPyGeneral + | Self::CallStr1 + | Self::CallTuple1 + | Self::InstrumentedCall + | Self::InstrumentedCallFunctionEx + | Self::InstrumentedJumpBackward + | Self::InstrumentedResume + | Self::JumpBackward + | Self::JumpBackwardJit + | Self::JumpBackwardNoJit + | Self::Resume + ) + } + #[must_use] pub const fn is_block_push(&self) -> bool { false @@ -714,6 +801,35 @@ impl PseudoOpcode { matches!(self, Self::Jump | Self::JumpNoInterrupt) } + /// Does this pseudo opcode have CPython's `HAS_EVAL_BREAK_FLAG` set. + #[must_use] + pub const fn has_eval_break(&self) -> bool { + matches!(self, Self::Jump) + } + + #[must_use] + pub const fn is_assembler(&self) -> bool { + false + } + + /// CPython's `IS_TERMINATOR_OPCODE`. + #[must_use] + pub const fn is_terminator(&self) -> bool { + self.has_jump() + } + + /// CPython's `IS_SCOPE_EXIT_OPCODE || IS_UNCONDITIONAL_JUMP_OPCODE`. + #[must_use] + pub const fn is_no_fallthrough(&self) -> bool { + self.is_unconditional_jump() + } + + /// CPython's `HAS_TARGET`. + #[must_use] + pub const fn has_target(&self) -> bool { + self.has_jump() || self.is_block_push() + } + /// Handler entry effect for SETUP_* pseudo ops. /// /// Fallthrough effect is 0 (NOPs), but when the branch is taken the @@ -778,6 +894,36 @@ impl AnyInstruction { pub const fn is_scope_exit(&self) -> bool ); + either_real_pseudo!( + #[must_use] + pub const fn is_terminator(&self) -> bool + ); + + either_real_pseudo!( + #[must_use] + pub const fn is_no_fallthrough(&self) -> bool + ); + + either_real_pseudo!( + #[must_use] + pub const fn has_target(&self) -> bool + ); + + either_real_pseudo!( + #[must_use] + pub const fn has_jump(&self) -> bool + ); + + either_real_pseudo!( + #[must_use] + pub const fn has_eval_break(&self) -> bool + ); + + either_real_pseudo!( + #[must_use] + pub const fn is_assembler(&self) -> bool + ); + either_real_pseudo!( #[must_use] pub fn stack_effect(&self, oparg: u32) -> i32 @@ -1179,3 +1325,80 @@ impl fmt::Debug for Arg { // breaks the VM:/ const _: () = assert!(core::mem::size_of::() == 1); const _: () = assert!(core::mem::size_of::() == 2); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn eval_break_flags_match_cpython_jump_metadata() { + assert!(Opcode::JumpBackward.has_eval_break()); + assert!(!Opcode::JumpBackwardNoInterrupt.has_eval_break()); + assert!(!Opcode::JumpForward.has_eval_break()); + + assert!(PseudoOpcode::Jump.has_eval_break()); + assert!(!PseudoOpcode::JumpIfFalse.has_eval_break()); + assert!(!PseudoOpcode::JumpIfTrue.has_eval_break()); + assert!(!PseudoOpcode::JumpNoInterrupt.has_eval_break()); + + assert!(AnyInstruction::from(PseudoOpcode::Jump).has_eval_break()); + } + + #[test] + fn terminator_flags_match_cpython_opcode_utils() { + assert!(Opcode::JumpForward.is_terminator()); + assert!(Opcode::PopJumpIfFalse.is_terminator()); + assert!(Opcode::ReturnValue.is_terminator()); + assert!(!Opcode::Nop.is_terminator()); + + assert!(PseudoOpcode::JumpIfTrue.is_terminator()); + assert!(PseudoOpcode::JumpNoInterrupt.is_terminator()); + assert!(!PseudoOpcode::SetupWith.is_terminator()); + assert!(!PseudoOpcode::PopBlock.is_terminator()); + + assert!(AnyInstruction::from(PseudoOpcode::JumpIfFalse).is_terminator()); + } + + #[test] + fn assembler_flags_match_cpython_opcode_utils() { + assert!(Opcode::JumpForward.is_assembler()); + assert!(Opcode::JumpBackward.is_assembler()); + assert!(Opcode::JumpBackwardNoInterrupt.is_assembler()); + assert!(!Opcode::PopJumpIfFalse.is_assembler()); + assert!(!Opcode::Nop.is_assembler()); + + assert!(!PseudoOpcode::Jump.is_assembler()); + assert!(!PseudoOpcode::JumpNoInterrupt.is_assembler()); + assert!(!AnyInstruction::from(PseudoOpcode::Jump).is_assembler()); + } + + #[test] + fn target_flags_match_cpython_opcode_utils() { + assert!(Opcode::JumpForward.has_target()); + assert!(Opcode::ForIter.has_target()); + assert!(!Opcode::ReturnValue.has_target()); + assert!(!Opcode::Nop.has_target()); + + assert!(PseudoOpcode::Jump.has_target()); + assert!(PseudoOpcode::SetupWith.has_target()); + assert!(PseudoOpcode::SetupCleanup.has_target()); + assert!(!PseudoOpcode::PopBlock.has_target()); + + assert!(AnyInstruction::from(PseudoOpcode::SetupFinally).has_target()); + } + + #[test] + fn no_fallthrough_flags_match_cpython_basicblock_nofallthrough() { + assert!(Opcode::JumpForward.is_no_fallthrough()); + assert!(Opcode::ReturnValue.is_no_fallthrough()); + assert!(!Opcode::PopJumpIfFalse.is_no_fallthrough()); + assert!(!Opcode::Nop.is_no_fallthrough()); + + assert!(PseudoOpcode::Jump.is_no_fallthrough()); + assert!(PseudoOpcode::JumpNoInterrupt.is_no_fallthrough()); + assert!(!PseudoOpcode::JumpIfFalse.is_no_fallthrough()); + assert!(!PseudoOpcode::SetupWith.is_no_fallthrough()); + + assert!(AnyInstruction::from(PseudoOpcode::Jump).is_no_fallthrough()); + } +} diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index ffe8b871852..0c28a83e72a 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -575,6 +575,13 @@ pub trait MarshalBag: Copy { } fn constant_bag(self) -> Self::ConstantBag; + + fn constant_ref_from_value( + &self, + _value: &Self::Value, + ) -> Option<::Constant> { + None + } } impl MarshalBag for Bag { @@ -669,6 +676,13 @@ impl MarshalBag for Bag { fn constant_bag(self) -> Self::ConstantBag { self } + + fn constant_ref_from_value( + &self, + value: &Self::Value, + ) -> Option<::Constant> { + Some(value.clone()) + } } pub const MAX_MARSHAL_STACK_DEPTH: usize = 2000; @@ -727,16 +741,18 @@ fn deserialize_value_after_header( }; let typ = Type::try_from(type_code)?; - // Code-objects keep their own inner ref table because Bag::Value (the - // outer marshal value) and the constant-bag's Constant type are not - // in general the same. When the outer header carried FLAG_REF, the - // code object occupies slot 0 of CPython's single global ref space, - // so we mirror that by reserving slot 0 of the inner table. + // CPython's r_object() uses one global ref table: TYPE_CODE reserves its + // slot before reading code fields, and those fields may use later TYPE_REF + // indexes. Keep the same indexes even when Bag::Value and Constant differ. let value = if matches!(typ, Type::Code) { - let mut inner_refs: Vec::Constant>> = Vec::new(); - if flag { - inner_refs.push(None); - } + let mut inner_refs: Vec::Constant>> = refs + .iter() + .map(|value| { + value + .as_ref() + .and_then(|value| bag.constant_ref_from_value(value)) + }) + .collect(); let code = deserialize_code_inner(rdr, bag.constant_bag(), depth - 1, &mut inner_refs)?; bag.make_code(code) } else { @@ -1501,6 +1517,15 @@ mod tests { } } + fn decode_tuple(hex: &str) -> Vec { + let bytes = hex_to_bytes(hex); + let value = deserialize_value(&mut &bytes[..], BasicBag).expect("decode failed"); + match value { + ConstantData::Tuple { elements } => elements, + other => panic!("expected Tuple, got {other:?}"), + } + } + /// CPython 3.14 marshal output for: `compile("x = 1", "", "exec")`. /// Exercises FLAG_REF on the code object and TYPE_REF for qualname /// pointing back at the obj_name slot. @@ -1562,4 +1587,27 @@ mod tests { )); assert!(matches!(consts[2], ConstantData::None)); } + + /// CPython 3.14 marshal output for: + /// `(compile("x = 1", "", "exec"),)`. + /// The outer tuple occupies ref slot 0 and the code object occupies + /// slot 1, so code-object fields must preserve that global ref offset. + #[test] + fn cpython_314_code_inside_tuple_preserves_ref_indexes() { + let hex = "a901630000000000000000000000000100000000000000f30a00000080005e017400\ + 520123002902e9010000004e2901da0178a900f300000000da033c743eda083c6d6f\ + 64756c653e720700000001000000730a000000f003010101d8040582017205000000"; + let tuple = decode_tuple(hex); + assert_eq!(tuple.len(), 1); + let code = match &tuple[0] { + ConstantData::Code { code } => code, + other => panic!("expected nested Code, got {other:?}"), + }; + assert_eq!(code.obj_name.as_str(), ""); + assert_eq!(code.qualname.as_str(), ""); + assert_eq!(code.source_path.as_str(), ""); + assert_eq!(code.names.len(), 1); + assert_eq!(code.names[0].as_str(), "x"); + assert_eq!(code.constants.len(), 2); + } } From d4af69b3f94d92f54db4f89b4b26e1988bdd0a90 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 25 May 2026 22:55:15 +0900 Subject: [PATCH 009/131] align compiler to CPython --- crates/codegen/src/compile.rs | 584 +++++++++++++++++++-------------- crates/codegen/src/ir.rs | 117 +++---- crates/vm/src/vm/python_run.rs | 25 ++ 3 files changed, 412 insertions(+), 314 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 5c9e13dbeb8..c406a42beed 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -1637,32 +1637,7 @@ impl Compiler { ); // Body label - continue with annotation evaluation - self.use_cpython_transient_label_block(body_block); - } - - /// Push a new fblock - // = compiler_push_fblock - fn push_fblock( - &mut self, - fb_type: FBlockType, - fb_block_direct: BlockIdx, - fb_exit_direct: BlockIdx, - ) -> CompileResult<()> { - self.push_fblock_full(fb_type, fb_block_direct, fb_exit_direct, FBlockDatum::None) - } - - /// Direct-CFG bridge for `_PyCompile_PushFBlock()`. - fn push_fblock_full( - &mut self, - fb_type: FBlockType, - fb_block_direct: BlockIdx, - fb_exit_direct: BlockIdx, - fb_datum: FBlockDatum, - ) -> CompileResult<()> { - let code = self.current_code_info(); - let fb_block = code.instr_sequence_label_for_block(fb_block_direct); - let fb_exit = code.instr_sequence_label_for_block(fb_exit_direct); - self.push_fblock_labels(fb_type, fb_block, fb_exit, fb_datum) + self.use_cpython_label_block(body_block); } /// CPython `_PyCompile_PushFBlock()`: store the active label targets on the @@ -1691,19 +1666,6 @@ impl Compiler { Ok(()) } - /// Pop an fblock - // = compiler_pop_fblock - fn pop_fblock( - &mut self, - expected_type: FBlockType, - expected_block_direct: BlockIdx, - ) -> FBlockInfo { - let expected_block = self - .current_code_info() - .instr_sequence_label_for_block(expected_block_direct); - self.pop_fblock_label(expected_type, expected_block) - } - /// CPython `_PyCompile_PopFBlock()`: assert the popped type and label. fn pop_fblock_label( &mut self, @@ -1968,14 +1930,19 @@ impl Compiler { // Push PopValue fblock if preserving tos if preserve_tos { - self.push_fblock(FBlockType::PopValue, BlockIdx::NULL, BlockIdx::NULL)?; + self.push_fblock_labels( + FBlockType::PopValue, + None, + None, + FBlockDatum::None, + )?; } self.compile_statements(&body)?; unwind_loc = None; if preserve_tos { - self.pop_fblock(FBlockType::PopValue, BlockIdx::NULL); + self.pop_fblock_label(FBlockType::PopValue, None); } // Restore the fblock @@ -2294,25 +2261,58 @@ impl Compiler { self.symbol_table_stack.push(symbol_table); self.emit_resume_for_scope(CompilerScope::Module, 1); - self.compile_statements(body)?; - - if let Some(last_statement) = body.last() { + if let Some((last_statement, statements)) = body.split_last() { + self.compile_statements(statements)?; match last_statement { ast::Stmt::Expr(ast::StmtExpr { value, .. }) => { - if !self.interactive && Self::is_const_expression(value) { - self.compile_expression(value)?; - } else { - self.pop_last_emitted_instruction(); // pop Instruction::PopTop - } + self.compile_expression(value)?; } - ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_) => { - let pop_instructions = self.pop_last_emitted_instruction(); - let store_inst = compiler_unwrap_option(self, pop_instructions); // pop Instruction::Store - emit!(self, Instruction::Copy { i: 1 }); - self.push_emitted_instruction(store_inst); + ast::Stmt::FunctionDef(ast::StmtFunctionDef { + name, + parameters, + body, + decorator_list, + returns, + type_params, + is_async, + .. + }) => { + validate_duplicate_params(parameters).map_err(|e| self.error(e))?; + self.compile_function_def( + name.as_str(), + parameters, + body, + decorator_list, + returns.as_deref(), + *is_async, + type_params.as_deref(), + true, + )?; + } + ast::Stmt::ClassDef(ast::StmtClassDef { + name, + body, + decorator_list, + type_params, + arguments, + .. + }) => { + self.compile_class_def( + name.as_str(), + body, + decorator_list, + type_params.as_deref(), + arguments.as_deref(), + true, + )?; + } + _ => { + self.compile_statement(last_statement)?; + self.emit_load_const(ConstantData::None); } - _ => self.emit_load_const(ConstantData::None), } + } else { + self.emit_load_const(ConstantData::None); } self.emit_return_value(); @@ -2356,12 +2356,7 @@ impl Compiler { } /// CPython `codegen_with_except_finish()`. - /// - /// CPython creates a helper-local `exit` label followed immediately by the - /// caller's `exit` label. `_PyInstructionSequence_ApplyLabelMap()` resolves - /// both labels to the same next instruction, so the direct-CFG equivalent is - /// to target the caller exit block. - fn compile_with_except_finish(&mut self, cleanup_block: BlockIdx, exit_block: BlockIdx) { + fn compile_with_except_finish(&mut self, cleanup_block: BlockIdx) { let suppress_block = self.new_block(); emit!(self, Instruction::ToBool); @@ -2389,6 +2384,7 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::PopTop); self.set_no_location(); + let exit_block = self.new_block(); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: exit_block } @@ -2402,6 +2398,8 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::Reraise { depth: 1 }); self.set_no_location(); + + self.use_cpython_label_block(exit_block); } fn compile_loop_body_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { @@ -2914,10 +2912,6 @@ impl Compiler { }; self.set_source_range(*range); emit!(self, Instruction::RaiseVarargs { argc: kind }); - // Start a new block so dead code after raise doesn't - // corrupt the except stack in label_exception_targets - let dead = self.new_block(); - self.switch_to_block(dead); } ast::Stmt::Try(ast::StmtTry { body, @@ -2955,6 +2949,7 @@ impl Compiler { returns.as_deref(), *is_async, type_params.as_deref(), + false, )? } ast::Stmt::ClassDef(ast::StmtClassDef { @@ -2970,6 +2965,7 @@ impl Compiler { decorator_list, type_params.as_deref(), arguments.as_deref(), + false, )?, ast::Stmt::Assert(ast::StmtAssert { test, msg, range, .. @@ -2997,7 +2993,7 @@ impl Compiler { argc: bytecode::RaiseKind::Raise, } ); - self.switch_to_block(after_block); + self.use_cpython_label_block(after_block); } else { // Optimized-out asserts still need to consume any nested // scope symbol tables they contain so later nested scopes @@ -3011,14 +3007,10 @@ impl Compiler { ast::Stmt::Break(_) => { // Unwind fblock stack until we find a loop, emitting cleanup for each fblock self.compile_break_continue(statement.range(), true)?; - let dead = self.new_block(); - self.switch_to_block(dead); } ast::Stmt::Continue(_) => { // Unwind fblock stack until we find a loop, emitting cleanup for each fblock self.compile_break_continue(statement.range(), false)?; - let dead = self.new_block(); - self.switch_to_block(dead); } ast::Stmt::Return(ast::StmtReturn { value, .. }) => { if !self.ctx.in_func() { @@ -3097,8 +3089,6 @@ impl Compiler { } } self.set_source_range(prev_source_range); - let dead = self.new_block(); - self.switch_to_block(dead); } ast::Stmt::Assign(ast::StmtAssign { targets, @@ -3596,11 +3586,18 @@ impl Compiler { delta: finally_except_block } ); - self.use_cpython_transient_label_block(body_block); - self.push_fblock_full( + self.use_cpython_label_block(body_block); + let (body_label, finally_except_label) = { + let code = self.current_code_info(); + ( + code.instr_sequence_label_for_block(body_block), + code.instr_sequence_label_for_block(finally_except_block), + ) + }; + self.push_fblock_labels( FBlockType::FinallyTry, - body_block, - finally_except_block, + body_label, + finally_except_label, FBlockDatum::FinallyBody(finalbody.to_vec()), )?; @@ -3614,7 +3611,7 @@ impl Compiler { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::FinallyTry, body_block); + self.pop_fblock_label(FBlockType::FinallyTry, body_label); let sub_table_cursor = self.symbol_table_stack.last().map(|t| t.next_sub_table); @@ -3641,9 +3638,14 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::PushExcInfo); self.set_no_location(); - self.push_fblock(FBlockType::FinallyEnd, finally_except_block, BlockIdx::NULL)?; + self.push_fblock_labels( + FBlockType::FinallyEnd, + finally_except_label, + None, + FBlockDatum::None, + )?; self.compile_statements(finalbody)?; - self.pop_fblock(FBlockType::FinallyEnd, finally_except_block); + self.pop_fblock_label(FBlockType::FinallyEnd, finally_except_label); emit!(self, Instruction::Reraise { depth: 0 }); self.set_no_location(); @@ -3686,10 +3688,13 @@ impl Compiler { delta: handler_block } ); - self.use_cpython_transient_label_block(body_block); - self.push_fblock(FBlockType::TryExcept, body_block, BlockIdx::NULL)?; + self.use_cpython_label_block(body_block); + let body_label = self + .current_code_info() + .instr_sequence_label_for_block(body_block); + self.push_fblock_labels(FBlockType::TryExcept, body_label, None, FBlockDatum::None)?; self.compile_statements(body)?; - self.pop_fblock(FBlockType::TryExcept, body_block); + self.pop_fblock_label(FBlockType::TryExcept, body_label); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); self.compile_statements(orelse)?; @@ -3709,7 +3714,7 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::PushExcInfo); self.set_no_location(); - self.push_fblock(FBlockType::ExceptionHandler, BlockIdx::NULL, BlockIdx::NULL)?; + self.push_fblock_labels(FBlockType::ExceptionHandler, None, None, FBlockDatum::None)?; for (i, handler) in handlers.iter().enumerate() { let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -3740,22 +3745,26 @@ impl Compiler { } if let Some(alias) = name { + let cleanup_end = self.new_block(); + let cleanup_body = self.new_block(); + self.store_name(alias.as_str())?; - let cleanup_end = self.new_block(); emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup_end }); - let cleanup_body = self.new_block(); self.use_cpython_label_block(cleanup_body); - self.push_fblock_full( + let cleanup_body_label = self + .current_code_info() + .instr_sequence_label_for_block(cleanup_body); + self.push_fblock_labels( FBlockType::HandlerCleanup, - cleanup_body, - BlockIdx::NULL, + cleanup_body_label, + None, FBlockDatum::ExceptionName(alias.as_str().to_owned()), )?; self.compile_statements(body)?; - self.pop_fblock(FBlockType::HandlerCleanup, cleanup_body); + self.pop_fblock_label(FBlockType::HandlerCleanup, cleanup_body_label); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); emit!(self, PseudoInstruction::PopBlock); @@ -3776,14 +3785,23 @@ impl Compiler { emit!(self, Instruction::Reraise { depth: 1 }); self.set_no_location(); } else { - emit!(self, Instruction::PopTop); let cleanup_body = self.new_block(); + + emit!(self, Instruction::PopTop); self.use_cpython_label_block(cleanup_body); - self.push_fblock(FBlockType::HandlerCleanup, cleanup_body, BlockIdx::NULL)?; + let cleanup_body_label = self + .current_code_info() + .instr_sequence_label_for_block(cleanup_body); + self.push_fblock_labels( + FBlockType::HandlerCleanup, + cleanup_body_label, + None, + FBlockDatum::None, + )?; self.compile_statements(body)?; - self.pop_fblock(FBlockType::HandlerCleanup, cleanup_body); + self.pop_fblock_label(FBlockType::HandlerCleanup, cleanup_body_label); emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); emit!(self, Instruction::PopExcept); @@ -3800,7 +3818,7 @@ impl Compiler { emit!(self, Instruction::Reraise { depth: 0 }); self.set_no_location(); - self.pop_fblock(FBlockType::ExceptionHandler, BlockIdx::NULL); + self.pop_fblock_label(FBlockType::ExceptionHandler, None); self.use_cpython_label_block(cleanup_block); emit!(self, Instruction::Copy { i: 3 }); @@ -3836,11 +3854,18 @@ impl Compiler { delta: finally_except_block } ); - self.use_cpython_transient_label_block(body_block); - self.push_fblock_full( + self.use_cpython_label_block(body_block); + let (body_label, finally_except_label) = { + let code = self.current_code_info(); + ( + code.instr_sequence_label_for_block(body_block), + code.instr_sequence_label_for_block(finally_except_block), + ) + }; + self.push_fblock_labels( FBlockType::FinallyTry, - body_block, - finally_except_block, + body_label, + finally_except_label, FBlockDatum::FinallyBody(finalbody.to_vec()), )?; @@ -3852,7 +3877,7 @@ impl Compiler { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::FinallyTry, body_block); + self.pop_fblock_label(FBlockType::FinallyTry, body_label); let sub_table_cursor = self.symbol_table_stack.last().map(|t| t.next_sub_table); self.compile_statements(finalbody)?; @@ -3879,9 +3904,14 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::PushExcInfo); self.set_no_location(); - self.push_fblock(FBlockType::FinallyEnd, finally_except_block, BlockIdx::NULL)?; + self.push_fblock_labels( + FBlockType::FinallyEnd, + finally_except_label, + None, + FBlockDatum::None, + )?; self.compile_statements(finalbody)?; - self.pop_fblock(FBlockType::FinallyEnd, finally_except_block); + self.pop_fblock_label(FBlockType::FinallyEnd, finally_except_label); emit!(self, Instruction::Reraise { depth: 0 }); self.set_no_location(); @@ -3921,12 +3951,15 @@ impl Compiler { delta: handler_block } ); - self.use_cpython_transient_label_block(body_block); - self.push_fblock(FBlockType::TryExcept, body_block, BlockIdx::NULL)?; + self.use_cpython_label_block(body_block); + let body_label = self + .current_code_info() + .instr_sequence_label_for_block(body_block); + self.push_fblock_labels(FBlockType::TryExcept, body_label, None, FBlockDatum::None)?; self.compile_statements(body)?; emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::TryExcept, body_block); + self.pop_fblock_label(FBlockType::TryExcept, body_label); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: else_block } @@ -3951,10 +3984,11 @@ impl Compiler { // Stack: [prev_exc, exc] // Push EXCEPTION_GROUP_HANDLER fblock - self.push_fblock( + self.push_fblock_labels( FBlockType::ExceptionGroupHandler, - BlockIdx::NULL, - BlockIdx::NULL, + None, + None, + FBlockDatum::None, )?; let n = handlers.len(); @@ -4008,7 +4042,8 @@ impl Compiler { // Handler matched // Stack: [prev_exc, orig, list, new_rest, match] // Note: CheckEgMatch already sets the matched exception as current exception - let handler_except_block = self.new_block(); + let cleanup_end_block = self.new_block(); + let cleanup_body_block = self.new_block(); // Store match to name or pop if let Some(alias) = name { @@ -4019,18 +4054,20 @@ impl Compiler { // Stack: [prev_exc, orig, list, new_rest] // HANDLER_CLEANUP fblock for handler body - let cleanup_body_block = self.new_block(); emit!( self, PseudoInstruction::SetupCleanup { - delta: handler_except_block + delta: cleanup_end_block } ); self.use_cpython_label_block(cleanup_body_block); - self.push_fblock_full( + let cleanup_body_label = self + .current_code_info() + .instr_sequence_label_for_block(cleanup_body_block); + self.push_fblock_labels( FBlockType::HandlerCleanup, - cleanup_body_block, - BlockIdx::NULL, + cleanup_body_label, + None, if let Some(alias) = name { FBlockDatum::ExceptionName(alias.as_str().to_owned()) } else { @@ -4044,7 +4081,7 @@ impl Compiler { // Handler body completed normally emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); - self.pop_fblock(FBlockType::HandlerCleanup, cleanup_body_block); + self.pop_fblock_label(FBlockType::HandlerCleanup, cleanup_body_label); // Cleanup name binding if let Some(alias) = name { @@ -4060,7 +4097,7 @@ impl Compiler { self.set_no_location(); // Handler raised an exception (cleanup_end label) - self.use_cpython_label_block(handler_except_block); + self.use_cpython_label_block(cleanup_end_block); // Stack: [prev_exc, orig, list, new_rest, lasti, raised_exc] // (lasti is pushed because push_lasti=true in HANDLER_CLEANUP fblock) @@ -4123,7 +4160,7 @@ impl Compiler { } // Pop EXCEPTION_GROUP_HANDLER fblock - self.pop_fblock(FBlockType::ExceptionGroupHandler, BlockIdx::NULL); + self.pop_fblock_label(FBlockType::ExceptionGroupHandler, None); // Reraise star block self.use_cpython_label_block(reraise_star_block); @@ -4302,13 +4339,19 @@ impl Compiler { // Set qualname self.set_qualname(); + let start_label = self.use_cpython_function_start_label(); // PEP 479: Wrap generator/coroutine body with StopIteration handler let is_gen = is_async || self.current_symbol_table().is_generator; let stop_iteration_block = if is_gen { let handler_block = self.new_block(); self.insert_cpython_stopiteration_setup_cleanup(handler_block); - self.push_fblock(FBlockType::StopIteration, handler_block, handler_block)?; + self.push_fblock_labels( + FBlockType::StopIteration, + Some(start_label), + None, + FBlockDatum::None, + )?; Some(handler_block) } else { None @@ -4347,9 +4390,7 @@ impl Compiler { // Close StopIteration handler and emit handler code if let Some(handler_block) = stop_iteration_block { - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.pop_fblock(FBlockType::StopIteration, handler_block); + self.pop_fblock_label(FBlockType::StopIteration, Some(start_label)); self.use_cpython_label_block(handler_block); emit!( self, @@ -4691,7 +4732,7 @@ impl Compiler { simple_idx += 1; if let Some(not_set_block) = not_set_block { - self.switch_to_block(not_set_block); + self.use_cpython_label_block(not_set_block); } } @@ -4741,6 +4782,7 @@ impl Compiler { returns: Option<&ast::Expr>, // TODO: use type hint somehow.. is_async: bool, type_params: Option<&ast::TypeParams>, + preserve_value_before_store: bool, ) -> CompileResult<()> { // CPython's FunctionDef/AsyncFunctionDef LOC(s) starts at the // definition line even when decorators are present. @@ -4892,6 +4934,9 @@ impl Compiler { // Store the function self.set_source_range(def_source_range); + if preserve_value_before_store { + emit!(self, Instruction::Copy { i: 1 }); + } self.store_name(name)?; Ok(()) @@ -5300,6 +5345,7 @@ impl Compiler { decorator_list: &[ast::Decorator], type_params: Option<&ast::TypeParams>, arguments: Option<&ast::Arguments>, + preserve_value_before_store: bool, ) -> CompileResult<()> { // CPython's ClassDef LOC(s) starts at the class line even when // decorators are present. @@ -5547,6 +5593,9 @@ impl Compiler { // Step 4: Apply decorators and store (common to both paths) self.apply_decorators(decorator_list); self.set_source_range(class_source_range); + if preserve_value_before_store { + emit!(self, Instruction::Copy { i: 1 }); + } self.store_name(name) } @@ -5602,14 +5651,26 @@ impl Compiler { let loop_block = self.start_cpython_label_block(); let end_block = self.new_block(); let anchor_block = self.new_block(); - self.push_fblock(FBlockType::WhileLoop, loop_block, end_block)?; + let (loop_label, end_label) = { + let code = self.current_code_info(); + ( + code.instr_sequence_label_for_block(loop_block), + code.instr_sequence_label_for_block(end_block), + ) + }; + self.push_fblock_labels( + FBlockType::WhileLoop, + loop_label, + end_label, + FBlockDatum::None, + )?; self.compile_jump_if(test, false, anchor_block)?; self.compile_loop_body_statements(body)?; emit!(self, PseudoInstruction::Jump { delta: loop_block }); self.set_no_location(); - self.pop_fblock(FBlockType::WhileLoop, loop_block); + self.pop_fblock_label(FBlockType::WhileLoop, loop_label); self.use_cpython_label_block(anchor_block); self.compile_statements(orelse)?; self.use_cpython_label_block(end_block); @@ -5658,8 +5719,10 @@ impl Compiler { }; let with_range = item.context_expr.range(); + let body_block = self.new_block(); let exc_handler_block = self.new_block(); let after_block = self.new_block(); + let cleanup_block = self.new_block(); // Compile context expression and load __enter__/__exit__ methods self.compile_expression(&item.context_expr)?; @@ -5719,16 +5782,24 @@ impl Compiler { delta: exc_handler_block } ); - let body_block = self.new_block(); - self.use_cpython_transient_label_block(body_block); - self.push_fblock( - if is_async { - FBlockType::AsyncWith - } else { - FBlockType::With - }, - body_block, - exc_handler_block, + self.use_cpython_label_block(body_block); + let fblock_type = if is_async { + FBlockType::AsyncWith + } else { + FBlockType::With + }; + let (body_label, exc_handler_label) = { + let code = self.current_code_info(); + ( + code.instr_sequence_label_for_block(body_block), + code.instr_sequence_label_for_block(exc_handler_block), + ) + }; + self.push_fblock_labels( + fblock_type, + body_label, + exc_handler_label, + FBlockDatum::None, )?; // Store or pop the enter result @@ -5754,22 +5825,17 @@ impl Compiler { self.compile_with(items, body, is_async)?; } - let fblock_type = if is_async { - FBlockType::AsyncWith - } else { - FBlockType::With - }; // CPython pops the async-with fblock before emitting POP_BLOCK, but // sync with emits the artificial POP_BLOCK before popping the fblock. if is_async { - self.pop_fblock(fblock_type, body_block); + self.pop_fblock_label(fblock_type, body_label); self.set_source_range(with_range); emit!(self, PseudoInstruction::PopBlock); } else { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); self.set_source_range(with_range); - self.pop_fblock(fblock_type, body_block); + self.pop_fblock_label(fblock_type, body_label); } self.compile_call_exit_with_nones(); @@ -5787,8 +5853,6 @@ impl Compiler { // PUSH_EXC_INFO -> [..., exit_func, self_exit, lasti, prev_exc, exc] self.use_cpython_label_block(exc_handler_block); - let cleanup_block = self.new_block(); - emit!( self, PseudoInstruction::SetupCleanup { @@ -5808,10 +5872,9 @@ impl Compiler { let _ = self.compile_yield_from_sequence(true); } - self.compile_with_except_finish(cleanup_block, after_block); + self.compile_with_except_finish(cleanup_block); - // ===== After block ===== - self.switch_to_block(after_block); + self.use_cpython_label_block(after_block); self.leave_conditional_block(); Ok(()) @@ -5830,14 +5893,31 @@ impl Compiler { // Start loop let for_block = self.new_block(); + let send_block = if is_async { + self.new_block() + } else { + BlockIdx::NULL + }; let else_block = self.new_block(); let after_block = self.new_block(); + let (for_label, after_label) = { + let code = self.current_code_info(); + ( + code.instr_sequence_label_for_block(for_block), + code.instr_sequence_label_for_block(after_block), + ) + }; let mut end_async_for_target = BlockIdx::NULL; if !is_async { // CPython codegen_for() pushes the loop fblock before compiling // the iterable expression. - self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + self.push_fblock_labels( + FBlockType::ForLoop, + for_label, + after_label, + FBlockDatum::None, + )?; } // The thing iterated: @@ -5854,13 +5934,20 @@ impl Compiler { self.set_source_range(for_range); // codegen_async_for: push fblock BEFORE SETUP_FINALLY - self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + self.push_fblock_labels( + FBlockType::ForLoop, + for_label, + after_label, + FBlockDatum::None, + )?; // SETUP_FINALLY to guard the __anext__ call emit!(self, PseudoInstruction::SetupFinally { delta: else_block }); emit!(self, Instruction::GetAnext); self.emit_load_const(ConstantData::None); - end_async_for_target = self.compile_yield_from_sequence(true); + self.use_cpython_label_block(send_block); + let _ = self.compile_yield_from_sequence(true); + end_async_for_target = send_block; // POP_BLOCK for SETUP_FINALLY - only GetANext/yield_from are protected emit!(self, PseudoInstruction::PopBlock); emit!(self, Instruction::NotTaken); @@ -5905,7 +5992,7 @@ impl Compiler { self.set_no_location(); } // No PopBlock here - for async, POP_BLOCK is already in for_block - self.pop_fblock(FBlockType::ForLoop, for_block); + self.pop_fblock_label(FBlockType::ForLoop, for_label); self.compile_statements(orelse)?; self.use_cpython_label_block(after_block); @@ -7668,11 +7755,9 @@ impl Compiler { let (last_value, prefix_values) = values.split_last().unwrap(); for value in prefix_values { - let continue_block = self.new_block(); self.compile_expression(value)?; self.set_source_range(boolop_range); self.emit_short_circuit_test(op, after_block); - self.switch_to_block(continue_block); self.set_source_range(boolop_range); emit!(self, Instruction::PopTop); } @@ -9398,12 +9483,12 @@ impl Compiler { let return_none = init_collection.is_none(); - // PEP 479: Wrap generator/coroutine body with StopIteration handler - let is_gen_scope = self.current_symbol_table().is_generator || is_async; - let stop_iteration_block = if is_gen_scope { + // CPython codegen_comprehension() wraps only generator expressions + // with codegen_wrap_in_stopiteration_handler(); unlike function bodies, + // it does not push a COMPILE_FBLOCK_STOP_ITERATION fblock. + let stop_iteration_block = if comprehension_type == ComprehensionType::Generator { let handler_block = self.new_block(); self.insert_cpython_stopiteration_setup_cleanup(handler_block); - self.push_fblock(FBlockType::StopIteration, handler_block, handler_block)?; Some(handler_block) } else { None @@ -9430,16 +9515,22 @@ impl Compiler { for if_condition in &generator.ifs { self.compile_jump_if(if_condition, false, if_cleanup_block)?; } - let body_block = self.new_block(); - self.switch_to_block(body_block); loop_labels.push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); } continue; } let loop_block = self.new_block(); - let if_cleanup_block = self.new_block(); - let after_block = self.new_block(); + let (send_block, after_block, if_cleanup_block) = if generator.is_async { + let send_block = self.new_block(); + let after_block = self.new_block(); + let if_cleanup_block = self.new_block(); + (send_block, after_block, if_cleanup_block) + } else { + let if_cleanup_block = self.new_block(); + let after_block = self.new_block(); + (BlockIdx::NULL, after_block, if_cleanup_block) + }; if gen_index == 0 { // Load iterator onto stack (passed as first argument): @@ -9452,19 +9543,24 @@ impl Compiler { self.use_cpython_label_block(loop_block); let mut end_async_for_target = BlockIdx::NULL; if generator.is_async { - emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); - emit!(self, Instruction::GetAnext); - self.push_fblock( + let loop_label = self + .current_code_info() + .instr_sequence_label_for_block(loop_block); + self.push_fblock_labels( FBlockType::AsyncComprehensionGenerator, - loop_block, - after_block, + loop_label, + None, + FBlockDatum::None, )?; + emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); + emit!(self, Instruction::GetAnext); self.emit_load_const(ConstantData::None); - end_async_for_target = self.compile_yield_from_sequence(true); + self.use_cpython_label_block(send_block); + let _ = self.compile_yield_from_sequence(true); + end_async_for_target = send_block; // POP_BLOCK before store: only __anext__/yield_from are // protected by SetupFinally targeting END_ASYNC_FOR. emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::AsyncComprehensionGenerator, loop_block); self.compile_store(&generator.target)?; } else { let saved_range = self.current_source_range; @@ -9494,10 +9590,6 @@ impl Compiler { for if_condition in &generator.ifs { self.compile_jump_if(if_condition, false, if_cleanup_block)?; } - if !generator.ifs.is_empty() { - let body_block = self.new_block(); - self.switch_to_block(body_block); - } } compile_element(self, real_loop_depth + 1)?; @@ -9520,6 +9612,13 @@ impl Compiler { self.set_source_range(backedge_range); emit!(self, PseudoInstruction::Jump { delta: loop_block }); + if is_async { + let loop_label = self + .current_code_info() + .instr_sequence_label_for_block(loop_block); + self.pop_fblock_label(FBlockType::AsyncComprehensionGenerator, loop_label); + } + self.use_cpython_label_block(after_block); if is_async { self.set_source_range(comprehension_range); @@ -9547,9 +9646,6 @@ impl Compiler { // Close StopIteration handler and emit handler code if let Some(handler_block) = stop_iteration_block { - emit!(self, PseudoInstruction::PopBlock); - self.set_no_location(); - self.pop_fblock(FBlockType::StopIteration, handler_block); self.use_cpython_label_block(handler_block); emit!( self, @@ -9804,8 +9900,6 @@ impl Compiler { for if_condition in &generator.ifs { self.compile_jump_if(if_condition, false, if_cleanup_block)?; } - let body_block = self.new_block(); - self.switch_to_block(body_block); loop_labels .push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); } @@ -9813,8 +9907,16 @@ impl Compiler { } let loop_block = self.new_block(); - let if_cleanup_block = self.new_block(); - let after_block = self.new_block(); + let (send_block, after_block, if_cleanup_block) = if generator.is_async { + let send_block = self.new_block(); + let after_block = self.new_block(); + let if_cleanup_block = self.new_block(); + (send_block, after_block, if_cleanup_block) + } else { + let if_cleanup_block = self.new_block(); + let after_block = self.new_block(); + (BlockIdx::NULL, after_block, if_cleanup_block) + }; if i > 0 { self.compile_comprehension_iter(generator)?; @@ -9824,17 +9926,22 @@ impl Compiler { let mut end_async_for_target = BlockIdx::NULL; if generator.is_async { - emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); - emit!(self, Instruction::GetAnext); - self.push_fblock( + let loop_label = self + .current_code_info() + .instr_sequence_label_for_block(loop_block); + self.push_fblock_labels( FBlockType::AsyncComprehensionGenerator, - loop_block, - after_block, + loop_label, + None, + FBlockDatum::None, )?; + emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); + emit!(self, Instruction::GetAnext); self.emit_load_const(ConstantData::None); - end_async_for_target = self.compile_yield_from_sequence(true); + self.use_cpython_label_block(send_block); + let _ = self.compile_yield_from_sequence(true); + end_async_for_target = send_block; emit!(self, PseudoInstruction::PopBlock); - self.pop_fblock(FBlockType::AsyncComprehensionGenerator, loop_block); self.compile_store(&generator.target)?; } else { let saved_range = self.current_source_range; @@ -9886,6 +9993,16 @@ impl Compiler { self.set_source_range(backedge_range); emit!(self, PseudoInstruction::Jump { delta: loop_block }); + if is_async { + let loop_label = self + .current_code_info() + .instr_sequence_label_for_block(loop_block); + self.pop_fblock_label( + FBlockType::AsyncComprehensionGenerator, + loop_label, + ); + } + self.use_cpython_label_block(after_block); if is_async { self.set_source_range(comprehension_range); @@ -10029,31 +10146,6 @@ impl Compiler { self.current_block().instructions.push(info); } - fn pop_last_emitted_instruction(&mut self) -> Option { - let sequence_info = self.current_code_info().pop_instr_sequence(); - let block_info = self.current_block().instructions.pop(); - match (sequence_info, block_info) { - (Some(sequence_info), Some(block_info)) => { - debug_assert_eq!( - format!("{:?}", sequence_info.instr), - format!("{:?}", block_info.instr), - "direct-CFG shadow must pop the same opcode as the CPython instruction sequence" - ); - debug_assert_eq!(u32::from(sequence_info.arg), u32::from(block_info.arg)); - debug_assert_eq!(sequence_info.target, block_info.target); - Some(block_info) - } - (None, None) => None, - _ => { - debug_assert!( - false, - "direct-CFG shadow and CPython instruction sequence must pop together" - ); - block_info - } - } - } - fn last_emitted_instruction_mut(&mut self) -> Option<&mut ir::InstructionInfo> { self.current_block().instructions.last_mut() } @@ -10102,12 +10194,11 @@ impl Compiler { } /// CPython `codegen_addop_j()`: emit a HAS_TARGET instruction with a - /// jump_target_label oparg. The direct target is only the shadow CFG bridge. + /// jump_target_label oparg. fn emit_jump_label>( &mut self, instr: I, target_label: ir::InstructionSequenceLabel, - target_direct: BlockIdx, ) { if self.do_not_emit_bytecode > 0 { return; @@ -10125,12 +10216,15 @@ impl Compiler { let source = self.source_file.to_source_code(); let location = source.source_location(range.start(), PositionEncoding::Utf8); let end_location = source.source_location(range.end(), PositionEncoding::Utf8); + let target = self + .current_code_info() + .block_for_instr_sequence_label(Some(target_label)); self.maybe_start_cpython_cfg_addop_block(); self.push_emitted_instruction_with_target_label( ir::InstructionInfo { instr, arg: OpArg::NULL, - target: target_direct, + target, location, end_location, except_handler: None, @@ -10922,9 +11016,6 @@ impl Compiler { .fb_block .expect("loop fblock must have a CPython block label") }; - let target_direct = self - .current_code_info() - .block_for_instr_sequence_label(Some(target_label)); if let Some(loc) = unwind_loc { self.set_source_range(loc); } else { @@ -10935,7 +11026,6 @@ impl Compiler { delta: OpArgMarker::marker(), }, target_label, - target_direct, ); if unwind_loc.is_none() { self.set_no_location(); @@ -10966,8 +11056,10 @@ impl Compiler { } debug_assert_eq!(code.blocks[cur.idx()].next, BlockIdx::NULL); - let block = self.new_block(); - self.switch_to_block(block); + let block = self.new_unlabeled_block(); + let code = self.current_code_info(); + code.blocks[cur.idx()].next = block; + code.current_block = block; } fn cpython_cfg_builder_current_block_is_terminated_for_label(block: &ir::Block) -> bool { @@ -10993,6 +11085,17 @@ impl Compiler { block } + /// CPython `codegen_funcbody()` emits `NEW_JUMP_TARGET_LABEL(start)` and + /// `USE_LABEL(start)` after scope entry. That label is part of the + /// instruction-sequence label map, but it does not become a CFG + /// `basicblock.b_label` unless some emitted instruction targets it. + fn use_cpython_function_start_label(&mut self) -> ir::InstructionSequenceLabel { + let code = self.current_code_info(); + let label = code.new_instr_sequence_label(); + code.use_raw_instr_sequence_label(label); + label + } + /// Switch to a block as CPython instruction-sequence labels would resolve. /// /// Consecutive `USE_LABEL()` calls can map multiple labels to the same @@ -11029,40 +11132,19 @@ impl Compiler { .use_instr_sequence_label_at_block(block, cur); } - /// Enter a compile-time-only `USE_LABEL()` site that CPython drops before - /// the CFG exists. - /// - /// Labels such as try-body entries can matter while codegen records fblock - /// targets, but `_PyInstructionSequence_ApplyLabelMap()` and - /// `_PyCfg_FromInstructionSequence()` only keep labels that are reached by - /// `HAS_TARGET` operands. Consecutive dropped labels at an already-retained - /// label offset still alias to the same next instruction, so reusing a - /// labeled empty current block is valid here. - fn use_cpython_transient_label_block(&mut self, block: BlockIdx) { + fn new_block(&mut self) -> BlockIdx { let code = self.current_code_info(); - code.use_instr_sequence_label(block); - let block = code.resolve_instr_sequence_label(block); - let cur = code.current_block; - let can_reuse_current = cur != block - && code.blocks[cur.idx()].instructions.is_empty() - && code.blocks[cur.idx()].next == BlockIdx::NULL - && code.blocks[block.idx()].instructions.is_empty() - && code.blocks[block.idx()].next == BlockIdx::NULL; - - if !can_reuse_current { - self.switch_to_block(block); - return; - } - - self.current_code_info() - .use_instr_sequence_label_at_block(block, cur); + let idx = BlockIdx::new(code.blocks.len().to_u32()); + code.blocks.push(ir::Block::default()); + code.instr_sequence_label_map.push_unmapped_label(); + idx } - fn new_block(&mut self) -> BlockIdx { + fn new_unlabeled_block(&mut self) -> BlockIdx { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); - code.instr_sequence_label_map.push_unmapped_label(); + code.instr_sequence_label_map.push_unlabeled_block(); idx } @@ -12411,12 +12493,18 @@ def f(x, y, z): in_async_scope: is_async, }; compiler.set_qualname(); + let start_label = compiler.use_cpython_function_start_label(); let is_gen = is_async || compiler.current_symbol_table().is_generator; let stop_iteration_block = if is_gen { let handler_block = compiler.new_block(); compiler.insert_cpython_stopiteration_setup_cleanup(handler_block); compiler - .push_fblock(FBlockType::StopIteration, handler_block, handler_block) + .push_fblock_labels( + FBlockType::StopIteration, + Some(start_label), + None, + FBlockDatum::None, + ) .unwrap(); Some(handler_block) } else { @@ -12432,9 +12520,7 @@ def f(x, y, z): compiler.arg_constant(ConstantData::None); } if let Some(handler_block) = stop_iteration_block { - emit!(compiler, PseudoInstruction::PopBlock); - compiler.set_no_location(); - compiler.pop_fblock(FBlockType::StopIteration, handler_block); + compiler.pop_fblock_label(FBlockType::StopIteration, Some(start_label)); compiler.use_cpython_label_block(handler_block); emit!( compiler, diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a2f5b224be8..ffa0b2b5bbf 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -373,40 +373,6 @@ impl InstructionSequence { )); } - fn debug_check_pop_preserves_cpython_label_map(&self) { - let Some((popped_index, entry)) = self.instrs.len().checked_sub(1).and_then(|index| { - self.instrs.last().map(|entry| { - ( - isize::try_from(index).expect("too many instructions"), - entry, - ) - }) - }) else { - return; - }; - debug_assert!( - !entry.info.instr.has_target() - && entry.info.target == BlockIdx::NULL - && entry.target_label.is_none() - && entry.target_offset.is_none() - && entry.except_handler.is_none(), - "RustPython-only instruction-sequence pop must not remove CPython label/target state" - ); - let InstructionSequenceLabelOffsets::Active(label_map) = &self.label_map else { - debug_assert!(false, "cannot pop after CPython label map application"); - return; - }; - debug_assert!( - label_map.iter().all(|&target| target < popped_index), - "RustPython-only instruction-sequence pop must not change CPython label offsets" - ); - } - - fn pop(&mut self) -> Option { - self.debug_check_pop_preserves_cpython_label_map(); - self.instrs.pop().map(|entry| entry.info) - } - fn last_info_mut(&mut self) -> Option<&mut InstructionInfo> { self.instrs.last_mut().map(|entry| &mut entry.info) } @@ -611,7 +577,7 @@ impl Block { #[derive(Clone, Debug)] pub(crate) struct InstructionSequenceLabelMap { next_free_label: usize, - block_labels: Vec, + block_labels: Vec>, /// Direct-CFG shadow for CPython labels that map to the same instruction /// offset in `_PyInstructionSequence_UseLabel()`. direct_block_by_label: Vec>, @@ -621,8 +587,8 @@ impl InstructionSequenceLabelMap { pub(crate) fn new() -> Self { Self { next_free_label: 0, - block_labels: vec![InstructionSequenceLabel(0)], - direct_block_by_label: vec![Some(BlockIdx::new(0))], + block_labels: vec![None], + direct_block_by_label: Vec::new(), } } @@ -636,6 +602,10 @@ impl InstructionSequenceLabelMap { label } + pub(crate) fn push_unlabeled_block(&mut self) { + self.block_labels.push(None); + } + pub(crate) fn push_unmapped_label(&mut self) { let label = self.new_label(); let block = BlockIdx( @@ -645,15 +615,23 @@ impl InstructionSequenceLabelMap { .expect("too many direct-CFG blocks"), ); self.direct_block_by_label[label.idx()] = Some(block); - self.block_labels.push(label); + self.block_labels.push(Some(label)); } - fn label_for_block(&self, block: BlockIdx) -> InstructionSequenceLabel { + fn ensure_label_for_block(&mut self, block: BlockIdx) -> InstructionSequenceLabel { debug_assert_ne!(block, BlockIdx::NULL); - self.block_labels - .get(block.idx()) - .copied() - .expect("basic block must have an instruction-sequence label") + if let Some(label) = self.block_labels[block.idx()] { + return label; + } + let label = self.new_label(); + self.direct_block_by_label[label.idx()] = Some(block); + self.block_labels[block.idx()] = Some(label); + label + } + + fn label_for_block(&self, block: BlockIdx) -> Option { + debug_assert_ne!(block, BlockIdx::NULL); + self.block_labels.get(block.idx()).copied().flatten() } fn block_for_label(&self, label: InstructionSequenceLabel) -> Option { @@ -667,11 +645,13 @@ impl InstructionSequenceLabelMap { if block == BlockIdx::NULL { return BlockIdx::NULL; } - self.block_for_label(self.label_for_block(block)) - .unwrap_or_else(|| { - debug_assert!(false, "CPython label must map to a direct-CFG block"); - BlockIdx::NULL - }) + let Some(label) = self.label_for_block(block) else { + return block; + }; + self.block_for_label(label).unwrap_or_else(|| { + debug_assert!(false, "CPython label must map to a direct-CFG block"); + BlockIdx::NULL + }) } pub(crate) fn resolve_label_to_block( @@ -691,7 +671,7 @@ impl InstructionSequenceLabelMap { if from == BlockIdx::NULL || from == to { return; } - let from_label = self.label_for_block(from); + let from_label = self.ensure_label_for_block(from); let to_block = self.resolve_label(to); if to_block == BlockIdx::NULL { debug_assert!(false, "CPython label target must map to a direct-CFG block"); @@ -704,12 +684,11 @@ impl InstructionSequenceLabelMap { debug_assert_eq!( self.block_labels.len(), blocks_len, - "every direct-CFG block must have a CPython instruction-sequence label" + "every direct-CFG block must have an instruction-sequence label slot" ); - debug_assert_eq!(self.block_labels[0], InstructionSequenceLabel(0)); - debug_assert!(self.next_free_label + 1 >= self.block_labels.len()); + debug_assert!(self.direct_block_by_label.len() <= self.next_free_label + 1); let mut seen_labels = vec![false; self.next_free_label + 1]; - for &label in &self.block_labels { + for &label in self.block_labels.iter().flatten() { debug_assert!( label.idx() <= self.next_free_label, "direct-CFG block labels must come from _PyInstructionSequence_NewLabel()" @@ -728,7 +707,7 @@ impl InstructionSequenceLabelMap { ); } } - for &label in &self.block_labels { + for &label in self.block_labels.iter().flatten() { debug_assert!( self.block_for_label(label) .is_some_and(|block| block.idx() < blocks_len), @@ -758,7 +737,7 @@ impl InstructionSequenceLabelMap { if label_offset < 0 { continue; } - let Some(block_label) = self.block_labels.get(block.idx()).copied() else { + let Some(block_label) = self.block_labels.get(block.idx()).copied().flatten() else { debug_assert!( false, "CPython label must map to an existing direct-CFG block" @@ -819,7 +798,9 @@ impl CodeInfo { mut info: InstructionInfo, ) -> crate::InternalResult<()> { let target_label = if info.instr.has_target() && info.target != BlockIdx::NULL { - let label = self.instr_sequence_label_map.label_for_block(info.target); + let label = self + .instr_sequence_label_map + .ensure_label_for_block(info.target); info.arg = OpArg::new( label .idx() @@ -854,10 +835,6 @@ impl CodeInfo { Ok(()) } - pub(crate) fn pop_instr_sequence(&mut self) -> Option { - self.instr_sequence.pop() - } - pub(crate) fn set_last_instr_sequence_lineno_override(&mut self, lineno_override: i32) { if let Some(last) = self.instr_sequence.last_info_mut() { last.lineno_override = Some(lineno_override); @@ -865,12 +842,20 @@ impl CodeInfo { } pub(crate) fn use_instr_sequence_label(&mut self, block: BlockIdx) { - let label = self.instr_sequence_label_map.label_for_block(block); + let label = self.instr_sequence_label_map.ensure_label_for_block(block); + self.instr_sequence.use_label(label); + } + + pub(crate) fn new_instr_sequence_label(&mut self) -> InstructionSequenceLabel { + self.instr_sequence_label_map.new_label() + } + + pub(crate) fn use_raw_instr_sequence_label(&mut self, label: InstructionSequenceLabel) { self.instr_sequence.use_label(label); } pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) { - let label = self.instr_sequence_label_map.label_for_block(block); + let label = self.instr_sequence_label_map.ensure_label_for_block(block); self.blocks[block.idx()].cpython_label_id = Some(label); } @@ -890,18 +875,20 @@ impl CodeInfo { } pub(crate) fn instr_sequence_label_for_block( - &self, + &mut self, block: BlockIdx, ) -> Option { if block == BlockIdx::NULL { None } else { - Some(self.instr_sequence_label_map.label_for_block(block)) + Some(self.instr_sequence_label_map.ensure_label_for_block(block)) } } pub(crate) fn insert_start_setup_cleanup(&mut self, handler_block: BlockIdx) { - let handler_label = self.instr_sequence_label_map.label_for_block(handler_block); + let handler_label = self + .instr_sequence_label_map + .ensure_label_for_block(handler_block); self.instr_sequence.insert_instruction( 0, InstructionInfo { diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index 8d17c71d85e..a4142deb371 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -177,6 +177,7 @@ mod file_run { #[cfg(test)] mod tests { + use crate::object::AsObject; use rustpython_vm::Interpreter; fn interpreter() -> Interpreter { @@ -205,4 +206,28 @@ mod tests { assert_eq!(value, 5); }) } + + #[test] + fn test_block_expr_return_function_def() { + interpreter().enter(|vm| { + let scope = vm.new_scope_with_builtins(); + let value = + vm.unwrap_pyresult(vm.run_block_expr(scope.clone(), "def f():\n return 7")); + vm.unwrap_pyresult(scope.globals.set_item("returned", value, vm)); + let value = vm.unwrap_pyresult(vm.run_block_expr(scope, "returned is f")); + assert!(value.is(&vm.ctx.true_value)); + }) + } + + #[test] + fn test_block_expr_return_class_def() { + interpreter().enter(|vm| { + let scope = vm.new_scope_with_builtins(); + let value = + vm.unwrap_pyresult(vm.run_block_expr(scope.clone(), "class C:\n value = 11")); + vm.unwrap_pyresult(scope.globals.set_item("returned", value, vm)); + let value = vm.unwrap_pyresult(vm.run_block_expr(scope, "returned is C")); + assert!(value.is(&vm.ctx.true_value)); + }) + } } From 3dc37d817d56ba4d45c0a88a9c1a2055f987a274 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 26 May 2026 09:27:38 +0900 Subject: [PATCH 010/131] Align codegen with CPython compile.c Rename CFG helpers and accessors to the names used in CPython's compile.c (basicblock_next_instr, basicblock_last_instr, basicblock_append_instructions, bb_has_fallthrough, is_jump, make_cfg_traversal_stack, mark_warm/mark_cold, etc.). Drop the unused boolop-folding gate, mark_cpython_cfg_label_block helper, and ComprehensionLoopControl::iter_range field. Track an is_coroutine flag on SymbolTable, set in async def, await, and async comprehensions, and propagate it through non-generator comprehensions per symtable_handle_comprehension(). Mark SetupCleanup/SetupFinally/SetupWith as has_arg pseudo-ops, mark ForIter as a terminator, and add has_arg/has_const on AnyInstruction. Fix Instruction::stack_effect_jump to delegate to the opcode's stack_effect_jump rather than stack_effect. --- crates/codegen/src/compile.rs | 646 +-- crates/codegen/src/ir.rs | 4255 +++++++++-------- crates/codegen/src/symboltable.rs | 18 + .../compiler-core/src/bytecode/instruction.rs | 56 + 4 files changed, 2821 insertions(+), 2154 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c406a42beed..c16216aedb4 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -169,9 +169,6 @@ struct Compiler { /// When > 0, the compiler walks AST (consuming sub_tables) but emits no bytecode. /// Mirrors CPython's `c_do_not_emit_bytecode`. do_not_emit_bytecode: u32, - /// Disable constant BoolOp folding in contexts where CPython preserves - /// short-circuit structure, such as starred unpack expressions. - disable_const_boolop_folding: bool, /// Disable constant tuple/list/set collection folding in contexts where /// CPython keeps the builder form for later assignment lowering. disable_const_collection_folding: bool, @@ -237,7 +234,6 @@ enum ComprehensionLoopControl { loop_block: BlockIdx, if_cleanup_block: BlockIdx, after_block: BlockIdx, - iter_range: TextRange, backedge_range: TextRange, is_async: bool, end_async_for_target: BlockIdx, @@ -478,33 +474,6 @@ impl Compiler { } } - fn boolop_fast_fold_literal(expr: &ast::Expr) -> bool { - matches!( - expr, - ast::Expr::NumberLiteral(_) - | ast::Expr::StringLiteral(_) - | ast::Expr::BytesLiteral(_) - | ast::Expr::BooleanLiteral(_) - | ast::Expr::NoneLiteral(_) - | ast::Expr::EllipsisLiteral(_) - ) - } - - /// Mark a direct-CFG block as carrying CPython's `basicblock.b_label`. - /// - /// In CPython, codegen records labels in the instruction sequence with - /// `USE_LABEL()`, then `_PyCfg_FromInstructionSequence()` emits - /// `_PyCfgBuilder_UseLabel()` only for instruction offsets reached by - /// `HAS_TARGET` operands. The direct-CFG backend uses this marker where the - /// CPython CFG builder would retain a materialized label boundary. - fn mark_cpython_cfg_label_block(&mut self, block: BlockIdx) { - let code = self.current_code_info(); - let block = code.resolve_instr_sequence_label(block); - if block != BlockIdx::NULL { - code.mark_cpython_cfg_label(block); - } - } - fn new(opts: CompileOpts, source_file: SourceFile, code_name: &str) -> Self { let module_code = ir::CodeInfo { // CPython convention: top-level module / interactive / @@ -513,7 +482,7 @@ impl Compiler { // scope.) This matches the per-scope mapping at // enter_scope::CompilerScope::Module below, which also returns // empty flags. frame.rs:725-731 then binds locals to globals - // for module/REPL frames whose `scope.locals` is None — the + // for module/REPL frames whose `scope.locals` is None - the // correct semantics for `exec(code, globals)` and module init. flags: bytecode::CodeFlags::empty(), source_path: source_file.name().to_owned(), @@ -562,22 +531,10 @@ impl Compiler { in_annotation: false, interactive: false, do_not_emit_bytecode: 0, - disable_const_boolop_folding: false, disable_const_collection_folding: false, } } - fn compile_expression_without_const_boolop_folding( - &mut self, - expression: &ast::Expr, - ) -> CompileResult<()> { - let previous = self.disable_const_boolop_folding; - self.disable_const_boolop_folding = true; - let result = self.compile_expression(expression); - self.disable_const_boolop_folding = previous; - result.map(|_| ()) - } - fn compile_expression_without_const_collection_folding( &mut self, expression: &ast::Expr, @@ -823,7 +780,7 @@ impl Compiler { } // Compile the starred expression and extend - self.compile_expression_without_const_boolop_folding(value)?; + self.compile_expression(value)?; self.set_source_range(collection_range); match collection_type { CollectionType::List => { @@ -1229,7 +1186,7 @@ impl Compiler { &mut self, name: &str, scope_type: CompilerScope, - key: usize, // In RustPython, we use the index in symbol_table_stack as key + key: usize, // Symbol table stack index used like CPython's scope key. lineno: u32, ) -> CompileResult<()> { // Allocate a new compiler unit @@ -2815,6 +2772,7 @@ impl Compiler { } ); emit!(self, Instruction::PopTop); + self.set_no_location(); } else { // from mod import a, b as c @@ -3605,8 +3563,7 @@ impl Compiler { self.compile_statements(body)?; self.compile_statements(orelse)?; } else { - let try_except_end = self.new_block(); - self.compile_try_except_no_finally_with_end(body, handlers, orelse, try_except_end)?; + self.compile_try_except_no_finally(body, handlers, orelse)?; } emit!(self, PseudoInstruction::PopBlock); @@ -3667,20 +3624,10 @@ impl Compiler { body: &[ast::Stmt], handlers: &[ast::ExceptHandler], orelse: &[ast::Stmt], - ) -> CompileResult<()> { - let end_block = self.new_block(); - self.compile_try_except_no_finally_with_end(body, handlers, orelse, end_block) - } - - fn compile_try_except_no_finally_with_end( - &mut self, - body: &[ast::Stmt], - handlers: &[ast::ExceptHandler], - orelse: &[ast::Stmt], - end_block: BlockIdx, ) -> CompileResult<()> { let body_block = self.new_block(); let handler_block = self.new_block(); + let end_block = self.new_block(); let cleanup_block = self.new_block(); emit!( self, @@ -3942,7 +3889,6 @@ impl Compiler { let end_block = self.new_block(); let cleanup_block = self.new_block(); let reraise_star_block = self.new_block(); - let reraise_block = self.new_block(); // SETUP_FINALLY for try body emit!( @@ -4161,6 +4107,7 @@ impl Compiler { // Pop EXCEPTION_GROUP_HANDLER fblock self.pop_fblock_label(FBlockType::ExceptionGroupHandler, None); + let reraise_block = self.new_block(); // Reraise star block self.use_cpython_label_block(reraise_star_block); @@ -4339,6 +4286,21 @@ impl Compiler { // Set qualname self.set_qualname(); + + // Handle docstring - store in co_consts[0] if present + let (doc_info, body) = split_doc_with_range(body, &self.opts); + let doc_str = doc_info.as_ref().map(|(doc, _)| doc); + if let Some(doc) = &doc_str { + // Docstring present: store in co_consts[0] and set HAS_DOCSTRING flag + self.current_code_info() + .metadata + .consts + .insert_full(ConstantData::Str { + value: (*doc).to_string().into(), + }); + self.current_code_info().flags |= bytecode::CodeFlags::HAS_DOCSTRING; + } + let start_label = self.use_cpython_function_start_label(); // PEP 479: Wrap generator/coroutine body with StopIteration handler @@ -4356,20 +4318,6 @@ impl Compiler { } else { None }; - - // Handle docstring - store in co_consts[0] if present - let (doc_info, body) = split_doc_with_range(body, &self.opts); - let doc_str = doc_info.as_ref().map(|(doc, _)| doc); - if let Some(doc) = &doc_str { - // Docstring present: store in co_consts[0] and set HAS_DOCSTRING flag - self.current_code_info() - .metadata - .consts - .insert_full(ConstantData::Str { - value: (*doc).to_string().into(), - }); - self.current_code_info().flags |= bytecode::CodeFlags::HAS_DOCSTRING; - } // Compile body statements self.compile_statements(body)?; @@ -4571,41 +4519,7 @@ impl Compiler { &mut self, annotation: &ast::Expr, ) -> CompileResult<()> { - let code_stack_len = self.code_stack.len(); - let code_info = self.current_code_info(); - let saved_blocks = code_info.blocks.clone(); - let saved_current_block = code_info.current_block; - let saved_instr_sequence = code_info.instr_sequence.clone(); - let saved_instr_sequence_label_map = code_info.instr_sequence_label_map.clone(); - let saved_annotations_instr_sequence = code_info.annotations_instr_sequence.clone(); - let saved_metadata = code_info.metadata.clone(); - let saved_static_attributes = code_info.static_attributes.clone(); - let saved_in_inlined_comp = code_info.in_inlined_comp; - let saved_fblock = code_info.fblock.clone(); - let saved_in_conditional_block = code_info.in_conditional_block; - let saved_next_conditional_annotation_index = code_info.next_conditional_annotation_index; - let saved_source_range = self.current_source_range; - - self.do_not_emit_bytecode += 1; - let result = self.compile_annotation(annotation); - self.do_not_emit_bytecode -= 1; - - debug_assert_eq!(self.code_stack.len(), code_stack_len); - let code_info = self.current_code_info(); - code_info.blocks = saved_blocks; - code_info.current_block = saved_current_block; - code_info.instr_sequence = saved_instr_sequence; - code_info.instr_sequence_label_map = saved_instr_sequence_label_map; - code_info.annotations_instr_sequence = saved_annotations_instr_sequence; - code_info.metadata = saved_metadata; - code_info.static_attributes = saved_static_attributes; - code_info.in_inlined_comp = saved_in_inlined_comp; - code_info.fblock = saved_fblock; - code_info.in_conditional_block = saved_in_conditional_block; - code_info.next_conditional_annotation_index = saved_next_conditional_annotation_index; - self.current_source_range = saved_source_range; - - result + self.consume_skipped_nested_scopes_in_expr(annotation) } /// Compile module-level __annotate__ function (PEP 649) @@ -4693,6 +4607,8 @@ impl Compiler { } let not_set_block = has_conditional.then(|| self.new_block()); + let not_set_label = + (!has_conditional).then(|| self.current_code_info().new_instr_sequence_label()); let name = simple_name.expect("missing simple annotation name"); if has_conditional { @@ -4733,6 +4649,9 @@ impl Compiler { if let Some(not_set_block) = not_set_block { self.use_cpython_label_block(not_set_block); + } else if let Some(not_set_label) = not_set_label { + self.current_code_info() + .use_raw_instr_sequence_label(not_set_label); } } @@ -5648,7 +5567,7 @@ impl Compiler { ) -> CompileResult<()> { self.enter_conditional_block(); - let loop_block = self.start_cpython_label_block(); + let loop_block = self.new_block(); let end_block = self.new_block(); let anchor_block = self.new_block(); let (loop_label, end_label) = { @@ -5658,6 +5577,7 @@ impl Compiler { code.instr_sequence_label_for_block(end_block), ) }; + self.use_cpython_label_block(loop_block); self.push_fblock_labels( FBlockType::WhileLoop, loop_label, @@ -5893,10 +5813,11 @@ impl Compiler { // Start loop let for_block = self.new_block(); - let send_block = if is_async { - self.new_block() + let (body_label, send_block) = if is_async { + (None, self.new_block()) } else { - BlockIdx::NULL + let body_label = self.current_code_info().new_instr_sequence_label(); + (Some(body_label), BlockIdx::NULL) }; let else_block = self.new_block(); let after_block = self.new_block(); @@ -5965,6 +5886,8 @@ impl Compiler { let saved_range = self.current_source_range; self.set_source_range(target.range()); emit!(self, Instruction::Nop); + self.current_code_info() + .use_raw_instr_sequence_label(body_label.expect("sync for must have body label")); self.compile_store(target)?; self.set_source_range(saved_range); }; @@ -7212,9 +7135,8 @@ impl Compiler { let (last_op, mid_ops) = ops.split_last().unwrap(); let (last_comparator, mid_comparators) = comparators.split_last().unwrap(); - self.compile_expression(left)?; - if mid_comparators.is_empty() { + self.compile_expression(left)?; self.compile_expression(last_comparator)?; self.set_source_range(compare_range); self.compile_addcompare(last_op); @@ -7223,7 +7145,7 @@ impl Compiler { } let cleanup = self.new_block(); - let end = self.new_block(); + self.compile_expression(left)?; for (op, comparator) in mid_ops.iter().zip(mid_comparators) { self.compile_expression(comparator)?; @@ -7240,6 +7162,7 @@ impl Compiler { self.compile_addcompare(last_op); emit!(self, Instruction::ToBool); self.emit_pop_jump_by_condition(condition, target_block); + let end = self.new_block(); emit!(self, PseudoInstruction::JumpNoInterrupt { delta: end }); self.set_no_location(); @@ -7931,7 +7854,7 @@ impl Compiler { /// SEND exit /// SETUP_FINALLY fail (via exception table) /// YIELD_VALUE 1 - /// POP_BLOCK (implicit) + /// POP_BLOCK (NO_LOCATION) /// RESUME /// JUMP send /// fail: @@ -7956,6 +7879,7 @@ impl Compiler { // POP_BLOCK before RESUME emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); // RESUME emit!( @@ -8018,55 +7942,6 @@ impl Compiler { return Ok(()); } - if !self.disable_const_boolop_folding - && let ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) = expression - { - let mut simplified_prefix = 0usize; - let mut last_constant = None; - let mut last_constant_range = None; - for value in values { - let Some(constant) = self.try_fold_constant_expr(value)? else { - break; - }; - if !Self::boolop_fast_fold_literal(value) { - break; - } - // CPython codegen_boolop() emits each literal with - // ADDOP_LOAD_CONST before flowgraph.c folds the constant - // branch away. Register it here so remove_unused_consts() - // preserves the same first-constant ordering. - self.arg_constant(constant.clone()); - let is_truthy = Self::constant_truthiness(&constant); - last_constant = Some(constant); - last_constant_range = Some(value.range()); - match op { - ast::BoolOp::Or if is_truthy => { - self.set_source_range(last_constant_range.expect("missing boolop range")); - self.emit_load_const(last_constant.expect("missing boolop constant")); - return Ok(()); - } - ast::BoolOp::And if !is_truthy => { - self.set_source_range(last_constant_range.expect("missing boolop range")); - self.emit_load_const(last_constant.expect("missing boolop constant")); - return Ok(()); - } - ast::BoolOp::Or | ast::BoolOp::And => { - simplified_prefix += 1; - } - } - } - - if simplified_prefix == values.len() { - self.set_source_range(last_constant_range.expect("missing boolop range")); - self.emit_load_const(last_constant.expect("missing folded boolop constant")); - return Ok(()); - } - if simplified_prefix > 0 { - self.compile_bool_op(op, values)?; - return Ok(()); - } - } - match &expression { ast::Expr::Call(ast::ExprCall { func, arguments, .. @@ -8169,13 +8044,16 @@ impl Compiler { // ast::Expr::Constant(ExprConstant { value, .. }) => { // self.emit_load_const(compile_constant(value)); // } - ast::Expr::List(ast::ExprList { elts, .. }) => { + ast::Expr::List(ast::ExprList { elts, range, .. }) => { + self.set_source_range(*range); self.starunpack_helper(elts, 0, CollectionType::List)?; } - ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { + ast::Expr::Tuple(ast::ExprTuple { elts, range, .. }) => { + self.set_source_range(*range); self.starunpack_helper(elts, 0, CollectionType::Tuple)?; } - ast::Expr::Set(ast::ExprSet { elts, .. }) => { + ast::Expr::Set(ast::ExprSet { elts, range, .. }) => { + self.set_source_range(*range); self.starunpack_helper(elts, 0, CollectionType::Set)?; } ast::Expr::Dict(ast::ExprDict { items, range, .. }) => { @@ -8499,8 +8377,8 @@ impl Compiler { ast::Expr::If(ast::ExprIf { test, body, orelse, .. }) => { - let else_block = self.new_block(); let after_block = self.new_block(); + let else_block = self.new_block(); self.compile_jump_if(test, false, else_block)?; // True case @@ -8585,21 +8463,42 @@ impl Compiler { Ok(()) } - fn detect_builtin_generator_call( + fn cpython_sync_genexpr_call_name<'a>( &self, - func: &ast::Expr, + func: &'a ast::Expr, args: &ast::Arguments, - ) -> Option { + ) -> Option<&'a str> { let ast::Expr::Name(ast::ExprName { id, .. }) = func else { return None; }; - if args.args.len() != 1 - || !args.keywords.is_empty() - || !matches!(args.args[0], ast::Expr::Generator(_)) - { + let [ + ast::Expr::Generator(ast::ExprGenerator { + elt: _, + generators: _, + .. + }), + ] = &args.args[..] + else { + return None; + }; + if !args.keywords.is_empty() || { + let table = self.current_symbol_table(); + table + .sub_tables + .get(table.next_sub_table) + .is_none_or(|generator_entry| generator_entry.is_coroutine) + } { return None; } - match id.as_str() { + Some(id.as_str()) + } + + fn detect_builtin_generator_call( + &self, + func: &ast::Expr, + args: &ast::Arguments, + ) -> Option { + match self.cpython_sync_genexpr_call_name(func, args)? { "tuple" => Some(BuiltinGeneratorCallKind::Tuple), "all" => Some(BuiltinGeneratorCallKind::All), "any" => Some(BuiltinGeneratorCallKind::Any), @@ -8610,7 +8509,7 @@ impl Compiler { /// Emit the optimized inline loop for builtin(genexpr) calls. /// /// Stack on entry: `[func]` where `func` is the builtin candidate. - /// On return the compiler is positioned at the fallback block so the + /// On return the compiler is positioned at the skip-optimization block so the /// normal call path can compile the original generator argument again. fn optimize_builtin_generator_call( &mut self, @@ -8625,11 +8524,9 @@ impl Compiler { BuiltinGeneratorCallKind::Any => bytecode::CommonConstant::BuiltinAny, }; - let fallback = self.new_block(); - let loop_block = self.new_block(); - let cleanup = self.new_block(); + let skip_optimization = self.new_block(); - // Stack: [func] — copy function for identity check + // Stack: [func] - copy function for identity check self.set_source_range(loc); emit!(self, Instruction::Copy { i: 1 }); emit!( @@ -8639,7 +8536,12 @@ impl Compiler { } ); emit!(self, Instruction::IsOp { invert: Invert::No }); - emit!(self, Instruction::PopJumpIfFalse { delta: fallback }); + emit!( + self, + Instruction::PopJumpIfFalse { + delta: skip_optimization + } + ); emit!(self, Instruction::PopTop); if matches!(kind, BuiltinGeneratorCallKind::Tuple) { @@ -8658,6 +8560,9 @@ impl Compiler { { current_table.next_sub_table = cursor; } + + let loop_block = self.new_block(); + let cleanup = self.new_block(); self.use_cpython_label_block(loop_block); self.set_source_range(loc); emit!(self, Instruction::ForIter { delta: cleanup }); @@ -8675,7 +8580,7 @@ impl Compiler { emit!(self, Instruction::PopJumpIfTrue { delta: loop_block }); self.set_source_range(loc); emit!(self, Instruction::PopIter); - self.set_source_range(loc); + self.set_no_location(); self.emit_load_const(ConstantData::Boolean { value: false }); self.set_source_range(loc); emit!(self, PseudoInstruction::Jump { delta: end }); @@ -8686,7 +8591,7 @@ impl Compiler { emit!(self, Instruction::PopJumpIfFalse { delta: loop_block }); self.set_source_range(loc); emit!(self, Instruction::PopIter); - self.set_source_range(loc); + self.set_no_location(); self.emit_load_const(ConstantData::Boolean { value: true }); self.set_source_range(loc); emit!(self, PseudoInstruction::Jump { delta: end }); @@ -8696,8 +8601,10 @@ impl Compiler { self.use_cpython_label_block(cleanup); self.set_source_range(loc); emit!(self, Instruction::EndFor); + self.set_no_location(); self.set_source_range(loc); emit!(self, Instruction::PopIter); + self.set_no_location(); match kind { BuiltinGeneratorCallKind::Tuple => { self.set_source_range(loc); @@ -8720,7 +8627,7 @@ impl Compiler { self.set_source_range(loc); emit!(self, PseudoInstruction::Jump { delta: end }); - self.use_cpython_label_block(fallback); + self.use_cpython_label_block(skip_optimization); Ok(()) } @@ -8825,19 +8732,42 @@ impl Compiler { .then(|| self.detect_builtin_generator_call(func, args)) .flatten() { - let end = self.new_block(); + let skip_normal_call = self.new_block(); self.compile_expression(func)?; - self.optimize_builtin_generator_call(kind, &args.args[0], func.range(), end)?; + self.optimize_builtin_generator_call( + kind, + &args.args[0], + func.range(), + skip_normal_call, + )?; self.set_source_range(func.range()); emit!(self, Instruction::PushNull); self.codegen_call_helper(0, args, call_range, None)?; - self.use_cpython_label_block(end); + self.use_cpython_label_block(skip_normal_call); } else { // Regular call: push func, then NULL for self_or_null slot // Stack layout: [func, NULL, args...] - same as method call [func, self, args...] + // CPython `codegen_call()` always creates and uses + // `skip_normal_call`, even when `maybe_optimize_function_call()` + // leaves it untargeted. + let skip_normal_call = self.current_code_info().new_instr_sequence_label(); + let sync_genexpr_call_name = (!uses_ex_call) + .then(|| self.cpython_sync_genexpr_call_name(func, args)) + .flatten() + .is_some(); self.compile_expression(func)?; + if sync_genexpr_call_name { + // CPython `maybe_optimize_function_call()` creates and uses + // `skip_optimization` for every sync name(genexpr) shape after + // loading the function, even when the name is not all/any/tuple. + let skip_optimization = self.current_code_info().new_instr_sequence_label(); + self.current_code_info() + .use_raw_instr_sequence_label(skip_optimization); + } emit!(self, Instruction::PushNull); self.codegen_call_helper(0, args, call_range, None)?; + self.current_code_info() + .use_raw_instr_sequence_label(skip_normal_call); } Ok(()) } @@ -8970,35 +8900,20 @@ impl Compiler { // Single starred arg: pass value directly to CallFunctionEx. // Runtime will convert to tuple and validate with function name. if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = &arguments.args[0] { - self.compile_expression_without_const_boolop_folding(value)?; - } - } else if !has_starred { - for arg in &arguments.args { - self.compile_expression(arg)?; + self.compile_expression(value)?; } - self.set_source_range(call_range); - let positional_count = additional_positional + nelts.to_u32(); - emit!( - self, - Instruction::BuildTuple { - count: positional_count - } - ); } else { - // Use starunpack_helper to build a list, then convert to tuple + // CPython `codegen_call_helper_impl()` sends every other + // CALL_FUNCTION_EX positional shape through + // `starunpack_helper_impl(..., BUILD_LIST, LIST_APPEND, + // LIST_EXTEND, tuple=1)`, even when the only reason for the + // ex-call path is too many non-starred positional arguments. self.set_source_range(call_range); self.starunpack_helper( &arguments.args, additional_positional, - CollectionType::List, + CollectionType::Tuple, )?; - self.set_source_range(call_range); - emit!( - self, - Instruction::CallIntrinsic1 { - func: IntrinsicFunction1::ListToTuple - } - ); } self.compile_call_function_ex_keywords(&arguments.keywords, call_range)?; @@ -9042,7 +8957,7 @@ impl Compiler { have_dict = true; } - self.compile_expression_without_const_boolop_folding(&keyword.value)?; + self.compile_expression(&keyword.value)?; self.set_source_range(call_range); emit!(self, Instruction::DictMerge { i: 1 }); } else { @@ -9507,16 +9422,20 @@ impl Compiler { && let Some(singleton_iter) = Self::singleton_comprehension_assignment_iter(&generator.iter) { + // CPython allocates start/if_cleanup/anchor labels before the + // singleton sub-iterator fast path sets start = NO_LABEL. + let _start_label = self.current_code_info().new_instr_sequence_label(); + let if_cleanup_block = self.new_block(); + let _anchor_label = self.current_code_info().new_instr_sequence_label(); self.compile_expression(singleton_iter)?; self.compile_store(&generator.target)?; if !generator.ifs.is_empty() { - let if_cleanup_block = self.new_block(); for if_condition in &generator.ifs { self.compile_jump_if(if_condition, false, if_cleanup_block)?; } - loop_labels.push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); } + loop_labels.push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); continue; } @@ -9579,7 +9498,6 @@ impl Compiler { loop_block, if_cleanup_block, after_block, - iter_range: generator.iter.range(), backedge_range, is_async: generator.is_async, end_async_for_target, @@ -9600,7 +9518,6 @@ impl Compiler { loop_block, if_cleanup_block, after_block, - iter_range, backedge_range, is_async, end_async_for_target, @@ -9626,10 +9543,7 @@ impl Compiler { // (handler depth is before GetANext, so aiter is at handler depth) self.emit_end_async_for(end_async_for_target); } else { - self.set_source_range(iter_range); - // END_FOR + POP_ITER pattern (CPython 3.14) - emit!(self, Instruction::EndFor); - emit!(self, Instruction::PopIter); + self.emit_sync_comprehension_end_for(); } } ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block } => { @@ -9639,11 +9553,11 @@ impl Compiler { } if return_none { - self.emit_load_const(ConstantData::None) + self.emit_return_const_no_location(ConstantData::None); + } else { + self.emit_return_value(); } - self.emit_return_value(); - // Close StopIteration handler and emit handler code if let Some(handler_block) = stop_iteration_block { self.use_cpython_label_block(handler_block); @@ -9862,16 +9776,15 @@ impl Compiler { // CPython's codegen_push_inlined_comprehension_locals() // installs the virtual cleanup before codegen_comprehension() // emits BUILD_LIST/BUILD_SET/BUILD_MAP for the result object. - let cleanup_blocks = if !pushed_locals.is_empty() { + let cleanup_block = if !pushed_locals.is_empty() { let cleanup_block = self.new_block(); - let end_block = self.new_block(); emit!( self, PseudoInstruction::SetupFinally { delta: cleanup_block } ); - Some((cleanup_block, end_block)) + Some(cleanup_block) } else { None }; @@ -9892,17 +9805,20 @@ impl Compiler { && let Some(singleton_iter) = Self::singleton_comprehension_assignment_iter(&generator.iter) { + // CPython allocates start/if_cleanup/anchor labels before + // the singleton sub-iterator fast path sets start = NO_LABEL. + let _start_label = self.current_code_info().new_instr_sequence_label(); + let if_cleanup_block = self.new_block(); + let _anchor_label = self.current_code_info().new_instr_sequence_label(); self.compile_expression(singleton_iter)?; self.compile_store(&generator.target)?; if !generator.ifs.is_empty() { - let if_cleanup_block = self.new_block(); for if_condition in &generator.ifs { self.compile_jump_if(if_condition, false, if_cleanup_block)?; } - loop_labels - .push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); } + loop_labels.push(ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block }); continue; } @@ -9961,7 +9877,6 @@ impl Compiler { loop_block, if_cleanup_block, after_block, - iter_range: generator.iter.range(), backedge_range, is_async: generator.is_async, end_async_for_target, @@ -9984,7 +9899,6 @@ impl Compiler { loop_block, if_cleanup_block, after_block, - iter_range, backedge_range, is_async, end_async_for_target, @@ -10008,9 +9922,7 @@ impl Compiler { self.set_source_range(comprehension_range); self.emit_end_async_for(end_async_for_target); } else { - self.set_source_range(iter_range); - emit!(self, Instruction::EndFor); - emit!(self, Instruction::PopIter); + self.emit_sync_comprehension_end_for(); } } ComprehensionLoopControl::IfCleanupOnly { if_cleanup_block } => { @@ -10021,7 +9933,7 @@ impl Compiler { // Step 8: Clean up - restore saved locals (and cell values) self.set_source_range(comprehension_range); - if let Some((cleanup_block, end_block)) = cleanup_blocks { + if let Some(cleanup_block) = cleanup_block { emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); @@ -10029,6 +9941,7 @@ impl Compiler { // the synthetic jump that skips the exception cleanup uses // JUMP_NO_INTERRUPT, which becomes JUMP_BACKWARD_NO_INTERRUPT // when the cleanup tail sits above the final restore block. + let end_block = self.new_block(); emit!( self, PseudoInstruction::JumpNoInterrupt { delta: end_block } @@ -10241,6 +10154,16 @@ impl Compiler { self.set_last_emitted_lineno_override(-1); } + /// CPython `codegen_sync_comprehension_generator()` emits END_FOR/POP_ITER + /// with NO_LOCATION and lets `flowgraph.c::propagate_line_numbers()` copy + /// the FOR_ITER location onto the cleanup block. + fn emit_sync_comprehension_end_for(&mut self) { + emit!(self, Instruction::EndFor); + self.set_no_location(); + emit!(self, Instruction::PopIter); + self.set_no_location(); + } + /// CPython `codegen_wrap_in_stopiteration_handler()` inserts /// `SETUP_CLEANUP` into the instruction sequence at index 0 with /// `_PyInstructionSequence_InsertInstruction()`. Generator and cell/free @@ -11056,33 +10979,8 @@ impl Compiler { } debug_assert_eq!(code.blocks[cur.idx()].next, BlockIdx::NULL); - let block = self.new_unlabeled_block(); - let code = self.current_code_info(); - code.blocks[cur.idx()].next = block; - code.current_block = block; - } - - fn cpython_cfg_builder_current_block_is_terminated_for_label(block: &ir::Block) -> bool { - !block.instructions.is_empty() || block.has_cpython_cfg_label() - } - - /// Start a label block using CPython's `cfg_builder_current_block_is_terminated()` - /// rule: if the current block is empty and unlabeled, `_PyCfgBuilder_UseLabel()` - /// attaches the pending label to that block instead of allocating a new one. - fn start_cpython_label_block(&mut self) -> BlockIdx { - let cur = self.current_code_info().current_block; - let block = &self.current_code_info().blocks[cur.idx()]; - let block = if Self::cpython_cfg_builder_current_block_is_terminated_for_label(block) { - let b = self.new_block(); - self.switch_to_block(b); - b - } else { - debug_assert_eq!(block.next, BlockIdx::NULL); - cur - }; - self.mark_cpython_cfg_label_block(block); - self.current_code_info().use_instr_sequence_label(block); - block + let block = self.cpython_cfg_builder_new_block(); + self.cpython_cfg_builder_use_next_block(block); } /// CPython `codegen_funcbody()` emits `NEW_JUMP_TARGET_LABEL(start)` and @@ -11101,7 +10999,7 @@ impl Compiler { /// Consecutive `USE_LABEL()` calls can map multiple labels to the same /// instruction offset before `_PyCfg_FromInstructionSequence()` builds a /// CFG. Reuse an empty current block even if it already carries a label so - /// direct CFG codegen preserves that aliasing. + /// codegen CFG path preserves that aliasing. fn use_cpython_label_block(&mut self, block: BlockIdx) { let code = self.current_code_info(); code.use_instr_sequence_label(block); @@ -11148,6 +11046,19 @@ impl Compiler { idx } + /// flowgraph.c cfg_builder_new_block + fn cpython_cfg_builder_new_block(&mut self) -> BlockIdx { + self.new_unlabeled_block() + } + + /// flowgraph.c cfg_builder_use_next_block + fn cpython_cfg_builder_use_next_block(&mut self, block: BlockIdx) { + let code = self.current_code_info(); + let cur = code.current_block; + code.blocks[cur.idx()].next = block; + code.current_block = block; + } + fn switch_to_block(&mut self, block: BlockIdx) { let code = self.current_code_info(); code.use_instr_sequence_label(block); @@ -12410,6 +12321,16 @@ def f(x, y, z): .unwrap() } + fn find_symbol_table<'a>(table: &'a SymbolTable, name: &str) -> Option<&'a SymbolTable> { + if table.name == name { + return Some(table); + } + table + .sub_tables + .iter() + .find_map(|sub_table| find_symbol_table(sub_table, name)) + } + fn compile_exec_late_cfg_trace(source: &str) -> Vec<(String, String)> { let opts = CompileOpts::default(); let source_file = SourceFileBuilder::new("source_path", source).finish(); @@ -12493,6 +12414,7 @@ def f(x, y, z): in_async_scope: is_async, }; compiler.set_qualname(); + let (_doc_str, body) = split_doc(body, &compiler.opts); let start_label = compiler.use_cpython_function_start_label(); let is_gen = is_async || compiler.current_symbol_table().is_generator; let stop_iteration_block = if is_gen { @@ -12510,7 +12432,6 @@ def f(x, y, z): } else { None }; - let (_doc_str, body) = split_doc(body, &compiler.opts); compiler.compile_statements(body).unwrap(); match body.last() { Some(ast::Stmt::Return(_)) => {} @@ -12538,6 +12459,43 @@ def f(x, y, z): stack_top.debug_late_cfg_trace().unwrap() } + #[test] + fn try_else_nested_try_const_list_keeps_setup_finally_nop() { + let trace = compile_single_function_late_cfg_trace( + r#" +def f(arch): + try: + [arch, *_] = g() + except OSError: + pass + else: + try: + arch = ['x86', 'MIPS', 'Alpha', 'PowerPC', None, + 'ARM', 'ia64', None, None, + 'AMD64', None, None, 'ARM64', + ][int(arch)] + except (ValueError, IndexError): + pass + else: + if arch: + return arch +"#, + "f", + ); + let (_, dump) = trace + .iter() + .find(|(label, _)| label == "after_convert_pseudo_ops") + .expect("missing convert_pseudo_ops trace"); + assert!( + dump.contains("[disp=8:9 raw=8:9-17:28 override=None] Real(Nop)"), + "SETUP_FINALLY should survive as a line-bearing NOP like CPython" + ); + assert!( + dump.contains("[disp=9:20 raw=9:20-12:14 override=None] Real(BuildList"), + "CPython optimize_lists_and_sets() restores the literal location to BUILD_LIST" + ); + } + #[test] fn debug_trace_make_dataclass_borrow_tail() { let trace = compile_single_function_late_cfg_trace( @@ -12612,7 +12570,7 @@ def f(proc, unittest): "f", ); for (label, dump) in trace { - if label == "after_raw_optimize_load_fast_borrow" + if label == "after_optimize_load_fast" || label.contains("deoptimize_borrow_in_protected_conditional_tail") { eprintln!("=== {label} ===\n{dump}"); @@ -12641,9 +12599,8 @@ def f(sys, os, file): "f", ); for (label, dump) in trace { - if label == "after_raw_optimize_load_fast_borrow" + if label == "after_optimize_load_fast" || label == "after_deoptimize_borrow_after_protected_import" - || label == "after_optimize_load_fast_borrow" || label == "after_borrow_deopts" { eprintln!("=== {label} ===\n{dump}"); @@ -17152,6 +17109,48 @@ def f(g): ); } + #[test] + fn too_large_plain_call_uses_cpython_tuple_ex_call_path() { + let args = (0..=STACK_USE_GUIDELINE) + .map(|i| format!("'v{i}'")) + .collect::>() + .join(", "); + let code = compile_exec(&format!("def f(g):\n return g({args})\n")); + let f = find_code(&code, "f").expect("missing function code"); + let ops: Vec<_> = f + .instructions + .iter() + .map(|unit| unit.op) + .filter(|op| !matches!(op, Instruction::Cache)) + .collect(); + + assert!( + ops.iter() + .any(|op| matches!(op, Instruction::CallFunctionEx)), + "CPython routes calls over _PY_STACK_USE_GUIDELINE through CALL_FUNCTION_EX, got ops={ops:?}" + ); + assert!( + !ops.iter().any(|op| matches!( + op, + Instruction::BuildTuple { .. } + | Instruction::ListAppend { .. } + | Instruction::CallIntrinsic1 { .. } + )), + "CPython flowgraph.c folds the starunpack tuple for constant too-large calls, got ops={ops:?}" + ); + assert!( + f.constants.iter().any(|constant| { + matches!( + constant, + ConstantData::Tuple { elements } + if elements.len() == usize::try_from(STACK_USE_GUIDELINE + 1).unwrap() + ) + }), + "expected CPython folded tuple constant for too-large call args, got constants={:?}", + f.constants.iter().collect::>() + ); + } + #[test] fn simple_attribute_call_keeps_method_load() { let code = compile_exec( @@ -17264,6 +17263,44 @@ def f(xs): ); } + #[test] + fn builtin_any_async_genexpr_call_is_not_optimized() { + let code = compile_exec( + "\ +async def f(xs): + return any(x async for x in xs) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !has_common_constant(f, bytecode::CommonConstant::BuiltinAny), + "CPython maybe_optimize_function_call() skips coroutine generator expressions" + ); + assert!( + f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::Call { .. })), + "async genexpr any() should stay on the normal call path" + ); + } + + #[test] + fn builtin_any_genexpr_outermost_await_is_optimized_like_cpython() { + let code = compile_exec( + "\ +async def f(get_xs): + return any(x for x in await get_xs()) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + has_common_constant(f, bytecode::CommonConstant::BuiltinAny), + "CPython checks the generator expression symtable entry, so await in the outermost iterator does not make the genexpr coroutine" + ); + } + #[test] fn builtin_tuple_genexpr_call_is_optimized_but_list_set_are_not() { let code = compile_exec( @@ -17775,24 +17812,8 @@ def f(obj): ] ) }); - let has_conservative_shape = ops.windows(9).any(|window| { - matches!( - window, - [ - Instruction::PopJumpIfNone { .. }, - Instruction::NotTaken, - Instruction::LoadFastBorrow { .. } | Instruction::LoadFast { .. }, - Instruction::Swap { .. }, - Instruction::PopTop, - Instruction::ReturnValue, - Instruction::Nop, - Instruction::JumpBackward { .. }, - Instruction::EndFor, - ] - ) - }); assert!( - has_cpython_shape || has_conservative_shape, + has_cpython_shape, "expected loop return null-check to keep the backedge adjacent to the return cleanup, got ops={ops:?}" ); @@ -19727,7 +19748,7 @@ def f(self): let prev = f.instructions[key_load_idx - 1].op; assert!( matches!(prev, Instruction::LoadFast { .. }), - "CPython optimize_load_fast() leaves Keys plain here because MATCH_KEYS' no-input pseudo-ref uses the inner produced index; got ops={:?}", + "CPython optimize_load_fast() records MATCH_KEYS' no-input pseudo-ref with the produced-value loop index, so this consumed Keys load stays strong; got ops={:?}", f.instructions .iter() .map(|unit| unit.op) @@ -25129,7 +25150,7 @@ def f(self): !tail .iter() .any(|op| matches!(op, Instruction::LoadFast { .. })), - "CPython codegen_try_except() uses USE_LABEL(end), so the empty RustPython label must be a passthrough and post-handler closure/tail loads should borrow; got tail={tail:?}", + "CPython codegen_try_except() uses USE_LABEL(end), so the handler continuation should be a shared passthrough and post-handler closure/tail loads should borrow; got tail={tail:?}", ); } @@ -27720,7 +27741,7 @@ def func[T](a: T = 'a', *, b: T = 'b'): ] ) }), - "generic defaults call should not use RustPython-specific PUSH_NULL reshuffle, got ops={ops:?}" + "CPython generic defaults use SWAP/CALL after codegen_make_closure(), not a PUSH_NULL reshuffle, got ops={ops:?}" ); } @@ -29213,6 +29234,24 @@ async def f(): ); } + #[test] + fn async_comprehension_propagates_coroutine_to_enclosing_genexpr_like_cpython() { + let symbol_table = scan_program_symbol_table( + "\ +async def f(): + gen = ([i async for i in asynciter([1, 2])] for j in [10, 20]) + return [x async for x in gen] +", + ); + let genexpr = + find_symbol_table(&symbol_table, "").expect("missing genexpr symbol table"); + assert!(genexpr.is_generator, "expected genexpr symbol table"); + assert!( + genexpr.is_coroutine, + "CPython symtable_handle_comprehension() propagates non-generator async comprehension ste_coroutine to the enclosing genexpr" + ); + } + #[test] fn nested_module_scope_dictcomp_symbols_are_local() { let symbol_table = scan_program_symbol_table( @@ -34005,7 +34044,7 @@ def f(x): assert!( matches!(ops[rpartition_idx - 1].op, Instruction::LoadFast { .. }), - "CPython keeps the conditional-store join receiver strong before IMPORT_FROM, got ops={ops:?}" + "CPython optimize_load_fast() keeps the conditional-store join receiver strong before IMPORT_FROM, got ops={ops:?}" ); } @@ -34260,6 +34299,35 @@ def f(self, quoted): ); } + #[test] + fn backward_jump_extended_arg_accounts_for_jump_cache() { + let mut source = String::from("def f(x, items):\n while x:\n"); + for _ in 0..10 { + source.push_str(" x = len(items[-1])\n"); + } + for _ in 0..6 { + source.push_str(" len(items)\n"); + } + source.push_str(" continue\n"); + + let code = compile_exec(&source); + let f = find_code(&code, "f").expect("missing f code"); + assert!( + f.instructions.windows(2).any(|window| { + matches!( + (&window[0].op, &window[1].op), + ( + Instruction::ExtendedArg, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) + ) + }), + "CPython assemble.c resolves unconditional jumps before jump offsets, so the first offset pass must include JUMP_BACKWARD's inline cache and emit EXTENDED_ARG at this boundary; got instructions={:?}", + f.instructions + ); + } + #[test] fn for_continue_before_return_orders_backedge_before_return_body() { let code = compile_exec( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index ffa0b2b5bbf..19ed452d392 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1,3 +1,4 @@ +use alloc::collections::VecDeque; use core::ops; use crate::{IndexMap, IndexSet, error::InternalError}; @@ -29,7 +30,7 @@ struct LineTableLocation { pub(crate) const LINE_ONLY_LOCATION_OVERRIDE: i32 = -4; pub(crate) const NEXT_LOCATION_OVERRIDE: i32 = -2; -const MAX_INT_SIZE_BITS: u64 = 128; +const MAX_INT_SIZE: u64 = 128; const MAX_COLLECTION_SIZE: usize = 256; const MAX_TOTAL_ITEMS: isize = 1024; const MAX_STR_SIZE: usize = 4096; @@ -216,7 +217,6 @@ pub struct ExceptHandlerInfo { fn set_to_nop(info: &mut InstructionInfo) { info.instr = Instruction::Nop.into(); info.arg = OpArg::new(0); - info.target = BlockIdx::NULL; info.cache_entries = 0; } @@ -225,10 +225,24 @@ fn nop_out_no_location(info: &mut InstructionInfo) { info.lineno_override = Some(-1); } +/// flowgraph.c basicblock_next_instr +fn basicblock_next_instr(block: &mut Block) -> Option { + block.cpython_spare_instr_slots.pop_front() +} + +/// flowgraph.c basicblock_last_instr +fn basicblock_last_instr(block: &Block) -> Option<&InstructionInfo> { + block.instructions.last() +} + +/// flowgraph.c basicblock_last_instr +fn basicblock_last_instr_mut(block: &mut Block) -> Option<&mut InstructionInfo> { + block.instructions.last_mut() +} + /// flowgraph.c basicblock_addop fn basicblock_addop(block: &mut Block, mut info: InstructionInfo) { - if let Some(stale) = block.cpython_spare_instr_slots.first().copied() { - block.cpython_spare_instr_slots.remove(0); + if let Some(stale) = basicblock_next_instr(block) { info.except_handler = stale.except_handler; } info.target = BlockIdx::NULL; @@ -253,14 +267,52 @@ fn basicblock_add_jump_op( Ok(()) } +fn cpython_target_label_arg(blocks: &[Block], target: BlockIdx) -> OpArg { + debug_assert!(target != BlockIdx::NULL); + let label = blocks[target.idx()] + .cpython_label_id + .expect("target block has a CPython CFG label"); + OpArg::new(label.idx().to_u32().expect("too many CPython CFG labels")) +} + /// flowgraph.c basicblock_insert_instruction fn basicblock_insert_instruction(block: &mut Block, pos: usize, info: InstructionInfo) { - if !block.cpython_spare_instr_slots.is_empty() { - block.cpython_spare_instr_slots.remove(0); - } + basicblock_next_instr(block); block.instructions.insert(pos, info); } +/// flowgraph.c basicblock_append_instructions +fn basicblock_append_instructions(to: &mut Block, from: &[InstructionInfo]) { + for info in from { + basicblock_next_instr(to); + to.instructions.push(*info); + } +} + +/// flowgraph.c direct `b_iused = 0` +fn basicblock_clear(block: &mut Block) { + let instructions = core::mem::take(&mut block.instructions); + if !instructions.is_empty() { + let spare_instr_slots = core::mem::take(&mut block.cpython_spare_instr_slots); + block.cpython_spare_instr_slots = instructions.into(); + block.cpython_spare_instr_slots.extend(spare_instr_slots); + } +} + +/// CPython direct `b_instr[0]` access. Some passes set `b_iused = 0` +/// without clearing the backing array, so an empty basic block can still have +/// a first raw instruction slot. +fn basicblock_raw_first_instr_mut(block: &mut Block) -> &mut InstructionInfo { + if !block.instructions.is_empty() { + &mut block.instructions[0] + } else { + block + .cpython_spare_instr_slots + .front_mut() + .expect("CPython basicblock has a backing instruction slot") + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct InstructionSequenceLabel(usize); @@ -437,62 +489,53 @@ impl InstructionSequence { Ok(()) } - fn mark_targets(&mut self) -> crate::InternalResult<()> { + fn mark_targets(&mut self) { for entry in &mut self.instrs { entry.is_target = false; } - let targets: Vec = self - .instrs - .iter() - .filter_map(|entry| entry.target_offset) - .collect(); - for target_offset in targets { - let target = self - .instrs - .get_mut(target_offset) - .ok_or(InternalError::MalformedControlFlowGraph)?; - target.is_target = true; + for i in 0..self.instrs.len() { + if let Some(target_offset) = self.instrs[i].target_offset { + let target = self + .instrs + .get_mut(target_offset) + .expect("instruction-sequence target offset is in range"); + target.is_target = true; + } } - Ok(()) } } /// flowgraph.c _PyCfg_ToInstructionSequence -fn cfg_to_instruction_sequence( - blocks: &mut [Block], - block_order: &[BlockIdx], -) -> crate::InternalResult { +fn cfg_to_instruction_sequence(blocks: &mut [Block]) -> crate::InternalResult { for block in blocks.iter_mut() { block.cpython_label_id = None; } - for (label_id, block_idx) in block_order.iter().copied().enumerate() { + let mut label_id = 0; + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { blocks[block_idx.idx()].cpython_label_id = Some(InstructionSequenceLabel(label_id)); - } - - let mut block_to_label = vec![None; blocks.len()]; - for block_idx in block_order.iter().copied() { - block_to_label[block_idx.idx()] = blocks[block_idx.idx()].cpython_label_id; + label_id += 1; + block_idx = blocks[block_idx.idx()].next; } let mut instr_sequence = InstructionSequence::new(); - for block_idx in block_order.iter().copied() { + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { let block_label = blocks[block_idx.idx()] .cpython_label_id - .ok_or(InternalError::MalformedControlFlowGraph)?; + .expect("block has a CPython CFG label"); instr_sequence.use_label(block_label); - for info in &blocks[block_idx.idx()].instructions { - let mut info = *info; - let target_label = if info.target != BlockIdx::NULL { - if !info.instr.has_target() { - return Err(InternalError::MalformedControlFlowGraph); - } - let label_id = block_to_label - .get(info.target.idx()) - .copied() - .flatten() - .ok_or(InternalError::MalformedControlFlowGraph)?; + let instr_count = blocks[block_idx.idx()].instructions.len(); + for i in 0..instr_count { + let mut info = blocks[block_idx.idx()].instructions[i]; + let target_label = if info.instr.has_target() { + let target_block = info.target; + debug_assert!(target_block != BlockIdx::NULL); + let label_id = blocks[target_block.idx()] + .cpython_label_id + .expect("target block has a CPython CFG label"); info.arg = OpArg::new( label_id .idx() @@ -506,11 +549,10 @@ fn cfg_to_instruction_sequence( }; let except_handler = if let Some(handler) = info.except_handler.take() { - let label_id = block_to_label - .get(handler.handler_block.idx()) - .copied() - .flatten() - .ok_or(InternalError::MalformedControlFlowGraph)?; + debug_assert!(handler.handler_block != BlockIdx::NULL); + let label_id = blocks[handler.handler_block.idx()] + .cpython_label_id + .expect("exception handler block has a CPython CFG label"); Some(InstructionSequenceExceptHandlerInfo { target_label: Some(label_id), target_offset: None, @@ -523,20 +565,19 @@ fn cfg_to_instruction_sequence( instr_sequence.addop(info, target_label, except_handler); } + block_idx = blocks[block_idx.idx()].next; } instr_sequence.apply_label_map()?; Ok(instr_sequence) } -// spell-checker:ignore petgraph -// TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it -// might enable more analysis/optimizations #[derive(Debug, Clone)] pub struct Block { pub instructions: Vec, pub next: BlockIdx, - // Post-codegen analysis fields (set by label_exception_targets) + /// Exception stack at start of block, used by label_exception_targets (b_exceptstack) + except_stack: Option>, /// Whether this block is an exception handler target (b_except_handler) pub except_handler: bool, /// Whether to preserve lasti for this handler block (b_preserve_lasti) @@ -545,12 +586,20 @@ pub struct Block { pub start_depth: Option, /// Whether this block is only reachable via exception table (b_cold) pub cold: bool, + /// Temporary traversal mark used by CFG passes (b_visited) + visited: bool, + /// Definitely reachable outside exception-only paths (b_warm) + warm: bool, + /// Number of incoming CFG edges from reachable blocks (b_predecessors) + predecessors: i32, /// CPython `basicblock.b_label` used by translate_jump_labels_to_targets. cpython_label_id: Option, + /// Potentially uninitialized locals mask for local-check analysis (b_unsafe_locals_mask) + unsafe_locals_mask: u64, /// CPython keeps `b_instr` allocated beyond `b_iused`. Instructions removed /// by compaction remain in those spare slots until `basicblock_next_instr()` /// reuses them. - cpython_spare_instr_slots: Vec, + cpython_spare_instr_slots: VecDeque, } impl Default for Block { @@ -558,12 +607,17 @@ impl Default for Block { Self { instructions: Vec::new(), next: BlockIdx::NULL, + except_stack: None, except_handler: false, preserve_lasti: false, start_depth: None, cold: false, + visited: false, + warm: false, + predecessors: 0, cpython_label_id: None, - cpython_spare_instr_slots: Vec::new(), + unsafe_locals_mask: 0, + cpython_spare_instr_slots: VecDeque::new(), } } } @@ -578,9 +632,13 @@ impl Block { pub(crate) struct InstructionSequenceLabelMap { next_free_label: usize, block_labels: Vec>, - /// Direct-CFG shadow for CPython labels that map to the same instruction - /// offset in `_PyInstructionSequence_UseLabel()`. - direct_block_by_label: Vec>, + /// Codegen-side shadow of CPython's instruction-sequence label map. + /// + /// `_PyInstructionSequence_UseLabel()` can map multiple labels to the same + /// instruction offset before `_PyCfg_FromInstructionSequence()` materializes + /// CFG blocks. The codegen CFG path keeps the same aliasing by resolving + /// those labels to the block that owns the shared offset. + cpython_block_by_label: Vec>, } impl InstructionSequenceLabelMap { @@ -588,7 +646,7 @@ impl InstructionSequenceLabelMap { Self { next_free_label: 0, block_labels: vec![None], - direct_block_by_label: Vec::new(), + cpython_block_by_label: Vec::new(), } } @@ -596,8 +654,8 @@ impl InstructionSequenceLabelMap { fn new_label(&mut self) -> InstructionSequenceLabel { self.next_free_label += 1; let label = InstructionSequenceLabel(self.next_free_label); - if self.direct_block_by_label.len() <= label.idx() { - self.direct_block_by_label.resize(label.idx() + 1, None); + if self.cpython_block_by_label.len() <= label.idx() { + self.cpython_block_by_label.resize(label.idx() + 1, None); } label } @@ -612,9 +670,9 @@ impl InstructionSequenceLabelMap { self.block_labels .len() .to_u32() - .expect("too many direct-CFG blocks"), + .expect("too many codegen CFG blocks"), ); - self.direct_block_by_label[label.idx()] = Some(block); + self.cpython_block_by_label[label.idx()] = Some(block); self.block_labels.push(Some(label)); } @@ -624,7 +682,7 @@ impl InstructionSequenceLabelMap { return label; } let label = self.new_label(); - self.direct_block_by_label[label.idx()] = Some(block); + self.cpython_block_by_label[label.idx()] = Some(block); self.block_labels[block.idx()] = Some(label); label } @@ -635,7 +693,7 @@ impl InstructionSequenceLabelMap { } fn block_for_label(&self, label: InstructionSequenceLabel) -> Option { - self.direct_block_by_label + self.cpython_block_by_label .get(label.idx()) .copied() .flatten() @@ -649,7 +707,10 @@ impl InstructionSequenceLabelMap { return block; }; self.block_for_label(label).unwrap_or_else(|| { - debug_assert!(false, "CPython label must map to a direct-CFG block"); + debug_assert!( + false, + "CPython instruction-sequence label must map to a codegen CFG block" + ); BlockIdx::NULL }) } @@ -662,7 +723,10 @@ impl InstructionSequenceLabelMap { return BlockIdx::NULL; }; self.block_for_label(label).unwrap_or_else(|| { - debug_assert!(false, "CPython label must map to a direct-CFG block"); + debug_assert!( + false, + "CPython instruction-sequence label must map to a codegen CFG block" + ); BlockIdx::NULL }) } @@ -674,36 +738,39 @@ impl InstructionSequenceLabelMap { let from_label = self.ensure_label_for_block(from); let to_block = self.resolve_label(to); if to_block == BlockIdx::NULL { - debug_assert!(false, "CPython label target must map to a direct-CFG block"); + debug_assert!( + false, + "CPython label target must map to a codegen CFG block" + ); return; } - self.direct_block_by_label[from_label.idx()] = Some(to_block); + self.cpython_block_by_label[from_label.idx()] = Some(to_block); } fn debug_check_for_blocks(&self, blocks_len: usize) { debug_assert_eq!( self.block_labels.len(), blocks_len, - "every direct-CFG block must have an instruction-sequence label slot" + "every codegen CFG block must have an instruction-sequence label slot" ); - debug_assert!(self.direct_block_by_label.len() <= self.next_free_label + 1); + debug_assert!(self.cpython_block_by_label.len() <= self.next_free_label + 1); let mut seen_labels = vec![false; self.next_free_label + 1]; for &label in self.block_labels.iter().flatten() { debug_assert!( label.idx() <= self.next_free_label, - "direct-CFG block labels must come from _PyInstructionSequence_NewLabel()" + "codegen CFG block labels must come from _PyInstructionSequence_NewLabel()" ); debug_assert!( !seen_labels[label.idx()], - "direct-CFG blocks must not share a CPython instruction-sequence label" + "codegen CFG blocks must not share a CPython instruction-sequence label" ); seen_labels[label.idx()] = true; } - for &block in &self.direct_block_by_label { + for &block in &self.cpython_block_by_label { if let Some(block) = block { debug_assert!( block.idx() < blocks_len, - "CPython label must map to an existing direct-CFG block" + "CPython label must map to an existing codegen CFG block" ); } } @@ -711,7 +778,7 @@ impl InstructionSequenceLabelMap { debug_assert!( self.block_for_label(label) .is_some_and(|block| block.idx() < blocks_len), - "direct-CFG block label must map to a direct-CFG block" + "codegen CFG block label must map to a codegen CFG block" ); } } @@ -723,11 +790,11 @@ impl InstructionSequenceLabelMap { let InstructionSequenceLabelOffsets::Active(label_map) = &instr_sequence.label_map else { debug_assert!( false, - "direct-CFG label map must be checked before CPython label-map application" + "codegen CFG label shadow must be checked before CPython label-map application" ); return; }; - for (label_idx, block) in self.direct_block_by_label.iter().copied().enumerate() { + for (label_idx, block) in self.cpython_block_by_label.iter().copied().enumerate() { let Some(block) = block else { continue; }; @@ -740,7 +807,7 @@ impl InstructionSequenceLabelMap { let Some(block_label) = self.block_labels.get(block.idx()).copied().flatten() else { debug_assert!( false, - "CPython label must map to an existing direct-CFG block" + "CPython label must map to an existing codegen CFG block" ); continue; }; @@ -752,7 +819,7 @@ impl InstructionSequenceLabelMap { } debug_assert!( label_offset == block_offset, - "direct-CFG labels may share a block only when CPython maps them to the same instruction offset" + "codegen CFG labels may share a block only when CPython maps them to the same instruction offset" ); } } @@ -954,9 +1021,9 @@ impl CodeInfo { ) -> crate::InternalResult<()> { // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) self.blocks = cfg_from_instruction_sequence(instr_sequence)?; - translate_jump_labels_to_targets(&mut self.blocks)?; - mark_except_handlers(&mut self.blocks)?; - label_exception_targets(&mut self.blocks)?; + translate_jump_labels_to_targets(&mut self.blocks); + mark_except_handlers(&mut self.blocks); + label_exception_targets(&mut self.blocks); // CPython optimize_cfg() starts with check_cfg() and raises // SystemError if a jump or scope exit is not the last instruction in // its block. @@ -965,22 +1032,25 @@ impl CodeInfo { // CPython does not re-run instruction-sequence label-map/CFG conversion // after this point. Unreferenced label blocks left by jump inlining // remain block boundaries and can preserve line-marker NOPs. - self.remove_unreachable_blocks(); + Self::remove_unreachable(&mut self.blocks); // CPython optimize_cfg resolves line numbers before local checks and // superinstruction insertion, so fusion decisions see propagated // source locations. resolve_line_numbers(&mut self.blocks); // CPython optimize_cfg() runs optimize_load_const() and then // optimize_basic_block() after line numbers are resolved. - self.convert_to_load_small_int(); - self.peephole_optimize(); - self.convert_to_load_small_int(); - self.optimize_basic_blocks()?; + self.optimize_load_const(); + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = self.blocks[block_idx.idx()].next; + Self::optimize_basic_block(&mut self.blocks, &mut self.metadata, block_idx)?; + block_idx = next_block; + } self.remove_redundant_nops_and_pairs(); // CPython optimize_cfg() removes newly-unreachable blocks and // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes // unused constants. - self.remove_unreachable_blocks(); + Self::remove_unreachable(&mut self.blocks); remove_redundant_nops_and_jumps(&mut self.blocks)?; debug_assert!(no_redundant_jumps(&self.blocks)); self.remove_unused_consts(); @@ -998,7 +1068,7 @@ impl CodeInfo { fn optimized_cfg_to_instruction_sequence(&mut self) -> crate::InternalResult<(u32, usize)> { // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) convert_pseudo_conditional_jumps(&mut self.blocks); - let max_stackdepth = self.max_stackdepth()?; + let max_stackdepth = self.calculate_stackdepth()?; debug_assert!( !self.flags.intersects( CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR @@ -1011,7 +1081,7 @@ impl CodeInfo { normalize_jumps(&mut self.blocks)?; debug_assert!(no_redundant_jumps(&self.blocks)); // optimize_load_fast: after normalize_jumps - self.optimize_load_fast_borrow(); + self.optimize_load_fast(); Ok((max_stackdepth, nlocalsplus)) } @@ -1079,8 +1149,48 @@ impl CodeInfo { } } - let block_order = layout_block_order(&blocks); - let mut instr_sequence = cfg_to_instruction_sequence(&mut blocks, &block_order)?; + let mut instr_sequence = cfg_to_instruction_sequence(&mut blocks)?; + // CPython assemble.c::resolve_unconditional_jumps() runs before + // resolve_jump_offsets(), so the first offset pass already uses the + // concrete JUMP_* opcode and its inline-cache size. + for (current_instr_index, entry) in instr_sequence.instrs.iter_mut().enumerate() { + let op = match entry.info.instr { + AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { + if entry + .target_offset + .ok_or(InternalError::MalformedControlFlowGraph)? + > current_instr_index + { + Instruction::JumpForward { + delta: Arg::marker(), + } + } else { + Instruction::JumpBackward { + delta: Arg::marker(), + } + } + } + AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) => { + if entry + .target_offset + .ok_or(InternalError::MalformedControlFlowGraph)? + > current_instr_index + { + Instruction::JumpForward { + delta: Arg::marker(), + } + } else { + Instruction::JumpBackwardNoInterrupt { + delta: Arg::marker(), + } + } + } + _ => continue, + }; + entry.info.instr = op.into(); + entry.info.cache_entries = op.cache_entries() as u32; + } + let mut instruction_offsets = vec![0u32; instr_sequence.instrs.len()]; let mut end_offset; // The offset (in code units) of END_SEND from SEND in the yield-from sequence. @@ -1107,45 +1217,7 @@ impl CodeInfo { // Keep offsets fixed within this pass: changes in jump // arg/cache sizes only take effect in the next iteration. let offset_after = current_offset + old_arg_size as u32 + old_cache_entries; - let op = match info.instr { - AnyInstruction::Pseudo( - PseudoInstruction::Jump { .. } - | PseudoInstruction::JumpNoInterrupt { .. }, - ) if entry.target_offset.is_none() => { - return Err(InternalError::MalformedControlFlowGraph); - } - // CPython assemble.c::resolve_unconditional_jumps() - // resolves pseudo JUMP/JUMP_NO_INTERRUPT after label-map - // application, using instruction indexes rather than CFG - // block order or byte offsets. - AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { - if entry.target_offset.expect("missing jump target") - > current_instr_index - { - Instruction::JumpForward { - delta: Arg::marker(), - } - } else { - Instruction::JumpBackward { - delta: Arg::marker(), - } - } - } - AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) => { - if entry.target_offset.expect("missing jump target") - > current_instr_index - { - Instruction::JumpForward { - delta: Arg::marker(), - } - } else { - Instruction::JumpBackwardNoInterrupt { - delta: Arg::marker(), - } - } - } - _ => info.instr.expect_real(), - }; + let op = info.instr.expect_real(); if let Some(target_index) = entry.target_offset { let target_offset = instruction_offsets[target_index]; @@ -1455,41 +1527,24 @@ impl CodeInfo { } /// flowgraph.c remove_unreachable - fn remove_unreachable_blocks(&mut self) { - let mut reachable = vec![false; self.blocks.len()]; - reachable[0] = true; - let mut stack = vec![BlockIdx(0)]; - while let Some(block_idx) = stack.pop() { - let idx = block_idx.idx(); - let block = &self.blocks[idx]; - let next = block.next; - if next != BlockIdx::NULL && block_has_fallthrough(block) && !reachable[next.idx()] { - reachable[next.idx()] = true; - stack.push(next); - } - for ins in &block.instructions { - if (is_jump_instruction(ins) || ins.instr.is_block_push()) - && ins.target != BlockIdx::NULL - && !reachable[ins.target.idx()] - { - reachable[ins.target.idx()] = true; - stack.push(ins.target); - } - } - } + fn remove_unreachable(blocks: &mut [Block]) { + compute_reachable_predecessors(blocks); - for block_idx in self.block_next_order() { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { let i = block_idx.idx(); - let is_reachable = reachable[i]; - if !is_reachable { - let block = &mut self.blocks[i]; - block.instructions.clear(); + let next = blocks[i].next; + if blocks[i].predecessors == 0 { + let block = &mut blocks[i]; + basicblock_clear(block); block.except_handler = false; } + block_idx = next; } } - fn eval_unary_constant( + /// flowgraph.c eval_const_unaryop + fn eval_const_unaryop( operand: &ConstantData, op: Instruction, intrinsic: Option, @@ -1513,6 +1568,9 @@ impl CodeInfo { Some(ConstantData::Integer { value: !value }) } (ConstantData::Boolean { .. }, Instruction::UnaryInvert, None) => None, + (_, Instruction::UnaryNot, None) => Some(ConstantData::Boolean { + value: !Self::constant_truthiness(operand), + }), ( ConstantData::Integer { value }, Instruction::CallIntrinsic1 { .. }, @@ -1541,16 +1599,43 @@ impl CodeInfo { } } + fn constant_truthiness(constant: &ConstantData) -> bool { + match constant { + ConstantData::Tuple { elements } | ConstantData::Frozenset { elements } => { + !elements.is_empty() + } + ConstantData::Integer { value } => !value.is_zero(), + ConstantData::Float { value } => *value != 0.0, + ConstantData::Complex { value } => value.re != 0.0 || value.im != 0.0, + ConstantData::Boolean { value } => *value, + ConstantData::Str { value } => !value.is_empty(), + ConstantData::Bytes { value } => !value.is_empty(), + ConstantData::Code { .. } | ConstantData::Slice { .. } | ConstantData::Ellipsis => true, + ConstantData::None => false, + } + } + + fn load_const_truthiness( + instr: Instruction, + arg: OpArg, + metadata: &CodeUnitMetadata, + ) -> Option { + match instr { + Instruction::LoadConst { consti } => { + let constant = &metadata.consts[consti.get(arg).as_usize()]; + Some(Self::constant_truthiness(constant)) + } + Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), + _ => None, + } + } + fn instr_make_load_const( metadata: &mut CodeUnitMetadata, instr: &mut InstructionInfo, constant: ConstantData, ) { - if let ConstantData::Integer { value } = &constant - && let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) - { - instr.instr = Opcode::LoadSmallInt.into(); - instr.arg = OpArg::new(small as u32); + if Self::maybe_instr_make_load_smallint(instr, &constant) { return; } @@ -1562,17 +1647,13 @@ impl CodeInfo { instr.arg = OpArg::new(const_idx as u32); } - /// Try to fold a single unary instruction at position `i` in `block`. - /// Returns true if folded. Mirrors CPython fold_const_unaryop(). - fn fold_unary_constant_at( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, - ) -> bool { + /// flowgraph.c fold_const_unaryop + fn fold_const_unaryop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { let instr = &block.instructions[i]; let (op, intrinsic) = match instr.instr.real() { Some(Instruction::UnaryNegative) => (Instruction::UnaryNegative, None), Some(Instruction::UnaryInvert) => (Instruction::UnaryInvert, None), + Some(Instruction::UnaryNot) => (Instruction::UnaryNot, None), Some(Instruction::CallIntrinsic1 { func }) if matches!( func.get(instr.arg), @@ -1590,33 +1671,25 @@ impl CodeInfo { }; let Some(operand_index) = i .checked_sub(1) - .and_then(|start| Self::get_const_loading_instr_indices(block, start, 1)) + .and_then(|start| Self::get_const_loading_instrs(block, start, 1)) .and_then(|indices| indices.into_iter().next()) else { return false; }; - let operand = Self::get_const_value_from(metadata, &block.instructions[operand_index]); + let operand = Self::get_const_value(metadata, &block.instructions[operand_index]); let Some(operand) = operand else { return false; }; - let Some(folded_const) = Self::eval_unary_constant(&operand, op, intrinsic) else { + let Some(folded_const) = Self::eval_const_unaryop(&operand, op, intrinsic) else { return false; }; - nop_out_no_location(&mut block.instructions[operand_index]); - let mut prev = operand_index; - while let Some(idx) = prev.checked_sub(1) { - if !matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { - break; - } - block.instructions[idx].location = block.instructions[i].location; - block.instructions[idx].end_location = block.instructions[i].end_location; - prev = idx; - } + Self::nop_out(block, &[operand_index]); Self::instr_make_load_const(metadata, &mut block.instructions[i], folded_const); true } - fn get_const_loading_instr_indices( + /// flowgraph.c get_const_loading_instrs + fn get_const_loading_instrs( block: &Block, mut start: usize, size: usize, @@ -1625,7 +1698,9 @@ impl CodeInfo { loop { let instr = block.instructions.get(start)?; if !matches!(instr.instr.real(), Some(Instruction::Nop)) { - Self::get_const_value_from_dummy(instr)?; + if !Self::loads_const(instr) { + return None; + } indices.push(start); if indices.len() == size { break; @@ -1637,48 +1712,15 @@ impl CodeInfo { Some(indices) } - fn get_const_sequence( - metadata: &CodeUnitMetadata, - block: &Block, - build_index: usize, - size: usize, - ) -> Option<(Vec, Vec)> { - if size == 0 { - return Some((Vec::new(), Vec::new())); - } - - let operand_indices = build_index - .checked_sub(1) - .and_then(|start| Self::get_const_loading_instr_indices(block, start, size))?; - let mut elements = Vec::with_capacity(size); - - for &j in &operand_indices { - elements.push(Self::get_const_value_from( - metadata, - &block.instructions[j], - )?); - } - - Some((operand_indices, elements)) - } - - fn block_next_order(&self) -> Vec { - let mut order = Vec::new(); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - order.push(current); - current = self.blocks[current.idx()].next; + /// flowgraph.c nop_out + fn nop_out(block: &mut Block, instrs: &[usize]) { + for &i in instrs { + nop_out_no_location(&mut block.instructions[i]); } - order } - /// Try to fold a single BINARY_OP instruction at position `i` in `block`. - /// Returns true if folded. Mirrors CPython fold_const_binop(). - fn fold_binop_constant_at( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, - ) -> bool { + /// flowgraph.c fold_const_binop + fn fold_const_binop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { use oparg::BinaryOperator as BinOp; let Some(Instruction::BinaryOp { .. }) = block.instructions[i].instr.real() else { @@ -1686,7 +1728,7 @@ impl CodeInfo { }; let Some(operand_indices) = i .checked_sub(1) - .and_then(|start| Self::get_const_loading_instr_indices(block, start, 2)) + .and_then(|start| Self::get_const_loading_instrs(block, start, 2)) else { return false; }; @@ -1694,47 +1736,46 @@ impl CodeInfo { let Ok(op) = BinOp::try_from(op_raw) else { return false; }; - let left = Self::get_const_value_from(metadata, &block.instructions[operand_indices[0]]); - let right = Self::get_const_value_from(metadata, &block.instructions[operand_indices[1]]); + let left = Self::get_const_value(metadata, &block.instructions[operand_indices[0]]); + let right = Self::get_const_value(metadata, &block.instructions[operand_indices[1]]); let (Some(left_val), Some(right_val)) = (left, right) else { return false; }; - let Some(result_const) = Self::eval_binop(&left_val, &right_val, op) else { + let Some(result_const) = Self::eval_const_binop(&left_val, &right_val, op) else { return false; }; - for &idx in &operand_indices { - nop_out_no_location(&mut block.instructions[idx]); - } + Self::nop_out(block, &operand_indices); Self::instr_make_load_const(metadata, &mut block.instructions[i], result_const); true } - fn get_const_value_from_dummy(info: &InstructionInfo) -> Option<()> { - match info.instr.real() { - Some(Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }) => Some(()), - _ => None, - } + /// flowgraph.c loads_const + fn loads_const(info: &InstructionInfo) -> bool { + info.instr.has_const() + || matches!(info.instr.real(), Some(Instruction::LoadSmallInt { .. })) } - fn get_const_value_from( + /// flowgraph.c get_const_value + fn get_const_value( metadata: &CodeUnitMetadata, info: &InstructionInfo, ) -> Option { match info.instr.real() { - Some(Instruction::LoadConst { .. }) => { - let idx = u32::from(info.arg) as usize; - metadata.consts.get_index(idx).cloned() - } Some(Instruction::LoadSmallInt { .. }) => { let v = u32::from(info.arg) as i32; Some(ConstantData::Integer { value: BigInt::from(v), }) } + _ if info.instr.has_const() => { + let idx = u32::from(info.arg) as usize; + metadata.consts.get_index(idx).cloned() + } _ => None, } } + /// flowgraph.c const_folding_check_complexity fn const_folding_check_complexity(obj: &ConstantData, mut limit: isize) -> Option { if let ConstantData::Tuple { elements } = obj { limit -= isize::try_from(elements.len()).ok()?; @@ -1748,29 +1789,202 @@ impl CodeInfo { Some(limit) } - fn eval_binop( + fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Wtf8Buf { + let mut result = Wtf8Buf::with_capacity(value.len().saturating_mul(n)); + for _ in 0..n { + result.push_wtf8(value); + } + result + } + + fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { + let n = n.to_isize()?; + if item_size != 0 && (n < 0 || n as usize > MAX_STR_SIZE / item_size) { + return None; + } + Some(n.max(0) as usize) + } + + /// flowgraph.c const_folding_safe_multiply + fn const_folding_safe_multiply( left: &ConstantData, right: &ConstantData, - op: oparg::BinaryOperator, ) -> Option { - use oparg::BinaryOperator as BinOp; + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if !l.is_zero() && !r.is_zero() && l.bits() + r.bits() > MAX_INT_SIZE { + return None; + } + Some(ConstantData::Integer { value: l * r }) + } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + Some(ConstantData::Float { value: l * r }) + } + (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) => { + let n = Self::checked_repeat_count(n, s.code_points().count())?; + Some(ConstantData::Str { + value: Self::repeat_wtf8(s, n), + }) + } + (ConstantData::Integer { .. }, ConstantData::Str { .. }) => { + Self::const_folding_safe_multiply(right, left) + } + (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) => { + let n = Self::checked_repeat_count(n, b.len())?; + Some(ConstantData::Bytes { value: b.repeat(n) }) + } + (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) => { + Self::const_folding_safe_multiply(right, left) + } + (ConstantData::Tuple { elements }, ConstantData::Integer { value: n }) => { + let n = n.to_usize()?; + if n != 0 && !elements.is_empty() { + if n > MAX_COLLECTION_SIZE / elements.len() { + return None; + } + Self::const_folding_check_complexity( + &ConstantData::Tuple { + elements: elements.clone(), + }, + MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, + )?; + } + let mut result = Vec::with_capacity(elements.len() * n); + for _ in 0..n { + result.extend(elements.iter().cloned()); + } + Some(ConstantData::Tuple { elements: result }) + } + (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) => { + Self::const_folding_safe_multiply(right, left) + } + _ => None, + } + } - fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Wtf8Buf { - let mut result = Wtf8Buf::with_capacity(value.len().saturating_mul(n)); - for _ in 0..n { - result.push_wtf8(value); + /// flowgraph.c const_folding_safe_power + fn const_folding_safe_power(left: &ConstantData, right: &ConstantData) -> Option { + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if r < &BigInt::from(0) { + if l.is_zero() { + return None; + } + let base = l.to_f64()?; + if !base.is_finite() { + return None; + } + let result = if let Some(exp) = r.to_i32() { + base.powi(exp) + } else { + base.powf(r.to_f64()?) + }; + if !result.is_finite() { + return None; + } + return Some(ConstantData::Float { value: result }); + } + let exp: u64 = r.try_into().ok()?; + let exp_usize = usize::try_from(exp).ok()?; + if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE / exp { + return None; + } + Some(ConstantData::Integer { + value: num_traits::pow::pow(l.clone(), exp_usize), + }) + } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let result = l.powf(*r); + result + .is_finite() + .then_some(ConstantData::Float { value: result }) } - result + _ => None, } + } - fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { - let n = n.to_isize()?; - if item_size != 0 && (n < 0 || n as usize > MAX_STR_SIZE / item_size) { - return None; + /// flowgraph.c const_folding_safe_lshift + fn const_folding_safe_lshift( + left: &ConstantData, + right: &ConstantData, + ) -> Option { + let (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) = + (left, right) + else { + return None; + }; + let shift: u64 = r.try_into().ok()?; + let shift_usize = usize::try_from(shift).ok()?; + if shift > MAX_INT_SIZE || (!l.is_zero() && l.bits() > MAX_INT_SIZE - shift) { + return None; + } + Some(ConstantData::Integer { + value: l << shift_usize, + }) + } + + /// flowgraph.c const_folding_safe_mod + fn const_folding_safe_mod(left: &ConstantData, right: &ConstantData) -> Option { + if matches!(left, ConstantData::Str { .. } | ConstantData::Bytes { .. }) { + return None; + } + + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if r.is_zero() { + return None; + } + let rem = l.clone() % r.clone(); + let value = if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { + rem + r + } else { + rem + }; + Some(ConstantData::Integer { value }) + } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let (_, modulo) = Self::float_div_mod(*l, *r)?; + Some(ConstantData::Float { value: modulo }) } - Some(n.max(0) as usize) + _ => None, + } + } + + fn float_div_mod(left: f64, right: f64) -> Option<(f64, f64)> { + if right == 0.0 { + return None; } + let mut modulo = left % right; + let div = (left - modulo) / right; + let floordiv = if modulo != 0.0 { + let div = if (right < 0.0) != (modulo < 0.0) { + modulo += right; + div - 1.0 + } else { + div + }; + let mut floordiv = div.floor(); + if div - floordiv > 0.5 { + floordiv += 1.0; + } + floordiv + } else { + modulo = 0.0f64.copysign(right); + 0.0f64.copysign(left / right) + }; + + Some((floordiv, modulo)) + } + + /// flowgraph.c eval_const_binop + fn eval_const_binop( + left: &ConstantData, + right: &ConstantData, + op: oparg::BinaryOperator, + ) -> Option { + use oparg::BinaryOperator as BinOp; + fn eval_complex_binop( left: Complex, right: Complex, @@ -1834,33 +2048,6 @@ impl CodeInfo { complex_const(value) } - fn float_div_mod(left: f64, right: f64) -> Option<(f64, f64)> { - if right == 0.0 { - return None; - } - - let mut modulo = left % right; - let div = (left - modulo) / right; - let floordiv = if modulo != 0.0 { - let div = if (right < 0.0) != (modulo < 0.0) { - modulo += right; - div - 1.0 - } else { - div - }; - let mut floordiv = div.floor(); - if div - floordiv > 0.5 { - floordiv += 1.0; - } - floordiv - } else { - modulo = 0.0f64.copysign(right); - 0.0f64.copysign(left / right) - }; - - Some((floordiv, modulo)) - } - fn constant_as_index(value: &ConstantData) -> Option { match value { ConstantData::Integer { value } => value.to_i64().or_else(|| { @@ -2045,7 +2232,7 @@ impl CodeInfo { } } - return Self::eval_binop( + return Self::eval_const_binop( &ConstantData::Integer { value: left_int }, &ConstantData::Integer { value: right_int }, op, @@ -2058,10 +2245,7 @@ impl CodeInfo { BinOp::Add => l + r, BinOp::Subtract => l - r, BinOp::Multiply => { - if !l.is_zero() && !r.is_zero() && l.bits() + r.bits() > MAX_INT_SIZE_BITS { - return None; - } - l * r + return Self::const_folding_safe_multiply(left, right); } BinOp::TrueDivide => { if r.is_zero() { @@ -2087,54 +2271,9 @@ impl CodeInfo { q } } - BinOp::Remainder => { - if r.is_zero() { - return None; - } - // Python modulo: result has same sign as divisor - let rem = l.clone() % r.clone(); - if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { - rem + r - } else { - rem - } - } - BinOp::Power => { - if r < &BigInt::from(0) { - if l.is_zero() { - return None; - } - let base = l.to_f64()?; - if !base.is_finite() { - return None; - } - let result = if let Some(exp) = r.to_i32() { - base.powi(exp) - } else { - base.powf(r.to_f64()?) - }; - if !result.is_finite() { - return None; - } - return Some(ConstantData::Float { value: result }); - } - let exp: u64 = r.try_into().ok()?; - let exp_usize = usize::try_from(exp).ok()?; - if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE_BITS / exp { - return None; - } - num_traits::pow::pow(l.clone(), exp_usize) - } - BinOp::Lshift => { - let shift: u64 = r.try_into().ok()?; - let shift_usize = usize::try_from(shift).ok()?; - if shift > MAX_INT_SIZE_BITS - || (!l.is_zero() && l.bits() > MAX_INT_SIZE_BITS - shift) - { - return None; - } - l << shift_usize - } + BinOp::Remainder => return Self::const_folding_safe_mod(left, right), + BinOp::Power => return Self::const_folding_safe_power(left, right), + BinOp::Lshift => return Self::const_folding_safe_lshift(left, right), BinOp::Rshift => { let shift: u32 = r.try_into().ok()?; l >> (shift as usize) @@ -2150,7 +2289,7 @@ impl CodeInfo { let result = match op { BinOp::Add => l + r, BinOp::Subtract => l - r, - BinOp::Multiply => l * r, + BinOp::Multiply => return Self::const_folding_safe_multiply(left, right), BinOp::TrueDivide => { if *r == 0.0 { return None; @@ -2158,14 +2297,11 @@ impl CodeInfo { l / r } BinOp::FloorDivide => { - let (floordiv, _) = float_div_mod(*l, *r)?; + let (floordiv, _) = Self::float_div_mod(*l, *r)?; floordiv } - BinOp::Remainder => { - let (_, modulo) = float_div_mod(*l, *r)?; - modulo - } - BinOp::Power => l.powf(*r), + BinOp::Remainder => return Self::const_folding_safe_mod(left, right), + BinOp::Power => return Self::const_folding_safe_power(left, right), _ => return None, }; if matches!(op, BinOp::Power) && !result.is_finite() { @@ -2176,7 +2312,7 @@ impl CodeInfo { // Int op Float or Float op Int → Float (ConstantData::Integer { value: l }, ConstantData::Float { value: r }) => { let l_f = l.to_f64()?; - Self::eval_binop( + Self::eval_const_binop( &ConstantData::Float { value: l_f }, &ConstantData::Float { value: *r }, op, @@ -2184,7 +2320,7 @@ impl CodeInfo { } (ConstantData::Float { value: l }, ConstantData::Integer { value: r }) => { let r_f = r.to_f64()?; - Self::eval_binop( + Self::eval_const_binop( &ConstantData::Float { value: *l }, &ConstantData::Float { value: r_f }, op, @@ -2213,12 +2349,10 @@ impl CodeInfo { result.push_wtf8(r); Some(ConstantData::Str { value: result }) } - (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) + (ConstantData::Str { .. }, ConstantData::Integer { .. }) if matches!(op, BinOp::Multiply) => { - let n = checked_repeat_count(n, s.code_points().count())?; - let result = repeat_wtf8(s, n); - Some(ConstantData::Str { value: result }) + Self::const_folding_safe_multiply(left, right) } (ConstantData::Tuple { elements: l }, ConstantData::Tuple { elements: r }) if matches!(op, BinOp::Add) => @@ -2227,54 +2361,20 @@ impl CodeInfo { result.extend(r.iter().cloned()); Some(ConstantData::Tuple { elements: result }) } - (ConstantData::Tuple { elements }, ConstantData::Integer { value: n }) + (ConstantData::Tuple { .. }, ConstantData::Integer { .. }) if matches!(op, BinOp::Multiply) => { - let n = n.to_usize()?; - if n != 0 && !elements.is_empty() { - if n > MAX_COLLECTION_SIZE / elements.len() { - return None; - } - Self::const_folding_check_complexity( - &ConstantData::Tuple { - elements: elements.clone(), - }, - MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, - )?; - } - let mut result = Vec::with_capacity(elements.len() * n); - for _ in 0..n { - result.extend(elements.iter().cloned()); - } - Some(ConstantData::Tuple { elements: result }) + Self::const_folding_safe_multiply(left, right) } - (ConstantData::Integer { value: n }, ConstantData::Tuple { elements }) + (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) if matches!(op, BinOp::Multiply) => { - let n = n.to_usize()?; - if n != 0 && !elements.is_empty() { - if n > MAX_COLLECTION_SIZE / elements.len() { - return None; - } - Self::const_folding_check_complexity( - &ConstantData::Tuple { - elements: elements.clone(), - }, - MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, - )?; - } - let mut result = Vec::with_capacity(elements.len() * n); - for _ in 0..n { - result.extend(elements.iter().cloned()); - } - Some(ConstantData::Tuple { elements: result }) + Self::const_folding_safe_multiply(left, right) } - (ConstantData::Integer { value: n }, ConstantData::Str { value: s }) + (ConstantData::Integer { .. }, ConstantData::Str { .. }) if matches!(op, BinOp::Multiply) => { - let n = checked_repeat_count(n, s.code_points().count())?; - let result = repeat_wtf8(s, n); - Some(ConstantData::Str { value: result }) + Self::const_folding_safe_multiply(left, right) } (ConstantData::Bytes { value: l }, ConstantData::Bytes { value: r }) if matches!(op, BinOp::Add) => @@ -2283,23 +2383,22 @@ impl CodeInfo { result.extend_from_slice(r); Some(ConstantData::Bytes { value: result }) } - (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) + (ConstantData::Bytes { .. }, ConstantData::Integer { .. }) if matches!(op, BinOp::Multiply) => { - let n = checked_repeat_count(n, b.len())?; - Some(ConstantData::Bytes { value: b.repeat(n) }) + Self::const_folding_safe_multiply(left, right) } - (ConstantData::Integer { value: n }, ConstantData::Bytes { value: b }) + (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) if matches!(op, BinOp::Multiply) => { - let n = checked_repeat_count(n, b.len())?; - Some(ConstantData::Bytes { value: b.repeat(n) }) + Self::const_folding_safe_multiply(left, right) } _ => None, } } - fn fold_tuple_constant_at( + /// flowgraph.c fold_tuple_of_constants + fn fold_tuple_of_constants( metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize, @@ -2309,51 +2408,39 @@ impl CodeInfo { }; let tuple_size = u32::from(block.instructions[i].arg) as usize; - if tuple_size <= 3 - && block - .instructions - .get(i + 1) - .and_then(|next| next.instr.real()) - .is_some_and(|next| { - matches!( - next, - Instruction::UnpackSequence { .. } - if usize::try_from(u32::from(block.instructions[i + 1].arg)).ok() - == Some(tuple_size) - ) - }) - { + if tuple_size > STACK_USE_GUIDELINE { return false; } - if tuple_size == 0 { - let (const_idx, _) = metadata.consts.insert_full(ConstantData::Tuple { - elements: Vec::new(), - }); - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - return true; - } - let Some((operand_indices, elements)) = - Self::get_const_sequence(metadata, block, i, tuple_size) - else { + let Some(operand_indices) = (if tuple_size == 0 { + Some(Vec::new()) + } else { + i.checked_sub(1) + .and_then(|start| Self::get_const_loading_instrs(block, start, tuple_size)) + }) else { return false; }; + let mut elements = Vec::with_capacity(tuple_size); + for &j in &operand_indices { + let Some(element) = Self::get_const_value(metadata, &block.instructions[j]) else { + return false; + }; + elements.push(element); + } + let (const_idx, _) = metadata .consts .insert_full(ConstantData::Tuple { elements }); - for &j in &operand_indices { - nop_out_no_location(&mut block.instructions[j]); - } + Self::nop_out(block, &operand_indices); block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); true } - fn fold_constant_intrinsic_list_to_tuple_at( + fn fold_constant_intrinsic_list_to_tuple( metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize, @@ -2364,14 +2451,6 @@ impl CodeInfo { if func.get(block.instructions[i].arg) != IntrinsicFunction1::ListToTuple { return false; } - if block - .instructions - .get(i + 1) - .and_then(|instr| instr.instr.real()) - .is_some_and(|instr| matches!(instr, Instruction::GetIter)) - { - return false; - } let mut consts_found = 0usize; let mut expect_append = true; @@ -2390,35 +2469,31 @@ impl CodeInfo { return false; } - let mut elements = Vec::with_capacity(consts_found); - let mut expect_load = true; - for idx in pos + 1..i { - let instr = &block.instructions[idx]; - if matches!(instr.instr.real(), Some(Instruction::Nop)) { + let mut elements = vec![None; consts_found]; + let mut remaining = consts_found; + for idx in (pos..i).rev() { + if matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { continue; } - if expect_load { - let Some(value) = Self::get_const_value_from(metadata, instr) else { + if Self::loads_const(&block.instructions[idx]) { + let Some(value) = Self::get_const_value(metadata, &block.instructions[idx]) + else { return false; }; - elements.push(value); - } else if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) - || u32::from(instr.arg) != 1 - { - return false; + debug_assert!(remaining > 0); + remaining -= 1; + elements[remaining] = Some(value); } - expect_load = !expect_load; - } - if !expect_load || elements.len() != consts_found { - return false; + nop_out_no_location(&mut block.instructions[idx]); } - + debug_assert_eq!(remaining, 0); + let elements = elements + .into_iter() + .map(|element| element.expect("constant list item was collected")) + .collect(); let (const_idx, _) = metadata .consts .insert_full(ConstantData::Tuple { elements }); - for idx in pos..i { - nop_out_no_location(&mut block.instructions[idx]); - } block.instructions[i].instr = Instruction::LoadConst { consti: Arg::marker(), } @@ -2434,7 +2509,7 @@ impl CodeInfo { return false; } } else { - if Self::get_const_value_from_dummy(instr).is_none() { + if !Self::loads_const(instr) { return false; } consts_found += 1; @@ -2445,66 +2520,8 @@ impl CodeInfo { false } - fn fold_list_constant_at(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { - let Some(Instruction::BuildList { .. }) = block.instructions[i].instr.real() else { - return false; - }; - - let list_size = u32::from(block.instructions[i].arg) as usize; - if list_size == 0 || list_size > STACK_USE_GUIDELINE { - return false; - } - - let Some((operand_indices, elements)) = - Self::get_const_sequence(metadata, block, i, list_size) - else { - return false; - }; - if list_size < MIN_CONST_SEQUENCE_SIZE { - return false; - } - - let (const_idx, _) = metadata - .consts - .insert_full(ConstantData::Tuple { elements }); - - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - let eh = block.instructions[i].except_handler; - - let build_idx = operand_indices[0]; - let const_idx_slot = operand_indices[1]; - - block.instructions[build_idx].instr = Instruction::BuildList { - count: Arg::marker(), - } - .into(); - block.instructions[build_idx].arg = OpArg::new(0); - block.instructions[build_idx].location = folded_loc; - block.instructions[build_idx].end_location = end_loc; - block.instructions[build_idx].except_handler = eh; - - block.instructions[const_idx_slot].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); - block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); - block.instructions[const_idx_slot].location = folded_loc; - block.instructions[const_idx_slot].end_location = end_loc; - block.instructions[const_idx_slot].except_handler = eh; - - for &j in &operand_indices[2..] { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - } - - block.instructions[i].instr = Opcode::ListExtend.into(); - block.instructions[i].arg = OpArg::new(1); - true - } - /// Port of CPython's flowgraph.c optimize_lists_and_sets(). - fn optimize_lists_and_sets_at( + fn optimize_lists_and_sets( metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize, @@ -2530,9 +2547,12 @@ impl CodeInfo { return false; } - let Some((operand_indices, elements)) = - Self::get_const_sequence(metadata, block, i, seq_size) - else { + let Some(operand_indices) = (if seq_size == 0 { + Some(Vec::new()) + } else { + i.checked_sub(1) + .and_then(|start| Self::get_const_loading_instrs(block, start, seq_size)) + }) else { if contains_or_iter && is_list { block.instructions[i].instr = Opcode::BuildTuple.into(); return true; @@ -2540,12 +2560,12 @@ impl CodeInfo { return false; }; - if !contains_or_iter { - return if is_list { - Self::fold_list_constant_at(metadata, block, i) - } else { - Self::fold_set_constant_at(metadata, block, i) + let mut elements = Vec::with_capacity(seq_size); + for &j in &operand_indices { + let Some(element) = Self::get_const_value(metadata, &block.instructions[j]) else { + return false; }; + elements.push(element); } let const_data = if is_list { @@ -2554,75 +2574,50 @@ impl CodeInfo { ConstantData::Frozenset { elements } }; let (const_idx, _) = metadata.consts.insert_full(const_data); - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - let eh = block.instructions[i].except_handler; - - for &j in &operand_indices { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - block.instructions[j].end_location = end_loc; - } - - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - block.instructions[i].location = folded_loc; - block.instructions[i].end_location = end_loc; - block.instructions[i].except_handler = eh; - true - } - - fn fold_set_constant_at(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { - let Some(Instruction::BuildSet { .. }) = block.instructions[i].instr.real() else { - return false; - }; - let set_size = u32::from(block.instructions[i].arg) as usize; - if !(3..=STACK_USE_GUIDELINE).contains(&set_size) { - return false; - } - - let Some((operand_indices, elements)) = - Self::get_const_sequence(metadata, block, i, set_size) - else { - return false; - }; - let (const_idx, _) = metadata - .consts - .insert_full(ConstantData::Frozenset { elements }); + if !contains_or_iter { + debug_assert!(i >= 2); + let folded_loc = block.instructions[i].location; + let end_loc = block.instructions[i].end_location; - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - let eh = block.instructions[i].except_handler; + Self::nop_out(block, &operand_indices); - let build_idx = operand_indices[0]; - let const_idx_slot = operand_indices[1]; + if is_list { + block.instructions[i - 2].instr = Instruction::BuildList { + count: Arg::marker(), + } + .into(); + } else { + block.instructions[i - 2].instr = Instruction::BuildSet { + count: Arg::marker(), + } + .into(); + } + block.instructions[i - 2].arg = OpArg::new(0); + block.instructions[i - 2].location = folded_loc; + block.instructions[i - 2].end_location = end_loc; + block.instructions[i - 2].lineno_override = None; - block.instructions[build_idx].instr = Instruction::BuildSet { - count: Arg::marker(), - } - .into(); - block.instructions[build_idx].arg = OpArg::new(0); - block.instructions[build_idx].location = folded_loc; - block.instructions[build_idx].end_location = end_loc; - block.instructions[build_idx].except_handler = eh; + block.instructions[i - 1].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[i - 1].arg = OpArg::new(const_idx as u32); - block.instructions[const_idx_slot].instr = Instruction::LoadConst { - consti: Arg::marker(), + block.instructions[i].instr = if is_list { + Opcode::ListExtend + } else { + Opcode::SetUpdate + } + .into(); + block.instructions[i].arg = OpArg::new(1); + return true; } - .into(); - block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); - block.instructions[const_idx_slot].location = folded_loc; - block.instructions[const_idx_slot].end_location = end_loc; - block.instructions[const_idx_slot].except_handler = eh; - for &j in &operand_indices[2..] { - set_to_nop(&mut block.instructions[j]); - block.instructions[j].location = folded_loc; - } + Self::nop_out(block, &operand_indices); - block.instructions[i].instr = Opcode::SetUpdate.into(); - block.instructions[i].arg = OpArg::new(1); + block.instructions[i].instr = Opcode::LoadConst.into(); + block.instructions[i].arg = OpArg::new(const_idx as u32); true } @@ -2671,8 +2666,8 @@ impl CodeInfo { return None; } let info = &instructions[i]; - let info_lineno = info.location.line.get() as i32; - if lineno >= 0 && info_lineno > 0 && info_lineno != lineno { + let info_lineno = instruction_lineno(info); + if lineno >= 0 && info_lineno != lineno { return None; } if matches!(info.instr, AnyInstruction::Real(Instruction::Nop)) { @@ -2685,75 +2680,70 @@ impl CodeInfo { } } - fn optimize_swap_block(instructions: &mut [InstructionInfo]) { - let mut i = 0usize; - while i < instructions.len() { - let AnyInstruction::Real(Instruction::Swap { .. }) = instructions[i].instr else { - i += 1; - continue; - }; - - let mut len = 0usize; - let mut depth = 0usize; - let mut more = false; - while i + len < instructions.len() { - let info = &instructions[i + len]; - match info.instr.real() { - Some(Instruction::Swap { .. }) => { - let oparg = u32::from(info.arg) as usize; - depth = depth.max(oparg); - more |= len > 0; - len += 1; - } - Some(Instruction::Nop) => { - len += 1; - } - _ => break, + fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) { + debug_assert!(matches!( + instructions[*ix].instr.real(), + Some(Instruction::Swap { .. }) + )); + let mut depth = u32::from(instructions[*ix].arg) as usize; + let mut len = 1usize; + let mut more = false; + let limit = instructions.len() - *ix; + while len < limit { + match instructions[*ix + len].instr.real() { + Some(Instruction::Swap { .. }) => { + depth = depth.max(u32::from(instructions[*ix + len].arg) as usize); + more = true; + len += 1; + } + Some(Instruction::Nop) => { + len += 1; } + _ => break, } + } - if !more { - i += len.max(1); - continue; - } + if !more { + return; + } - let mut stack: Vec = (0..depth as i32).collect(); - for info in &instructions[i..i + len] { - if matches!(info.instr.real(), Some(Instruction::Swap { .. })) { - let oparg = u32::from(info.arg) as usize; - stack.swap(0, oparg - 1); - } + let mut stack: Vec = (0..depth as i32).collect(); + for info in &instructions[*ix..*ix + len] { + if matches!(info.instr.real(), Some(Instruction::Swap { .. })) { + let oparg = u32::from(info.arg) as usize; + stack.swap(0, oparg - 1); } + } - let mut current = len as isize - 1; - for slot in 0..depth { - if stack[slot] == VISITED || stack[slot] == slot as i32 { - continue; + let mut current = len as isize - 1; + for i in 0..depth { + if stack[i] == VISITED || stack[i] == i as i32 { + continue; + } + let mut j = i; + loop { + if j != 0 { + debug_assert!(current >= 0); + let out = &mut instructions[*ix + current as usize]; + out.instr = Opcode::Swap.into(); + out.arg = OpArg::new((j + 1) as u32); + current -= 1; } - let mut j = slot; - loop { - if j != 0 { - let out = &mut instructions[i + current as usize]; - out.instr = Opcode::Swap.into(); - out.arg = OpArg::new((j + 1) as u32); - out.target = BlockIdx::NULL; - current -= 1; - } - if stack[j] == VISITED { - debug_assert_eq!(j, slot); - break; - } - let next_j = stack[j] as usize; - stack[j] = VISITED; - j = next_j; + if stack[j] == VISITED { + debug_assert_eq!(j, i); + break; } + let next_j = stack[j] as usize; + stack[j] = VISITED; + j = next_j; } - while current >= 0 { - set_to_nop(&mut instructions[i + current as usize]); - current -= 1; - } - i += len; } + + while current >= 0 { + set_to_nop(&mut instructions[*ix + current as usize]); + current -= 1; + } + *ix += len - 1; } fn apply_from(instructions: &mut [InstructionInfo], mut i: isize) { @@ -2785,7 +2775,7 @@ impl CodeInfo { let Some(j) = next_swappable(instructions, idx, -1) else { return; }; - let lineno = instructions[j].location.line.get() as i32; + let lineno = instruction_lineno(&instructions[j]); let mut k = j; for _ in 1..swap_arg { let Some(next) = next_swappable(instructions, k, lineno) else { @@ -2812,388 +2802,389 @@ impl CodeInfo { } } - instructions[idx].instr = Opcode::Nop.into(); - instructions[idx].arg = OpArg::new(0); + set_to_nop(&mut instructions[idx]); instructions.swap(j, k); i -= 1; } } - optimize_swap_block(&mut block.instructions); - let len = block.instructions.len(); - for i in 0..len { + let mut i = 0; + while i < block.instructions.len() { if matches!( block.instructions[i].instr.real(), Some(Instruction::Swap { .. }) ) { + swaptimize(&mut block.instructions, &mut i); apply_from(&mut block.instructions, i as isize); } + i += 1; } } - /// Peephole optimization: combine consecutive instructions into super-instructions - fn peephole_optimize(&mut self) { - let const_truthiness = - |instr: Instruction, arg: OpArg, metadata: &CodeUnitMetadata| match instr { - Instruction::LoadConst { consti } => { - let constant = &metadata.consts[consti.get(arg).as_usize()]; - match constant { - ConstantData::Tuple { .. } => None, - ConstantData::Integer { value } => Some(!value.is_zero()), - ConstantData::Float { value } => Some(*value != 0.0), - ConstantData::Complex { value } => Some(value.re != 0.0 || value.im != 0.0), - ConstantData::Boolean { value } => Some(*value), - ConstantData::Str { value } => Some(!value.is_empty()), - ConstantData::Bytes { value } => Some(!value.is_empty()), - ConstantData::Code { .. } => Some(true), - ConstantData::Slice { .. } => Some(true), - ConstantData::Frozenset { elements } => Some(!elements.is_empty()), - ConstantData::None => Some(false), - ConstantData::Ellipsis => Some(true), - } - } - Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), - _ => None, + /// flowgraph.c maybe_instr_make_load_smallint + fn maybe_instr_make_load_smallint( + instr: &mut InstructionInfo, + constant: &ConstantData, + ) -> bool { + if let ConstantData::Integer { value } = constant + && let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) + { + instr.instr = Opcode::LoadSmallInt.into(); + instr.arg = OpArg::new(small as u32); + return true; + } + false + } + + /// flowgraph.c basicblock_optimize_load_const + fn basicblock_optimize_load_const(metadata: &mut CodeUnitMetadata, block: &mut Block) { + let mut i = 0; + let mut effective_opcode = None; + let mut effective_oparg = OpArg::new(0); + while i < block.instructions.len() { + if matches!( + block.instructions[i].instr.real(), + Some(Instruction::LoadConst { .. }) + ) && let Some(constant) = Self::get_const_value(metadata, &block.instructions[i]) + { + Self::maybe_instr_make_load_smallint(&mut block.instructions[i], &constant); + } + + let curr = block.instructions[i]; + let curr_arg = curr.arg; + + // Only combine if the source is a real instruction. + let Some(curr_instr) = curr.instr.real() else { + i += 1; + continue; }; - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i + 1 < block.instructions.len() { - let curr = &block.instructions[i]; - let next = &block.instructions[i + 1]; - let curr_arg = curr.arg; - let next_arg = next.arg; - - // Only combine if the source is a real instruction. - let Some(curr_instr) = curr.instr.real() else { - i += 1; - continue; - }; - if let Some(is_true) = const_truthiness(curr_instr, curr.arg, &self.metadata) { - let jump_if_true = match next.instr.pseudo() { - Some(PseudoInstruction::JumpIfTrue { .. }) => Some(true), - Some(PseudoInstruction::JumpIfFalse { .. }) => Some(false), - _ => None, - }; - if let Some(jump_if_true) = jump_if_true { - // CPython flowgraph.c::basicblock_optimize_load_const() - // folds LOAD_CONST/LOAD_SMALL_INT followed by - // JUMP_IF_TRUE/FALSE. Unlike POP_JUMP_IF_*, these - // pseudo jumps do not consume the condition, so keep - // the constant for the following POP_TOP pair removal. - if is_true == jump_if_true { - block.instructions[i + 1].instr = PseudoInstruction::Jump { - delta: Arg::marker(), - } - .into(); - } else { - set_to_nop(&mut block.instructions[i + 1]); + let is_copy_of_load_const = matches!( + (effective_opcode, curr_instr), + (Some(Instruction::LoadConst { .. }), Instruction::Copy { i }) if i.get(curr_arg) == 1 + ); + if !is_copy_of_load_const { + effective_opcode = Some(curr_instr); + effective_oparg = curr_arg; + } + let Some(const_instr) = effective_opcode else { + i += 1; + continue; + }; + let const_arg = effective_oparg; + + if i + 1 >= block.instructions.len() { + i += 1; + continue; + } + + let next = block.instructions[i + 1]; + let next_arg = next.arg; + + if let Some(is_true) = Self::load_const_truthiness(const_instr, const_arg, metadata) { + let const_jump = match (next.instr.real(), next.instr.pseudo()) { + (_, Some(PseudoInstruction::JumpIfTrue { .. })) => Some((true, false)), + (_, Some(PseudoInstruction::JumpIfFalse { .. })) => Some((false, false)), + (Some(Instruction::PopJumpIfTrue { .. }), _) => Some((true, true)), + (Some(Instruction::PopJumpIfFalse { .. }), _) => Some((false, true)), + _ => None, + }; + if let Some((jump_if_true, pops_condition)) = const_jump { + if pops_condition { + set_to_nop(&mut block.instructions[i]); + } + if is_true == jump_if_true { + block.instructions[i + 1].instr = PseudoInstruction::Jump { + delta: Arg::marker(), } - i += 1; - continue; + .into(); + if let Some(next_instr) = next.instr.real() + && let Instruction::PopJumpIfTrue { delta } + | Instruction::PopJumpIfFalse { delta } = next_instr + { + block.instructions[i + 1].arg = + OpArg::new(u32::from(delta.get(next.arg))); + } + } else { + set_to_nop(&mut block.instructions[i + 1]); } - } - - // The remaining combinations require both instructions to be real. - let Some(next_instr) = next.instr.real() else { i += 1; continue; - }; + } + } - if let Some(is_true) = const_truthiness(curr_instr, curr.arg, &self.metadata) { - let jump_if_true = match next_instr { - Instruction::PopJumpIfTrue { .. } => Some(true), - Instruction::PopJumpIfFalse { .. } => Some(false), - _ => None, - }; - if let Some(jump_if_true) = jump_if_true { - let target = match next_instr { - Instruction::PopJumpIfTrue { delta } - | Instruction::PopJumpIfFalse { delta } => delta.get(next.arg), - _ => unreachable!(), - }; - set_to_nop(&mut block.instructions[i]); - if is_true == jump_if_true { - block.instructions[i + 1].instr = PseudoInstruction::Jump { - delta: Arg::marker(), - } - .into(); - block.instructions[i + 1].arg = OpArg::new(u32::from(target)); - } else { - set_to_nop(&mut block.instructions[i + 1]); - } + // The remaining combinations require both instructions to be real. + let Some(next_instr) = next.instr.real() else { + i += 1; + continue; + }; + + if let Instruction::LoadConst { consti } = const_instr { + let constant = &metadata.consts[consti.get(const_arg).as_usize()]; + if matches!(constant, ConstantData::None) + && let Instruction::IsOp { invert } = next_instr + { + let mut jump_idx = i + 2; + if jump_idx >= block.instructions.len() { i += 1; continue; } - } - if let Instruction::LoadConst { consti } = curr_instr { - let constant = &self.metadata.consts[consti.get(curr_arg).as_usize()]; - if matches!(constant, ConstantData::None) - && let Instruction::IsOp { invert } = next_instr - { - let mut jump_idx = i + 2; + if matches!( + block.instructions[jump_idx].instr.real(), + Some(Instruction::ToBool) + ) { + set_to_nop(&mut block.instructions[jump_idx]); + jump_idx += 1; if jump_idx >= block.instructions.len() { i += 1; continue; } + } - if matches!( - block.instructions[jump_idx].instr.real(), - Some(Instruction::ToBool) - ) { - set_to_nop(&mut block.instructions[jump_idx]); - jump_idx += 1; - if jump_idx >= block.instructions.len() { - i += 1; - continue; - } - } + let Some(jump_instr) = block.instructions[jump_idx].instr.real() else { + i += 1; + continue; + }; - let Some(jump_instr) = block.instructions[jump_idx].instr.real() else { + let mut invert = matches!( + invert.get(next_arg), + rustpython_compiler_core::bytecode::Invert::Yes + ); + let delta = match jump_instr { + Instruction::PopJumpIfFalse { delta } => { + invert = !invert; + delta.get(block.instructions[jump_idx].arg) + } + Instruction::PopJumpIfTrue { delta } => { + delta.get(block.instructions[jump_idx].arg) + } + _ => { i += 1; continue; - }; - - let mut invert = matches!( - invert.get(next_arg), - rustpython_compiler_core::bytecode::Invert::Yes - ); - let delta = match jump_instr { - Instruction::PopJumpIfFalse { delta } => { - invert = !invert; - delta.get(block.instructions[jump_idx].arg) - } - Instruction::PopJumpIfTrue { delta } => { - delta.get(block.instructions[jump_idx].arg) - } - _ => { - i += 1; - continue; - } - }; - - set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - block.instructions[jump_idx].instr = if invert { - Instruction::PopJumpIfNotNone { - delta: Arg::marker(), - } - } else { - Instruction::PopJumpIfNone { - delta: Arg::marker(), - } } - .into(); - block.instructions[jump_idx].arg = OpArg::new(u32::from(delta)); - i = jump_idx; - continue; - } - } + }; - if matches!( - curr_instr, - Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } - ) && matches!(next_instr, Instruction::ToBool) - && let Some(value) = const_truthiness(curr_instr, curr.arg, &self.metadata) - { - let (const_idx, _) = self - .metadata - .consts - .insert_full(ConstantData::Boolean { value }); set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = Instruction::LoadConst { - consti: Arg::marker(), + set_to_nop(&mut block.instructions[i + 1]); + block.instructions[jump_idx].instr = if invert { + Instruction::PopJumpIfNotNone { + delta: Arg::marker(), + } + } else { + Instruction::PopJumpIfNone { + delta: Arg::marker(), + } } .into(); - block.instructions[i + 1].arg = OpArg::new(const_idx as u32); - i += 1; + block.instructions[jump_idx].arg = OpArg::new(u32::from(delta)); + i = jump_idx; continue; } + } - if let (Instruction::LoadConst { consti }, Instruction::UnaryNot) = - (curr_instr, next_instr) - { - let constant = &self.metadata.consts[consti.get(curr.arg).as_usize()]; - if let ConstantData::Boolean { value } = constant { - let (const_idx, _) = self - .metadata - .consts - .insert_full(ConstantData::Boolean { value: !value }); - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); - block.instructions[i + 1].arg = OpArg::new(const_idx as u32); - i += 1; - continue; - } + if matches!( + const_instr, + Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } + ) && matches!(next_instr, Instruction::ToBool) + && let Some(value) = Self::load_const_truthiness(const_instr, const_arg, metadata) + { + let (const_idx, _) = metadata.consts.insert_full(ConstantData::Boolean { value }); + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = Instruction::LoadConst { + consti: Arg::marker(), } - + .into(); + block.instructions[i + 1].arg = OpArg::new(const_idx as u32); i += 1; + continue; } + + i += 1; } } - /// flowgraph.c optimize_basic_block - fn optimize_basic_blocks(&mut self) -> crate::InternalResult<()> { - for block_idx in self.block_next_order() { + /// flowgraph.c optimize_load_const + fn optimize_load_const(&mut self) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = self.blocks[block_idx.idx()].next; { let metadata = &mut self.metadata; let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - let inst = block.instructions[i]; - let Some(opcode) = inst.instr.real() else { - i += 1; - continue; - }; - let nextop = block - .instructions - .get(i + 1) - .and_then(|next| next.instr.real()); - - match opcode { - Instruction::BuildTuple { .. } => { - let oparg = u32::from(inst.arg); - if matches!(nextop, Some(Instruction::UnpackSequence { .. })) - && u32::from(block.instructions[i + 1].arg) == oparg - { - match oparg { - 1 => { - set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; - continue; - } - 2 | 3 => { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = - Instruction::Swap { i: Arg::marker() }.into(); - block.instructions[i + 1].arg = OpArg::new(oparg); - i += 1; - continue; - } - _ => {} - } - } - Self::fold_tuple_constant_at(metadata, block, i); - } - Instruction::BuildList { .. } | Instruction::BuildSet { .. } => { - Self::optimize_lists_and_sets_at(metadata, block, i, nextop); - } - Instruction::StoreFast { .. } - if matches!(nextop, Some(Instruction::StoreFast { .. })) - && u32::from(inst.arg) - == u32::from(block.instructions[i + 1].arg) - && instruction_lineno(&block.instructions[i]) - == instruction_lineno(&block.instructions[i + 1]) => - { - block.instructions[i].instr = Instruction::PopTop.into(); - block.instructions[i].arg = OpArg::NULL; - block.instructions[i].target = BlockIdx::NULL; - } - Instruction::Swap { .. } if u32::from(inst.arg) == 1 => { - set_to_nop(&mut block.instructions[i]); - } - Instruction::LoadGlobal { .. } - if matches!(nextop, Some(Instruction::PushNull)) - && (u32::from(inst.arg) & 1) == 0 => - { - block.instructions[i].arg = OpArg::new(u32::from(inst.arg) | 1); - set_to_nop(&mut block.instructions[i + 1]); - } - Instruction::CompareOp { .. } - if matches!(nextop, Some(Instruction::ToBool)) => + Self::basicblock_optimize_load_const(metadata, block); + } + block_idx = next_block; + } + } + + /// flowgraph.c optimize_basic_block + fn optimize_basic_block( + blocks: &mut [Block], + metadata: &mut CodeUnitMetadata, + block_idx: BlockIdx, + ) -> crate::InternalResult<()> { + for inst in &blocks[block_idx.idx()].instructions { + if inst.instr.has_target() { + debug_assert!(inst.target != BlockIdx::NULL); + debug_assert!(!blocks[inst.target.idx()].instructions.is_empty()); + debug_assert!( + !blocks[inst.target.idx()].instructions[0] + .instr + .is_assembler() + ); + } + } + { + let block = &mut blocks[block_idx]; + let mut i = 0; + while i < block.instructions.len() { + let inst = block.instructions[i]; + let Some(opcode) = inst.instr.real() else { + i += 1; + continue; + }; + let nextop = block + .instructions + .get(i + 1) + .and_then(|next| next.instr.real()); + + match opcode { + Instruction::BuildTuple { .. } => { + let oparg = u32::from(inst.arg); + if matches!(nextop, Some(Instruction::UnpackSequence { .. })) + && u32::from(block.instructions[i + 1].arg) == oparg { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = Instruction::CompareOp { - opname: Arg::marker(), + match oparg { + 1 => { + set_to_nop(&mut block.instructions[i]); + set_to_nop(&mut block.instructions[i + 1]); + i += 1; + continue; + } + 2 | 3 => { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = + Instruction::Swap { i: Arg::marker() }.into(); + block.instructions[i + 1].arg = OpArg::new(oparg); + i += 1; + continue; + } + _ => {} } - .into(); - block.instructions[i + 1].arg = - OpArg::new(u32::from(inst.arg) | oparg::COMPARE_OP_BOOL_MASK); - i += 1; - continue; } - Instruction::ContainsOp { .. } | Instruction::IsOp { .. } - if matches!(nextop, Some(Instruction::ToBool)) => - { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = opcode.into(); - block.instructions[i + 1].arg = inst.arg; - i += 1; - continue; + Self::fold_tuple_of_constants(metadata, block, i); + } + Instruction::BuildList { .. } | Instruction::BuildSet { .. } => { + Self::optimize_lists_and_sets(metadata, block, i, nextop); + } + Instruction::StoreFast { .. } + if matches!(nextop, Some(Instruction::StoreFast { .. })) + && u32::from(inst.arg) == u32::from(block.instructions[i + 1].arg) + && instruction_lineno(&block.instructions[i]) + == instruction_lineno(&block.instructions[i + 1]) => + { + block.instructions[i].instr = Instruction::PopTop.into(); + block.instructions[i].arg = OpArg::NULL; + } + Instruction::Swap { .. } if u32::from(inst.arg) == 1 => { + set_to_nop(&mut block.instructions[i]); + } + Instruction::LoadGlobal { .. } + if matches!(nextop, Some(Instruction::PushNull)) + && (u32::from(inst.arg) & 1) == 0 => + { + block.instructions[i].arg = OpArg::new(u32::from(inst.arg) | 1); + set_to_nop(&mut block.instructions[i + 1]); + } + Instruction::CompareOp { .. } + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = Instruction::CompareOp { + opname: Arg::marker(), } - Instruction::ContainsOp { .. } | Instruction::IsOp { .. } - if matches!(nextop, Some(Instruction::UnaryNot)) => - { + .into(); + block.instructions[i + 1].arg = + OpArg::new(u32::from(inst.arg) | oparg::COMPARE_OP_BOOL_MASK); + i += 1; + continue; + } + Instruction::ContainsOp { .. } | Instruction::IsOp { .. } + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = opcode.into(); + block.instructions[i + 1].arg = inst.arg; + i += 1; + continue; + } + Instruction::ContainsOp { .. } | Instruction::IsOp { .. } + if matches!(nextop, Some(Instruction::UnaryNot)) => + { + set_to_nop(&mut block.instructions[i]); + block.instructions[i + 1].instr = opcode.into(); + block.instructions[i + 1].arg = OpArg::new(u32::from(inst.arg) ^ 1); + i += 1; + continue; + } + Instruction::ToBool if matches!(nextop, Some(Instruction::ToBool)) => { + set_to_nop(&mut block.instructions[i]); + i += 1; + continue; + } + Instruction::UnaryNot => { + if matches!(nextop, Some(Instruction::ToBool)) { set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = opcode.into(); - block.instructions[i + 1].arg = OpArg::new(u32::from(inst.arg) ^ 1); + block.instructions[i + 1].instr = Instruction::UnaryNot.into(); + block.instructions[i + 1].arg = OpArg::new(0); i += 1; continue; } - Instruction::ToBool if matches!(nextop, Some(Instruction::ToBool)) => { + if matches!(nextop, Some(Instruction::UnaryNot)) { set_to_nop(&mut block.instructions[i]); + set_to_nop(&mut block.instructions[i + 1]); i += 1; continue; } - Instruction::UnaryNot => { - if matches!(nextop, Some(Instruction::ToBool)) { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = Instruction::UnaryNot.into(); - block.instructions[i + 1].arg = OpArg::new(0); - i += 1; - continue; - } - if matches!(nextop, Some(Instruction::UnaryNot)) { + Self::fold_const_unaryop(metadata, block, i); + } + Instruction::UnaryInvert | Instruction::UnaryNegative => { + Self::fold_const_unaryop(metadata, block, i); + } + Instruction::CallIntrinsic1 { func } => match func.get(inst.arg) { + IntrinsicFunction1::ListToTuple => { + if matches!(nextop, Some(Instruction::GetIter)) { set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; - continue; + } else { + Self::fold_constant_intrinsic_list_to_tuple(metadata, block, i); } - Self::fold_unary_constant_at(metadata, block, i); } - Instruction::UnaryInvert | Instruction::UnaryNegative => { - Self::fold_unary_constant_at(metadata, block, i); - } - Instruction::CallIntrinsic1 { func } => match func.get(inst.arg) { - IntrinsicFunction1::ListToTuple => { - if matches!(nextop, Some(Instruction::GetIter)) { - set_to_nop(&mut block.instructions[i]); - } else { - Self::fold_constant_intrinsic_list_to_tuple_at( - metadata, block, i, - ); - } - } - IntrinsicFunction1::UnaryPositive => { - Self::fold_unary_constant_at(metadata, block, i); - } - _ => {} - }, - Instruction::BinaryOp { .. } => { - Self::fold_binop_constant_at(metadata, block, i); + IntrinsicFunction1::UnaryPositive => { + Self::fold_const_unaryop(metadata, block, i); } _ => {} + }, + Instruction::BinaryOp { .. } => { + Self::fold_const_binop(metadata, block, i); } - - i += 1; + _ => {} } + + i += 1; } - jump_threading_block(&mut self.blocks, block_idx)?; - Self::apply_static_swaps_block(&mut self.blocks[block_idx]); } + jump_threading_block(blocks, block_idx)?; + Self::apply_static_swaps_block(&mut blocks[block_idx]); Ok(()) } /// flowgraph.c remove_redundant_nops_and_pairs fn remove_redundant_nops_and_pairs(&mut self) { - loop { - let mut changed = false; + let mut done = false; + + while !done { + done = true; let mut prev: Option<(BlockIdx, usize)> = None; let mut block_idx = BlockIdx::new(0); @@ -3229,59 +3220,17 @@ impl CodeInfo { let (prev_block, prev_instr) = prev.expect("redundant pair has previous"); set_to_nop(&mut self.blocks[prev_block.idx()].instructions[prev_instr]); set_to_nop(&mut self.blocks[block_idx.idx()].instructions[instr_idx]); - changed = true; + done = false; } prev = Some((block_idx, instr_idx)); } let block = &self.blocks[block_idx.idx()]; - if block - .instructions - .last() - .is_some_and(|info| info.instr.is_unconditional_jump()) - || !block_has_fallthrough(block) - { + if block.instructions.last().is_some_and(is_jump) || !bb_has_fallthrough(block) { prev = None; } block_idx = block.next; } - - if !changed { - break; - } - } - } - - /// Convert LOAD_CONST for small integers to LOAD_SMALL_INT - /// maybe_instr_make_load_smallint - fn convert_to_load_small_int(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - for instr in &mut block.instructions { - // Check if it's a LOAD_CONST instruction - let Some(Instruction::LoadConst { .. }) = instr.instr.real() else { - continue; - }; - - // Get the constant value - let const_idx = u32::from(instr.arg) as usize; - let Some(constant) = self.metadata.consts.get_index(const_idx) else { - continue; - }; - - // Check if it's a small integer - let ConstantData::Integer { value } = constant else { - continue; - }; - - // LOAD_SMALL_INT oparg is unsigned, so only 0..=255 can be encoded - if let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) { - // Convert LOAD_CONST to LOAD_SMALL_INT - instr.instr = Opcode::LoadSmallInt.into(); - // The arg is the i32 value stored as u32 (two's complement) - instr.arg = OpArg::new(small as u32); - } - } } } @@ -3293,137 +3242,161 @@ impl CodeInfo { return; } - // Mark used constants - // The first constant (index 0) is always kept (may be docstring) - let mut used = vec![false; nconsts]; - used[0] = true; + let mut index_map = vec![-1isize; nconsts]; + // The first constant may be docstring; keep it always. + index_map[0] = 0; - for block_idx in self.block_next_order() { + // Mark used consts. + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { let block = &self.blocks[block_idx]; for instr in &block.instructions { - if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { - let idx = u32::from(instr.arg) as usize; - if idx < nconsts { - used[idx] = true; - } + if instr.instr.has_const() { + let index = u32::from(instr.arg) as usize; + debug_assert!(index < nconsts); + index_map[index] = index as isize; } } + block_idx = block.next; } - // Check if any constants can be removed - let n_used: usize = used.iter().filter(|&&u| u).count(); - if n_used == nconsts { - return; // Nothing to remove + // Condense consts. + let mut n_used_consts = 0; + for i in 0..nconsts { + if index_map[i] != -1 { + debug_assert_eq!(index_map[i], i as isize); + index_map[n_used_consts] = index_map[i]; + n_used_consts += 1; + } } - // Build old_to_new index mapping - let mut old_to_new = vec![0usize; nconsts]; - let mut new_idx = 0usize; - for (old_idx, &is_used) in used.iter().enumerate() { - if is_used { - old_to_new[old_idx] = new_idx; - new_idx += 1; - } + if n_used_consts == nconsts { + return; } - // Build new consts list - let old_consts: Vec<_> = self.metadata.consts.iter().cloned().collect(); - self.metadata.consts.clear(); - for (old_idx, constant) in old_consts.into_iter().enumerate() { - if used[old_idx] { - self.metadata.consts.insert(constant); - } + // Move all used consts to the beginning of the consts list. + let old_consts = core::mem::take(&mut self.metadata.consts.constants); + debug_assert!(n_used_consts < nconsts); + for &old_index in &index_map[..n_used_consts] { + let old_index = old_index as usize; + debug_assert!(old_index < nconsts); + self.metadata + .consts + .constants + .push(old_consts[old_index].clone()); } - // Update LOAD_CONST instruction arguments - for block_idx in self.block_next_order() { + // Adjust const indices in the bytecode. + let mut reverse_index_map = vec![-1isize; nconsts]; + for (i, &old_index) in index_map[..n_used_consts].iter().enumerate() { + debug_assert!(old_index != -1); + let old_index = old_index as usize; + debug_assert_eq!(reverse_index_map[old_index], -1); + reverse_index_map[old_index] = i as isize; + } + + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = self.blocks[block_idx.idx()].next; let block = &mut self.blocks[block_idx]; for instr in &mut block.instructions { - if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { - let old_idx = u32::from(instr.arg) as usize; - if old_idx < nconsts { - instr.arg = OpArg::new(old_to_new[old_idx] as u32); - } + if instr.instr.has_const() { + let index = u32::from(instr.arg) as usize; + debug_assert!(reverse_index_map[index] >= 0); + debug_assert!(reverse_index_map[index] < n_used_consts as isize); + instr.arg = OpArg::new(reverse_index_map[index] as u32); } } + block_idx = next_block; } } - - /// insert_superinstructions (flowgraph.c): combine adjacent same-line - /// LOAD_FAST / STORE_FAST pairs before later flowgraph passes change - /// block layout. - fn insert_superinstructions(&mut self) { - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - let mut i = 0; - while i + 1 < block.instructions.len() { - let curr = &block.instructions[i]; - let next = &block.instructions[i + 1]; - let curr_line = instruction_lineno(curr); - let next_line = instruction_lineno(next); - if curr_line >= 0 && next_line >= 0 && curr_line != next_line { - i += 1; - continue; - } - - match (curr.instr.real(), next.instr.real()) { - (Some(Instruction::LoadFast { .. }), Some(Instruction::LoadFast { .. })) => { - let idx1 = u32::from(curr.arg); - let idx2 = u32::from(next.arg); - if idx1 >= 16 || idx2 >= 16 { - i += 1; - continue; - } - let packed = (idx1 << 4) | idx2; - block.instructions[i].instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - block.instructions[i].arg = OpArg::new(packed); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; - } - (Some(Instruction::StoreFast { .. }), Some(Instruction::LoadFast { .. })) => { - let store_idx = u32::from(curr.arg); - let load_idx = u32::from(next.arg); - if store_idx >= 16 || load_idx >= 16 { - i += 1; - continue; - } - let packed = (store_idx << 4) | load_idx; - block.instructions[i].instr = Instruction::StoreFastLoadFast { - var_nums: Arg::marker(), + + /// flowgraph.c make_super_instruction + fn make_super_instruction( + inst1: &mut InstructionInfo, + inst2: &mut InstructionInfo, + super_op: AnyInstruction, + ) { + let line1 = instruction_lineno(inst1); + let line2 = instruction_lineno(inst2); + if line1 >= 0 && line2 >= 0 && line1 != line2 { + return; + } + let arg1 = u32::from(inst1.arg); + let arg2 = u32::from(inst2.arg); + if arg1 >= 16 || arg2 >= 16 { + return; + } + inst1.instr = super_op; + inst1.arg = OpArg::new((arg1 << 4) | arg2); + set_to_nop(inst2); + } + + /// flowgraph.c insert_superinstructions + fn insert_superinstructions(&mut self) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = self.blocks[block_idx.idx()].next; + let block = &mut self.blocks[block_idx]; + for i in 0..block.instructions.len() { + let nextop = if i + 1 < block.instructions.len() { + block.instructions[i + 1].instr.real() + } else { + None + }; + match block.instructions[i].instr.real() { + Some(Instruction::LoadFast { .. }) => { + if matches!(nextop, Some(Instruction::LoadFast { .. })) { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + Self::make_super_instruction( + &mut inst1[0], + &mut rest[0], + Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(), + ); } - .into(); - block.instructions[i].arg = OpArg::new(packed); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; } - (Some(Instruction::StoreFast { .. }), Some(Instruction::StoreFast { .. })) => { - let idx1 = u32::from(curr.arg); - let idx2 = u32::from(next.arg); - if idx1 >= 16 || idx2 >= 16 { - i += 1; - continue; + Some(Instruction::StoreFast { .. }) => match nextop { + Some(Instruction::LoadFast { .. }) => { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + Self::make_super_instruction( + &mut inst1[0], + &mut rest[0], + Instruction::StoreFastLoadFast { + var_nums: Arg::marker(), + } + .into(), + ); } - let packed = (idx1 << 4) | idx2; - block.instructions[i].instr = Instruction::StoreFastStoreFast { - var_nums: Arg::marker(), + Some(Instruction::StoreFast { .. }) => { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + Self::make_super_instruction( + &mut inst1[0], + &mut rest[0], + Instruction::StoreFastStoreFast { + var_nums: Arg::marker(), + } + .into(), + ); } - .into(); - block.instructions[i].arg = OpArg::new(packed); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; - } - _ => i += 1, + _ => {} + }, + _ => {} } } + block_idx = next_block; } remove_redundant_nops(&mut self.blocks); - debug_assert!(no_redundant_nops(&self.blocks)); + #[cfg(debug_assertions)] + { + let no_redundant = no_redundant_nops(&mut self.blocks); + debug_assert!(no_redundant); + } } - fn optimize_load_fast_borrow(&mut self) { + fn optimize_load_fast(&mut self) { // NOT_LOCAL marker: instruction didn't come from a LOAD_FAST const NOT_LOCAL: usize = usize::MAX; const DUMMY_INSTR: isize = -1; @@ -3475,50 +3448,49 @@ impl CodeInfo { (((packed >> 4) & 0xF) as usize, (packed & 0xF) as usize) } - fn push_block( + fn load_fast_push_block( worklist: &mut Vec, - visited: &mut [bool], - blocks: &[Block], - source: BlockIdx, + blocks: &mut [Block], target: BlockIdx, start_depth: usize, ) { debug_assert!(target != BlockIdx::NULL); - let expected = blocks[target.idx()].start_depth.map(|depth| depth as usize); - debug_assert!( - expected == Some(start_depth), - "optimize_load_fast_borrow start_depth mismatch: source={source:?} target={target:?} expected={expected:?} actual={:?} source_last={:?} target_instrs={:?}", - Some(start_depth), - blocks[source.idx()] - .instructions - .last() - .and_then(|info| info.instr.real()), - blocks[target.idx()] - .instructions - .iter() - .map(|info| info.instr) - .collect::>(), + debug_assert_eq!( + blocks[target.idx()].start_depth.map(|depth| depth as usize), + Some(start_depth) ); - if !visited[target.idx()] { - visited[target.idx()] = true; + if !blocks[target.idx()].visited { + blocks[target.idx()].visited = true; worklist.push(target); } } - let mut visited = vec![false; self.blocks.len()]; - let mut worklist = vec![BlockIdx(0)]; - visited[0] = true; + let mut max_instrs = 0; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + max_instrs = max_instrs.max(self.blocks[current.idx()].instructions.len()); + current = self.blocks[current.idx()].next; + } + let mut instr_flags = vec![0u8; max_instrs]; + let mut refs = Vec::new(); + let mut worklist = make_cfg_traversal_stack(&mut self.blocks); + worklist.push(BlockIdx(0)); + self.blocks[0].start_depth = Some(0); + self.blocks[0].visited = true; while let Some(block_idx) = worklist.pop() { - let block = &self.blocks[block_idx]; + let block_i = block_idx.idx(); - let mut instr_flags = vec![0u8; block.instructions.len()]; - let start_depth = block.start_depth.unwrap_or(0) as usize; - let mut refs = Vec::with_capacity(block.instructions.len() + start_depth + 2); + let instr_count = self.blocks[block_i].instructions.len(); + instr_flags[..instr_count].fill(0); + let start_depth = self.blocks[block_i].start_depth.unwrap_or(0) as usize; + refs.clear(); + refs.reserve(instr_count + start_depth + 2); for _ in 0..start_depth { push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL); } - for (i, info) in block.instructions.iter().enumerate() { + for i in 0..instr_count { + let info = self.blocks[block_i].instructions[i]; let instr = info.instr; let arg_u32 = u32::from(info.arg); @@ -3548,10 +3520,6 @@ impl CodeInfo { r, ); } - AnyInstruction::Pseudo(PseudoInstruction::StoreFastMaybeNull { var_num }) => { - let r = pop_ref(&mut refs); - store_local(&mut instr_flags, &refs, var_num.get(info.arg) as usize, r); - } AnyInstruction::Real(Instruction::StoreFastLoadFast { .. }) => { let (store_local_idx, load_local_idx) = decode_packed_fast_locals(info.arg); let r = pop_ref(&mut refs); @@ -3592,8 +3560,8 @@ impl CodeInfo { let effect = instr.stack_effect_info(arg_u32); let net_pushed = effect.pushed() as isize - effect.popped() as isize; debug_assert!(net_pushed >= 0); - // CPython optimize_load_fast() records the produced - // pseudo-ref with the inner loop index here. + // CPython optimize_load_fast() shadows the outer + // instruction index in this produced-value loop. for produced in 0..net_pushed { push_ref(&mut refs, produced, NOT_LOCAL); } @@ -3618,6 +3586,9 @@ impl CodeInfo { AnyInstruction::Real( Instruction::EndSend | Instruction::SetFunctionAttribute { .. }, ) => { + let effect = instr.stack_effect_info(arg_u32); + debug_assert_eq!(effect.popped(), 2); + debug_assert_eq!(effect.pushed(), 1); let tos = pop_ref(&mut refs); let _ = pop_ref(&mut refs); push_ref(&mut refs, tos.instr, tos.local); @@ -3628,16 +3599,13 @@ impl CodeInfo { } AnyInstruction::Real(Instruction::ForIter { .. }) => { let target = info.target; - if target != BlockIdx::NULL { - push_block( - &mut worklist, - &mut visited, - &self.blocks, - block_idx, - target, - refs.len() + 1, - ); - } + debug_assert!(target != BlockIdx::NULL); + load_fast_push_block( + &mut worklist, + &mut self.blocks, + target, + refs.len() + 1, + ); push_ref(&mut refs, i as isize, NOT_LOCAL); } AnyInstruction::Real(Instruction::LoadAttr { namei }) => { @@ -3665,16 +3633,8 @@ impl CodeInfo { } AnyInstruction::Real(Instruction::Send { .. }) => { let target = info.target; - if target != BlockIdx::NULL { - push_block( - &mut worklist, - &mut visited, - &self.blocks, - block_idx, - target, - refs.len(), - ); - } + debug_assert!(target != BlockIdx::NULL); + load_fast_push_block(&mut worklist, &mut self.blocks, target, refs.len()); let _ = pop_ref(&mut refs); push_ref(&mut refs, i as isize, NOT_LOCAL); } @@ -3683,21 +3643,18 @@ impl CodeInfo { let num_popped = effect.popped() as usize; let num_pushed = effect.pushed() as usize; let target = info.target; - if instr.has_target() && target != BlockIdx::NULL { - let target_depth = refs - .len() - .saturating_sub(num_popped) - .saturating_add(num_pushed); - push_block( + if instr.has_target() { + debug_assert!(target != BlockIdx::NULL); + debug_assert!(refs.len() >= num_popped); + let target_depth = refs.len() - num_popped + num_pushed; + load_fast_push_block( &mut worklist, - &mut visited, - &self.blocks, - block_idx, + &mut self.blocks, target, target_depth, ); } - if !instr.is_block_push() { + if !is_block_push(&info) { for _ in 0..num_popped { let _ = pop_ref(&mut refs); } @@ -3709,23 +3666,18 @@ impl CodeInfo { } } - if let Some(term) = block.instructions.last() - && block.next != BlockIdx::NULL + let fallthrough = self.blocks[block_i].next; + let term = self.blocks[block_i].instructions.last().copied(); + if let Some(term) = term + && fallthrough != BlockIdx::NULL && !term.instr.is_unconditional_jump() && !term.instr.is_scope_exit() { - debug_assert!(block_has_fallthrough(block)); - push_block( - &mut worklist, - &mut visited, - &self.blocks, - block_idx, - block.next, - refs.len(), - ); + debug_assert!(bb_has_fallthrough(&self.blocks[block_i])); + load_fast_push_block(&mut worklist, &mut self.blocks, fallthrough, refs.len()); } - for r in refs { + for r in refs.iter().copied() { if r.instr != DUMMY_INSTR { instr_flags[r.instr as usize] |= REF_UNCONSUMED; } @@ -3763,45 +3715,33 @@ impl CodeInfo { while current != BlockIdx::NULL { blocknum += 1; for info in &mut self.blocks[current.idx()].instructions { + let arg = u32::from(info.arg) as usize; + if arg < 64 { + continue; + } match info.instr.real() { - Some( - Instruction::DeleteFast { var_num } - | Instruction::LoadFastAndClear { var_num }, - ) => { - let idx = usize::from(var_num.get(info.arg)); - if idx >= 64 && idx < nlocals { - states[idx - 64] = blocknum - 1; - } + Some(Instruction::DeleteFast { .. } | Instruction::LoadFastAndClear { .. }) => { + debug_assert!(arg < nlocals); + states[arg - 64] = blocknum - 1; } None if matches!( info.instr.pseudo(), Some(PseudoInstruction::StoreFastMaybeNull { .. }) ) => { - let Some(PseudoInstruction::StoreFastMaybeNull { var_num }) = - info.instr.pseudo() - else { - unreachable!(); - }; - let idx = var_num.get(info.arg) as usize; - if idx >= 64 && idx < nlocals { - states[idx - 64] = blocknum - 1; - } + debug_assert!(arg < nlocals); + states[arg - 64] = blocknum - 1; } - Some(Instruction::StoreFast { var_num }) => { - let idx = usize::from(var_num.get(info.arg)); - if idx >= 64 && idx < nlocals { - states[idx - 64] = blocknum; - } + Some(Instruction::StoreFast { .. }) => { + debug_assert!(arg < nlocals); + states[arg - 64] = blocknum; } - Some(Instruction::LoadFast { var_num }) => { - let idx = usize::from(var_num.get(info.arg)); - if idx >= 64 && idx < nlocals && states[idx - 64] != blocknum { + Some(Instruction::LoadFast { .. }) => { + debug_assert!(arg < nlocals); + if states[arg - 64] != blocknum { info.instr = Opcode::LoadFastCheck.into(); } - if idx >= 64 && idx < nlocals { - states[idx - 64] = blocknum; - } + states[arg - 64] = blocknum; } _ => {} } @@ -3830,59 +3770,48 @@ impl CodeInfo { nlocals = 64; } - let mut unsafe_masks = vec![0u64; self.blocks.len()]; - let mut on_stack = vec![false; self.blocks.len()]; - let mut worklist = Vec::with_capacity(self.blocks.len()); + let mut worklist = make_cfg_traversal_stack(&mut self.blocks); let mut start_mask = 0u64; for i in nparams..nlocals { start_mask |= 1u64 << i; } - maybe_push_local_block( - &mut worklist, - &mut on_stack, - &mut unsafe_masks, - BlockIdx(0), - start_mask, - ); + maybe_push_local_block(&mut self.blocks, &mut worklist, BlockIdx(0), start_mask); let mut current = BlockIdx(0); while current != BlockIdx::NULL { - scan_block_for_locals( - &mut self.blocks, - current, - &mut worklist, - &mut on_stack, - &mut unsafe_masks, - ); + scan_block_for_locals(&mut self.blocks, current, &mut worklist); current = self.blocks[current.idx()].next; } while let Some(block_idx) = worklist.pop() { - on_stack[block_idx.idx()] = false; - scan_block_for_locals( - &mut self.blocks, - block_idx, - &mut worklist, - &mut on_stack, - &mut unsafe_masks, - ); + self.blocks[block_idx.idx()].visited = false; + scan_block_for_locals(&mut self.blocks, block_idx, &mut worklist); } } - fn max_stackdepth(&mut self) -> crate::InternalResult { + /// flowgraph.c calculate_stackdepth + fn calculate_stackdepth(&mut self) -> crate::InternalResult { let mut maxdepth = 0u32; - let mut stack = Vec::with_capacity(self.blocks.len()); - let mut start_depths = vec![u32::MAX; self.blocks.len()]; - stackdepth_push(&mut stack, &mut start_depths, BlockIdx(0), 0)?; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + self.blocks[current.idx()].start_depth = None; + current = self.blocks[current.idx()].next; + } + let mut stack = make_cfg_traversal_stack(&mut self.blocks); + stackdepth_push(&mut stack, &mut self.blocks, BlockIdx(0), 0)?; const DEBUG: bool = false; - 'process_blocks: while let Some(block_idx) = stack.pop() { + while let Some(block_idx) = stack.pop() { let idx = block_idx.idx(); - let mut depth = start_depths[idx]; + let mut depth = self.blocks[idx] + .start_depth + .expect("stackdepth stack contains only started blocks"); + let mut next = self.blocks[idx].next; if DEBUG { eprintln!("===BLOCK {}===", block_idx.0); } - let block = &self.blocks[block_idx]; - for ins in &block.instructions { + let instr_count = self.blocks[idx].instructions.len(); + for i in 0..instr_count { + let ins = self.blocks[idx].instructions[i]; let instr = &ins.instr; let effect = instr.stack_effect(ins.arg.into()); if DEBUG { @@ -3905,10 +3834,8 @@ impl CodeInfo { } maxdepth = maxdepth.max(depth); // Process target blocks for branching instructions - if instr.has_target() - && ins.target != BlockIdx::NULL - && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) - { + if instr.has_target() && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) { + debug_assert!(ins.target != BlockIdx::NULL); let jump_effect = instr.stack_effect_jump(ins.arg.into()); let target_depth = depth.checked_add_signed(jump_effect).ok_or({ if jump_effect < 0 { @@ -3918,44 +3845,47 @@ impl CodeInfo { } })?; maxdepth = maxdepth.max(depth); - stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth)?; + stackdepth_push(&mut stack, &mut self.blocks, ins.target, target_depth)?; } depth = new_depth; + debug_assert!(!instr.is_assembler()); if instr.is_no_fallthrough() { - continue 'process_blocks; + next = BlockIdx::NULL; + break; } } - // Only push next block if it's not NULL - if block.next != BlockIdx::NULL { - stackdepth_push(&mut stack, &mut start_depths, block.next, depth)?; + if next != BlockIdx::NULL { + debug_assert!(bb_has_fallthrough(&self.blocks[idx])); + stackdepth_push(&mut stack, &mut self.blocks, next, depth)?; } } if DEBUG { eprintln!("DONE: {maxdepth}"); } - for block_idx in self.block_next_order() { - let start_depth = start_depths[block_idx.idx()]; - self.blocks[block_idx].start_depth = (start_depth != u32::MAX).then_some(start_depth); - } - - // Fix up handler stack_depth in ExceptHandlerInfo using start_depths + // Fix up handler stack_depth in ExceptHandlerInfo using b_startdepth // computed above: depth = start_depth - 1 - preserve_lasti - for block_idx in self.block_next_order() { - let block = &mut self.blocks[block_idx]; - for ins in &mut block.instructions { - if let Some(ref mut handler) = ins.except_handler { - let h_start = start_depths[handler.handler_block.idx()]; - if h_start != u32::MAX { - let adjustment = 1 + handler.preserve_lasti as u32; - debug_assert!( - h_start >= adjustment, - "handler start depth {h_start} too shallow for adjustment {adjustment}" - ); - handler.stack_depth = h_start.saturating_sub(adjustment); - } + current = BlockIdx(0); + while current != BlockIdx::NULL { + let next = self.blocks[current.idx()].next; + let instr_count = self.blocks[current.idx()].instructions.len(); + for i in 0..instr_count { + let h_start = self.blocks[current.idx()].instructions[i] + .except_handler + .and_then(|handler| self.blocks[handler.handler_block.idx()].start_depth); + if let (Some(handler), Some(h_start)) = ( + &mut self.blocks[current.idx()].instructions[i].except_handler, + h_start, + ) { + let adjustment = 1 + handler.preserve_lasti as u32; + debug_assert!( + h_start >= adjustment, + "handler start depth {h_start} too shallow for adjustment {adjustment}" + ); + handler.stack_depth = h_start.saturating_sub(adjustment); } } + current = next; } Ok(maxdepth) @@ -3966,11 +3896,18 @@ impl CodeInfo { impl CodeInfo { fn debug_block_dump(&self) -> String { let mut out = String::new(); - for (block_idx, block) in iter_blocks(&self.blocks) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { use core::fmt::Write; + let block = &self.blocks[block_idx.idx()]; + let block_return = if basicblock_returns(block) { + " return" + } else { + "" + }; let _ = writeln!( out, - "block {} next={} cold={} except={} preserve_lasti={} start_depth={}", + "block {} next={} cold={} except={} preserve_lasti={} start_depth={}{}", u32::from(block_idx), if block.next == BlockIdx::NULL { String::from("NULL") @@ -3983,6 +3920,7 @@ impl CodeInfo { block .start_depth .map_or_else(|| String::from("None"), |depth| depth.to_string()), + block_return, ); for info in &block.instructions { let lineno = instruction_lineno(info); @@ -4005,6 +3943,7 @@ impl CodeInfo { } ); } + block_idx = block.next; } out } @@ -4019,31 +3958,34 @@ impl CodeInfo { "after_cfg_from_instruction_sequence".to_owned(), self.debug_block_dump(), )); - translate_jump_labels_to_targets(&mut self.blocks)?; - mark_except_handlers(&mut self.blocks)?; - label_exception_targets(&mut self.blocks)?; + translate_jump_labels_to_targets(&mut self.blocks); + mark_except_handlers(&mut self.blocks); + label_exception_targets(&mut self.blocks); check_cfg(&self.blocks)?; inline_small_or_no_lineno_blocks(&mut self.blocks); trace.push(( "after_inline_small_or_no_lineno_blocks".to_owned(), self.debug_block_dump(), )); - self.remove_unreachable_blocks(); + Self::remove_unreachable(&mut self.blocks); resolve_line_numbers(&mut self.blocks); - self.convert_to_load_small_int(); - self.peephole_optimize(); + self.optimize_load_const(); trace.push(( - "after_peephole_optimize".to_owned(), + "after_optimize_load_const".to_owned(), self.debug_block_dump(), )); - self.convert_to_load_small_int(); - self.optimize_basic_blocks()?; + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = self.blocks[block_idx.idx()].next; + Self::optimize_basic_block(&mut self.blocks, &mut self.metadata, block_idx)?; + block_idx = next_block; + } trace.push(( "after_optimize_basic_block".to_owned(), self.debug_block_dump(), )); self.remove_redundant_nops_and_pairs(); - self.remove_unreachable_blocks(); + Self::remove_unreachable(&mut self.blocks); remove_redundant_nops_and_jumps(&mut self.blocks)?; debug_assert!(no_redundant_jumps(&self.blocks)); self.remove_unused_consts(); @@ -4075,7 +4017,7 @@ impl CodeInfo { self.debug_block_dump(), )); - let _max_stackdepth = self.max_stackdepth()?; + let _max_stackdepth = self.calculate_stackdepth()?; let _nlocalsplus = self.prepare_localsplus(); convert_pseudo_ops(&mut self.blocks)?; trace.push(( @@ -4086,9 +4028,9 @@ impl CodeInfo { normalize_jumps(&mut self.blocks)?; debug_assert!(no_redundant_jumps(&self.blocks)); trace.push(("after_normalize_jumps".to_owned(), self.debug_block_dump())); - self.optimize_load_fast_borrow(); + self.optimize_load_fast(); trace.push(( - "after_raw_optimize_load_fast_borrow".to_owned(), + "after_optimize_load_fast".to_owned(), self.debug_block_dump(), )); @@ -4129,34 +4071,22 @@ impl InstrDisplayContext for CodeInfo { fn stackdepth_push( stack: &mut Vec, - start_depths: &mut [u32], + blocks: &mut [Block], target: BlockIdx, depth: u32, ) -> crate::InternalResult<()> { let idx = target.idx(); - let block_depth = &mut start_depths[idx]; - if *block_depth != u32::MAX && *block_depth != depth { + let block_depth = &mut blocks[idx].start_depth; + if block_depth.is_some_and(|block_depth| block_depth != depth) { return Err(InternalError::InconsistentStackDepth); } - if *block_depth == u32::MAX { - *block_depth = depth; + if block_depth.is_none() { + *block_depth = Some(depth); stack.push(target); } Ok(()) } -fn iter_blocks(blocks: &[Block]) -> impl Iterator + '_ { - let mut next = BlockIdx(0); - core::iter::from_fn(move || { - if next == BlockIdx::NULL { - return None; - } - let (idx, b) = (next, &blocks[next]); - next = b.next; - Some((idx, b)) - }) -} - /// Generate Python 3.11+ format linetable from source locations fn generate_linetable( locations: &[LineTableLocation], @@ -4410,28 +4340,26 @@ fn generate_exception_table( /// Mark exception handler target blocks. /// flowgraph.c mark_except_handlers -pub(crate) fn mark_except_handlers(blocks: &mut [Block]) -> crate::InternalResult<()> { - let block_order = layout_block_order(blocks); - for block_idx in block_order.iter().copied() { +pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { debug_assert!(!blocks[block_idx.idx()].except_handler); + block_idx = blocks[block_idx.idx()].next; } - let mut targets = Vec::new(); - for block_idx in block_order.iter().copied() { - for instr in &blocks[block_idx.idx()].instructions { - if instr.instr.is_block_push() { - if instr.target == BlockIdx::NULL { - return Err(InternalError::MalformedControlFlowGraph); - } - targets.push(instr.target.idx()); + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next = blocks[block_idx.idx()].next; + let instr_count = blocks[block_idx.idx()].instructions.len(); + for i in 0..instr_count { + let instr = blocks[block_idx.idx()].instructions[i]; + if is_block_push(&instr) { + debug_assert!(instr.target != BlockIdx::NULL); + blocks[instr.target.idx()].except_handler = true; } } + block_idx = next; } - - for idx in targets { - blocks[idx].except_handler = true; - } - Ok(()) } /// flowgraph.c mark_cold (two-pass to match CPython). @@ -4449,172 +4377,187 @@ pub(crate) fn mark_except_handlers(blocks: &mut [Block]) -> crate::InternalResul /// their original chain position (e.g. between entry and the post-try /// continuation for a nested try/except whose inner_end was emptied by /// optimize_cfg). This matches CPython's behavior and is necessary for -/// optimize_load_fast_borrow to terminate fall-through at those placeholders. -fn mark_cold(blocks: &mut [Block]) -> Vec { - let n = blocks.len(); - let block_order = layout_block_order(blocks); - for block_idx in block_order.iter().copied() { - let block = &blocks[block_idx.idx()]; - debug_assert!(!block.cold); - } - - let mut warm = vec![false; n]; - let mut stack = Vec::new(); - warm[0] = true; +/// optimize_load_fast to terminate fall-through at those placeholders. +/// flowgraph.c mark_warm +fn mark_warm(blocks: &mut [Block]) { + let mut stack = make_cfg_traversal_stack(blocks); stack.push(BlockIdx(0)); - + blocks[0].visited = true; while let Some(block_idx) = stack.pop() { - let block = &blocks[block_idx.idx()]; - debug_assert!(!block.except_handler); + let idx = block_idx.idx(); + debug_assert!(!blocks[idx].except_handler); + blocks[idx].warm = true; - if block_has_fallthrough(block) && block.next != BlockIdx::NULL { - let next_idx = block.next.idx(); - if !warm[next_idx] { - warm[next_idx] = true; - stack.push(block.next); - } + let next = blocks[idx].next; + if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) && !blocks[next.idx()].visited + { + blocks[next.idx()].visited = true; + stack.push(next); } - for instr in &block.instructions { - if is_jump_instruction(instr) && instr.target != BlockIdx::NULL { - let target_idx = instr.target.idx(); - if !warm[target_idx] { - warm[target_idx] = true; - stack.push(instr.target); + let instr_count = blocks[idx].instructions.len(); + for i in 0..instr_count { + let instr = blocks[idx].instructions[i]; + if is_jump(&instr) { + let target = instr.target; + debug_assert!(target != BlockIdx::NULL); + if !blocks[target.idx()].visited { + blocks[target.idx()].visited = true; + stack.push(target); } } } } +} + +fn mark_cold(blocks: &mut [Block]) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let block = &mut blocks[block_idx.idx()]; + debug_assert!(!block.cold); + debug_assert!(!block.warm); + block_idx = block.next; + } - let mut cold = vec![false; n]; - let mut cold_visited = vec![false; n]; - let mut cold_stack = Vec::new(); - for block_idx in block_order.iter().copied() { + mark_warm(blocks); + + let mut cold_stack = make_cfg_traversal_stack(blocks); + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { let i = block_idx.idx(); + let next = blocks[i].next; let block = &blocks[i]; if block.except_handler { - debug_assert!(!warm[i]); + debug_assert!(!block.warm); cold_stack.push(block_idx); - cold_visited[i] = true; + blocks[i].visited = true; } + block_idx = next; } while let Some(block_idx) = cold_stack.pop() { let idx = block_idx.idx(); - cold[idx] = true; - let block = &blocks[idx]; - if block_has_fallthrough(block) && block.next != BlockIdx::NULL { - let next_idx = block.next.idx(); - if !warm[next_idx] && !cold_visited[next_idx] { - cold_visited[next_idx] = true; - cold_stack.push(block.next); - } - } - let instr_count = block.instructions.len(); - for (i, instr) in block.instructions.iter().enumerate() { - if is_jump_instruction(instr) && instr.target != BlockIdx::NULL { + blocks[idx].cold = true; + let next = blocks[idx].next; + if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { + let next_idx = next.idx(); + if !blocks[next_idx].warm && !blocks[next_idx].visited { + blocks[next_idx].visited = true; + cold_stack.push(next); + } + } + + let instr_count = blocks[idx].instructions.len(); + for i in 0..instr_count { + let instr = blocks[idx].instructions[i]; + if is_jump(&instr) { debug_assert_eq!(i, instr_count - 1); - let target_idx = instr.target.idx(); - if !warm[target_idx] && !cold_visited[target_idx] { - cold_visited[target_idx] = true; - cold_stack.push(instr.target); + let target = instr.target; + debug_assert!(target != BlockIdx::NULL); + let target_idx = target.idx(); + if !blocks[target_idx].warm && !blocks[target_idx].visited { + blocks[target_idx].visited = true; + cold_stack.push(target); } } } } - - for block_idx in block_order { - let i = block_idx.idx(); - blocks[i].cold = cold[i]; - } - warm } /// flowgraph.c push_cold_blocks_to_end fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> { - if blocks.len() <= 1 { + if blocks.is_empty() || blocks[0].next == BlockIdx::NULL { return Ok(()); } - let warm = mark_cold(blocks); + mark_cold(blocks); + let mut next_label = next_cpython_cfg_label(blocks); // If a cold block falls through to a warm block, add an explicit jump - let fixups: Vec<(BlockIdx, BlockIdx)> = iter_blocks(blocks) - .filter(|(_, block)| { - block.cold - && block.next != BlockIdx::NULL - && warm[block.next.idx()] - && block_has_fallthrough(block) - }) - .map(|(idx, block)| (idx, block.next)) - .collect(); - - for (cold_idx, warm_next) in fixups { - let jump_block_idx = BlockIdx(blocks.len() as u32); - let mut jump_block = Block { - cold: true, - ..Block::default() - }; - basicblock_add_jump_op( - &mut jump_block, - InstructionInfo { - instr: PseudoOpcode::JumpNoInterrupt.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - lineno_override: Some(-1), - cache_entries: 0, - }, - warm_next, - )?; - jump_block.next = blocks[cold_idx.idx()].next; - blocks[cold_idx.idx()].next = jump_block_idx; - blocks.push(jump_block); + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next = blocks[block_idx.idx()].next; + if blocks[block_idx.idx()].cold + && bb_has_fallthrough(&blocks[block_idx.idx()]) + && next != BlockIdx::NULL + && blocks[next.idx()].warm + { + if blocks[next.idx()].cpython_label_id.is_none() { + blocks[next.idx()].cpython_label_id = Some(InstructionSequenceLabel(next_label)); + next_label += 1; + } + let jump_arg = cpython_target_label_arg(blocks, next); + let jump_block_idx = BlockIdx(blocks.len() as u32); + let mut jump_block = Block { + cold: true, + predecessors: 1, + ..Block::default() + }; + basicblock_add_jump_op( + &mut jump_block, + InstructionInfo { + instr: PseudoOpcode::JumpNoInterrupt.into(), + arg: jump_arg, + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(-1), + cache_entries: 0, + }, + next, + )?; + jump_block.next = next; + blocks[block_idx.idx()].next = jump_block_idx; + blocks.push(jump_block); + } + block_idx = blocks[block_idx.idx()].next; } - // Extract cold block streaks and append at the end - let mut cold_head: BlockIdx = BlockIdx::NULL; - let mut cold_tail: BlockIdx = BlockIdx::NULL; - let mut current = BlockIdx(0); + let mut cold_blocks: BlockIdx = BlockIdx::NULL; + let mut cold_blocks_tail: BlockIdx = BlockIdx::NULL; assert!(!blocks[0].cold); + let mut block_idx = BlockIdx(0); - while current != BlockIdx::NULL { - let next = blocks[current.idx()].next; - if next == BlockIdx::NULL { + while blocks[block_idx.idx()].next != BlockIdx::NULL { + debug_assert!(!blocks[block_idx.idx()].cold); + while blocks[block_idx.idx()].next != BlockIdx::NULL + && !blocks[blocks[block_idx.idx()].next.idx()].cold + { + block_idx = blocks[block_idx.idx()].next; + } + if blocks[block_idx.idx()].next == BlockIdx::NULL { break; } - if blocks[next.idx()].cold { - let cold_start = next; - let mut cold_end = next; - while blocks[cold_end.idx()].next != BlockIdx::NULL - && blocks[blocks[cold_end.idx()].next.idx()].cold - { - cold_end = blocks[cold_end.idx()].next; - } + let mut block_end = blocks[block_idx.idx()].next; + while blocks[block_end.idx()].next != BlockIdx::NULL + && blocks[blocks[block_end.idx()].next.idx()].cold + { + block_end = blocks[block_end.idx()].next; + } - let after_cold = blocks[cold_end.idx()].next; - blocks[current.idx()].next = after_cold; - blocks[cold_end.idx()].next = BlockIdx::NULL; + debug_assert!(!blocks[block_idx.idx()].cold); + debug_assert!(blocks[blocks[block_idx.idx()].next.idx()].cold); + debug_assert!(blocks[block_end.idx()].cold); + debug_assert!( + blocks[block_end.idx()].next == BlockIdx::NULL + || !blocks[blocks[block_end.idx()].next.idx()].cold + ); - if cold_head == BlockIdx::NULL { - cold_head = cold_start; - } else { - blocks[cold_tail.idx()].next = cold_start; - } - cold_tail = cold_end; + if cold_blocks == BlockIdx::NULL { + cold_blocks = blocks[block_idx.idx()].next; } else { - current = next; + blocks[cold_blocks_tail.idx()].next = blocks[block_idx.idx()].next; } + cold_blocks_tail = block_end; + blocks[block_idx.idx()].next = blocks[block_end.idx()].next; + blocks[block_end.idx()].next = BlockIdx::NULL; } - if cold_head != BlockIdx::NULL { - let mut last = current; - while blocks[last.idx()].next != BlockIdx::NULL { - last = blocks[last.idx()].next; - } - blocks[last.idx()].next = cold_head; + debug_assert!(blocks[block_idx.idx()].next == BlockIdx::NULL); + blocks[block_idx.idx()].next = cold_blocks; + + if cold_blocks != BlockIdx::NULL { remove_redundant_nops_and_jumps(blocks)?; } Ok(()) @@ -4622,13 +4565,16 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> /// flowgraph.c check_cfg fn check_cfg(blocks: &[Block]) -> crate::InternalResult<()> { - for (_, block) in iter_blocks(blocks) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let block = &blocks[block_idx.idx()]; for (i, ins) in block.instructions.iter().enumerate() { debug_assert!(!ins.instr.is_assembler()); if ins.instr.is_terminator() && i + 1 != block.instructions.len() { return Err(InternalError::MalformedControlFlowGraph); } } + block_idx = block.next; } Ok(()) } @@ -4649,16 +4595,8 @@ fn jump_thread_kind(instr: AnyInstruction) -> Option { }) } -fn threaded_jump_instr( - source: AnyInstruction, - target: AnyInstruction, - conditional: bool, -) -> Option { +fn threaded_jump_instr(source: AnyInstruction, target: AnyInstruction) -> Option { let target_kind = jump_thread_kind(target)?; - if conditional { - return (target_kind == JumpThreadKind::Plain).then_some(source); - } - let source_kind = jump_thread_kind(source)?; let result_kind = if source_kind == JumpThreadKind::NoInterrupt && target_kind == JumpThreadKind::NoInterrupt @@ -4691,16 +4629,12 @@ fn jump_threading_block(blocks: &mut [Block], block_idx: BlockIdx) -> crate::Int let bi = block_idx.idx(); while let Some(last_idx) = blocks[bi].instructions.len().checked_sub(1) { let ins = blocks[bi].instructions[last_idx]; - let target = ins.target; - if target == BlockIdx::NULL { - return Ok(()); - } - if !(ins.instr.is_unconditional_jump() || is_conditional_jump(&ins.instr)) { - return Ok(()); - } - if blocks[target.idx()].instructions.is_empty() { + if !(ins.instr.is_unconditional_jump() || is_cfg_conditional_jump(&ins.instr)) { return Ok(()); } + let target = ins.target; + debug_assert!(target != BlockIdx::NULL); + debug_assert!(!blocks[target.idx()].instructions.is_empty()); let target_ins = blocks[target.idx()].instructions[0]; match ( ins.instr.pseudo().map(Into::into), @@ -4713,40 +4647,42 @@ fn jump_threading_block(blocks: &mut [Block], block_idx: BlockIdx) -> crate::Int | (Some(source @ PseudoOpcode::JumpIfFalse), Some(PseudoOpcode::JumpIfFalse)) | (Some(source @ PseudoOpcode::JumpIfTrue), Some(PseudoOpcode::JumpIfTrue)) => { let final_target = target_ins.target; - if final_target == BlockIdx::NULL || final_target == ins.target { + debug_assert!(final_target != BlockIdx::NULL); + if final_target == ins.target { return Ok(()); } set_to_nop(&mut blocks[bi].instructions[last_idx]); basicblock_add_jump(blocks, block_idx, source.into(), final_target, target_ins)?; - return Ok(()); + continue; } (Some(PseudoOpcode::JumpIfFalse), Some(PseudoOpcode::JumpIfTrue)) | (Some(PseudoOpcode::JumpIfTrue), Some(PseudoOpcode::JumpIfFalse)) => { let next = blocks[target.idx()].next; - if next == BlockIdx::NULL || next == target { - return Ok(()); - } + debug_assert!(next != BlockIdx::NULL); + debug_assert!(next != target); blocks[bi].instructions[last_idx].target = next; continue; } _ => {} } - if !target_ins.instr.is_unconditional_jump() - || target_ins.target == BlockIdx::NULL - || target_ins.target == target - { + if !target_ins.instr.is_unconditional_jump() || target_ins.target == target { return Ok(()); } - let conditional = is_conditional_jump(&ins.instr); - let final_target = target_ins.target; - let Some(threaded_instr) = (if conditional { - match jump_thread_kind(target_ins.instr) { - Some(JumpThreadKind::Plain) => Some(ins.instr), - _ => None, + debug_assert!(target_ins.target != BlockIdx::NULL); + if is_cfg_conditional_jump(&ins.instr) { + if jump_thread_kind(target_ins.instr) != Some(JumpThreadKind::Plain) { + return Ok(()); } - } else { - threaded_jump_instr(ins.instr, target_ins.instr, false) - }) else { + let final_target = target_ins.target; + if ins.target == final_target { + return Ok(()); + } + set_to_nop(&mut blocks[bi].instructions[last_idx]); + basicblock_add_jump(blocks, block_idx, ins.instr, final_target, target_ins)?; + continue; + } + let final_target = target_ins.target; + let Some(threaded_instr) = threaded_jump_instr(ins.instr, target_ins.instr) else { return Ok(()); }; if ins.target == final_target { @@ -4754,7 +4690,7 @@ fn jump_threading_block(blocks: &mut [Block], block_idx: BlockIdx) -> crate::Int } set_to_nop(&mut blocks[bi].instructions[last_idx]); basicblock_add_jump(blocks, block_idx, threaded_instr, final_target, target_ins)?; - return Ok(()); + continue; } Ok(()) } @@ -4768,11 +4704,12 @@ fn basicblock_add_jump( loc_source: InstructionInfo, ) -> crate::InternalResult<()> { let bi = block_idx.idx(); + let arg = cpython_target_label_arg(blocks, target); basicblock_add_jump_op( &mut blocks[bi], InstructionInfo { instr, - arg: OpArg::new(0), + arg, target: BlockIdx::NULL, location: loc_source.location, end_location: loc_source.end_location, @@ -4784,7 +4721,8 @@ fn basicblock_add_jump( ) } -pub(crate) fn is_conditional_jump(instr: &AnyInstruction) -> bool { +/// pycore_opcode_utils.h IS_CONDITIONAL_JUMP_OPCODE +fn is_conditional_jump_opcode(instr: &AnyInstruction) -> bool { matches!( instr.real().map(Into::into), Some( @@ -4793,16 +4731,24 @@ pub(crate) fn is_conditional_jump(instr: &AnyInstruction) -> bool { | Opcode::PopJumpIfNone | Opcode::PopJumpIfNotNone ) - ) || matches!( - instr.pseudo(), - Some(PseudoInstruction::JumpIfFalse { .. } | PseudoInstruction::JumpIfTrue { .. }) ) } +/// flowgraph.c optimize_basic_block handles pseudo `JUMP_IF_*` before +/// convert_pseudo_conditional_jumps lowers them to POP_JUMP_IF_*. +fn is_cfg_conditional_jump(instr: &AnyInstruction) -> bool { + is_conditional_jump_opcode(instr) + || matches!( + instr.pseudo(), + Some(PseudoInstruction::JumpIfFalse { .. } | PseudoInstruction::JumpIfTrue { .. }) + ) +} + /// flowgraph.c convert_pseudo_conditional_jumps fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { - let block_order = layout_block_order(blocks); - for block_idx in block_order { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; let mut i = 0; while i < block.instructions.len() { @@ -4829,23 +4775,25 @@ fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { } }; - let jump_info = InstructionInfo { - instr: jump.into(), - ..block.instructions[i] - }; - block.instructions[i].instr = Instruction::Copy { i: Arg::marker() }.into(); - block.instructions[i].arg = OpArg::new(1); - block.instructions[i].target = BlockIdx::NULL; + let original = block.instructions[i]; + block.instructions[i].instr = jump.into(); - let mut to_bool = block.instructions[i]; + let mut copy = original; + copy.instr = Instruction::Copy { i: Arg::marker() }.into(); + copy.arg = OpArg::new(1); + copy.target = BlockIdx::NULL; + + let mut to_bool = original; to_bool.instr = Instruction::ToBool.into(); to_bool.arg = OpArg::new(0); to_bool.target = BlockIdx::NULL; - basicblock_insert_instruction(block, i + 1, to_bool); - basicblock_insert_instruction(block, i + 2, jump_info); - i += 3; + basicblock_insert_instruction(block, i, copy); + i += 1; + basicblock_insert_instruction(block, i, to_bool); + i += 2; } + block_idx = next; } } @@ -4864,18 +4812,19 @@ fn reversed_conditional(instr: &AnyInstruction) -> Option { fn normalize_jumps_in_block( blocks: &mut Vec, block_idx: BlockIdx, - visited: &mut Vec, ) -> crate::InternalResult<()> { let idx = block_idx.idx(); - let Some(last_ins) = blocks[idx].instructions.last().copied() else { + let Some(last_ins) = basicblock_last_instr(&blocks[idx]).copied() else { return Ok(()); }; - if !is_conditional_jump(&last_ins.instr) || last_ins.target == BlockIdx::NULL { + if !is_conditional_jump_opcode(&last_ins.instr) { return Ok(()); } + debug_assert!(!last_ins.instr.is_assembler()); let target = last_ins.target; - let is_forward = !visited[target.idx()]; + debug_assert!(target != BlockIdx::NULL); + let is_forward = !blocks[target.idx()].visited; if is_forward { // Insert NOT_TAKEN after forward conditional jump. @@ -4898,70 +4847,74 @@ fn normalize_jumps_in_block( // Into: `reversed_cond_jump b_next` + new block [NOT_TAKEN, JUMP T] let loc = last_ins.location; let end_loc = last_ins.end_location; + let reversed = + reversed_conditional(&last_ins.instr).expect("conditional jump has reverse opcode"); + + let old_next = blocks[idx].next; + debug_assert!(old_next != BlockIdx::NULL); + let is_cold = blocks[idx].cold; + let target_arg = cpython_target_label_arg(blocks, target); + + // Create new block with NOT_TAKEN + JUMP to original backward target + let new_block_idx = BlockIdx(blocks.len() as u32); + let mut new_block = Block { + cold: is_cold, + start_depth: blocks[target.idx()].start_depth, + ..Block::default() + }; + basicblock_addop( + &mut new_block, + InstructionInfo { + instr: Opcode::NotTaken.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: loc, + end_location: end_loc, + except_handler: None, + lineno_override: last_ins.lineno_override, + cache_entries: 0, + }, + ); + basicblock_add_jump_op( + &mut new_block, + InstructionInfo { + instr: PseudoOpcode::Jump.into(), + arg: target_arg, + target: BlockIdx::NULL, + location: loc, + end_location: end_loc, + except_handler: None, + lineno_override: last_ins.lineno_override, + cache_entries: 0, + }, + target, + )?; + new_block.next = old_next; - if let Some(reversed) = reversed_conditional(&last_ins.instr) { - let old_next = blocks[idx].next; - let is_cold = blocks[idx].cold; - - // Create new block with NOT_TAKEN + JUMP to original backward target - let new_block_idx = BlockIdx(blocks.len() as u32); - let mut new_block = Block { - cold: is_cold, - start_depth: blocks[target.idx()].start_depth, - ..Block::default() - }; - basicblock_addop( - &mut new_block, - InstructionInfo { - instr: Opcode::NotTaken.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location: loc, - end_location: end_loc, - except_handler: None, - lineno_override: last_ins.lineno_override, - cache_entries: 0, - }, - ); - basicblock_add_jump_op( - &mut new_block, - InstructionInfo { - instr: PseudoOpcode::Jump.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location: loc, - end_location: end_loc, - except_handler: None, - lineno_override: last_ins.lineno_override, - cache_entries: 0, - }, - target, - )?; - new_block.next = old_next; - - // Update the conditional jump: invert opcode, target = old next block - let last_mut = blocks[idx].instructions.last_mut().unwrap(); - last_mut.instr = reversed; - last_mut.target = old_next; - - // Splice new block between current and old next - blocks[idx].next = new_block_idx; - blocks.push(new_block); + // Update the conditional jump: invert opcode, target = old next block + let last_mut = blocks[idx].instructions.last_mut().unwrap(); + last_mut.instr = reversed; + last_mut.target = old_next; - // Extend visited array and update visit order - visited.push(true); - } + // Splice new block between current and old next + blocks[idx].next = new_block_idx; + blocks.push(new_block); Ok(()) } /// flowgraph.c normalize_jumps fn normalize_jumps(blocks: &mut Vec) -> crate::InternalResult<()> { - let mut visited = vec![false; blocks.len()]; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + blocks[current.idx()].visited = false; + current = blocks[current.idx()].next; + } + let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); - visited[idx] = true; - normalize_jumps_in_block(blocks, current, &mut visited)?; + blocks[idx].visited = true; + normalize_jumps_in_block(blocks, current)?; current = blocks[idx].next; } Ok(()) @@ -4971,33 +4924,32 @@ fn normalize_jumps(blocks: &mut Vec) -> crate::InternalResult<()> { fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: BlockIdx) -> bool { const MAX_COPY_SIZE: usize = 4; - let Some(last) = blocks[block_idx.idx()].instructions.last().copied() else { + let Some(last) = basicblock_last_instr(&blocks[block_idx.idx()]).copied() else { return false; }; - if !last.instr.is_unconditional_jump() || last.target == BlockIdx::NULL { + if !last.instr.is_unconditional_jump() { return false; } let target = last.target; - let small_exit_block = is_scope_exit_block(&blocks[target.idx()]) + debug_assert!(target != BlockIdx::NULL); + let small_exit_block = basicblock_exits_scope(&blocks[target.idx()]) && blocks[target.idx()].instructions.len() <= MAX_COPY_SIZE; - let no_lineno_no_fallthrough = - block_has_no_lineno(&blocks[target.idx()]) && !block_has_fallthrough(&blocks[target.idx()]); + let no_lineno_no_fallthrough = basicblock_has_no_lineno(&blocks[target.idx()]) + && !bb_has_fallthrough(&blocks[target.idx()]); if !small_exit_block && !no_lineno_no_fallthrough { return false; } - let removed_jump_kind = jump_thread_kind(last.instr); + let removed_jump_was_jump = matches!(last.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)); let target_instructions = blocks[target.idx()].instructions.clone(); - if let Some(last_instr) = blocks[block_idx.idx()].instructions.last_mut() { + if let Some(last_instr) = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]) { set_to_nop(last_instr); } - blocks[block_idx.idx()] - .instructions - .extend(target_instructions); + basicblock_append_instructions(&mut blocks[block_idx.idx()], &target_instructions); if no_lineno_no_fallthrough - && removed_jump_kind == Some(JumpThreadKind::Plain) - && let Some(last) = blocks[block_idx.idx()].instructions.last_mut() + && removed_jump_was_jump + && let Some(last) = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]) && jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) { last.instr = match last.instr.into() { @@ -5006,6 +4958,7 @@ fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: _ => last.instr, }; } + blocks[target.idx()].predecessors -= 1; true } @@ -5029,7 +4982,6 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { /// flowgraph.c basicblock_remove_redundant_nops fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) -> usize { - let mut changes = 0; let bi = block_idx.idx(); let mut instructions = core::mem::take(&mut blocks[bi].instructions); let spare_instr_slots = core::mem::take(&mut blocks[bi].cpython_spare_instr_slots); @@ -5039,82 +4991,79 @@ fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) - for src in 0..instructions.len() { let instr = instructions[src]; let lineno = instruction_lineno(&instr); - let mut remove = false; if matches!(instr.instr.real(), Some(Instruction::Nop)) { - if lineno < 0 || prev_lineno == lineno { - remove = true; - } else if src < instructions.len() - 1 { + if lineno < 0 { + continue; + } + if prev_lineno == lineno { + continue; + } + if src < instructions.len() - 1 { let next_lineno = instruction_lineno(&instructions[src + 1]); if next_lineno == lineno { - remove = true; + continue; } else if next_lineno < 0 { copy_instruction_location(instr, &mut instructions[src + 1]); - remove = true; + continue; } } else { let next = next_nonempty_block(blocks, blocks[bi].next); if next != BlockIdx::NULL { - let mut first_next = None; - for (next_i, next_instr) in blocks[next.idx()].instructions.iter().enumerate() { + let mut next_location = None; + for next_instr in &blocks[next.idx()].instructions { let next_lineno = instruction_lineno(next_instr); if matches!(next_instr.instr.real(), Some(Instruction::Nop)) && next_lineno < 0 { continue; } - first_next = Some((next_i, next_lineno)); + next_location = Some(next_lineno); break; } - if let Some((_next_i, next_lineno)) = first_next + if let Some(next_lineno) = next_location && next_lineno == lineno { - remove = true; + continue; } } } } - if remove { - changes += 1; - } else { - if dest != src { - instructions[dest] = instructions[src]; - } - dest += 1; - prev_lineno = lineno; + if dest != src { + instructions[dest] = instructions[src]; } + dest += 1; + prev_lineno = lineno; } + let num_removed = instructions.len() - dest; blocks[bi] .cpython_spare_instr_slots - .extend_from_slice(&instructions[dest..]); + .extend(instructions[dest..].iter().copied()); blocks[bi] .cpython_spare_instr_slots - .extend_from_slice(&spare_instr_slots); + .extend(spare_instr_slots); instructions.truncate(dest); blocks[bi].instructions = instructions; - changes + num_removed } /// flowgraph.c remove_redundant_nops fn remove_redundant_nops(blocks: &mut [Block]) -> usize { - let mut block_order = Vec::new(); + let mut changes = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { - block_order.push(current); - current = blocks[current.idx()].next; + let next = blocks[current.idx()].next; + changes += basicblock_remove_redundant_nops(blocks, current); + current = next; } - block_order - .into_iter() - .map(|block_idx| basicblock_remove_redundant_nops(blocks, block_idx)) - .sum() + changes } - /// flowgraph.c no_redundant_nops -fn no_redundant_nops(blocks: &[Block]) -> bool { - let mut blocks = blocks.to_vec(); - remove_redundant_nops(&mut blocks) == 0 +#[cfg(debug_assertions)] +fn no_redundant_nops(blocks: &mut [Block]) -> bool { + remove_redundant_nops(blocks) == 0 } /// flowgraph.c remove_redundant_jumps @@ -5123,7 +5072,7 @@ fn remove_redundant_jumps(blocks: &mut [Block]) -> crate::InternalResult let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); - let Some(last_instr) = blocks[idx].instructions.last().copied() else { + let Some(last_instr) = basicblock_last_instr(&blocks[idx]).copied() else { current = blocks[idx].next; continue; }; @@ -5135,11 +5084,9 @@ fn remove_redundant_jumps(blocks: &mut [Block]) -> crate::InternalResult } let next = next_nonempty_block(blocks, blocks[idx].next); if jump_target == next { - let last_instr = blocks[idx].instructions.last_mut().unwrap(); + let last_instr = basicblock_last_instr_mut(&mut blocks[idx]).unwrap(); set_to_nop(last_instr); changes += 1; - current = blocks[idx].next; - continue; } } current = blocks[idx].next; @@ -5152,7 +5099,7 @@ fn no_redundant_jumps(blocks: &[Block]) -> bool { let mut current = BlockIdx(0); while current != BlockIdx::NULL { let block = &blocks[current.idx()]; - let Some(last) = block.instructions.last() else { + let Some(last) = basicblock_last_instr(block) else { current = block.next; continue; }; @@ -5160,12 +5107,11 @@ fn no_redundant_jumps(blocks: &[Block]) -> bool { let next = next_nonempty_block(blocks, block.next); let jump_target = next_nonempty_block(blocks, last.target); if jump_target == next { - if next == BlockIdx::NULL { - return false; - } - if let Some(first_next) = blocks[next.idx()].instructions.first() - && instruction_lineno(last) == instruction_lineno(first_next) + debug_assert!(next != BlockIdx::NULL); + if instruction_lineno(last) + == instruction_lineno(&blocks[next.idx()].instructions[0]) { + debug_assert!(false, "redundant jump has same line as fallthrough target"); return false; } } @@ -5176,24 +5122,31 @@ fn no_redundant_jumps(blocks: &[Block]) -> bool { } fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResult<()> { + let mut removed_nops; + let mut removed_jumps; loop { - let removed_nops = remove_redundant_nops(blocks); - let removed_jumps = remove_redundant_jumps(blocks)?; - if removed_nops + removed_jumps == 0 { - break; + removed_nops = remove_redundant_nops(blocks); + removed_jumps = remove_redundant_jumps(blocks)?; + if removed_nops + removed_jumps > 0 { + continue; } + break; } Ok(()) } -fn layout_block_order(blocks: &[Block]) -> Vec { - let mut order = Vec::new(); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - order.push(current); - current = blocks[current.idx()].next; +/// flowgraph.c make_cfg_traversal_stack +fn make_cfg_traversal_stack(blocks: &mut [Block]) -> Vec { + let mut nblocks = 0; + if !blocks.is_empty() { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + blocks[current.idx()].visited = false; + nblocks += 1; + current = blocks[current.idx()].next; + } } - order + Vec::with_capacity(nblocks) } /// flowgraph.c struct _PyCfgBuilder @@ -5267,40 +5220,37 @@ impl CfgBuilder { } /// flowgraph.c translate_jump_labels_to_targets -fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResult<()> { - let max_label = iter_blocks(blocks) - .filter_map(|(_, block)| block.cpython_label_id) - .map(InstructionSequenceLabel::idx) - .max(); - let mut label_to_block = max_label.map_or_else(Vec::new, |max_label| { - vec![BlockIdx::NULL; max_label.saturating_add(1)] - }); - for (block_idx, block) in iter_blocks(blocks) { +fn translate_jump_labels_to_targets(blocks: &mut [Block]) { + let max_label = get_max_cpython_cfg_label(blocks); + let mapsize = (max_label + 1) as usize; + let mut label_to_block = vec![BlockIdx::NULL; mapsize]; + + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let block = &blocks[block_idx.idx()]; if let Some(label_id) = block.cpython_label_id { - let slot = label_to_block - .get_mut(label_id.idx()) - .ok_or(InternalError::MalformedControlFlowGraph)?; - *slot = block_idx; + debug_assert!(label_id.idx() <= max_label as usize); + label_to_block[label_id.idx()] = block_idx; } + block_idx = block.next; } - let mut block_idx = BlockIdx(0); + block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; for info in &mut blocks[block_idx.idx()].instructions { if info.instr.has_target() { let label_id = InstructionSequenceLabel(u32::from(info.arg) as usize); - let target = label_to_block - .get(label_id.idx()) - .copied() - .filter(|target| *target != BlockIdx::NULL) - .ok_or(InternalError::MalformedControlFlowGraph)?; + debug_assert!(max_label >= 0); + debug_assert!(label_id.idx() <= max_label as usize); + let target = label_to_block[label_id.idx()]; + debug_assert!(target != BlockIdx::NULL); + debug_assert_eq!(blocks[target.idx()].cpython_label_id, Some(label_id)); info.target = target; } } block_idx = next; } - Ok(()) } /// flowgraph.c _PyCfg_FromInstructionSequence @@ -5308,7 +5258,7 @@ fn cfg_from_instruction_sequence( mut instr_sequence: InstructionSequence, ) -> crate::InternalResult> { instr_sequence.apply_label_map()?; - instr_sequence.mark_targets()?; + instr_sequence.mark_targets(); let InstructionSequence { instrs, label_map, @@ -5336,9 +5286,7 @@ fn cfg_from_instruction_sequence( )); debug_assert!(annotations_code.annotations_code.is_none()); for ann_entry in annotations_code.instrs.iter().copied() { - if ann_entry.info.instr.has_target() { - return Err(InternalError::MalformedControlFlowGraph); - } + debug_assert!(!ann_entry.info.instr.has_target()); let mut info = ann_entry.info; info.target = BlockIdx::NULL; builder.addop(info); @@ -5389,36 +5337,27 @@ fn cfg_from_instruction_sequence( } fn maybe_push_local_block( + blocks: &mut [Block], worklist: &mut Vec, - on_stack: &mut [bool], - unsafe_masks: &mut [u64], block: BlockIdx, unsafe_mask: u64, ) { - if block == BlockIdx::NULL { - return; - } + debug_assert!(block != BlockIdx::NULL); let idx = block.idx(); - let both = unsafe_masks[idx] | unsafe_mask; - if unsafe_masks[idx] != both { - unsafe_masks[idx] = both; - if !on_stack[idx] { + let both = blocks[idx].unsafe_locals_mask | unsafe_mask; + if blocks[idx].unsafe_locals_mask != both { + blocks[idx].unsafe_locals_mask = both; + if !blocks[idx].visited { worklist.push(block); - on_stack[idx] = true; + blocks[idx].visited = true; } } } -fn scan_block_for_locals( - blocks: &mut [Block], - block_idx: BlockIdx, - worklist: &mut Vec, - on_stack: &mut [bool], - unsafe_masks: &mut [u64], -) { +fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &mut Vec) { let idx = block_idx.idx(); - let mut unsafe_mask = unsafe_masks[idx]; + let mut unsafe_mask = blocks[idx].unsafe_locals_mask; let instr_count = blocks[idx].instructions.len(); for i in 0..instr_count { @@ -5432,63 +5371,52 @@ fn scan_block_for_locals( }; if let Some(handler_block) = except_handler { - maybe_push_local_block(worklist, on_stack, unsafe_masks, handler_block, unsafe_mask); + maybe_push_local_block(blocks, worklist, handler_block, unsafe_mask); } - let (local_idx, action) = match instr { - AnyInstruction::Pseudo(PseudoInstruction::StoreFastMaybeNull { var_num }) => { - (var_num.get(arg) as usize, LocalScanAction::SetUnsafe) - } - AnyInstruction::Real( - Instruction::DeleteFast { var_num } | Instruction::LoadFastAndClear { var_num }, - ) => (usize::from(var_num.get(arg)), LocalScanAction::SetUnsafe), - AnyInstruction::Real(Instruction::StoreFast { var_num }) => { - (usize::from(var_num.get(arg)), LocalScanAction::ClearUnsafe) - } - AnyInstruction::Real(Instruction::LoadFastCheck { var_num }) => { - (usize::from(var_num.get(arg)), LocalScanAction::ClearUnsafe) - } - AnyInstruction::Real(Instruction::LoadFast { var_num }) => { - (usize::from(var_num.get(arg)), LocalScanAction::CheckLoad) - } - _ => continue, - }; - if local_idx >= 64 { + let oparg = u32::from(arg) as usize; + if oparg >= 64 { continue; } - let bit = 1u64 << local_idx; - match action { - LocalScanAction::SetUnsafe => unsafe_mask |= bit, - LocalScanAction::ClearUnsafe => unsafe_mask &= !bit, - LocalScanAction::CheckLoad => { + let bit = 1u64 << oparg; + match instr { + AnyInstruction::Real( + Instruction::DeleteFast { .. } | Instruction::LoadFastAndClear { .. }, + ) + | AnyInstruction::Pseudo(PseudoInstruction::StoreFastMaybeNull { .. }) => { + unsafe_mask |= bit; + } + AnyInstruction::Real( + Instruction::StoreFast { .. } | Instruction::LoadFastCheck { .. }, + ) => { + unsafe_mask &= !bit; + } + AnyInstruction::Real(Instruction::LoadFast { .. }) => { if unsafe_mask & bit != 0 { blocks[idx].instructions[i].instr = Opcode::LoadFastCheck.into(); } unsafe_mask &= !bit; } + _ => {} } } - let block = &blocks[idx]; - if block.next != BlockIdx::NULL && block_has_fallthrough(block) { - maybe_push_local_block(worklist, on_stack, unsafe_masks, block.next, unsafe_mask); + let next = blocks[idx].next; + if blocks[idx].next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { + maybe_push_local_block(blocks, worklist, next, unsafe_mask); } - if let Some(last) = block.instructions.last() - && is_jump_instruction(last) - && last.target != BlockIdx::NULL - { - maybe_push_local_block(worklist, on_stack, unsafe_masks, last.target, unsafe_mask); + let jump_target = blocks[idx] + .instructions + .last() + .and_then(|last| is_jump(last).then_some(last.target)); + if let Some(target) = jump_target { + debug_assert!(target != BlockIdx::NULL); + maybe_push_local_block(blocks, worklist, target, unsafe_mask); } } -enum LocalScanAction { - SetUnsafe, - ClearUnsafe, - CheckLoad, -} - /// Follow chain of empty blocks to find first non-empty block. fn next_nonempty_block(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { while idx != BlockIdx::NULL && blocks[idx.idx()].instructions.is_empty() { @@ -5508,57 +5436,54 @@ fn instruction_has_lineno(instr: &InstructionInfo) -> bool { instruction_lineno(instr) >= 0 } +fn instruction_is_no_location(instr: &InstructionInfo) -> bool { + instruction_lineno(instr) == -1 +} + fn copy_instruction_location(source: InstructionInfo, target: &mut InstructionInfo) { target.location = source.location; target.end_location = source.end_location; target.lineno_override = source.lineno_override; } -fn propagation_location( - instr: &InstructionInfo, -) -> Option<(SourceLocation, SourceLocation, Option)> { - instruction_has_lineno(instr).then_some(( - instr.location, - instr.end_location, - instr.lineno_override, - )) -} - /// flowgraph.c basicblock_nofallthrough / BB_NO_FALLTHROUGH fn basicblock_nofallthrough(block: &Block) -> bool { - block - .instructions - .last() - .is_some_and(|ins| ins.instr.is_no_fallthrough()) + basicblock_last_instr(block).is_some_and(|ins| ins.instr.is_no_fallthrough()) } /// flowgraph.c BB_HAS_FALLTHROUGH -fn block_has_fallthrough(block: &Block) -> bool { +fn bb_has_fallthrough(block: &Block) -> bool { !basicblock_nofallthrough(block) } -fn is_jump_instruction(instr: &InstructionInfo) -> bool { +/// flowgraph.c is_jump +fn is_jump(instr: &InstructionInfo) -> bool { instr.instr.has_jump() } -fn last_jump_for_line_propagation(block: &Block) -> Option { - let last = block.instructions.last().copied()?; - is_jump_instruction(&last).then_some(last) +/// flowgraph.c is_block_push +fn is_block_push(instr: &InstructionInfo) -> bool { + debug_assert!(instr.instr.has_arg() || !instr.instr.is_block_push()); + instr.instr.is_block_push() +} + +/// flowgraph.c basicblock_returns +#[cfg(test)] +fn basicblock_returns(block: &Block) -> bool { + basicblock_last_instr(block) + .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::ReturnValue))) } /// flowgraph.c basicblock_exits_scope fn basicblock_exits_scope(block: &Block) -> bool { - block - .instructions - .last() - .is_some_and(|instr| instr.instr.is_scope_exit()) + basicblock_last_instr(block).is_some_and(|instr| instr.instr.is_scope_exit()) } /// flowgraph.c is_exit_or_eval_check_without_lineno fn is_exit_or_eval_check_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { let block = &blocks[block_idx.idx()]; if basicblock_exits_scope(block) || basicblock_has_eval_break(block) { - block_has_no_lineno(block) + basicblock_has_no_lineno(block) } else { false } @@ -5572,43 +5497,27 @@ fn basicblock_has_eval_break(block: &Block) -> bool { .any(|info| info.instr.has_eval_break()) } -fn block_has_no_lineno(block: &Block) -> bool { +/// flowgraph.c basicblock_has_no_lineno +fn basicblock_has_no_lineno(block: &Block) -> bool { block .instructions .iter() .all(|ins| !instruction_has_lineno(ins)) } -fn is_scope_exit_block(block: &Block) -> bool { - basicblock_exits_scope(block) -} - fn maybe_propagate_location( instr: &mut InstructionInfo, location: SourceLocation, end_location: SourceLocation, lineno_override: Option, ) { - if instr.lineno_override != Some(NEXT_LOCATION_OVERRIDE) && !instruction_has_lineno(instr) { + if instruction_is_no_location(instr) { instr.location = location; instr.end_location = end_location; instr.lineno_override = lineno_override; } } -fn propagate_line_numbers_in_block( - block: &mut Block, -) -> Option<(SourceLocation, SourceLocation, Option)> { - let mut prev_location = None; - for instr in &mut block.instructions { - if let Some((location, end_location, lineno_override)) = prev_location { - maybe_propagate_location(instr, location, end_location, lineno_override); - } - prev_location = propagation_location(instr); - } - prev_location -} - fn overwrite_location( instr: &mut InstructionInfo, location: SourceLocation, @@ -5620,48 +5529,85 @@ fn overwrite_location( instr.lineno_override = lineno_override; } -fn compute_predecessors(blocks: &[Block]) -> Vec { - let mut predecessors = vec![0u32; blocks.len()]; +/// flowgraph.c remove_unreachable computes `b_predecessors` by traversing only +/// blocks reachable from entry. +fn compute_reachable_predecessors(blocks: &mut [Block]) { if blocks.is_empty() { - return predecessors; + return; + } + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + blocks[block_idx.idx()].predecessors = 0; + block_idx = blocks[block_idx.idx()].next; } - predecessors[0] = 1; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let block = &blocks[current.idx()]; - if block_has_fallthrough(block) { - let next = block.next; - if next != BlockIdx::NULL { - predecessors[next.idx()] += 1; + let mut stack = make_cfg_traversal_stack(blocks); + blocks[0].predecessors = 1; + blocks[0].visited = true; + stack.push(BlockIdx(0)); + while let Some(current) = stack.pop() { + let idx = current.idx(); + let next = blocks[idx].next; + if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { + if !blocks[next.idx()].visited { + blocks[next.idx()].visited = true; + stack.push(next); } + blocks[next.idx()].predecessors += 1; } - for ins in &block.instructions { - if ins.instr.has_target() && ins.target != BlockIdx::NULL { - predecessors[ins.target.idx()] += 1; + + let instr_count = blocks[idx].instructions.len(); + for i in 0..instr_count { + let instr = blocks[idx].instructions[i]; + if is_jump(&instr) || is_block_push(&instr) { + let target = instr.target; + debug_assert!(target != BlockIdx::NULL); + if !blocks[target.idx()].visited { + blocks[target.idx()].visited = true; + stack.push(target); + } + blocks[target.idx()].predecessors += 1; } } - current = block.next; } - predecessors } /// flowgraph.c copy_basicblock fn copy_basicblock(blocks: &[Block], block_idx: BlockIdx) -> Block { let block = &blocks[block_idx.idx()]; - debug_assert!(!block_has_fallthrough(block)); - Block { - instructions: block.instructions.clone(), - ..Block::default() + debug_assert!(!bb_has_fallthrough(block)); + let mut result = Block::default(); + basicblock_append_instructions(&mut result, &block.instructions); + result +} + +/// flowgraph.c get_max_label +fn get_max_cpython_cfg_label(blocks: &[Block]) -> isize { + let mut label = -1; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + if let Some(cpython_label) = blocks[current.idx()].cpython_label_id { + label = label.max(cpython_label.idx() as isize); + } + current = blocks[current.idx()].next; } + label +} + +fn next_cpython_cfg_label(blocks: &[Block]) -> usize { + (get_max_cpython_cfg_label(blocks) + 1) as usize } -fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Vec) { +fn duplicate_exits_without_lineno(blocks: &mut Vec) { + let mut next_label = next_cpython_cfg_label(blocks); let mut current = BlockIdx(0); while current != BlockIdx::NULL { let block = &blocks[current.idx()]; let last = match block.instructions.last() { - Some(ins) if ins.target != BlockIdx::NULL && is_jump_instruction(ins) => ins, + Some(ins) if is_jump(ins) => { + debug_assert!(ins.target != BlockIdx::NULL); + ins + } _ => { current = blocks[current.idx()].next; continue; @@ -5669,9 +5615,9 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve }; let target = next_nonempty_block(blocks, last.target); - if target == BlockIdx::NULL - || !is_exit_or_eval_check_without_lineno(blocks, target) - || predecessors[target.idx()] <= 1 + debug_assert!(target != BlockIdx::NULL); + if !is_exit_or_eval_check_without_lineno(blocks, target) + || blocks[target.idx()].predecessors <= 1 { current = blocks[current.idx()].next; continue; @@ -5679,80 +5625,95 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve let new_idx = BlockIdx(blocks.len() as u32); let mut new_block = copy_basicblock(blocks, target); - if let Some(first) = new_block.instructions.first_mut() { - overwrite_location( - first, - last.location, - last.end_location, - last.lineno_override, - ); - } + overwrite_location( + &mut new_block.instructions[0], + last.location, + last.end_location, + last.lineno_override, + ); let old_next = blocks[target.idx()].next; new_block.next = old_next; + new_block.predecessors = 1; + new_block.cpython_label_id = Some(InstructionSequenceLabel(next_label)); + next_label += 1; blocks.push(new_block); blocks[target.idx()].next = new_idx; let last_mut = blocks[current.idx()].instructions.last_mut().unwrap(); last_mut.target = new_idx; - predecessors[target.idx()] -= 1; - predecessors.push(1); + blocks[target.idx()].predecessors -= 1; current = blocks[current.idx()].next; } current = BlockIdx(0); while current != BlockIdx::NULL { - let (next_block, last_location) = { - let block = &blocks[current.idx()]; - ( - block_has_fallthrough(block).then_some(block.next), - block - .instructions - .last() - .map(|last| (last.location, last.end_location, last.lineno_override)), - ) - }; - if let (Some(target), Some((location, end_location, lineno_override))) = - (next_block, last_location) - && target != BlockIdx::NULL - && is_exit_or_eval_check_without_lineno(blocks, target) - && let Some(first) = blocks[target.idx()].instructions.first_mut() + let block = &blocks[current.idx()]; + if bb_has_fallthrough(block) + && block.next != BlockIdx::NULL + && !block.instructions.is_empty() { - overwrite_location(first, location, end_location, lineno_override); + let target = block.next; + let last = *block.instructions.last().expect("block has instructions"); + if is_exit_or_eval_check_without_lineno(blocks, target) { + overwrite_location( + &mut blocks[target.idx()].instructions[0], + last.location, + last.end_location, + last.lineno_override, + ); + } } current = blocks[current.idx()].next; } } -fn propagate_line_numbers(blocks: &mut [Block], predecessors: &[u32]) { +fn propagate_line_numbers(blocks: &mut [Block]) { let mut current = BlockIdx(0); while current != BlockIdx::NULL { - if !blocks[current.idx()].instructions.is_empty() { - let (next_block, has_fallthrough) = { - let block = &blocks[current.idx()]; - (block.next, block_has_fallthrough(block)) - }; + let idx = current.idx(); + let Some(last) = blocks[idx].instructions.last().copied() else { + current = blocks[idx].next; + continue; + }; - let prev_location = propagate_line_numbers_in_block(&mut blocks[current.idx()]); - let last_jump = last_jump_for_line_propagation(&blocks[current.idx()]); + let mut prev_location = None; + for instr in &mut blocks[idx].instructions { + if let Some((location, end_location, lineno_override)) = prev_location { + maybe_propagate_location(instr, location, end_location, lineno_override); + } + if !instruction_is_no_location(instr) { + prev_location = Some((instr.location, instr.end_location, instr.lineno_override)); + } + } - if has_fallthrough - && next_block != BlockIdx::NULL - && predecessors[next_block.idx()] == 1 + let next = blocks[idx].next; + if bb_has_fallthrough(&blocks[idx]) { + debug_assert!(next != BlockIdx::NULL); + if blocks[next.idx()].predecessors == 1 + && !blocks[next.idx()].instructions.is_empty() && let Some((location, end_location, lineno_override)) = prev_location - && let Some(first) = blocks[next_block.idx()].instructions.first_mut() { - maybe_propagate_location(first, location, end_location, lineno_override); + maybe_propagate_location( + &mut blocks[next.idx()].instructions[0], + location, + end_location, + lineno_override, + ); } + } - if let Some(last_jump) = last_jump { - let target = last_jump.target; - if target != BlockIdx::NULL - && predecessors[target.idx()] == 1 - && let Some((location, end_location, lineno_override)) = prev_location - && let Some(first) = blocks[target.idx()].instructions.first_mut() - { - maybe_propagate_location(first, location, end_location, lineno_override); - } + if is_jump(&last) { + let target = last.target; + debug_assert!(target != BlockIdx::NULL); + if blocks[target.idx()].predecessors == 1 + && let Some((location, end_location, lineno_override)) = prev_location + { + maybe_propagate_location( + basicblock_raw_first_instr_mut(&mut blocks[target.idx()]), + location, + end_location, + lineno_override, + ); } } current = blocks[current.idx()].next; @@ -5760,12 +5721,11 @@ fn propagate_line_numbers(blocks: &mut [Block], predecessors: &[u32]) { } fn resolve_line_numbers(blocks: &mut Vec) { - let mut predecessors = compute_predecessors(blocks); - duplicate_exits_without_lineno(blocks, &mut predecessors); - propagate_line_numbers(blocks, &predecessors); + duplicate_exits_without_lineno(blocks); + propagate_line_numbers(blocks); } -pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalResult<()> { +pub(crate) fn label_exception_targets(blocks: &mut [Block]) { fn except_stack_top(stack: &[BlockIdx], blocks: &[Block]) -> Option { let handler_block = stack.last().copied()?; if handler_block == BlockIdx::NULL { @@ -5780,14 +5740,13 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe fn push_except_block( stack: &mut Vec, - instr: AnyInstruction, - target: BlockIdx, + setup: InstructionInfo, blocks: &mut [Block], - ) -> crate::InternalResult> { - debug_assert!(instr.is_block_push()); - if target == BlockIdx::NULL { - return Err(InternalError::MalformedControlFlowGraph); - } + ) -> Option { + debug_assert!(is_block_push(&setup)); + let instr = setup.instr; + let target = setup.target; + debug_assert!(target != BlockIdx::NULL); if matches!( instr.pseudo(), Some(PseudoInstruction::SetupWith { .. } | PseudoInstruction::SetupCleanup { .. }) @@ -5795,73 +5754,66 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe blocks[target.idx()].preserve_lasti = true; } stack.push(target); - Ok(except_stack_top(stack, blocks)) + except_stack_top(stack, blocks) } - fn pop_except_block( - stack: &mut Vec, - blocks: &[Block], - ) -> crate::InternalResult> { + fn pop_except_block(stack: &mut Vec, blocks: &[Block]) -> Option { debug_assert!(!stack.is_empty()); - if stack.is_empty() { - return Err(InternalError::MalformedControlFlowGraph); - } - stack.pop(); - Ok(except_stack_top(stack, blocks)) + stack.pop().expect("non-empty exception stack"); + except_stack_top(stack, blocks) } let num_blocks = blocks.len(); if num_blocks == 0 { - return Ok(()); + return; } - let mut visited = vec![false; num_blocks]; - let mut block_stacks: Vec>> = vec![None; num_blocks]; + let mut todo = make_cfg_traversal_stack(blocks); // Entry block - visited[0] = true; - block_stacks[0] = Some(Vec::new()); - - let mut todo = vec![BlockIdx(0)]; + blocks[0].visited = true; + blocks[0].except_stack = Some(Vec::new()); + todo.push(BlockIdx(0)); while let Some(block_idx) = todo.pop() { let bi = block_idx.idx(); - let mut stack = block_stacks[bi].take().unwrap_or_default(); + debug_assert!(blocks[bi].visited); + let mut stack = blocks[bi] + .except_stack + .take() + .expect("visited exception block has an except stack"); let mut handler = except_stack_top(&stack, blocks); let mut last_yield_except_depth: i32 = -1; let instr_count = blocks[bi].instructions.len(); for i in 0..instr_count { - let instr = blocks[bi].instructions[i].instr; - let target = blocks[bi].instructions[i].target; - let arg = blocks[bi].instructions[i].arg; - - if instr.is_block_push() { - if target == BlockIdx::NULL { - return Err(InternalError::MalformedControlFlowGraph); - } - if !visited[target.idx()] { - visited[target.idx()] = true; - block_stacks[target.idx()] = Some(stack.clone()); + let info = blocks[bi].instructions[i]; + let instr = info.instr; + let target = info.target; + let arg = info.arg; + + if is_block_push(&info) { + debug_assert!(target != BlockIdx::NULL); + if !blocks[target.idx()].visited { + blocks[target.idx()].visited = true; + blocks[target.idx()].except_stack = Some(stack.clone()); todo.push(target); } - handler = push_except_block(&mut stack, instr, target, blocks)?; + handler = push_except_block(&mut stack, info, blocks); } else if instr.is_pop_block() { - handler = pop_except_block(&mut stack, blocks)?; + handler = pop_except_block(&mut stack, blocks); set_to_nop(&mut blocks[bi].instructions[i]); - } else if is_jump_instruction(&blocks[bi].instructions[i]) { + } else if is_jump(&blocks[bi].instructions[i]) { blocks[bi].instructions[i].except_handler = handler; debug_assert_eq!(i, instr_count - 1); // CPython label_exception_targets(): copy the except stack // when this block can also fall through, otherwise transfer it // to the jump target. - if target == BlockIdx::NULL { - return Err(InternalError::MalformedControlFlowGraph); - } - if !visited[target.idx()] { - visited[target.idx()] = true; - block_stacks[target.idx()] = Some(if block_has_fallthrough(&blocks[bi]) { + debug_assert!(target != BlockIdx::NULL); + if !blocks[target.idx()].visited { + blocks[target.idx()].visited = true; + blocks[target.idx()].except_stack = Some(if bb_has_fallthrough(&blocks[bi]) { stack.clone() } else { core::mem::take(&mut stack) @@ -5888,21 +5840,32 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe } let next = blocks[bi].next; - if block_has_fallthrough(&blocks[bi]) && next != BlockIdx::NULL && !visited[next.idx()] { - visited[next.idx()] = true; - block_stacks[next.idx()] = Some(stack); - todo.push(next); + if bb_has_fallthrough(&blocks[bi]) { + debug_assert!(next != BlockIdx::NULL); + if !blocks[next.idx()].visited { + blocks[next.idx()].visited = true; + blocks[next.idx()].except_stack = Some(stack); + todo.push(next); + } + } + } + #[cfg(debug_assertions)] + { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let block = &blocks[block_idx.idx()]; + debug_assert!(block.except_stack.is_none()); + block_idx = block.next; } } - debug_assert!(block_stacks.iter().all(Option::is_none)); - Ok(()) } /// Convert remaining pseudo ops to real instructions or NOP. /// flowgraph.c convert_pseudo_ops pub(crate) fn convert_pseudo_ops(blocks: &mut [Block]) -> crate::InternalResult<()> { - let block_order = layout_block_order(blocks); - for block_idx in block_order { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; for info in &mut block.instructions { let Some(pseudo) = info.instr.pseudo() else { @@ -5935,6 +5898,7 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block]) -> crate::InternalResult< } } } + block_idx = next; } // CPython flowgraph.c::convert_pseudo_ops() finishes by calling // remove_redundant_nops_and_jumps(). @@ -5981,8 +5945,9 @@ pub(crate) fn fix_cell_offsets( cellfixedoffsets: &mut [u32], ) -> usize { let numdropped = fix_cellfixedoffsets(nlocals, cellfixedoffsets); - let block_order = layout_block_order(blocks); - for block_idx in block_order { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; for info in &mut block.instructions { debug_assert!( @@ -6005,6 +5970,566 @@ pub(crate) fn fix_cell_offsets( info.arg = OpArg::new(cellfixedoffsets[cell_relative]); } } + block_idx = next; } numdropped } + +#[cfg(test)] +mod tests { + use super::*; + + fn test_location(line: u32) -> SourceLocation { + SourceLocation { + line: OneIndexed::new(line as usize).expect("valid line number"), + character_offset: OneIndexed::MIN, + } + } + + fn test_instr(instr: Instruction, line: u32) -> InstructionInfo { + InstructionInfo { + instr: instr.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: test_location(line), + end_location: test_location(line), + except_handler: None, + lineno_override: None, + cache_entries: 0, + } + } + + fn test_jump(target: BlockIdx, line: u32) -> InstructionInfo { + let mut instr = test_instr(Instruction::Nop, line); + instr.instr = PseudoOpcode::Jump.into(); + instr.target = target; + instr + } + + fn test_cond_jump(target: BlockIdx, line: u32) -> InstructionInfo { + let mut instr = test_instr(Instruction::Nop, line); + instr.instr = PseudoOpcode::JumpIfFalse.into(); + instr.target = target; + instr + } + + fn test_code_info(block: Block) -> CodeInfo { + CodeInfo { + flags: CodeFlags::empty(), + source_path: "source_path".to_owned(), + private: None, + blocks: vec![block], + current_block: BlockIdx::new(0), + instr_sequence: InstructionSequence::new(), + instr_sequence_label_map: InstructionSequenceLabelMap::new(), + annotations_instr_sequence: None, + metadata: CodeUnitMetadata { + name: "".to_owned(), + qualname: Some("".to_owned()), + consts: Default::default(), + names: IndexSet::default(), + varnames: IndexSet::default(), + cellvars: IndexSet::default(), + freevars: IndexSet::default(), + fast_hidden: IndexMap::default(), + fast_hidden_final: IndexSet::default(), + argcount: 0, + posonlyargcount: 0, + kwonlyargcount: 0, + firstlineno: OneIndexed::MIN, + }, + static_attributes: None, + in_inlined_comp: false, + fblock: Vec::new(), + symbol_table_index: 0, + in_conditional_block: 0, + next_conditional_annotation_index: 0, + } + } + + #[test] + fn instruction_sequence_label_shadow_preserves_cpython_offset_aliases() { + let mut labels = InstructionSequenceLabelMap::new(); + labels.push_unmapped_label(); + labels.push_unmapped_label(); + + let first = BlockIdx::new(1); + let second = BlockIdx::new(2); + assert_ne!( + labels.label_for_block(first), + labels.label_for_block(second) + ); + + // CPython `_PyInstructionSequence_UseLabel()` can map consecutive + // labels to the same instruction offset. The codegen CFG shadow must + // resolve the later block label to the block owning that shared offset. + labels.use_label_at_block(second, first); + assert_eq!(labels.resolve_label(first), first); + assert_eq!(labels.resolve_label(second), first); + } + + #[test] + fn basicblock_addop_reuses_cpython_spare_except_handler_slot() { + let handler = ExceptHandlerInfo { + handler_block: BlockIdx::new(7), + stack_depth: 3, + preserve_lasti: true, + }; + let mut block = Block::default(); + let mut stale = test_instr(Instruction::Nop, 11); + stale.except_handler = Some(handler); + block.cpython_spare_instr_slots.push_back(stale); + + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 12)); + + // CPython `basicblock_addop()` writes opcode/oparg/target/location into + // the reused `b_instr[b_iused]` slot, but does not clear `i_except`. + assert_eq!(block.cpython_spare_instr_slots.len(), 0); + assert_eq!(block.instructions.len(), 1); + assert_eq!(block.instructions[0].except_handler, Some(handler)); + assert_eq!(block.instructions[0].target, BlockIdx::NULL); + } + + #[test] + fn basicblock_insert_instruction_consumes_spare_without_inheriting_except_handler() { + let handler = ExceptHandlerInfo { + handler_block: BlockIdx::new(9), + stack_depth: 1, + preserve_lasti: false, + }; + let mut block = Block::default(); + block.instructions.push(test_instr(Instruction::Nop, 21)); + let mut stale = test_instr(Instruction::Nop, 22); + stale.except_handler = Some(handler); + block.cpython_spare_instr_slots.push_back(stale); + + basicblock_insert_instruction(&mut block, 0, test_instr(Instruction::PopTop, 23)); + + // CPython `basicblock_insert_instruction()` also obtains a slot with + // `basicblock_next_instr()`, then overwrites the inserted position with + // the provided instruction copy, including its `i_except` value. + assert_eq!(block.cpython_spare_instr_slots.len(), 0); + assert_eq!(block.instructions.len(), 2); + assert_eq!(block.instructions[0].except_handler, None); + } + + #[test] + fn basicblock_clear_preserves_cpython_spare_slots() { + let handler = ExceptHandlerInfo { + handler_block: BlockIdx::new(3), + stack_depth: 2, + preserve_lasti: true, + }; + let mut block = Block::default(); + let mut stale = test_instr(Instruction::PopTop, 31); + stale.except_handler = Some(handler); + block.instructions.push(stale); + + basicblock_clear(&mut block); + basicblock_addop(&mut block, test_instr(Instruction::Nop, 32)); + + // CPython `remove_unreachable()` sets `b_iused = 0` without clearing the + // backing `b_instr` slot. A later `basicblock_addop()` reuses that slot + // and does not overwrite `i_except`. + assert_eq!(block.cpython_spare_instr_slots.len(), 0); + assert_eq!(block.instructions.len(), 1); + assert_eq!(block.instructions[0].except_handler, Some(handler)); + } + + #[test] + fn basicblock_append_instructions_overwrites_cpython_spare_slot() { + let handler = ExceptHandlerInfo { + handler_block: BlockIdx::new(5), + stack_depth: 4, + preserve_lasti: false, + }; + let mut to = Block::default(); + let mut stale = test_instr(Instruction::Nop, 41); + stale.except_handler = Some(handler); + to.cpython_spare_instr_slots.push_back(stale); + + let from = [test_instr(Instruction::PopTop, 42)]; + basicblock_append_instructions(&mut to, &from); + + // CPython `basicblock_append_instructions()` obtains a slot with + // `basicblock_next_instr()`, then overwrites it with the copied + // instruction, including `i_except`. + assert_eq!(to.cpython_spare_instr_slots.len(), 0); + assert_eq!(to.instructions.len(), 1); + assert_eq!(to.instructions[0].except_handler, None); + } + + #[test] + fn instr_set_op0_nop_preserves_cpython_stale_target() { + let mut info = test_jump(BlockIdx::new(1), 50); + set_to_nop(&mut info); + + assert_eq!(info.target, BlockIdx::new(1)); + + let mut blocks = vec![Block::default(), Block::default()]; + blocks[0].instructions.push(info); + blocks[0].next = BlockIdx::new(1); + + cfg_to_instruction_sequence(&mut blocks) + .expect("non-target NOP should ignore stale CPython i_target"); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic(expected = "target_block != BlockIdx::NULL")] + fn cfg_to_instruction_sequence_requires_target_for_target_opcodes() { + let mut block = Block::default(); + block.instructions.push(test_jump(BlockIdx::NULL, 51)); + let mut blocks = vec![block]; + + let _ = cfg_to_instruction_sequence(&mut blocks); + } + + #[test] + fn static_swaps_respect_cpython_no_location_line_boundary() { + let mut block = Block::default(); + let mut swap = test_instr(Instruction::Swap { i: Arg::marker() }, 60); + swap.arg = OpArg::new(2); + let mut store = test_instr( + Instruction::StoreFast { + var_num: Arg::marker(), + }, + 60, + ); + store.arg = OpArg::new(0); + let mut pop = test_instr(Instruction::PopTop, 60); + pop.lineno_override = Some(-1); + block.instructions.extend([swap, store, pop]); + + CodeInfo::apply_static_swaps_block(&mut block); + + // CPython `next_swappable_instruction()` compares `i_loc.lineno` + // directly, so a following NO_LOCATION swaperand does not match the + // first swaperand's positive line number. + assert!(matches!( + block.instructions[0].instr.real(), + Some(Instruction::Swap { .. }) + )); + assert!(matches!( + block.instructions[1].instr.real(), + Some(Instruction::StoreFast { .. }) + )); + assert!(matches!( + block.instructions[2].instr.real(), + Some(Instruction::PopTop) + )); + + let mut block = Block::default(); + let mut swap = test_instr(Instruction::Swap { i: Arg::marker() }, 70); + swap.arg = OpArg::new(2); + let mut store = test_instr( + Instruction::StoreFast { + var_num: Arg::marker(), + }, + 70, + ); + store.arg = OpArg::new(0); + store.lineno_override = Some(-1); + let pop = test_instr(Instruction::PopTop, 71); + block.instructions.extend([swap, store, pop]); + + CodeInfo::apply_static_swaps_block(&mut block); + + // Conversely, when the first swaperand has NO_LOCATION, CPython passes + // `-1` as the line filter and does not enforce a boundary. + assert!(matches!( + block.instructions[0].instr.real(), + Some(Instruction::Nop) + )); + assert!(matches!( + block.instructions[1].instr.real(), + Some(Instruction::PopTop) + )); + assert!(matches!( + block.instructions[2].instr.real(), + Some(Instruction::StoreFast { .. }) + )); + } + + #[test] + fn optimize_load_const_tracks_cpython_copy_of_load_const() { + let mut block = Block::default(); + block.instructions.push(test_instr( + Instruction::LoadConst { + consti: Arg::marker(), + }, + 80, + )); + let mut copy = test_instr(Instruction::Copy { i: Arg::marker() }, 80); + copy.arg = OpArg::new(1); + block.instructions.push(copy); + block.instructions.push(test_instr(Instruction::ToBool, 80)); + + let mut code = test_code_info(block); + let (const_idx, _) = code.metadata.consts.insert_full(ConstantData::Tuple { + elements: vec![ConstantData::Integer { + value: BigInt::from(1), + }], + }); + code.blocks[0].instructions[0].arg = OpArg::new(const_idx as u32); + + code.optimize_load_const(); + + // CPython `basicblock_optimize_load_const()` keeps the previous + // LOAD_CONST as the effective opcode for a following `COPY 1`, so the + // COPY is NOPed and TO_BOOL becomes LOAD_CONST True. + assert!(matches!( + code.blocks[0].instructions[0].instr.real(), + Some(Instruction::LoadConst { .. }) + )); + assert!(matches!( + code.blocks[0].instructions[1].instr.real(), + Some(Instruction::Nop) + )); + let load_bool = &code.blocks[0].instructions[2]; + assert!(matches!( + load_bool.instr.real(), + Some(Instruction::LoadConst { .. }) + )); + assert_eq!( + code.metadata.consts[u32::from(load_bool.arg) as usize], + ConstantData::Boolean { value: true } + ); + } + + #[test] + fn optimize_load_fast_records_no_input_opcode_ref_at_cpython_produced_index() { + let mut block = Block::default(); + block.instructions.push(test_instr( + Instruction::LoadFast { + var_num: Arg::marker(), + }, + 10, + )); + block.instructions.push(test_instr(Instruction::GetLen, 10)); + let mut swap = test_instr(Instruction::Swap { i: Arg::marker() }, 10); + swap.arg = OpArg::new(2); + block.instructions.push(swap); + block.instructions.push(test_instr(Instruction::PopTop, 10)); + + let mut code = test_code_info(block); + code.optimize_load_fast(); + + // CPython `optimize_load_fast()` shadows the outer instruction index in + // the produced-value loop for GET_LEN, so the produced ref is recorded + // with index 0 here. The original LOAD_FAST is therefore not considered + // the consumed producer. + assert!(matches!( + code.blocks[0].instructions[0].instr.real(), + Some(Instruction::LoadFast { .. }) + )); + } + + #[test] + fn constant_sequence_loads_use_cpython_opcode_has_const_metadata() { + let mut metadata = CodeUnitMetadata { + name: "".to_owned(), + qualname: Some("".to_owned()), + consts: Default::default(), + names: IndexSet::default(), + varnames: IndexSet::default(), + cellvars: IndexSet::default(), + freevars: IndexSet::default(), + fast_hidden: IndexMap::default(), + fast_hidden_final: IndexSet::default(), + argcount: 0, + posonlyargcount: 0, + kwonlyargcount: 0, + firstlineno: OneIndexed::MIN, + }; + let (left, _) = metadata + .consts + .insert_full(ConstantData::Str { value: "a".into() }); + let (right, _) = metadata + .consts + .insert_full(ConstantData::Str { value: "b".into() }); + + let mut immortal = test_instr(Instruction::Nop, 90); + immortal.instr = Opcode::LoadConstImmortal.into(); + immortal.arg = OpArg::new(left as u32); + let mut mortal = test_instr(Instruction::Nop, 90); + mortal.instr = Opcode::LoadConstMortal.into(); + mortal.arg = OpArg::new(right as u32); + let mut build = test_instr( + Instruction::BuildTuple { + count: Arg::marker(), + }, + 90, + ); + build.arg = OpArg::new(2); + let mut block = Block::default(); + block.instructions.extend([immortal, mortal, build]); + + assert!(CodeInfo::fold_tuple_of_constants( + &mut metadata, + &mut block, + 2 + )); + + // CPython `loads_const()` accepts every `OPCODE_HAS_CONST` opcode, not + // just canonical LOAD_CONST, so LOAD_CONST_IMMORTAL/MORTAL participate + // in constant-sequence folding. + assert!(matches!( + block.instructions[0].instr.real(), + Some(Instruction::Nop) + )); + assert!(matches!( + block.instructions[1].instr.real(), + Some(Instruction::Nop) + )); + let folded = &block.instructions[2]; + assert!(matches!( + folded.instr.real(), + Some(Instruction::LoadConst { .. }) + )); + assert!(matches!( + &metadata.consts[u32::from(folded.arg) as usize], + ConstantData::Tuple { elements } if elements.len() == 2 + )); + } + + #[test] + fn resolve_line_numbers_duplicates_exit_blocks_like_cpython() { + let exit = BlockIdx::new(2); + let mut blocks = vec![Block::default(), Block::default(), Block::default()]; + blocks[0].cpython_label_id = Some(InstructionSequenceLabel(0)); + blocks[1].cpython_label_id = Some(InstructionSequenceLabel(1)); + blocks[2].cpython_label_id = Some(InstructionSequenceLabel(2)); + blocks[0].next = BlockIdx::new(1); + blocks[0].instructions.push(test_cond_jump(exit, 10)); + blocks[1].next = exit; + blocks[1].instructions.push(test_jump(exit, 20)); + blocks[2] + .instructions + .push(test_instr(Instruction::ReturnValue, 30)); + blocks[2].instructions[0].lineno_override = Some(-1); + + compute_reachable_predecessors(&mut blocks); + resolve_line_numbers(&mut blocks); + + // CPython `duplicate_exits_without_lineno()` copies a shared exit block + // reached by jumps so each copy can inherit its sole predecessor's line. + let duplicate = blocks[0].instructions[0].target; + assert_ne!(duplicate, exit); + assert_eq!( + blocks[duplicate.idx()].cpython_label_id, + Some(InstructionSequenceLabel(3)) + ); + assert_eq!( + instruction_lineno(&blocks[duplicate.idx()].instructions[0]), + 10 + ); + assert_eq!(blocks[1].instructions[0].target, exit); + assert_eq!(instruction_lineno(&blocks[exit.idx()].instructions[0]), 20); + } + + #[test] + fn propagate_line_numbers_treats_next_location_like_cpython() { + let mut block = Block::default(); + block.instructions.push(test_instr(Instruction::Nop, 10)); + block.instructions.push(test_instr(Instruction::PopTop, 20)); + block.instructions[1].lineno_override = Some(NEXT_LOCATION_OVERRIDE); + block + .instructions + .push(test_instr(Instruction::ReturnValue, 30)); + block.instructions[2].lineno_override = Some(-1); + let mut blocks = vec![block]; + + compute_reachable_predecessors(&mut blocks); + propagate_line_numbers(&mut blocks); + + // CPython `propagate_line_numbers()` only copies over NO_LOCATION + // (`lineno == -1`). `NEXT_LOCATION` (`lineno == -2`) becomes the + // current previous location and is copied to following NO_LOCATION + // instructions for assemble.c to resolve later. + assert_eq!( + blocks[0].instructions[1].lineno_override, + Some(NEXT_LOCATION_OVERRIDE) + ); + assert_eq!( + blocks[0].instructions[2].lineno_override, + Some(NEXT_LOCATION_OVERRIDE) + ); + } + + #[test] + fn propagate_line_numbers_updates_empty_jump_target_raw_slot_like_cpython() { + let mut blocks = vec![Block::default(), Block::default(), Block::default()]; + blocks[0].next = BlockIdx::new(2); + blocks[0] + .instructions + .push(test_cond_jump(BlockIdx::new(1), 10)); + blocks[1] + .cpython_spare_instr_slots + .push_back(test_instr(Instruction::Nop, 20)); + blocks[1].cpython_spare_instr_slots[0].lineno_override = Some(-1); + blocks[2] + .instructions + .push(test_instr(Instruction::ReturnValue, 30)); + + compute_reachable_predecessors(&mut blocks); + propagate_line_numbers(&mut blocks); + + // CPython `propagate_line_numbers()` directly reads `target->b_instr[0]` + // for jump targets without checking `b_iused`. If + // `remove_redundant_nops()` emptied the target, that writes the stale + // backing slot rather than an active instruction. + assert_eq!( + instruction_lineno(&blocks[1].cpython_spare_instr_slots[0]), + 10 + ); + } + + #[test] + fn basicblock_has_no_lineno_treats_next_location_like_cpython() { + let mut block = Block::default(); + block.instructions.push(test_instr(Instruction::Nop, 10)); + block.instructions[0].lineno_override = Some(NEXT_LOCATION_OVERRIDE); + + // CPython `basicblock_has_no_lineno()` treats every negative lineno as + // no line number, including `NEXT_LOCATION` (`lineno == -2`). + assert!(basicblock_has_no_lineno(&block)); + + block.instructions.push(test_instr(Instruction::PopTop, 11)); + assert!(!basicblock_has_no_lineno(&block)); + } + + #[test] + fn jump_threading_rechecks_new_jump_like_cpython() { + let mut blocks = vec![ + Block::default(), + Block::default(), + Block::default(), + Block::default(), + ]; + for (i, block) in blocks.iter_mut().enumerate() { + block.cpython_label_id = Some(InstructionSequenceLabel(i)); + } + blocks[0].next = BlockIdx::new(1); + blocks[1].next = BlockIdx::new(2); + blocks[2].next = BlockIdx::new(3); + blocks[0].instructions.push(test_jump(BlockIdx::new(1), 10)); + blocks[1].instructions.push(test_jump(BlockIdx::new(2), 20)); + blocks[2].instructions.push(test_jump(BlockIdx::new(3), 30)); + blocks[3] + .instructions + .push(test_instr(Instruction::ReturnValue, 40)); + + jump_threading_block(&mut blocks, BlockIdx::new(0)).expect("valid jump chain"); + + // CPython `optimize_basic_block()` continues after `jump_thread()`, so + // the appended jump is immediately checked against the next jump target. + let threaded = blocks[0].instructions.last().expect("threaded jump"); + assert!(matches!( + threaded.instr.pseudo(), + Some(PseudoInstruction::Jump { .. }) + )); + assert_eq!(threaded.target, BlockIdx::new(3)); + assert_eq!(u32::from(threaded.arg), 3); + } +} diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 28133b1008b..10144597f46 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -60,6 +60,9 @@ pub struct SymbolTable { /// Whether this scope contains yield/yield from (is a generator function) pub is_generator: bool, + /// Whether this scope contains await or async comprehension machinery. + pub is_coroutine: bool, + /// Whether this comprehension scope should be inlined (PEP 709) /// True for list/set/dict comprehensions in non-generator expressions pub comp_inlined: bool, @@ -102,6 +105,7 @@ impl SymbolTable { needs_classdict: false, can_see_class_scope: false, is_generator: false, + is_coroutine: false, comp_inlined: false, annotation_block: None, skip_enclosing_function_scope: false, @@ -1532,6 +1536,9 @@ impl SymbolTableBuilder { }, has_type_params, // skip_defaults: already scanned above )?; + if *is_async { + self.tables.last_mut().unwrap().is_coroutine = true; + } self.scan_statements(body)?; self.leave_scope(); if type_params.is_some() { @@ -2026,6 +2033,7 @@ impl SymbolTableBuilder { range: _, }) => { self.scan_expression(value, context)?; + self.tables.last_mut().unwrap().is_coroutine = true; } Expr::Yield(ExprYield { value, @@ -2353,6 +2361,9 @@ impl SymbolTableBuilder { ); // Generator expressions need the is_generator flag self.tables.last_mut().unwrap().is_generator = is_generator; + if generators.iter().any(|generator| generator.is_async) { + self.tables.last_mut().unwrap().is_coroutine = true; + } // PEP 709: Mark non-generator comprehensions for inlining. // CPython's symtable marks all non-generator comprehensions for @@ -2389,7 +2400,14 @@ impl SymbolTableBuilder { } self.scan_expression(elt1, ExpressionContext::Load)?; + // CPython symtable_handle_comprehension(): non-generator async + // comprehensions propagate ste_coroutine to the enclosing scope after + // the comprehension block is exited. + let propagate_coroutine = self.tables.last().unwrap().is_coroutine && !is_generator; self.leave_scope(); + if propagate_coroutine { + self.tables.last_mut().unwrap().is_coroutine = true; + } Ok(()) } diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 3cc1cc98d08..802dd1ece2b 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -914,6 +914,22 @@ impl AnyInstruction { pub const fn has_jump(&self) -> bool ); + #[must_use] + pub const fn has_arg(&self) -> bool { + match self { + Self::Real(instr) => instr.as_opcode().has_arg(), + Self::Pseudo(instr) => instr.as_opcode().has_arg(), + } + } + + #[must_use] + pub const fn has_const(&self) -> bool { + match self { + Self::Real(instr) => instr.as_opcode().has_const(), + Self::Pseudo(instr) => instr.as_opcode().has_const(), + } + } + either_real_pseudo!( #[must_use] pub const fn has_eval_break(&self) -> bool @@ -1348,12 +1364,15 @@ mod tests { fn terminator_flags_match_cpython_opcode_utils() { assert!(Opcode::JumpForward.is_terminator()); assert!(Opcode::PopJumpIfFalse.is_terminator()); + assert!(Opcode::ForIter.is_terminator()); assert!(Opcode::ReturnValue.is_terminator()); assert!(!Opcode::Nop.is_terminator()); assert!(PseudoOpcode::JumpIfTrue.is_terminator()); assert!(PseudoOpcode::JumpNoInterrupt.is_terminator()); + assert!(!PseudoOpcode::SetupFinally.is_terminator()); assert!(!PseudoOpcode::SetupWith.is_terminator()); + assert!(!PseudoOpcode::SetupCleanup.is_terminator()); assert!(!PseudoOpcode::PopBlock.is_terminator()); assert!(AnyInstruction::from(PseudoOpcode::JumpIfFalse).is_terminator()); @@ -1380,6 +1399,7 @@ mod tests { assert!(!Opcode::Nop.has_target()); assert!(PseudoOpcode::Jump.has_target()); + assert!(PseudoOpcode::SetupFinally.has_target()); assert!(PseudoOpcode::SetupWith.has_target()); assert!(PseudoOpcode::SetupCleanup.has_target()); assert!(!PseudoOpcode::PopBlock.has_target()); @@ -1387,16 +1407,52 @@ mod tests { assert!(AnyInstruction::from(PseudoOpcode::SetupFinally).has_target()); } + #[test] + fn arg_flags_match_cpython_opcode_metadata() { + assert!(Opcode::LoadConst.has_arg()); + assert!(Opcode::YieldValue.has_arg()); + assert!(!Opcode::Nop.has_arg()); + assert!(!Opcode::ReturnValue.has_arg()); + + assert!(PseudoOpcode::Jump.has_arg()); + assert!(PseudoOpcode::JumpIfFalse.has_arg()); + assert!(PseudoOpcode::JumpIfTrue.has_arg()); + assert!(PseudoOpcode::JumpNoInterrupt.has_arg()); + assert!(PseudoOpcode::LoadClosure.has_arg()); + assert!(PseudoOpcode::SetupCleanup.has_arg()); + assert!(PseudoOpcode::SetupFinally.has_arg()); + assert!(PseudoOpcode::SetupWith.has_arg()); + assert!(PseudoOpcode::StoreFastMaybeNull.has_arg()); + assert!(!PseudoOpcode::AnnotationsPlaceholder.has_arg()); + assert!(!PseudoOpcode::PopBlock.has_arg()); + + assert!(AnyInstruction::from(PseudoOpcode::SetupFinally).has_arg()); + } + + #[test] + fn const_flags_match_cpython_opcode_metadata() { + assert!(Opcode::LoadConst.has_const()); + assert!(Opcode::LoadConstImmortal.has_const()); + assert!(Opcode::LoadConstMortal.has_const()); + assert!(!Opcode::LoadSmallInt.has_const()); + assert!(!Opcode::Nop.has_const()); + + assert!(!PseudoOpcode::LoadClosure.has_const()); + assert!(!AnyInstruction::from(PseudoOpcode::Jump).has_const()); + } + #[test] fn no_fallthrough_flags_match_cpython_basicblock_nofallthrough() { assert!(Opcode::JumpForward.is_no_fallthrough()); assert!(Opcode::ReturnValue.is_no_fallthrough()); assert!(!Opcode::PopJumpIfFalse.is_no_fallthrough()); + assert!(!Opcode::ForIter.is_no_fallthrough()); assert!(!Opcode::Nop.is_no_fallthrough()); assert!(PseudoOpcode::Jump.is_no_fallthrough()); assert!(PseudoOpcode::JumpNoInterrupt.is_no_fallthrough()); assert!(!PseudoOpcode::JumpIfFalse.is_no_fallthrough()); + assert!(!PseudoOpcode::SetupFinally.is_no_fallthrough()); assert!(!PseudoOpcode::SetupWith.is_no_fallthrough()); assert!(AnyInstruction::from(PseudoOpcode::Jump).is_no_fallthrough()); From 539f3ba63ed9b332aa319ac7c3301a7f3a599f13 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 10:37:29 +0900 Subject: [PATCH 011/131] Align codegen IR with CPython CFG structures --- crates/codegen/src/compile.rs | 117 +- crates/codegen/src/ir.rs | 8641 ++++++++++++++------------ crates/compiler-core/src/bytecode.rs | 7 +- 3 files changed, 4763 insertions(+), 4002 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c16216aedb4..5987183bb37 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -133,9 +133,9 @@ enum BuiltinGeneratorCallKind { pub struct FBlockInfo { pub fb_type: FBlockType, // CPython _PyCompile_FBlockInfo stores jump_target_label values here. - pub(crate) fb_block: Option, + pub(crate) fb_block: ir::InstructionSequenceLabel, // CPython's optional type-specific exit or cleanup jump_target_label. - pub(crate) fb_exit: Option, + pub(crate) fb_exit: ir::InstructionSequenceLabel, pub fb_range: TextRange, // additional data for fblock unwinding pub fb_datum: FBlockDatum, @@ -511,6 +511,7 @@ impl Compiler { in_inlined_comp: false, fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: 0, // Module is always the first symbol table + nparams: 0, in_conditional_block: 0, next_conditional_annotation_index: 0, }; @@ -1206,6 +1207,7 @@ impl Compiler { // Use varnames from symbol table (already collected in definition order) let varname_cache: IndexSet = ste.varnames.iter().cloned().collect(); + let nparams = ste.varnames.len(); // Build cellvars using dictbytype (CELL scope or COMP_CELL flag, sorted) let mut cellvar_cache = IndexSet::default(); @@ -1393,6 +1395,7 @@ impl Compiler { in_inlined_comp: false, fblock: Vec::with_capacity(MAXBLOCKS), symbol_table_index: key, + nparams, in_conditional_block: 0, next_conditional_annotation_index: 0, }; @@ -1443,7 +1446,6 @@ impl Compiler { end_location, except_handler, lineno_override, - cache_entries: 0, }); } @@ -1602,8 +1604,8 @@ impl Compiler { fn push_fblock_labels( &mut self, fb_type: FBlockType, - fb_block: Option, - fb_exit: Option, + fb_block: ir::InstructionSequenceLabel, + fb_exit: ir::InstructionSequenceLabel, fb_datum: FBlockDatum, ) -> CompileResult<()> { let fb_range = self.current_source_range; @@ -1627,7 +1629,7 @@ impl Compiler { fn pop_fblock_label( &mut self, expected_type: FBlockType, - expected_block: Option, + expected_block: ir::InstructionSequenceLabel, ) -> FBlockInfo { let code = self.current_code_info(); let fblock = code.fblock.pop().expect("fblock stack underflow"); @@ -1889,8 +1891,8 @@ impl Compiler { if preserve_tos { self.push_fblock_labels( FBlockType::PopValue, - None, - None, + ir::InstructionSequenceLabel::NO_LABEL, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; } @@ -1899,7 +1901,10 @@ impl Compiler { unwind_loc = None; if preserve_tos { - self.pop_fblock_label(FBlockType::PopValue, None); + self.pop_fblock_label( + FBlockType::PopValue, + ir::InstructionSequenceLabel::NO_LABEL, + ); } // Restore the fblock @@ -3598,7 +3603,7 @@ impl Compiler { self.push_fblock_labels( FBlockType::FinallyEnd, finally_except_label, - None, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; self.compile_statements(finalbody)?; @@ -3639,7 +3644,12 @@ impl Compiler { let body_label = self .current_code_info() .instr_sequence_label_for_block(body_block); - self.push_fblock_labels(FBlockType::TryExcept, body_label, None, FBlockDatum::None)?; + self.push_fblock_labels( + FBlockType::TryExcept, + body_label, + ir::InstructionSequenceLabel::NO_LABEL, + FBlockDatum::None, + )?; self.compile_statements(body)?; self.pop_fblock_label(FBlockType::TryExcept, body_label); emit!(self, PseudoInstruction::PopBlock); @@ -3661,7 +3671,12 @@ impl Compiler { self.set_no_location(); emit!(self, Instruction::PushExcInfo); self.set_no_location(); - self.push_fblock_labels(FBlockType::ExceptionHandler, None, None, FBlockDatum::None)?; + self.push_fblock_labels( + FBlockType::ExceptionHandler, + ir::InstructionSequenceLabel::NO_LABEL, + ir::InstructionSequenceLabel::NO_LABEL, + FBlockDatum::None, + )?; for (i, handler) in handlers.iter().enumerate() { let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -3705,7 +3720,7 @@ impl Compiler { self.push_fblock_labels( FBlockType::HandlerCleanup, cleanup_body_label, - None, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::ExceptionName(alias.as_str().to_owned()), )?; @@ -3742,7 +3757,7 @@ impl Compiler { self.push_fblock_labels( FBlockType::HandlerCleanup, cleanup_body_label, - None, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; @@ -3765,7 +3780,10 @@ impl Compiler { emit!(self, Instruction::Reraise { depth: 0 }); self.set_no_location(); - self.pop_fblock_label(FBlockType::ExceptionHandler, None); + self.pop_fblock_label( + FBlockType::ExceptionHandler, + ir::InstructionSequenceLabel::NO_LABEL, + ); self.use_cpython_label_block(cleanup_block); emit!(self, Instruction::Copy { i: 3 }); @@ -3854,7 +3872,7 @@ impl Compiler { self.push_fblock_labels( FBlockType::FinallyEnd, finally_except_label, - None, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; self.compile_statements(finalbody)?; @@ -3901,7 +3919,12 @@ impl Compiler { let body_label = self .current_code_info() .instr_sequence_label_for_block(body_block); - self.push_fblock_labels(FBlockType::TryExcept, body_label, None, FBlockDatum::None)?; + self.push_fblock_labels( + FBlockType::TryExcept, + body_label, + ir::InstructionSequenceLabel::NO_LABEL, + FBlockDatum::None, + )?; self.compile_statements(body)?; emit!(self, PseudoInstruction::PopBlock); self.set_no_location(); @@ -3932,8 +3955,8 @@ impl Compiler { // Push EXCEPTION_GROUP_HANDLER fblock self.push_fblock_labels( FBlockType::ExceptionGroupHandler, - None, - None, + ir::InstructionSequenceLabel::NO_LABEL, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; @@ -4013,7 +4036,7 @@ impl Compiler { self.push_fblock_labels( FBlockType::HandlerCleanup, cleanup_body_label, - None, + ir::InstructionSequenceLabel::NO_LABEL, if let Some(alias) = name { FBlockDatum::ExceptionName(alias.as_str().to_owned()) } else { @@ -4106,7 +4129,10 @@ impl Compiler { } // Pop EXCEPTION_GROUP_HANDLER fblock - self.pop_fblock_label(FBlockType::ExceptionGroupHandler, None); + self.pop_fblock_label( + FBlockType::ExceptionGroupHandler, + ir::InstructionSequenceLabel::NO_LABEL, + ); let reraise_block = self.new_block(); // Reraise star block @@ -4310,8 +4336,8 @@ impl Compiler { self.insert_cpython_stopiteration_setup_cleanup(handler_block); self.push_fblock_labels( FBlockType::StopIteration, - Some(start_label), - None, + start_label, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; Some(handler_block) @@ -4338,7 +4364,7 @@ impl Compiler { // Close StopIteration handler and emit handler code if let Some(handler_block) = stop_iteration_block { - self.pop_fblock_label(FBlockType::StopIteration, Some(start_label)); + self.pop_fblock_label(FBlockType::StopIteration, start_label); self.use_cpython_label_block(handler_block); emit!( self, @@ -4722,7 +4748,7 @@ impl Compiler { self.set_source_range(def_source_range); let is_generic = type_params.is_some(); - let mut num_typeparam_args = 0; + let mut num_typeparam_args = 0u32; // Save context before entering TypeParams scope let saved_ctx = self.ctx; @@ -4744,7 +4770,7 @@ impl Compiler { self.push_output( bytecode::CodeFlags::OPTIMIZED | bytecode::CodeFlags::NEWLOCALS, 0, - num_typeparam_args as u32, + num_typeparam_args, 0, &type_params_name, )?; @@ -4777,7 +4803,7 @@ impl Compiler { // Load defaults/kwdefaults with LOAD_FAST for i in 0..num_typeparam_args { - let var_num = oparg::VarNum::from(i as u32); + let var_num = oparg::VarNum::from(i); emit!(self, Instruction::LoadFast { var_num }); } } @@ -4818,7 +4844,8 @@ impl Compiler { emit!(self, Instruction::ReturnValue); // Set argcount for type params scope - self.current_code_info().metadata.argcount = num_typeparam_args as u32; + self.current_code_info().metadata.argcount = num_typeparam_args; + self.current_code_info().nparams = num_typeparam_args as usize; // Exit type params scope and create closure let type_params_code = self.exit_scope(); @@ -4831,13 +4858,13 @@ impl Compiler { emit!( self, Instruction::Swap { - i: num_typeparam_args as u32 + 1 + i: num_typeparam_args + 1 } ); emit!( self, Instruction::Call { - argc: num_typeparam_args as u32 - 1 + argc: num_typeparam_args - 1 } ); } else { @@ -9468,7 +9495,7 @@ impl Compiler { self.push_fblock_labels( FBlockType::AsyncComprehensionGenerator, loop_label, - None, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); @@ -9848,7 +9875,7 @@ impl Compiler { self.push_fblock_labels( FBlockType::AsyncComprehensionGenerator, loop_label, - None, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, )?; emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); @@ -10102,7 +10129,6 @@ impl Compiler { end_location, except_handler, lineno_override: None, - cache_entries: 0, }); } @@ -10131,7 +10157,7 @@ impl Compiler { let end_location = source.source_location(range.end(), PositionEncoding::Utf8); let target = self .current_code_info() - .block_for_instr_sequence_label(Some(target_label)); + .block_for_instr_sequence_label(target_label); self.maybe_start_cpython_cfg_addop_block(); self.push_emitted_instruction_with_target_label( ir::InstructionInfo { @@ -10142,7 +10168,6 @@ impl Compiler { end_location, except_handler: None, lineno_override: None, - cache_entries: 0, }, target_label, ); @@ -10931,13 +10956,11 @@ impl Compiler { // Jump to target let target_label = if is_break { - loop_fblock - .fb_exit - .expect("loop fblock must have a CPython exit label") + debug_assert!(loop_fblock.fb_exit.is_jump_target_label()); + loop_fblock.fb_exit } else { - loop_fblock - .fb_block - .expect("loop fblock must have a CPython block label") + debug_assert!(loop_fblock.fb_block.is_jump_target_label()); + loop_fblock.fb_block }; if let Some(loc) = unwind_loc { self.set_source_range(loc); @@ -11021,7 +11044,7 @@ impl Compiler { let target = &code.blocks[block.idx()]; debug_assert!(!target.except_handler); debug_assert!(!target.preserve_lasti); - debug_assert!(target.start_depth.is_none()); + debug_assert_eq!(target.start_depth, ir::START_DEPTH_UNSET); debug_assert!(!target.cold); } code.mark_cpython_cfg_label(cur); @@ -11034,7 +11057,7 @@ impl Compiler { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); - code.instr_sequence_label_map.push_unmapped_label(); + code.push_unmapped_instr_sequence_label(); idx } @@ -11042,7 +11065,7 @@ impl Compiler { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); - code.instr_sequence_label_map.push_unlabeled_block(); + code.push_unlabeled_instr_sequence_block(); idx } @@ -12423,8 +12446,8 @@ def f(x, y, z): compiler .push_fblock_labels( FBlockType::StopIteration, - Some(start_label), - None, + start_label, + ir::InstructionSequenceLabel::NO_LABEL, FBlockDatum::None, ) .unwrap(); @@ -12441,7 +12464,7 @@ def f(x, y, z): compiler.arg_constant(ConstantData::None); } if let Some(handler_block) = stop_iteration_block { - compiler.pop_fblock_label(FBlockType::StopIteration, Some(start_label)); + compiler.pop_fblock_label(FBlockType::StopIteration, start_label); compiler.use_cpython_label_block(handler_block); emit!( compiler, diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 19ed452d392..8118b971713 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1,4 +1,3 @@ -use alloc::collections::VecDeque; use core::ops; use crate::{IndexMap, IndexSet, error::InternalError}; @@ -10,10 +9,10 @@ use rustpython_wtf8::Wtf8Buf; use rustpython_compiler_core::{ OneIndexed, SourceLocation, bytecode::{ - AnyInstruction, AnyOpcode, Arg, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, - CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, ExceptionTableEntry, - InstrDisplayContext, Instruction, IntrinsicFunction1, OpArg, Opcode, PseudoInstruction, - PseudoOpcode, PyCodeLocationInfoKind, encode_exception_table, oparg, + AnyInstruction, AnyOpcode, Arg, CO_FAST_ARG_KW, CO_FAST_ARG_POS, CO_FAST_ARG_VAR, + CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, CodeFlags, CodeObject, CodeUnit, + CodeUnits, ConstantData, InstrDisplayContext, Instruction, IntrinsicFunction1, OpArg, + OpArgByte, Opcode, PseudoInstruction, PseudoOpcode, PyCodeLocationInfoKind, oparg, }, varint::{write_signed_varint, write_varint}, }; @@ -27,11 +26,22 @@ struct LineTableLocation { end_col: i32, } +#[derive(Clone, Copy)] +struct InstructionLocation { + location: SourceLocation, + end_location: SourceLocation, + lineno_override: Option, +} + pub(crate) const LINE_ONLY_LOCATION_OVERRIDE: i32 = -4; pub(crate) const NEXT_LOCATION_OVERRIDE: i32 = -2; +pub(crate) const NO_LOCATION_OVERRIDE: i32 = -1; const MAX_INT_SIZE: u64 = 128; const MAX_COLLECTION_SIZE: usize = 256; +const DEFAULT_BLOCK_SIZE: usize = 16; +const INITIAL_INSTR_SEQUENCE_SIZE: usize = 100; +const INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE: usize = 10; const MAX_TOTAL_ITEMS: isize = 1024; const MAX_STR_SIZE: usize = 4096; const MIN_CONST_SEQUENCE_SIZE: usize = 3; @@ -199,8 +209,6 @@ pub struct InstructionInfo { pub except_handler: Option, /// Override line number for linetable (e.g., line 0 for module RESUME) pub lineno_override: Option, - /// Number of CACHE code units emitted after this instruction - pub cache_entries: u32, } /// Exception handler information for an instruction. @@ -208,26 +216,129 @@ pub struct InstructionInfo { pub struct ExceptHandlerInfo { /// Block to jump to when exception occurs pub handler_block: BlockIdx, - /// Stack depth at handler entry - pub stack_depth: u32, /// Whether to push lasti before exception pub preserve_lasti: bool, } -fn set_to_nop(info: &mut InstructionInfo) { - info.instr = Instruction::Nop.into(); +/// flowgraph.c INSTR_SET_OP0 +fn instr_set_op0(info: &mut InstructionInfo, instr: AnyInstruction) { + debug_assert!(!AnyOpcode::from(instr).has_arg()); + info.instr = instr; info.arg = OpArg::new(0); - info.cache_entries = 0; +} + +/// flowgraph.c INSTR_SET_OP1 +fn instr_set_op1(info: &mut InstructionInfo, instr: AnyInstruction, arg: OpArg) { + debug_assert!(AnyOpcode::from(instr).has_arg()); + info.instr = instr; + info.arg = arg; +} + +/// flowgraph.c INSTR_SET_LOC +fn instr_set_loc( + info: &mut InstructionInfo, + location: SourceLocation, + end_location: SourceLocation, + lineno_override: Option, +) { + info.location = location; + info.end_location = end_location; + info.lineno_override = lineno_override; +} + +fn instr_location(info: &InstructionInfo) -> InstructionLocation { + InstructionLocation { + location: info.location, + end_location: info.end_location, + lineno_override: info.lineno_override, + } +} + +fn instr_set_location(info: &mut InstructionInfo, loc: InstructionLocation) { + instr_set_loc(info, loc.location, loc.end_location, loc.lineno_override); +} + +fn no_instruction_location() -> InstructionLocation { + InstructionLocation { + location: SourceLocation::default(), + end_location: SourceLocation::default(), + lineno_override: Some(NO_LOCATION_OVERRIDE), + } +} + +fn set_to_nop(info: &mut InstructionInfo) { + instr_set_op0(info, Instruction::Nop.into()); } fn nop_out_no_location(info: &mut InstructionInfo) { set_to_nop(info); - info.lineno_override = Some(-1); + instr_set_loc( + info, + SourceLocation::default(), + SourceLocation::default(), + Some(NO_LOCATION_OVERRIDE), + ); +} + +fn empty_instruction_info() -> InstructionInfo { + InstructionInfo { + instr: Instruction::Nop.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: None, + } +} + +/// codegen.c _Py_CArray_EnsureCapacity +fn c_array_ensure_capacity( + allocated_entries: usize, + idx: usize, + initial_num_entries: usize, +) -> usize { + if allocated_entries == 0 { + if idx >= initial_num_entries { + idx + initial_num_entries + } else { + initial_num_entries + } + } else if idx >= allocated_entries { + let doubled = allocated_entries + .checked_mul(2) + .expect("CPython C array allocation size overflow"); + if idx >= doubled { + idx + initial_num_entries + } else { + doubled + } + } else { + allocated_entries + } } /// flowgraph.c basicblock_next_instr -fn basicblock_next_instr(block: &mut Block) -> Option { - block.cpython_spare_instr_slots.pop_front() +fn basicblock_next_instr(block: &mut Block) -> usize { + let off = block.instructions.len(); + let new_allocation = + c_array_ensure_capacity(block.instruction_allocation, off + 1, DEFAULT_BLOCK_SIZE); + if new_allocation > block.instruction_allocation { + block.instruction_allocation = new_allocation; + if new_allocation > block.instructions.capacity() + block.cpython_spare_instr_slots.len() { + block.instructions.reserve_exact( + new_allocation + - block.instructions.capacity() + - block.cpython_spare_instr_slots.len(), + ); + } + } + let slot = block + .cpython_spare_instr_slots + .pop() + .unwrap_or_else(empty_instruction_info); + block.instructions.push(slot); + off } /// flowgraph.c basicblock_last_instr @@ -242,50 +353,38 @@ fn basicblock_last_instr_mut(block: &mut Block) -> Option<&mut InstructionInfo> /// flowgraph.c basicblock_addop fn basicblock_addop(block: &mut Block, mut info: InstructionInfo) { - if let Some(stale) = basicblock_next_instr(block) { - info.except_handler = stale.except_handler; - } + debug_assert!(!info.instr.is_assembler()); + debug_assert!( + info.instr.has_arg() || info.instr.has_target() || u32::from(info.arg) == 0, + "CPython basicblock_addop requires OPCODE_HAS_ARG, HAS_TARGET, or oparg == 0" + ); + debug_assert!( + u32::from(info.arg) < (1 << 30), + "CPython basicblock_addop requires 0 <= oparg < (1 << 30)" + ); + let off = basicblock_next_instr(block); + let except_handler = block.instructions[off].except_handler; info.target = BlockIdx::NULL; - block.instructions.push(info); -} - -/// flowgraph.c basicblock_add_jump -fn basicblock_add_jump_op( - block: &mut Block, - info: InstructionInfo, - target: BlockIdx, -) -> crate::InternalResult<()> { - if block - .instructions - .last() - .is_some_and(|last| last.instr.has_jump()) - { - return Err(InternalError::MalformedControlFlowGraph); - } - basicblock_addop(block, info); - block.instructions.last_mut().expect("missing jump").target = target; - Ok(()) -} - -fn cpython_target_label_arg(blocks: &[Block], target: BlockIdx) -> OpArg { - debug_assert!(target != BlockIdx::NULL); - let label = blocks[target.idx()] - .cpython_label_id - .expect("target block has a CPython CFG label"); - OpArg::new(label.idx().to_u32().expect("too many CPython CFG labels")) + info.except_handler = except_handler; + block.instructions[off] = info; } /// flowgraph.c basicblock_insert_instruction fn basicblock_insert_instruction(block: &mut Block, pos: usize, info: InstructionInfo) { + let old_len = block.instructions.len(); + debug_assert!(pos <= old_len); basicblock_next_instr(block); - block.instructions.insert(pos, info); + for i in (pos + 1..=old_len).rev() { + block.instructions[i] = block.instructions[i - 1]; + } + block.instructions[pos] = info; } /// flowgraph.c basicblock_append_instructions fn basicblock_append_instructions(to: &mut Block, from: &[InstructionInfo]) { for info in from { - basicblock_next_instr(to); - to.instructions.push(*info); + let off = basicblock_next_instr(to); + to.instructions[off] = *info; } } @@ -293,9 +392,9 @@ fn basicblock_append_instructions(to: &mut Block, from: &[InstructionInfo]) { fn basicblock_clear(block: &mut Block) { let instructions = core::mem::take(&mut block.instructions); if !instructions.is_empty() { - let spare_instr_slots = core::mem::take(&mut block.cpython_spare_instr_slots); - block.cpython_spare_instr_slots = instructions.into(); - block.cpython_spare_instr_slots.extend(spare_instr_slots); + block + .cpython_spare_instr_slots + .extend(instructions.into_iter().rev()); } } @@ -308,456 +407,990 @@ fn basicblock_raw_first_instr_mut(block: &mut Block) -> &mut InstructionInfo { } else { block .cpython_spare_instr_slots - .front_mut() + .last_mut() .expect("CPython basicblock has a backing instruction slot") } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct InstructionSequenceLabel(usize); +pub(crate) struct InstructionSequenceLabel(i32); + +/// flowgraph.c SAME_LABEL +fn same_label(a: InstructionSequenceLabel, b: InstructionSequenceLabel) -> bool { + a == b +} + +/// flowgraph.c IS_LABEL +fn is_label(label: InstructionSequenceLabel) -> bool { + !same_label(label, InstructionSequenceLabel::NO_LABEL) +} impl InstructionSequenceLabel { + pub(crate) const NO_LABEL: Self = Self(-1); + + pub(crate) fn from_index(index: usize) -> Self { + Self(i32::try_from(index).expect("instruction-sequence label id fits in int")) + } + + pub(crate) fn is_jump_target_label(self) -> bool { + is_label(self) + } + pub(crate) fn idx(self) -> usize { - self.0 + usize::try_from(self.0).expect("instruction-sequence label id is non-negative") } } #[derive(Clone, Copy)] struct InstructionSequenceExceptHandlerInfo { - target_label: Option, - target_offset: Option, - stack_depth: u32, - preserve_lasti: bool, + h_label: i32, + start_depth: i32, + preserve_lasti: i32, } +const NO_EXCEPTION_HANDLER_LABEL: i32 = -1; +const ZERO_EXCEPTION_HANDLER_INFO: InstructionSequenceExceptHandlerInfo = + InstructionSequenceExceptHandlerInfo { + h_label: 0, + start_depth: 0, + preserve_lasti: 0, + }; + #[derive(Clone, Copy)] struct InstructionSequenceEntry { info: InstructionInfo, - target_label: Option, - target_offset: Option, - except_handler: Option, - is_target: bool, + except_handler: InstructionSequenceExceptHandlerInfo, + i_target: i32, + i_offset: i32, } impl InstructionSequenceEntry { - fn new( - info: InstructionInfo, - target_label: Option, - except_handler: Option, - ) -> Self { + fn new(info: InstructionInfo, except_handler: InstructionSequenceExceptHandlerInfo) -> Self { Self { info, - target_label, - target_offset: None, except_handler, - is_target: false, + i_target: 0, + i_offset: 0, } } } -const INSTRUCTION_SEQUENCE_UNSET_LABEL: isize = -111; - -#[derive(Clone)] -enum InstructionSequenceLabelOffsets { - Active(Vec), - Applied, -} +const INSTRUCTION_SEQUENCE_UNSET_LABEL: i32 = -111; #[derive(Clone)] pub(crate) struct InstructionSequence { instrs: Vec, - label_map: InstructionSequenceLabelOffsets, + instr_allocation: usize, + next_free_label: usize, + label_map: Option>, + label_map_allocation: usize, annotations_code: Option>, } impl InstructionSequence { pub(crate) fn new() -> Self { - Self { - instrs: Vec::new(), - label_map: InstructionSequenceLabelOffsets::Active(Vec::new()), - annotations_code: None, - } + instruction_sequence_new() } +} - /// instruction_sequence.c _PyInstructionSequence_Addop asserts. - fn debug_check_addop(info: &InstructionInfo) { - let opcode = AnyOpcode::from(info.instr); - debug_assert!( - opcode.has_arg() || info.instr.has_target() || u32::from(info.arg) == 0, - "CPython _PyInstructionSequence_Addop requires either OPCODE_HAS_ARG, HAS_TARGET, or oparg == 0" - ); - debug_assert!( - u32::from(info.arg) < (1 << 30), - "CPython _PyInstructionSequence_Addop requires 0 <= oparg < (1 << 30)" - ); +/// instruction_sequence.c _PyInstructionSequence_New / inst_seq_create +fn instruction_sequence_new() -> InstructionSequence { + InstructionSequence { + instrs: Vec::new(), + instr_allocation: 0, + next_free_label: 0, + label_map: None, + label_map_allocation: 0, + annotations_code: None, } +} - fn set_annotations_code(&mut self, annotations_code: Option>) { - debug_assert!(self.annotations_code.is_none()); - self.annotations_code = annotations_code; +/// instruction_sequence.c instr_sequence_next_inst +fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> usize { + let idx = seq.instrs.len(); + let new_allocation = + c_array_ensure_capacity(seq.instr_allocation, idx + 1, INITIAL_INSTR_SEQUENCE_SIZE); + if new_allocation > seq.instr_allocation { + seq.instr_allocation = new_allocation; + if new_allocation > seq.instrs.capacity() { + seq.instrs + .reserve_exact(new_allocation - seq.instrs.capacity()); + } } + seq.instrs.push(InstructionSequenceEntry::new( + InstructionInfo { + instr: Instruction::Cache.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: None, + }, + ZERO_EXCEPTION_HANDLER_INFO, + )); + idx +} - fn use_label(&mut self, label: InstructionSequenceLabel) { - let InstructionSequenceLabelOffsets::Active(label_map) = &mut self.label_map else { - panic!("instruction sequence label map already applied"); - }; - let old_len = label_map.len(); - if label_map.len() <= label.idx() { - label_map.resize(label.idx() + 1, INSTRUCTION_SEQUENCE_UNSET_LABEL); - } - for slot in &mut label_map[old_len..] { - *slot = INSTRUCTION_SEQUENCE_UNSET_LABEL; +/// instruction_sequence.c _PyInstructionSequence_NewLabel +fn instruction_sequence_new_label(seq: &mut InstructionSequence) -> InstructionSequenceLabel { + seq.next_free_label += 1; + InstructionSequenceLabel::from_index(seq.next_free_label) +} + +/// instruction_sequence.c _PyInstructionSequence_Addop asserts. +fn instruction_sequence_debug_check_addop(info: &InstructionInfo) { + let opcode = AnyOpcode::from(info.instr); + debug_assert!( + opcode.has_arg() || info.instr.has_target() || u32::from(info.arg) == 0, + "CPython _PyInstructionSequence_Addop requires either OPCODE_HAS_ARG, HAS_TARGET, or oparg == 0" + ); + debug_assert!( + u32::from(info.arg) < (1 << 30), + "CPython _PyInstructionSequence_Addop requires 0 <= oparg < (1 << 30)" + ); +} + +/// instruction_sequence.c _PyInstructionSequence_SetAnnotationsCode +fn instruction_sequence_set_annotations_code( + seq: &mut InstructionSequence, + annotations_code: Option>, +) { + debug_assert!(seq.annotations_code.is_none()); + seq.annotations_code = annotations_code; +} + +/// instruction_sequence.c _PyInstructionSequence_UseLabel +#[allow(clippy::needless_range_loop)] +fn instruction_sequence_use_label(seq: &mut InstructionSequence, label: InstructionSequenceLabel) { + let label_map = seq.label_map.get_or_insert_with(Vec::new); + let old_len = label_map.len(); + let new_allocation = c_array_ensure_capacity( + seq.label_map_allocation, + label.idx(), + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + ); + if new_allocation > seq.label_map_allocation { + seq.label_map_allocation = new_allocation; + if new_allocation > label_map.capacity() { + label_map.reserve_exact(new_allocation - label_map.capacity()); } - label_map[label.idx()] = self.instrs.len() as isize; } - - fn addop( - &mut self, - info: InstructionInfo, - target_label: Option, - except_handler: Option, - ) { - Self::debug_check_addop(&info); - self.instrs.push(InstructionSequenceEntry::new( - info, - target_label, - except_handler, - )); + if label_map.len() < seq.label_map_allocation { + label_map.resize(seq.label_map_allocation, INSTRUCTION_SEQUENCE_UNSET_LABEL); } - - fn last_info_mut(&mut self) -> Option<&mut InstructionInfo> { - self.instrs.last_mut().map(|entry| &mut entry.info) + for i in old_len..label_map.len() { + label_map[i] = INSTRUCTION_SEQUENCE_UNSET_LABEL; } + label_map[label.idx()] = + i32::try_from(seq.instrs.len()).expect("instruction offset fits in int"); +} - fn insert_instruction( - &mut self, - pos: usize, - info: InstructionInfo, - target_label: Option, - ) { - debug_assert!(pos <= self.instrs.len()); - Self::debug_check_addop(&info); - self.instrs - .insert(pos, InstructionSequenceEntry::new(info, target_label, None)); - if let InstructionSequenceLabelOffsets::Active(label_map) = &mut self.label_map { - for target in label_map.iter_mut() { - if *target >= pos as isize { - *target += 1; - } +/// instruction_sequence.c _PyInstructionSequence_Addop +fn instruction_sequence_addop( + seq: &mut InstructionSequence, + info: InstructionInfo, +) -> &mut InstructionSequenceEntry { + instruction_sequence_debug_check_addop(&info); + let idx = instruction_sequence_next_inst(seq); + let entry = &mut seq.instrs[idx]; + entry.info = info; + entry +} + +fn instruction_sequence_last_info_mut( + seq: &mut InstructionSequence, +) -> Option<&mut InstructionInfo> { + seq.instrs.last_mut().map(|entry| &mut entry.info) +} + +/// instruction_sequence.c _PyInstructionSequence_InsertInstruction +#[allow(clippy::needless_range_loop)] +fn instruction_sequence_insert_instruction( + seq: &mut InstructionSequence, + pos: usize, + info: InstructionInfo, +) { + debug_assert!(pos <= seq.instrs.len()); + instruction_sequence_debug_check_addop(&info); + let last_idx = instruction_sequence_next_inst(seq); + for i in (pos..last_idx).rev() { + seq.instrs[i + 1] = seq.instrs[i]; + } + seq.instrs[pos].info = info; + if let Some(label_map) = &mut seq.label_map { + let pos = i32::try_from(pos).expect("instruction offset fits in int"); + for lbl in 0..label_map.len() { + if label_map[lbl] >= pos { + label_map[lbl] += 1; } } } +} - fn apply_label_map(&mut self) -> crate::InternalResult<()> { - let label_map = match core::mem::replace( - &mut self.label_map, - InstructionSequenceLabelOffsets::Applied, - ) { - InstructionSequenceLabelOffsets::Active(label_map) => label_map, - InstructionSequenceLabelOffsets::Applied => return Ok(()), - }; - let resolve_label = |label: InstructionSequenceLabel| -> crate::InternalResult { - label_map - .get(label.idx()) - .copied() - .filter(|target_index| *target_index >= 0) - .and_then(|target_index| usize::try_from(target_index).ok()) - .ok_or(InternalError::MalformedControlFlowGraph) - }; - for entry in &mut self.instrs { - if entry.info.instr.has_target() { - let label_id = entry - .target_label - .take() - .ok_or(InternalError::MalformedControlFlowGraph)?; - let target_index = resolve_label(label_id)?; - entry.info.arg = OpArg::new( - target_index - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); - entry.target_offset = Some(target_index); - } else if entry.target_label.take().is_some() { +/// instruction_sequence.c _PyInstructionSequence_ApplyLabelMap +#[allow(clippy::needless_range_loop)] +fn instruction_sequence_apply_label_map( + instrs: &mut InstructionSequence, +) -> crate::InternalResult<()> { + let Some(label_map) = instrs.label_map.take() else { + return Ok(()); + }; + instrs.label_map_allocation = 0; + for i in 0..instrs.instrs.len() { + let entry = &mut instrs.instrs[i]; + if entry.info.instr.has_target() { + let label = usize::try_from(u32::from(entry.info.arg)) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + let target = *label_map + .get(label) + .ok_or(InternalError::MalformedControlFlowGraph)?; + if target < 0 { return Err(InternalError::MalformedControlFlowGraph); } - if let Some(handler) = &mut entry.except_handler - && let Some(label_id) = handler.target_label.take() - { - handler.target_offset = Some(resolve_label(label_id)?); - } - } - Ok(()) - } - - fn mark_targets(&mut self) { - for entry in &mut self.instrs { - entry.is_target = false; + entry.info.arg = OpArg::new( + target + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); } - for i in 0..self.instrs.len() { - if let Some(target_offset) = self.instrs[i].target_offset { - let target = self - .instrs - .get_mut(target_offset) - .expect("instruction-sequence target offset is in range"); - target.is_target = true; - } + let handler = &mut entry.except_handler; + if handler.h_label >= 0 { + let label = usize::try_from(handler.h_label) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + handler.h_label = *label_map + .get(label) + .ok_or(InternalError::MalformedControlFlowGraph)?; } } + Ok(()) } /// flowgraph.c _PyCfg_ToInstructionSequence -fn cfg_to_instruction_sequence(blocks: &mut [Block]) -> crate::InternalResult { - for block in blocks.iter_mut() { - block.cpython_label_id = None; - } - +fn cfg_to_instruction_sequence( + blocks: &mut [Block], + instr_sequence: &mut InstructionSequence, +) -> crate::InternalResult<()> { let mut label_id = 0; let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { - blocks[block_idx.idx()].cpython_label_id = Some(InstructionSequenceLabel(label_id)); + blocks[block_idx.idx()].cpython_label = InstructionSequenceLabel::from_index(label_id); label_id += 1; block_idx = blocks[block_idx.idx()].next; } - let mut instr_sequence = InstructionSequence::new(); block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { - let block_label = blocks[block_idx.idx()] - .cpython_label_id - .expect("block has a CPython CFG label"); - instr_sequence.use_label(block_label); + let block_label = blocks[block_idx.idx()].cpython_label; + debug_assert!(is_label(block_label)); + instruction_sequence_use_label(instr_sequence, block_label); let instr_count = blocks[block_idx.idx()].instructions.len(); for i in 0..instr_count { - let mut info = blocks[block_idx.idx()].instructions[i]; - let target_label = if info.instr.has_target() { - let target_block = info.target; + if blocks[block_idx.idx()].instructions[i].instr.has_target() { + let target_block = blocks[block_idx.idx()].instructions[i].target; debug_assert!(target_block != BlockIdx::NULL); - let label_id = blocks[target_block.idx()] - .cpython_label_id - .expect("target block has a CPython CFG label"); - info.arg = OpArg::new( - label_id - .idx() + let lbl = blocks[target_block.idx()].cpython_label; + debug_assert!(is_label(lbl)); + blocks[block_idx.idx()].instructions[i].arg = OpArg::new( + lbl.idx() .to_u32() .ok_or(InternalError::MalformedControlFlowGraph)?, ); - info.target = BlockIdx::NULL; - Some(label_id) - } else { - None - }; + } - let except_handler = if let Some(handler) = info.except_handler.take() { + let mut info = blocks[block_idx.idx()].instructions[i]; + info.target = BlockIdx::NULL; + let except_handler = info.except_handler.take(); + let entry = instruction_sequence_addop(instr_sequence, info); + let hi = &mut entry.except_handler; + if let Some(handler) = except_handler { debug_assert!(handler.handler_block != BlockIdx::NULL); - let label_id = blocks[handler.handler_block.idx()] - .cpython_label_id - .expect("exception handler block has a CPython CFG label"); - Some(InstructionSequenceExceptHandlerInfo { - target_label: Some(label_id), - target_offset: None, - stack_depth: handler.stack_depth, - preserve_lasti: handler.preserve_lasti, - }) + let lbl = blocks[handler.handler_block.idx()].cpython_label; + debug_assert!(is_label(lbl)); + let start_depth = blocks[handler.handler_block.idx()].start_depth; + debug_assert!(start_depth >= 0); + hi.h_label = i32::try_from(lbl.idx()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + hi.start_depth = start_depth; + hi.preserve_lasti = i32::from(handler.preserve_lasti); } else { - None - }; - - instr_sequence.addop(info, target_label, except_handler); + hi.h_label = NO_EXCEPTION_HANDLER_LABEL; + } } block_idx = blocks[block_idx.idx()].next; } - instr_sequence.apply_label_map()?; - Ok(instr_sequence) + instruction_sequence_apply_label_map(instr_sequence)?; + Ok(()) } -#[derive(Debug, Clone)] -pub struct Block { - pub instructions: Vec, - pub next: BlockIdx, - /// Exception stack at start of block, used by label_exception_targets (b_exceptstack) - except_stack: Option>, - /// Whether this block is an exception handler target (b_except_handler) - pub except_handler: bool, - /// Whether to preserve lasti for this handler block (b_preserve_lasti) - pub preserve_lasti: bool, - /// Stack depth at block entry, set by stack depth analysis - pub start_depth: Option, - /// Whether this block is only reachable via exception table (b_cold) - pub cold: bool, - /// Temporary traversal mark used by CFG passes (b_visited) - visited: bool, - /// Definitely reachable outside exception-only paths (b_warm) - warm: bool, - /// Number of incoming CFG edges from reachable blocks (b_predecessors) - predecessors: i32, - /// CPython `basicblock.b_label` used by translate_jump_labels_to_targets. - cpython_label_id: Option, - /// Potentially uninitialized locals mask for local-check analysis (b_unsafe_locals_mask) - unsafe_locals_mask: u64, - /// CPython keeps `b_instr` allocated beyond `b_iused`. Instructions removed - /// by compaction remain in those spare slots until `basicblock_next_instr()` - /// reuses them. - cpython_spare_instr_slots: VecDeque, +/// assemble.c instr_size +fn instr_size(instr: &InstructionInfo) -> usize { + let opcode = instr.instr.expect_real(); + let oparg = i32::try_from(u32::from(instr.arg)).expect("oparg fits in int"); + debug_assert!( + instr.instr.has_arg() || oparg == 0, + "CPython assemble.c instr_size requires OPCODE_HAS_ARG or oparg == 0" + ); + let extended_args = + (0xFF_FFFF < oparg) as usize + (0xFF_FF < oparg) as usize + (0xFF < oparg) as usize; + let caches = opcode.cache_entries(); + extended_args + 1 + caches } -impl Default for Block { - fn default() -> Self { - Self { - instructions: Vec::new(), - next: BlockIdx::NULL, - except_stack: None, - except_handler: false, - preserve_lasti: false, - start_depth: None, - cold: false, - visited: false, - warm: false, - predecessors: 0, - cpython_label_id: None, - unsafe_locals_mask: 0, - cpython_spare_instr_slots: VecDeque::new(), +/// pycore_opcode_metadata.h is_pseudo_target +fn is_pseudo_target(pseudo: PseudoOpcode, target: Opcode) -> bool { + matches!( + (pseudo, target), + ( + PseudoOpcode::Jump, + Opcode::JumpForward | Opcode::JumpBackward + ) | ( + PseudoOpcode::JumpNoInterrupt, + Opcode::JumpForward | Opcode::JumpBackwardNoInterrupt + ) | (PseudoOpcode::LoadClosure, Opcode::LoadFast) + | (PseudoOpcode::StoreFastMaybeNull, Opcode::StoreFast) + ) +} + +/// assemble.c resolve_unconditional_jumps +#[allow(clippy::unnecessary_wraps)] +fn resolve_unconditional_jumps( + instr_sequence: &mut InstructionSequence, +) -> crate::InternalResult<()> { + for i in 0..instr_sequence.instrs.len() { + let instr = &mut instr_sequence.instrs[i].info; + let is_forward = i32::try_from(u32::from(instr.arg)) + .map_err(|_| InternalError::MalformedControlFlowGraph)? + > i.to_i32().ok_or(InternalError::MalformedControlFlowGraph)?; + match instr.instr { + AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { + debug_assert!(is_pseudo_target(PseudoOpcode::Jump, Opcode::JumpForward)); + debug_assert!(is_pseudo_target(PseudoOpcode::Jump, Opcode::JumpBackward)); + if is_forward { + instr.instr = Instruction::JumpForward { + delta: Arg::marker(), + } + .into(); + } else { + instr.instr = Instruction::JumpBackward { + delta: Arg::marker(), + } + .into(); + } + } + AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) => { + debug_assert!(is_pseudo_target( + PseudoOpcode::JumpNoInterrupt, + Opcode::JumpForward + )); + debug_assert!(is_pseudo_target( + PseudoOpcode::JumpNoInterrupt, + Opcode::JumpBackwardNoInterrupt + )); + if is_forward { + instr.instr = Instruction::JumpForward { + delta: Arg::marker(), + } + .into(); + } else { + instr.instr = Instruction::JumpBackwardNoInterrupt { + delta: Arg::marker(), + } + .into(); + } + } + _ => { + debug_assert!( + !(instr.instr.has_jump() && matches!(instr.instr, AnyInstruction::Pseudo(_))), + "CPython resolve_unconditional_jumps expects no remaining pseudo jump" + ); + } } } + Ok(()) } -impl Block { - pub(crate) fn has_cpython_cfg_label(&self) -> bool { - self.cpython_label_id.is_some() +/// assemble.c resolve_jump_offsets +#[allow(clippy::needless_range_loop)] +fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::InternalResult { + let mut end_offset; + // The offset (in code units) of END_SEND from SEND in the yield-from sequence. + const END_SEND_OFFSET: i32 = 5; + for i in 0..instr_sequence.instrs.len() { + let instr = &mut instr_sequence.instrs[i]; + let opcode = instr.info.instr.expect_real(); + if opcode.has_jump() { + instr.i_target = i32::try_from(u32::from(instr.info.arg)) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + } + } + let mut extended_arg_recompile; + loop { + let mut totsize = 0i32; + for i in 0..instr_sequence.instrs.len() { + let instr = &mut instr_sequence.instrs[i]; + instr.i_offset = totsize; + let isize = instr_size(&instr.info); + totsize += isize + .to_i32() + .ok_or(InternalError::MalformedControlFlowGraph)?; + } + end_offset = totsize + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?; + + extended_arg_recompile = false; + let mut offset = 0i32; + for i in 0..instr_sequence.instrs.len() { + let isize = instr_size(&instr_sequence.instrs[i].info); + // Jump offsets are computed relative to the instruction pointer + // after fetching the jump instruction. + offset += isize + .to_i32() + .ok_or(InternalError::MalformedControlFlowGraph)?; + + let opcode = instr_sequence.instrs[i].info.instr.expect_real(); + if opcode.has_jump() { + let target_index = usize::try_from(instr_sequence.instrs[i].i_target) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + let target_offset = instr_sequence + .instrs + .get(target_index) + .ok_or(InternalError::MalformedControlFlowGraph)? + .i_offset; + let info = &mut instr_sequence.instrs[i].info; + let op = opcode; + let mut oparg = target_offset; + info.arg = OpArg::new( + oparg + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); + if matches!(op, Instruction::EndAsyncFor) { + oparg = offset + .checked_sub(oparg + END_SEND_OFFSET) + .expect("END_ASYNC_FOR target must be before instruction") + } else if oparg < offset { + debug_assert!(matches!( + op.into(), + Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt + )); + oparg = offset + .checked_sub(oparg) + .expect("backward jump target must be before instruction") + } else { + debug_assert!(!matches!( + op.into(), + Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt + )); + oparg = oparg + .checked_sub(offset) + .expect("forward jump target must be after instruction") + } + info.arg = OpArg::new( + oparg + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); + if instr_size(info) != isize { + extended_arg_recompile = true; + } + } + } + + if !extended_arg_recompile { + break; + } } + + Ok(end_offset) } -#[derive(Clone, Debug)] -pub(crate) struct InstructionSequenceLabelMap { - next_free_label: usize, - block_labels: Vec>, - /// Codegen-side shadow of CPython's instruction-sequence label map. - /// - /// `_PyInstructionSequence_UseLabel()` can map multiple labels to the same - /// instruction offset before `_PyCfg_FromInstructionSequence()` materializes - /// CFG blocks. The codegen CFG path keeps the same aliasing by resolving - /// those labels to the block that owns the shared offset. - cpython_block_by_label: Vec>, +struct AssembledCode { + instructions: Vec, + linetable: Box<[u8]>, + exceptiontable: Box<[u8]>, } -impl InstructionSequenceLabelMap { - pub(crate) fn new() -> Self { - Self { - next_free_label: 0, - block_labels: vec![None], - cpython_block_by_label: Vec::new(), +struct LocalsPlusInfo { + cellvars: Box<[String]>, + kinds: Box<[u8]>, +} + +/// assemble.c same_location +fn same_location(a: LineTableLocation, b: LineTableLocation) -> bool { + a.line == b.line && a.end_line == b.end_line && a.col == b.col && a.end_col == b.end_col +} + +fn instruction_linetable_location(info: &InstructionInfo) -> LineTableLocation { + match info.lineno_override { + Some(NO_LOCATION_OVERRIDE) => LineTableLocation { + line: NO_LOCATION_OVERRIDE, + end_line: NO_LOCATION_OVERRIDE, + col: NO_LOCATION_OVERRIDE, + end_col: NO_LOCATION_OVERRIDE, + }, + Some(LINE_ONLY_LOCATION_OVERRIDE) => LineTableLocation { + line: info.location.line.get() as i32, + end_line: info.end_location.line.get() as i32, + col: -1, + end_col: -1, + }, + Some(NEXT_LOCATION_OVERRIDE) => next_linetable_location(), + Some(lineno) => LineTableLocation { + line: lineno, + end_line: info.end_location.line.get() as i32, + col: info.location.character_offset.to_zero_indexed() as i32, + end_col: info.end_location.character_offset.to_zero_indexed() as i32, + }, + None => LineTableLocation { + line: info.location.line.get() as i32, + end_line: info.end_location.line.get() as i32, + col: info.location.character_offset.to_zero_indexed() as i32, + end_col: info.end_location.character_offset.to_zero_indexed() as i32, + }, + } +} + +/// assemble.c write_instr +fn write_instr(instructions: &mut Vec, info: &InstructionInfo, ilen: usize) { + let opcode = info.instr.expect_real(); + let oparg = i32::try_from(u32::from(info.arg)).expect("oparg fits in int"); + debug_assert!( + info.instr.has_arg() || oparg == 0, + "CPython assemble.c write_instr requires OPCODE_HAS_ARG or oparg == 0" + ); + let caches = opcode.cache_entries(); + let non_cache_units = ilen - caches; + match non_cache_units { + 1..=4 => {} + _ => unreachable!("CPython write_instr expects 1 to 4 non-cache code units"), + } + if non_cache_units >= 4 { + instructions.push(CodeUnit::new( + Instruction::ExtendedArg, + OpArgByte::new(((oparg >> 24) & 0xff) as u8), + )); + } + if non_cache_units >= 3 { + instructions.push(CodeUnit::new( + Instruction::ExtendedArg, + OpArgByte::new(((oparg >> 16) & 0xff) as u8), + )); + } + if non_cache_units >= 2 { + instructions.push(CodeUnit::new( + Instruction::ExtendedArg, + OpArgByte::new(((oparg >> 8) & 0xff) as u8), + )); + } + instructions.push(CodeUnit::new(opcode, OpArgByte::new((oparg & 0xff) as u8))); + for _ in 0..caches { + instructions.push(CodeUnit::new(Instruction::Cache, OpArgByte::new(0))); + } +} + +/// assemble.c assemble_emit_instr +fn assemble_emit_instr(instructions: &mut Vec, info: &mut InstructionInfo) { + let size = instr_size(info); + write_instr(instructions, info, size); +} + +/// assemble.c assemble_location_info +#[allow(clippy::needless_range_loop)] +fn assemble_location_info( + instr_sequence: &mut InstructionSequence, + first_line: i32, + debug_ranges: bool, +) -> Box<[u8]> { + for i in (0..instr_sequence.instrs.len()).rev() { + let loc = instruction_linetable_location(&instr_sequence.instrs[i].info); + if same_location(loc, next_linetable_location()) { + if instr_sequence.instrs[i] + .info + .instr + .expect_real() + .is_terminator() + { + instr_sequence.instrs[i].info.lineno_override = Some(NO_LOCATION_OVERRIDE); + } else { + debug_assert!(i < instr_sequence.instrs.len() - 1); + let next = instr_sequence.instrs[i + 1].info; + instr_set_loc( + &mut instr_sequence.instrs[i].info, + next.location, + next.end_location, + next.lineno_override, + ); + } } } - /// instruction_sequence.c _PyInstructionSequence_NewLabel - fn new_label(&mut self) -> InstructionSequenceLabel { - self.next_free_label += 1; - let label = InstructionSequenceLabel(self.next_free_label); - if self.cpython_block_by_label.len() <= label.idx() { - self.cpython_block_by_label.resize(label.idx() + 1, None); + let mut linetable = Vec::new(); + let mut prev_line = first_line; + let mut loc = no_linetable_location(); + let mut size = 0; + for i in 0..instr_sequence.instrs.len() { + let entry = &instr_sequence.instrs[i]; + let instr_loc = instruction_linetable_location(&entry.info); + if !same_location(loc, instr_loc) { + assemble_emit_location(&mut linetable, loc, size, &mut prev_line, debug_ranges); + loc = instr_loc; + size = 0; + } + size += instr_size(&entry.info); + } + assemble_emit_location(&mut linetable, loc, size, &mut prev_line, debug_ranges); + linetable.into_boxed_slice() +} + +/// assemble.c assemble_emit +fn assemble_emit( + instr_sequence: &mut InstructionSequence, + end_offset: u32, + first_line: i32, + debug_ranges: bool, +) -> crate::InternalResult { + let num_instructions = + usize::try_from(end_offset).map_err(|_| InternalError::MalformedControlFlowGraph)?; + let mut instructions = Vec::new(); + instructions.reserve_exact(num_instructions); + + for i in 0..instr_sequence.instrs.len() { + let instr = &mut instr_sequence.instrs[i].info; + assemble_emit_instr(&mut instructions, instr); + } + + let linetable = assemble_location_info(instr_sequence, first_line, debug_ranges); + + let exceptiontable = assemble_exception_table(&instr_sequence.instrs); + + Ok(AssembledCode { + instructions, + linetable, + exceptiontable, + }) +} + +/// assemble.c compute_localsplus_info +fn compute_localsplus_info( + umd: &CodeUnitMetadata, + nlocalsplus: usize, + flags: CodeFlags, +) -> LocalsPlusInfo { + let nlocals = umd.varnames.len(); + let ncells = umd.cellvars.len(); + let nfrees = umd.freevars.len(); + let mut localspluskinds = vec![0u8; nlocalsplus]; + let mut cellvars = Vec::with_capacity(ncells); + + let argvarkinds = [ + (umd.posonlyargcount as usize, CO_FAST_ARG_POS), + (umd.argcount as usize, CO_FAST_ARG_POS | CO_FAST_ARG_KW), + (umd.kwonlyargcount as usize, CO_FAST_ARG_KW), + ( + usize::from(flags.contains(CodeFlags::VARARGS)), + CO_FAST_ARG_VAR | CO_FAST_ARG_POS, + ), + ( + usize::from(flags.contains(CodeFlags::VARKEYWORDS)), + CO_FAST_ARG_VAR | CO_FAST_ARG_KW, + ), + (usize::MAX, 0), + ]; + let mut pos = 0usize; + let mut max = 0usize; + for (count, argkind) in argvarkinds { + max = if count == usize::MAX { + usize::MAX + } else { + max.checked_add(count) + .expect("localsplus argument count fits in usize") + }; + while pos < max && pos < nlocals { + let name = umd + .varnames + .get_index(pos) + .expect("varname index is in range") + .as_str(); + let mut kind = CO_FAST_LOCAL | argkind; + if umd.fast_hidden.get(name).copied().unwrap_or(false) + || umd.fast_hidden_final.contains(name) + { + kind |= CO_FAST_HIDDEN; + } + if umd.cellvars.contains(name) { + kind |= CO_FAST_CELL; + cellvars.push(name.to_owned()); + } + localspluskinds[pos] = kind; + pos += 1; } - label } - pub(crate) fn push_unlabeled_block(&mut self) { - self.block_labels.push(None); + let mut numdropped = 0usize; + let mut cellvar_offset = -1i32; + for i in 0..ncells { + let name = umd + .cellvars + .get_index(i) + .expect("cellvar index is in range") + .as_str(); + if umd.varnames.contains(name) { + numdropped += 1; + continue; + } + let offset = i + nlocals - numdropped; + debug_assert!(offset < nlocalsplus); + cellvars.push(name.to_owned()); + localspluskinds[offset] = CO_FAST_CELL; + cellvar_offset = offset.to_i32().expect("localsplus cell offset fits in int"); } - pub(crate) fn push_unmapped_label(&mut self) { - let label = self.new_label(); - let block = BlockIdx( - self.block_labels - .len() - .to_u32() - .expect("too many codegen CFG blocks"), + for i in 0..nfrees { + let offset = ncells + i + nlocals - numdropped; + debug_assert!(offset < nlocalsplus); + debug_assert!( + offset.to_i32().expect("localsplus free offset fits in int") > cellvar_offset ); - self.cpython_block_by_label[label.idx()] = Some(block); - self.block_labels.push(Some(label)); + localspluskinds[offset] = CO_FAST_FREE; + } + + debug_assert_eq!( + nlocalsplus, + nlocals + ncells - numdropped + nfrees, + "CPython prepare_localsplus() result must match assemble.c localsplus sizing" + ); + debug_assert_eq!(cellvars.len(), ncells); + LocalsPlusInfo { + cellvars: cellvars.into_boxed_slice(), + kinds: localspluskinds.into_boxed_slice(), } +} + +#[derive(Debug, Clone)] +pub struct Block { + /// CPython `basicblock.b_list`, allocation-order list distinct from CFG `b_next`. + allocation_next: BlockIdx, + /// CPython `basicblock.b_label` used by translate_jump_labels_to_targets. + cpython_label: InstructionSequenceLabel, + /// CPython `basicblock.b_ialloc`, the allocated size of `b_instr`. + instruction_allocation: usize, + /// Exception stack at start of block, used by label_exception_targets (b_exceptstack) + except_stack: Option>, + pub instructions: Vec, + pub next: BlockIdx, + /// CPython keeps `b_instr` allocated beyond `b_iused`. Instructions removed + /// by compaction remain in those spare slots until `basicblock_next_instr()` + /// reuses them. The next CPython slot is stored at the end so reuse is O(1). + cpython_spare_instr_slots: Vec, + /// Potentially uninitialized locals mask for local-check analysis (b_unsafe_locals_mask) + unsafe_locals_mask: u64, + /// Number of incoming CFG edges from reachable blocks (b_predecessors) + predecessors: i32, + /// Stack depth at block entry, set by stack depth analysis + pub start_depth: i32, + /// Whether to preserve lasti for this handler block (b_preserve_lasti) + pub preserve_lasti: bool, + /// Temporary traversal mark used by CFG passes (b_visited) + visited: bool, + /// Whether this block is an exception handler target (b_except_handler) + pub except_handler: bool, + /// Whether this block is only reachable via exception table (b_cold) + pub cold: bool, + /// Definitely reachable outside exception-only paths (b_warm) + warm: bool, +} - fn ensure_label_for_block(&mut self, block: BlockIdx) -> InstructionSequenceLabel { - debug_assert_ne!(block, BlockIdx::NULL); - if let Some(label) = self.block_labels[block.idx()] { - return label; +impl Default for Block { + fn default() -> Self { + Self { + allocation_next: BlockIdx::NULL, + cpython_label: InstructionSequenceLabel::NO_LABEL, + instruction_allocation: 0, + except_stack: None, + instructions: Vec::new(), + next: BlockIdx::NULL, + cpython_spare_instr_slots: Vec::new(), + unsafe_locals_mask: 0, + predecessors: 0, + start_depth: START_DEPTH_UNSET, + preserve_lasti: false, + visited: false, + except_handler: false, + cold: false, + warm: false, } - let label = self.new_label(); - self.cpython_block_by_label[label.idx()] = Some(block); - self.block_labels[block.idx()] = Some(label); - label } +} + +pub(crate) const START_DEPTH_UNSET: i32 = i32::MIN; +const CO_MAXBLOCKS: usize = 20; + +#[derive(Clone, Debug)] +pub(crate) struct InstructionSequenceLabelMap { + block_labels: Vec, + /// Codegen-side shadow of CPython's instruction-sequence label map. + /// + /// `_PyInstructionSequence_UseLabel()` can map multiple labels to the same + /// instruction offset before `_PyCfg_FromInstructionSequence()` materializes + /// CFG blocks. The codegen CFG path keeps the same aliasing by resolving + /// those labels to the block that owns the shared offset. + cpython_block_by_label: Vec, +} - fn label_for_block(&self, block: BlockIdx) -> Option { - debug_assert_ne!(block, BlockIdx::NULL); - self.block_labels.get(block.idx()).copied().flatten() +fn instruction_sequence_label_map_register_label( + map: &mut InstructionSequenceLabelMap, + label: InstructionSequenceLabel, +) { + if map.cpython_block_by_label.len() <= label.idx() { + map.cpython_block_by_label + .resize(label.idx() + 1, BlockIdx::NULL); } +} + +fn instruction_sequence_label_map_ensure_label_for_block( + map: &mut InstructionSequenceLabelMap, + seq: &mut InstructionSequence, + block: BlockIdx, +) -> InstructionSequenceLabel { + debug_assert_ne!(block, BlockIdx::NULL); + let block_label = map.block_labels[block.idx()]; + if is_label(block_label) { + return block_label; + } + let label = instruction_sequence_new_label(seq); + instruction_sequence_label_map_register_label(map, label); + map.cpython_block_by_label[label.idx()] = block; + map.block_labels[block.idx()] = label; + label +} + +fn instruction_sequence_label_map_label_for_block( + map: &InstructionSequenceLabelMap, + block: BlockIdx, +) -> InstructionSequenceLabel { + debug_assert_ne!(block, BlockIdx::NULL); + map.block_labels + .get(block.idx()) + .copied() + .unwrap_or(InstructionSequenceLabel::NO_LABEL) +} + +fn instruction_sequence_label_map_block_for_label( + map: &InstructionSequenceLabelMap, + label: InstructionSequenceLabel, +) -> Option { + if !is_label(label) { + return None; + } + map.cpython_block_by_label + .get(label.idx()) + .copied() + .filter(|&block| block != BlockIdx::NULL) +} - fn block_for_label(&self, label: InstructionSequenceLabel) -> Option { - self.cpython_block_by_label - .get(label.idx()) - .copied() - .flatten() +fn instruction_sequence_label_map_resolve_label( + map: &InstructionSequenceLabelMap, + block: BlockIdx, +) -> BlockIdx { + if block == BlockIdx::NULL { + return BlockIdx::NULL; + } + let label = instruction_sequence_label_map_label_for_block(map, block); + if !is_label(label) { + return block; } + instruction_sequence_label_map_block_for_label(map, label).unwrap_or_else(|| { + debug_assert!( + false, + "CPython instruction-sequence label must map to a codegen CFG block" + ); + BlockIdx::NULL + }) +} - pub(crate) fn resolve_label(&self, block: BlockIdx) -> BlockIdx { - if block == BlockIdx::NULL { - return BlockIdx::NULL; - } - let Some(label) = self.label_for_block(block) else { - return block; - }; - self.block_for_label(label).unwrap_or_else(|| { - debug_assert!( - false, - "CPython instruction-sequence label must map to a codegen CFG block" - ); - BlockIdx::NULL - }) +fn instruction_sequence_label_map_resolve_label_to_block( + map: &InstructionSequenceLabelMap, + label: InstructionSequenceLabel, +) -> BlockIdx { + if !is_label(label) { + return BlockIdx::NULL; } + instruction_sequence_label_map_block_for_label(map, label).unwrap_or_else(|| { + debug_assert!( + false, + "CPython instruction-sequence label must map to a codegen CFG block" + ); + BlockIdx::NULL + }) +} - pub(crate) fn resolve_label_to_block( - &self, - label: Option, - ) -> BlockIdx { - let Some(label) = label else { - return BlockIdx::NULL; - }; - self.block_for_label(label).unwrap_or_else(|| { - debug_assert!( - false, - "CPython instruction-sequence label must map to a codegen CFG block" - ); - BlockIdx::NULL - }) +fn instruction_sequence_label_map_use_label_at_block( + map: &mut InstructionSequenceLabelMap, + seq: &mut InstructionSequence, + from: BlockIdx, + to: BlockIdx, +) { + if from == BlockIdx::NULL || from == to { + return; } + let from_label = instruction_sequence_label_map_ensure_label_for_block(map, seq, from); + let to_block = instruction_sequence_label_map_resolve_label(map, to); + if to_block == BlockIdx::NULL { + debug_assert!( + false, + "CPython label target must map to a codegen CFG block" + ); + return; + } + map.cpython_block_by_label[from_label.idx()] = to_block; +} - pub(crate) fn use_label_at_block(&mut self, from: BlockIdx, to: BlockIdx) { - if from == BlockIdx::NULL || from == to { - return; - } - let from_label = self.ensure_label_for_block(from); - let to_block = self.resolve_label(to); - if to_block == BlockIdx::NULL { - debug_assert!( - false, - "CPython label target must map to a codegen CFG block" - ); - return; +fn instruction_sequence_label_map_push_unlabeled_block(map: &mut InstructionSequenceLabelMap) { + map.block_labels.push(InstructionSequenceLabel::NO_LABEL); +} + +fn instruction_sequence_label_map_push_unmapped_label( + map: &mut InstructionSequenceLabelMap, + seq: &mut InstructionSequence, +) { + let label = instruction_sequence_new_label(seq); + instruction_sequence_label_map_register_label(map, label); + let block = BlockIdx( + map.block_labels + .len() + .to_u32() + .expect("too many codegen CFG blocks"), + ); + map.cpython_block_by_label[label.idx()] = block; + map.block_labels.push(label); +} + +impl InstructionSequenceLabelMap { + pub(crate) fn new() -> Self { + Self { + block_labels: vec![InstructionSequenceLabel::NO_LABEL], + cpython_block_by_label: Vec::new(), } - self.cpython_block_by_label[from_label.idx()] = Some(to_block); } - fn debug_check_for_blocks(&self, blocks_len: usize) { + fn debug_check_for_blocks(&self, blocks_len: usize, next_free_label: usize) { debug_assert_eq!( self.block_labels.len(), blocks_len, "every codegen CFG block must have an instruction-sequence label slot" ); - debug_assert!(self.cpython_block_by_label.len() <= self.next_free_label + 1); - let mut seen_labels = vec![false; self.next_free_label + 1]; - for &label in self.block_labels.iter().flatten() { + debug_assert!(self.cpython_block_by_label.len() <= next_free_label + 1); + let mut seen_labels = vec![false; next_free_label + 1]; + for &label in &self.block_labels { + if !is_label(label) { + continue; + } debug_assert!( - label.idx() <= self.next_free_label, + label.idx() <= next_free_label, "codegen CFG block labels must come from _PyInstructionSequence_NewLabel()" ); debug_assert!( @@ -767,16 +1400,19 @@ impl InstructionSequenceLabelMap { seen_labels[label.idx()] = true; } for &block in &self.cpython_block_by_label { - if let Some(block) = block { + if block != BlockIdx::NULL { debug_assert!( block.idx() < blocks_len, "CPython label must map to an existing codegen CFG block" ); } } - for &label in self.block_labels.iter().flatten() { + for &label in &self.block_labels { + if !is_label(label) { + continue; + } debug_assert!( - self.block_for_label(label) + instruction_sequence_label_map_block_for_label(self, label) .is_some_and(|block| block.idx() < blocks_len), "codegen CFG block label must map to a codegen CFG block" ); @@ -787,30 +1423,37 @@ impl InstructionSequenceLabelMap { &self, instr_sequence: &InstructionSequence, ) { - let InstructionSequenceLabelOffsets::Active(label_map) = &instr_sequence.label_map else { + let Some(label_map) = &instr_sequence.label_map else { debug_assert!( - false, - "codegen CFG label shadow must be checked before CPython label-map application" + self.cpython_block_by_label + .iter() + .all(|&block| block == BlockIdx::NULL), + "codegen CFG labels require a CPython instruction-sequence label map" ); return; }; for (label_idx, block) in self.cpython_block_by_label.iter().copied().enumerate() { - let Some(block) = block else { + if block == BlockIdx::NULL { continue; - }; + } let Some(&label_offset) = label_map.get(label_idx) else { continue; }; if label_offset < 0 { continue; } - let Some(block_label) = self.block_labels.get(block.idx()).copied().flatten() else { + let block_label = self + .block_labels + .get(block.idx()) + .copied() + .unwrap_or(InstructionSequenceLabel::NO_LABEL); + if !is_label(block_label) { debug_assert!( false, "CPython label must map to an existing codegen CFG block" ); continue; - }; + } let Some(&block_offset) = label_map.get(block_label.idx()) else { continue; }; @@ -849,6 +1492,9 @@ pub struct CodeInfo { // Reference to the symbol table for this scope pub symbol_table_index: usize, + // CPython compile.c uses PyList_GET_SIZE(u->u_ste->ste_varnames) + // when calling flowgraph.c _PyCfg_OptimizeCodeUnit(). + pub nparams: usize, // PEP 649: Track nesting depth inside conditional blocks (if/for/while/etc.) // u_in_conditional_block @@ -864,10 +1510,12 @@ impl CodeInfo { &mut self, mut info: InstructionInfo, ) -> crate::InternalResult<()> { - let target_label = if info.instr.has_target() && info.target != BlockIdx::NULL { - let label = self - .instr_sequence_label_map - .ensure_label_for_block(info.target); + if info.instr.has_target() && info.target != BlockIdx::NULL { + let label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + info.target, + ); info.arg = OpArg::new( label .idx() @@ -875,11 +1523,8 @@ impl CodeInfo { .ok_or(InternalError::MalformedControlFlowGraph)?, ); info.target = BlockIdx::NULL; - Some(label) - } else { - None - }; - self.instr_sequence.addop(info, target_label, None); + } + instruction_sequence_addop(&mut self.instr_sequence, info); Ok(()) } @@ -898,65 +1543,85 @@ impl CodeInfo { .ok_or(InternalError::MalformedControlFlowGraph)?, ); info.target = BlockIdx::NULL; - self.instr_sequence.addop(info, Some(target_label), None); + instruction_sequence_addop(&mut self.instr_sequence, info); Ok(()) } pub(crate) fn set_last_instr_sequence_lineno_override(&mut self, lineno_override: i32) { - if let Some(last) = self.instr_sequence.last_info_mut() { + if let Some(last) = instruction_sequence_last_info_mut(&mut self.instr_sequence) { last.lineno_override = Some(lineno_override); } } pub(crate) fn use_instr_sequence_label(&mut self, block: BlockIdx) { - let label = self.instr_sequence_label_map.ensure_label_for_block(block); - self.instr_sequence.use_label(label); + let label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + block, + ); + instruction_sequence_use_label(&mut self.instr_sequence, label); } pub(crate) fn new_instr_sequence_label(&mut self) -> InstructionSequenceLabel { - self.instr_sequence_label_map.new_label() + instruction_sequence_new_label(&mut self.instr_sequence) } pub(crate) fn use_raw_instr_sequence_label(&mut self, label: InstructionSequenceLabel) { - self.instr_sequence.use_label(label); + instruction_sequence_use_label(&mut self.instr_sequence, label); } pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) { - let label = self.instr_sequence_label_map.ensure_label_for_block(block); - self.blocks[block.idx()].cpython_label_id = Some(label); + let label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + block, + ); + self.blocks[block.idx()].cpython_label = label; } pub(crate) fn resolve_instr_sequence_label(&self, block: BlockIdx) -> BlockIdx { - self.instr_sequence_label_map.resolve_label(block) + instruction_sequence_label_map_resolve_label(&self.instr_sequence_label_map, block) } pub(crate) fn block_for_instr_sequence_label( &self, - label: Option, + label: InstructionSequenceLabel, ) -> BlockIdx { - self.instr_sequence_label_map.resolve_label_to_block(label) + instruction_sequence_label_map_resolve_label_to_block(&self.instr_sequence_label_map, label) } pub(crate) fn use_instr_sequence_label_at_block(&mut self, from: BlockIdx, to: BlockIdx) { - self.instr_sequence_label_map.use_label_at_block(from, to); + instruction_sequence_label_map_use_label_at_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + from, + to, + ); } pub(crate) fn instr_sequence_label_for_block( &mut self, block: BlockIdx, - ) -> Option { + ) -> InstructionSequenceLabel { if block == BlockIdx::NULL { - None + InstructionSequenceLabel::NO_LABEL } else { - Some(self.instr_sequence_label_map.ensure_label_for_block(block)) + instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + block, + ) } } pub(crate) fn insert_start_setup_cleanup(&mut self, handler_block: BlockIdx) { - let handler_label = self - .instr_sequence_label_map - .ensure_label_for_block(handler_block); - self.instr_sequence.insert_instruction( + let handler_label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + handler_block, + ); + instruction_sequence_insert_instruction( + &mut self.instr_sequence, 0, InstructionInfo { instr: PseudoInstruction::SetupCleanup { @@ -968,19 +1633,31 @@ impl CodeInfo { location: SourceLocation::default(), end_location: SourceLocation::default(), except_handler: None, - lineno_override: Some(-1), - cache_entries: 0, + lineno_override: Some(NO_LOCATION_OVERRIDE), }, - Some(handler_label), ); } + pub(crate) fn push_unmapped_instr_sequence_label(&mut self) { + instruction_sequence_label_map_push_unmapped_label( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + ); + } + + pub(crate) fn push_unlabeled_instr_sequence_block(&mut self) { + instruction_sequence_label_map_push_unlabeled_block(&mut self.instr_sequence_label_map); + } + fn take_recorded_instr_sequence(&mut self) -> crate::InternalResult { let mut instr_sequence = - core::mem::replace(&mut self.instr_sequence, InstructionSequence::new()); + core::mem::replace(&mut self.instr_sequence, instruction_sequence_new()); if let Some(mut annotations_instr_sequence) = self.annotations_instr_sequence.take() { - annotations_instr_sequence.apply_label_map()?; - instr_sequence.set_annotations_code(Some(Box::new(annotations_instr_sequence))); + instruction_sequence_apply_label_map(&mut annotations_instr_sequence)?; + instruction_sequence_set_annotations_code( + &mut instr_sequence, + Some(Box::new(annotations_instr_sequence)), + ); } Ok(instr_sequence) } @@ -992,7 +1669,7 @@ impl CodeInfo { debug_assert!(!self.instr_sequence.instrs.is_empty()); debug_assert!(self.current_block.idx() < self.blocks.len()); self.instr_sequence_label_map - .debug_check_for_blocks(self.blocks.len()); + .debug_check_for_blocks(self.blocks.len(), self.instr_sequence.next_free_label); self.instr_sequence_label_map .debug_check_label_blocks_match_instruction_sequence(&self.instr_sequence); for block in &self.blocks { @@ -1014,92 +1691,120 @@ impl CodeInfo { self.debug_check_recorded_cfg_builder(); self.take_recorded_instr_sequence() } +} - fn optimize_code_unit( - &mut self, - instr_sequence: InstructionSequence, - ) -> crate::InternalResult<()> { - // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) - self.blocks = cfg_from_instruction_sequence(instr_sequence)?; - translate_jump_labels_to_targets(&mut self.blocks); - mark_except_handlers(&mut self.blocks); - label_exception_targets(&mut self.blocks); - // CPython optimize_cfg() starts with check_cfg() and raises - // SystemError if a jump or scope exit is not the last instruction in - // its block. - check_cfg(&self.blocks)?; - inline_small_or_no_lineno_blocks(&mut self.blocks); - // CPython does not re-run instruction-sequence label-map/CFG conversion - // after this point. Unreferenced label blocks left by jump inlining - // remain block boundaries and can preserve line-marker NOPs. - Self::remove_unreachable(&mut self.blocks); - // CPython optimize_cfg resolves line numbers before local checks and - // superinstruction insertion, so fusion decisions see propagated - // source locations. - resolve_line_numbers(&mut self.blocks); - // CPython optimize_cfg() runs optimize_load_const() and then - // optimize_basic_block() after line numbers are resolved. - self.optimize_load_const(); - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = self.blocks[block_idx.idx()].next; - Self::optimize_basic_block(&mut self.blocks, &mut self.metadata, block_idx)?; - block_idx = next_block; - } - self.remove_redundant_nops_and_pairs(); - // CPython optimize_cfg() removes newly-unreachable blocks and - // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes - // unused constants. - Self::remove_unreachable(&mut self.blocks); - remove_redundant_nops_and_jumps(&mut self.blocks)?; - debug_assert!(no_redundant_jumps(&self.blocks)); - self.remove_unused_consts(); - self.add_checks_for_loads_of_uninitialized_variables(); - // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before - // later jump normalization / block reordering can create adjacencies - // that never exist at this stage in flowgraph.c. - self.insert_superinstructions(); - push_cold_blocks_to_end(&mut self.blocks)?; - // CPython resolves line numbers again after cold-block extraction. - resolve_line_numbers(&mut self.blocks); - Ok(()) - } +fn optimize_code_unit( + metadata: &mut CodeUnitMetadata, + blocks: &mut Vec, + instr_sequence: InstructionSequence, + nlocals: usize, + nparams: usize, +) -> crate::InternalResult<()> { + // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) + *blocks = cfg_from_instruction_sequence(instr_sequence)?; + optimize_code_unit_preprocess(blocks); + optimize_cfg(metadata, blocks, metadata.firstlineno)?; + remove_unused_consts(blocks, &mut metadata.consts); + add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams); + // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before + // later jump normalization / block reordering can create adjacencies + // that never exist at this stage in flowgraph.c. + insert_superinstructions(blocks); + push_cold_blocks_to_end(blocks)?; + // CPython resolves line numbers again after cold-block extraction. + resolve_line_numbers(blocks, metadata.firstlineno); + Ok(()) +} - fn optimized_cfg_to_instruction_sequence(&mut self) -> crate::InternalResult<(u32, usize)> { - // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) - convert_pseudo_conditional_jumps(&mut self.blocks); - let max_stackdepth = self.calculate_stackdepth()?; - debug_assert!( - !self.flags.intersects( - CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR - ) || max_stackdepth != 0 - ); - let nlocalsplus = self.prepare_localsplus(); - // Match CPython order: pseudo ops are lowered after stackdepth and - // localsplus preparation, before normalize_jumps. - convert_pseudo_ops(&mut self.blocks)?; - normalize_jumps(&mut self.blocks)?; - debug_assert!(no_redundant_jumps(&self.blocks)); - // optimize_load_fast: after normalize_jumps - self.optimize_load_fast(); +fn optimize_cfg( + metadata: &mut CodeUnitMetadata, + blocks: &mut Vec, + firstlineno: OneIndexed, +) -> crate::InternalResult<()> { + // flowgraph.c optimize_cfg + // CPython optimize_cfg() starts with check_cfg() and raises + // SystemError if a jump or scope exit is not the last instruction in + // its block. + check_cfg(blocks)?; + inline_small_or_no_lineno_blocks(blocks); + // CPython does not re-run instruction-sequence label-map/CFG conversion + // after this point. Unreferenced label blocks left by jump inlining + // remain block boundaries and can preserve line-marker NOPs. + remove_unreachable(blocks); + // CPython optimize_cfg resolves line numbers before local checks and + // superinstruction insertion, so fusion decisions see propagated + // source locations. + resolve_line_numbers(blocks, firstlineno); + // CPython optimize_cfg() runs optimize_load_const() and then + // optimize_basic_block() after line numbers are resolved. + optimize_load_const(metadata, blocks); + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = blocks[block_idx.idx()].next; + optimize_basic_block(blocks, metadata, block_idx)?; + block_idx = next_block; + } + remove_redundant_nops_and_pairs(blocks); + // CPython optimize_cfg() removes newly-unreachable blocks and + // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes + // unused constants. + remove_unreachable(blocks); + remove_redundant_nops_and_jumps(blocks)?; + #[cfg(debug_assertions)] + assert!(no_redundant_jumps(blocks)); + Ok(()) +} - Ok((max_stackdepth, nlocalsplus)) - } +fn optimized_cfg_to_instruction_sequence( + metadata: &CodeUnitMetadata, + flags: CodeFlags, + blocks: &mut Vec, +) -> crate::InternalResult<(u32, usize, InstructionSequence)> { + // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) + convert_pseudo_conditional_jumps(blocks); + let max_stackdepth = calculate_stackdepth(blocks)?; + debug_assert!(!is_generator(flags) || max_stackdepth != 0); + let nlocalsplus = prepare_localsplus(metadata, blocks, flags); + // Match CPython order: pseudo ops are lowered after stackdepth and + // localsplus preparation, before normalize_jumps. + convert_pseudo_ops(blocks)?; + normalize_jumps(blocks)?; + #[cfg(debug_assertions)] + assert!(no_redundant_jumps(blocks)); + // optimize_load_fast: after normalize_jumps + optimize_load_fast(blocks); + + let mut instr_sequence = instruction_sequence_new(); + cfg_to_instruction_sequence(blocks, &mut instr_sequence)?; + Ok((max_stackdepth, nlocalsplus, instr_sequence)) +} +impl CodeInfo { + #[allow(clippy::needless_range_loop)] pub fn finalize_code( mut self, opts: &crate::compile::CompileOpts, ) -> crate::InternalResult { let instr_sequence = self.prepare_cfg_from_codegen()?; - self.optimize_code_unit(instr_sequence)?; - let (max_stackdepth, nlocalsplus) = self.optimized_cfg_to_instruction_sequence()?; + let nlocals = self.metadata.varnames.len(); + let nparams = self.nparams; + optimize_code_unit( + &mut self.metadata, + &mut self.blocks, + instr_sequence, + nlocals, + nparams, + )?; + let (max_stackdepth, nlocalsplus, mut instr_sequence) = + optimized_cfg_to_instruction_sequence(&self.metadata, self.flags, &mut self.blocks)?; + let localsplusinfo = compute_localsplus_info(&self.metadata, nlocalsplus, self.flags); let Self { flags, source_path, private: _, // private is only used during compilation - mut blocks, + blocks: _, current_block: _, instr_sequence: _, instr_sequence_label_map: _, @@ -1109,6 +1814,7 @@ impl CodeInfo { in_inlined_comp: _, fblock: _, symbol_table_index: _, + nparams: _, in_conditional_block: _, next_conditional_annotation_index: _, } = self; @@ -1119,282 +1825,29 @@ impl CodeInfo { consts: constants, names: name_cache, varnames: varname_cache, - cellvars: cellvar_cache, + cellvars: _, freevars: freevar_cache, - fast_hidden, - fast_hidden_final, + fast_hidden: _, + fast_hidden_final: _, argcount: arg_count, posonlyargcount: posonlyarg_count, kwonlyargcount: kwonlyarg_count, firstlineno: first_line_number, } = metadata; - let mut instructions = Vec::new(); - let mut locations = Vec::new(); - let mut linetable_locations: Vec = Vec::new(); - - // Rebuild and adjust cellfixedoffsets for localsplus metadata. Pseudo - // lowering and deref operand fixups already ran in - // optimized_cfg_to_instruction_sequence(). - let mut cellfixedoffsets = - build_cellfixedoffsets(&varname_cache, &cellvar_cache, &freevar_cache); - let numdropped = fix_cellfixedoffsets(varname_cache.len(), &mut cellfixedoffsets); - - // Pre-compute cache_entries for real (non-pseudo) instructions - for block in &mut blocks { - for instr in &mut block.instructions { - if let AnyInstruction::Real(op) = instr.instr { - instr.cache_entries = op.cache_entries() as u32; - } - } - } - - let mut instr_sequence = cfg_to_instruction_sequence(&mut blocks)?; - // CPython assemble.c::resolve_unconditional_jumps() runs before - // resolve_jump_offsets(), so the first offset pass already uses the - // concrete JUMP_* opcode and its inline-cache size. - for (current_instr_index, entry) in instr_sequence.instrs.iter_mut().enumerate() { - let op = match entry.info.instr { - AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { - if entry - .target_offset - .ok_or(InternalError::MalformedControlFlowGraph)? - > current_instr_index - { - Instruction::JumpForward { - delta: Arg::marker(), - } - } else { - Instruction::JumpBackward { - delta: Arg::marker(), - } - } - } - AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) => { - if entry - .target_offset - .ok_or(InternalError::MalformedControlFlowGraph)? - > current_instr_index - { - Instruction::JumpForward { - delta: Arg::marker(), - } - } else { - Instruction::JumpBackwardNoInterrupt { - delta: Arg::marker(), - } - } - } - _ => continue, - }; - entry.info.instr = op.into(); - entry.info.cache_entries = op.cache_entries() as u32; - } - - let mut instruction_offsets = vec![0u32; instr_sequence.instrs.len()]; - let mut end_offset; - // The offset (in code units) of END_SEND from SEND in the yield-from sequence. - const END_SEND_OFFSET: u32 = 5; - loop { - let mut num_instructions = 0; - for (idx, instr) in instr_sequence.instrs.iter().enumerate() { - instruction_offsets[idx] = num_instructions as u32; - num_instructions += instr.info.arg.instr_size() + instr.info.cache_entries as usize; - } - end_offset = num_instructions as u32; - - instructions.reserve_exact(num_instructions); - locations.reserve_exact(num_instructions); - - let mut recompile = false; - for (current_instr_index, entry) in instr_sequence.instrs.iter_mut().enumerate() { - // Track current instruction offset for jump offset resolution. - let current_offset = instruction_offsets[current_instr_index]; - { - let info = &mut entry.info; - let old_arg_size = info.arg.instr_size(); - let old_cache_entries = info.cache_entries; - // Keep offsets fixed within this pass: changes in jump - // arg/cache sizes only take effect in the next iteration. - let offset_after = current_offset + old_arg_size as u32 + old_cache_entries; - let op = info.instr.expect_real(); - - if let Some(target_index) = entry.target_offset { - let target_offset = instruction_offsets[target_index]; - // CPython assemble.c::resolve_jump_offsets() only - // converts label/instruction indexes to bytecode - // offsets here. Direction selection for pseudo - // unconditional jumps has already happened in - // resolve_unconditional_jumps(), and redundant-jump - // removal belongs to flowgraph.c optimization. - info.instr = op.into(); - let updated_cache = op.cache_entries() as u32; - recompile |= updated_cache != old_cache_entries; - info.cache_entries = updated_cache; - let new_arg = if matches!(op, Instruction::EndAsyncFor) { - let arg = offset_after - .checked_sub(target_offset + END_SEND_OFFSET) - .expect("END_ASYNC_FOR target must be before instruction"); - OpArg::new(arg) - } else if matches!( - op.into(), - Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt - ) { - let arg = offset_after - .checked_sub(target_offset) - .expect("backward jump target must be before instruction"); - OpArg::new(arg) - } else { - let arg = target_offset - .checked_sub(offset_after) - .expect("forward jump target must be after instruction"); - OpArg::new(arg) - }; - recompile |= new_arg.instr_size() != old_arg_size; - info.arg = new_arg; - } - - let cache_count = info.cache_entries as usize; - let (extras, lo_arg) = info.arg.split(); - let loc_pair = (info.location, info.end_location); - locations.extend(core::iter::repeat_n( - loc_pair, - info.arg.instr_size() + cache_count, - )); - // Collect linetable locations with lineno_override support - let lt_loc = match info.lineno_override { - Some(-1) => LineTableLocation { - line: -1, - end_line: -1, - col: -1, - end_col: -1, - }, - Some(LINE_ONLY_LOCATION_OVERRIDE) => LineTableLocation { - line: info.location.line.get() as i32, - end_line: info.end_location.line.get() as i32, - col: -1, - end_col: -1, - }, - Some(NEXT_LOCATION_OVERRIDE) => LineTableLocation { - line: NEXT_LOCATION_OVERRIDE, - end_line: NEXT_LOCATION_OVERRIDE, - col: NEXT_LOCATION_OVERRIDE, - end_col: NEXT_LOCATION_OVERRIDE, - }, - Some(lineno) => LineTableLocation { - line: lineno, - end_line: info.end_location.line.get() as i32, - col: info.location.character_offset.to_zero_indexed() as i32, - end_col: info.end_location.character_offset.to_zero_indexed() as i32, - }, - None => LineTableLocation { - line: info.location.line.get() as i32, - end_line: info.end_location.line.get() as i32, - col: info.location.character_offset.to_zero_indexed() as i32, - end_col: info.end_location.character_offset.to_zero_indexed() as i32, - }, - }; - linetable_locations.extend(core::iter::repeat_n(lt_loc, info.arg.instr_size())); - // CACHE entries inherit parent instruction's location - if cache_count > 0 { - linetable_locations.extend(core::iter::repeat_n(lt_loc, cache_count)); - } - instructions.extend( - extras - .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) - .chain([CodeUnit { op, arg: lo_arg }]), - ); - // Emit CACHE code units after the instruction (all zeroed) - if cache_count > 0 { - instructions.extend(core::iter::repeat_n( - CodeUnit::new(Instruction::Cache, 0.into()), - cache_count, - )); - } - } - } - - if !recompile { - break; - } - - instructions.clear(); - locations.clear(); - linetable_locations.clear(); - } - - // CPython assemble.c::assemble_location_info() resolves NEXT_LOCATION - // after final instruction sizing, scanning backward through the - // instruction sequence. Non-terminators inherit the following - // instruction's location; terminators become NO_LOCATION. - resolve_next_locations(&instructions, &mut linetable_locations); - - // Generate linetable from linetable_locations (supports line 0 for RESUME) - let linetable = generate_linetable( - &linetable_locations, - first_line_number.get() as i32, - opts.debug_ranges, - ); - let locations = rustpython_compiler_core::marshal::linetable_to_locations( - &linetable, - first_line_number.get() as i32, - instructions.len(), - ); - - // Generate exception table before moving source_path - let exceptiontable = - generate_exception_table(&instr_sequence.instrs, &instruction_offsets, end_offset); - - // CPython builds u_cellvars in dictbytype() order, but the public - // co_cellvars tuple follows localsplus order from assemble.c: - // cell locals already present in varnames first, then remaining cells. - let final_cellvars = varname_cache - .iter() - .filter(|name| cellvar_cache.contains(name.as_str())) - .chain( - cellvar_cache - .iter() - .filter(|name| !varname_cache.contains(name.as_str())), - ) - .cloned() - .collect::>(); - - // Build localspluskinds with cell-local merging - let nlocals = varname_cache.len(); - let ncells = cellvar_cache.len(); - let nfrees = freevar_cache.len(); - debug_assert_eq!( - nlocalsplus, - nlocals + ncells - numdropped + nfrees, - "CPython prepare_localsplus() result must match assemble.c localsplus sizing" - ); - let mut localspluskinds = vec![0u8; nlocalsplus]; - // Mark locals - for kind in localspluskinds.iter_mut().take(nlocals) { - *kind = CO_FAST_LOCAL; - } - // Mark cells (merged and non-merged) - for (i, cellvar) in cellvar_cache.iter().enumerate() { - let idx = cellfixedoffsets[i] as usize; - if varname_cache.contains(cellvar.as_str()) { - localspluskinds[idx] |= CO_FAST_CELL; // merged: LOCAL | CELL - } else { - localspluskinds[idx] = CO_FAST_CELL; - } - } - // Mark frees - for i in 0..nfrees { - let idx = cellfixedoffsets[ncells + i] as usize; - localspluskinds[idx] = CO_FAST_FREE; - } - // Apply CO_FAST_HIDDEN for inlined comprehension variables - for (name, &hidden) in &fast_hidden { - if (hidden || fast_hidden_final.contains(name)) - && let Some(idx) = varname_cache.get_index_of(name.as_str()) - { - localspluskinds[idx] |= CO_FAST_HIDDEN; - } - } + resolve_unconditional_jumps(&mut instr_sequence)?; + let end_offset = resolve_jump_offsets(&mut instr_sequence)?; + let assembled = assemble_emit( + &mut instr_sequence, + end_offset, + first_line_number.get() as i32, + opts.debug_ranges, + )?; + let locations = rustpython_compiler_core::marshal::linetable_to_locations( + &assembled.linetable, + first_line_number.get() as i32, + assembled.instructions.len(), + ); Ok(CodeObject { flags, @@ -1407,2489 +1860,2341 @@ impl CodeInfo { qualname: qualname.unwrap_or(obj_name), max_stackdepth, - instructions: CodeUnits::from(instructions), + instructions: CodeUnits::from(assembled.instructions), locations, constants: constants.into_iter().collect(), names: name_cache.into_iter().collect(), varnames: varname_cache.into_iter().collect(), - cellvars: final_cellvars.into_boxed_slice(), + cellvars: localsplusinfo.cellvars, freevars: freevar_cache.into_iter().collect(), - localspluskinds: localspluskinds.into_boxed_slice(), - linetable, - exceptiontable, + localspluskinds: localsplusinfo.kinds, + linetable: assembled.linetable, + exceptiontable: assembled.exceptiontable, }) } +} - /// flowgraph.c insert_prefix_instructions - fn insert_prefix_instructions(&mut self, cellfixedoffsets: &[u32]) { - let Some(entry) = self.blocks.first_mut() else { - return; - }; - let ncells = self.metadata.cellvars.len(); - let nfrees = self.metadata.freevars.len(); - let firstlineno = self.metadata.firstlineno; +/// flowgraph.c IS_GENERATOR +fn is_generator(flags: CodeFlags) -> bool { + flags.intersects(CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR) +} - if self - .flags - .intersects(CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR) - { - let location = SourceLocation { - line: firstlineno, - character_offset: OneIndexed::MIN, - }; - basicblock_insert_instruction( - entry, - 0, - InstructionInfo { - instr: Instruction::ReturnGenerator.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location, - end_location: location, - except_handler: None, - lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), - cache_entries: 0, - }, - ); - basicblock_insert_instruction( - entry, - 1, - InstructionInfo { - instr: Instruction::PopTop.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location, - end_location: location, - except_handler: None, - lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), - cache_entries: 0, - }, - ); - } +/// flowgraph.c insert_prefix_instructions +fn insert_prefix_instructions( + metadata: &CodeUnitMetadata, + blocks: &mut [Block], + cellfixedoffsets: &[i32], + nfreevars: usize, + flags: CodeFlags, +) { + debug_assert!(!blocks.is_empty()); + let entry = &mut blocks[0]; + let ncellvars = metadata.cellvars.len(); + let firstlineno = metadata.firstlineno; + debug_assert!(firstlineno.get() > 0); + + if is_generator(flags) { + let location = SourceLocation { + line: firstlineno, + character_offset: OneIndexed::MIN, + }; + basicblock_insert_instruction( + entry, + 0, + InstructionInfo { + instr: Instruction::ReturnGenerator.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location, + end_location: location, + except_handler: None, + lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), + }, + ); + basicblock_insert_instruction( + entry, + 1, + InstructionInfo { + instr: Instruction::PopTop.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location, + end_location: location, + except_handler: None, + lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), + }, + ); + } - let mut sorted = vec![None; self.metadata.varnames.len() + ncells]; - for (oldindex, fixed) in cellfixedoffsets.iter().copied().take(ncells).enumerate() { - sorted[fixed as usize] = Some(oldindex + 1); + if ncellvars > 0 { + let nvars = metadata.varnames.len() + ncellvars; + let mut sorted = vec![0i32; nvars]; + for i in 0..ncellvars { + sorted[usize::try_from(cellfixedoffsets[i]) + .expect("localsplus offset is non-negative")] = i as i32 + 1; } - for (ncellsused, oldindex) in sorted.into_iter().flatten().enumerate() { + let mut ncellsused = 0; + let mut i = 0; + while ncellsused < ncellvars { + let oldindex = sorted[i] - 1; + i += 1; + if oldindex == -1 { + continue; + } basicblock_insert_instruction( entry, ncellsused, InstructionInfo { instr: Instruction::MakeCell { i: Arg::marker() }.into(), - arg: OpArg::new((oldindex - 1) as u32), - target: BlockIdx::NULL, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - lineno_override: Some(-1), - cache_entries: 0, - }, - ); - } - - if nfrees > 0 { - basicblock_insert_instruction( - entry, - 0, - InstructionInfo { - instr: Instruction::CopyFreeVars { n: Arg::marker() }.into(), - arg: OpArg::new(nfrees as u32), + arg: OpArg::new(oldindex.to_u32().expect("cell index is non-negative")), target: BlockIdx::NULL, location: SourceLocation::default(), end_location: SourceLocation::default(), except_handler: None, - lineno_override: Some(-1), - cache_entries: 0, + lineno_override: Some(NO_LOCATION_OVERRIDE), }, ); + ncellsused += 1; } } - /// flowgraph.c prepare_localsplus - fn prepare_localsplus(&mut self) -> usize { - let nlocals = self.metadata.varnames.len(); - let ncells = self.metadata.cellvars.len(); - let nfrees = self.metadata.freevars.len(); - let mut nlocalsplus = nlocals + ncells + nfrees; - let mut cellfixedoffsets = build_cellfixedoffsets( - &self.metadata.varnames, - &self.metadata.cellvars, - &self.metadata.freevars, + if nfreevars > 0 { + basicblock_insert_instruction( + entry, + 0, + InstructionInfo { + instr: Instruction::CopyFreeVars { n: Arg::marker() }.into(), + arg: OpArg::new(nfreevars as u32), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(NO_LOCATION_OVERRIDE), + }, ); + } +} - // This must be called before fix_cell_offsets(). - self.insert_prefix_instructions(&cellfixedoffsets); +/// flowgraph.c prepare_localsplus +fn prepare_localsplus( + metadata: &CodeUnitMetadata, + blocks: &mut [Block], + flags: CodeFlags, +) -> usize { + let nlocals = metadata.varnames.len(); + let ncellvars = metadata.cellvars.len(); + let nfreevars = metadata.freevars.len(); + let int_max = i32::MAX as usize; + debug_assert!(nlocals < int_max); + debug_assert!(ncellvars < int_max); + debug_assert!(nfreevars < int_max); + debug_assert!( + nlocals + .checked_add(ncellvars) + .is_some_and(|sum| sum < int_max) + ); + debug_assert!( + nlocals + .checked_add(ncellvars) + .and_then(|sum| sum.checked_add(nfreevars)) + .is_some_and(|sum| sum < int_max) + ); + let mut nlocalsplus = nlocals + ncellvars + nfreevars; + let mut cellfixedoffsets = build_cellfixedoffsets(metadata); - let numdropped = fix_cell_offsets(&mut self.blocks, nlocals, &mut cellfixedoffsets); - nlocalsplus -= numdropped; - nlocalsplus - } + // This must be called before fix_cell_offsets(). + insert_prefix_instructions(metadata, blocks, &cellfixedoffsets, nfreevars, flags); - /// flowgraph.c remove_unreachable - fn remove_unreachable(blocks: &mut [Block]) { - compute_reachable_predecessors(blocks); + let numdropped = fix_cell_offsets(metadata, blocks, &mut cellfixedoffsets); + nlocalsplus -= numdropped; + nlocalsplus +} - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let i = block_idx.idx(); - let next = blocks[i].next; - if blocks[i].predecessors == 0 { - let block = &mut blocks[i]; - basicblock_clear(block); - block.except_handler = false; - } - block_idx = next; - } - } +/// flowgraph.c remove_unreachable +fn remove_unreachable(blocks: &mut [Block]) { + compute_reachable_predecessors(blocks); - /// flowgraph.c eval_const_unaryop - fn eval_const_unaryop( - operand: &ConstantData, - op: Instruction, - intrinsic: Option, - ) -> Option { - match (operand, op, intrinsic) { - (ConstantData::Integer { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Integer { value: -value }) - } - (ConstantData::Float { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Float { value: -value }) - } - (ConstantData::Complex { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Complex { value: -value }) - } - (ConstantData::Boolean { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Integer { - value: BigInt::from(-i32::from(*value)), - }) - } - (ConstantData::Integer { value }, Instruction::UnaryInvert, None) => { - Some(ConstantData::Integer { value: !value }) - } - (ConstantData::Boolean { .. }, Instruction::UnaryInvert, None) => None, - (_, Instruction::UnaryNot, None) => Some(ConstantData::Boolean { - value: !Self::constant_truthiness(operand), - }), - ( - ConstantData::Integer { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Integer { - value: value.clone(), - }), - ( - ConstantData::Float { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Float { value: *value }), - ( - ConstantData::Boolean { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Integer { - value: BigInt::from(i32::from(*value)), - }), - ( - ConstantData::Complex { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Complex { value: *value }), - _ => None, + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let i = block_idx.idx(); + let next = blocks[i].next; + if blocks[i].predecessors == 0 { + let block = &mut blocks[i]; + basicblock_clear(block); + block.except_handler = false; } + block_idx = next; } +} - fn constant_truthiness(constant: &ConstantData) -> bool { - match constant { - ConstantData::Tuple { elements } | ConstantData::Frozenset { elements } => { - !elements.is_empty() - } - ConstantData::Integer { value } => !value.is_zero(), - ConstantData::Float { value } => *value != 0.0, - ConstantData::Complex { value } => value.re != 0.0 || value.im != 0.0, - ConstantData::Boolean { value } => *value, - ConstantData::Str { value } => !value.is_empty(), - ConstantData::Bytes { value } => !value.is_empty(), - ConstantData::Code { .. } | ConstantData::Slice { .. } | ConstantData::Ellipsis => true, - ConstantData::None => false, - } - } - - fn load_const_truthiness( - instr: Instruction, - arg: OpArg, - metadata: &CodeUnitMetadata, - ) -> Option { - match instr { - Instruction::LoadConst { consti } => { - let constant = &metadata.consts[consti.get(arg).as_usize()]; - Some(Self::constant_truthiness(constant)) - } - Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), - _ => None, - } +/// flowgraph.c eval_const_unaryop +fn eval_const_unaryop( + operand: &ConstantData, + op: Instruction, + intrinsic: Option, +) -> Option { + match (operand, op, intrinsic) { + (ConstantData::Integer { value }, Instruction::UnaryNegative, None) => { + Some(ConstantData::Integer { value: -value }) + } + (ConstantData::Float { value }, Instruction::UnaryNegative, None) => { + Some(ConstantData::Float { value: -value }) + } + (ConstantData::Complex { value }, Instruction::UnaryNegative, None) => { + Some(ConstantData::Complex { value: -value }) + } + (ConstantData::Boolean { value }, Instruction::UnaryNegative, None) => { + Some(ConstantData::Integer { + value: BigInt::from(-i32::from(*value)), + }) + } + (ConstantData::Integer { value }, Instruction::UnaryInvert, None) => { + Some(ConstantData::Integer { value: !value }) + } + (ConstantData::Boolean { .. }, Instruction::UnaryInvert, None) => None, + (_, Instruction::UnaryNot, None) => Some(ConstantData::Boolean { + value: !constant_truthiness(operand), + }), + ( + ConstantData::Integer { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Integer { + value: value.clone(), + }), + ( + ConstantData::Float { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Float { value: *value }), + ( + ConstantData::Boolean { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Integer { + value: BigInt::from(i32::from(*value)), + }), + ( + ConstantData::Complex { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Complex { value: *value }), + _ => None, } +} - fn instr_make_load_const( - metadata: &mut CodeUnitMetadata, - instr: &mut InstructionInfo, - constant: ConstantData, - ) { - if Self::maybe_instr_make_load_smallint(instr, &constant) { - return; - } - - let (const_idx, _) = metadata.consts.insert_full(constant); - instr.instr = Instruction::LoadConst { - consti: Arg::marker(), +fn constant_truthiness(constant: &ConstantData) -> bool { + match constant { + ConstantData::Tuple { elements } | ConstantData::Frozenset { elements } => { + !elements.is_empty() } - .into(); - instr.arg = OpArg::new(const_idx as u32); + ConstantData::Integer { value } => !value.is_zero(), + ConstantData::Float { value } => *value != 0.0, + ConstantData::Complex { value } => value.re != 0.0 || value.im != 0.0, + ConstantData::Boolean { value } => *value, + ConstantData::Str { value } => !value.is_empty(), + ConstantData::Bytes { value } => !value.is_empty(), + ConstantData::Code { .. } | ConstantData::Slice { .. } | ConstantData::Ellipsis => true, + ConstantData::None => false, } +} - /// flowgraph.c fold_const_unaryop - fn fold_const_unaryop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { - let instr = &block.instructions[i]; - let (op, intrinsic) = match instr.instr.real() { - Some(Instruction::UnaryNegative) => (Instruction::UnaryNegative, None), - Some(Instruction::UnaryInvert) => (Instruction::UnaryInvert, None), - Some(Instruction::UnaryNot) => (Instruction::UnaryNot, None), - Some(Instruction::CallIntrinsic1 { func }) - if matches!( - func.get(instr.arg), - oparg::IntrinsicFunction1::UnaryPositive - ) => - { - ( - Instruction::CallIntrinsic1 { - func: Arg::marker(), - }, - Some(func.get(instr.arg)), - ) - } - _ => return false, - }; - let Some(operand_index) = i - .checked_sub(1) - .and_then(|start| Self::get_const_loading_instrs(block, start, 1)) - .and_then(|indices| indices.into_iter().next()) - else { - return false; - }; - let operand = Self::get_const_value(metadata, &block.instructions[operand_index]); - let Some(operand) = operand else { - return false; - }; - let Some(folded_const) = Self::eval_const_unaryop(&operand, op, intrinsic) else { - return false; - }; - Self::nop_out(block, &[operand_index]); - Self::instr_make_load_const(metadata, &mut block.instructions[i], folded_const); - true - } - - /// flowgraph.c get_const_loading_instrs - fn get_const_loading_instrs( - block: &Block, - mut start: usize, - size: usize, - ) -> Option> { - let mut indices = Vec::with_capacity(size); - loop { - let instr = block.instructions.get(start)?; - if !matches!(instr.instr.real(), Some(Instruction::Nop)) { - if !Self::loads_const(instr) { - return None; - } - indices.push(start); - if indices.len() == size { - break; - } - } - start = start.checked_sub(1)?; +fn load_const_truthiness( + instr: Instruction, + arg: OpArg, + metadata: &CodeUnitMetadata, +) -> Option { + match instr { + Instruction::LoadConst { consti } => { + let constant = &metadata.consts[consti.get(arg).as_usize()]; + Some(constant_truthiness(constant)) } - indices.reverse(); - Some(indices) + Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), + _ => None, } +} - /// flowgraph.c nop_out - fn nop_out(block: &mut Block, instrs: &[usize]) { - for &i in instrs { - nop_out_no_location(&mut block.instructions[i]); - } +/// flowgraph.c add_const +fn add_const(metadata: &mut CodeUnitMetadata, constant: ConstantData) -> usize { + metadata.consts.insert_full(constant).0 +} + +fn instr_make_load_const( + metadata: &mut CodeUnitMetadata, + instr: &mut InstructionInfo, + constant: ConstantData, +) { + if maybe_instr_make_load_smallint(instr, &constant) { + return; } - /// flowgraph.c fold_const_binop - fn fold_const_binop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { - use oparg::BinaryOperator as BinOp; + let const_idx = add_const(metadata, constant); + instr_set_op1( + instr, + Instruction::LoadConst { + consti: Arg::marker(), + } + .into(), + OpArg::new(const_idx as u32), + ); +} - let Some(Instruction::BinaryOp { .. }) = block.instructions[i].instr.real() else { - return false; - }; - let Some(operand_indices) = i - .checked_sub(1) - .and_then(|start| Self::get_const_loading_instrs(block, start, 2)) - else { - return false; - }; - let op_raw = u32::from(block.instructions[i].arg); - let Ok(op) = BinOp::try_from(op_raw) else { - return false; - }; - let left = Self::get_const_value(metadata, &block.instructions[operand_indices[0]]); - let right = Self::get_const_value(metadata, &block.instructions[operand_indices[1]]); - let (Some(left_val), Some(right_val)) = (left, right) else { - return false; - }; - let Some(result_const) = Self::eval_const_binop(&left_val, &right_val, op) else { - return false; - }; - Self::nop_out(block, &operand_indices); - Self::instr_make_load_const(metadata, &mut block.instructions[i], result_const); - true - } - - /// flowgraph.c loads_const - fn loads_const(info: &InstructionInfo) -> bool { - info.instr.has_const() - || matches!(info.instr.real(), Some(Instruction::LoadSmallInt { .. })) - } - - /// flowgraph.c get_const_value - fn get_const_value( - metadata: &CodeUnitMetadata, - info: &InstructionInfo, - ) -> Option { - match info.instr.real() { - Some(Instruction::LoadSmallInt { .. }) => { - let v = u32::from(info.arg) as i32; - Some(ConstantData::Integer { - value: BigInt::from(v), - }) - } - _ if info.instr.has_const() => { - let idx = u32::from(info.arg) as usize; - metadata.consts.get_index(idx).cloned() - } - _ => None, +/// flowgraph.c fold_const_unaryop +fn fold_const_unaryop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { + let instr = &block.instructions[i]; + let (op, intrinsic) = match instr.instr.real() { + Some(Instruction::UnaryNegative) => (Instruction::UnaryNegative, None), + Some(Instruction::UnaryInvert) => (Instruction::UnaryInvert, None), + Some(Instruction::UnaryNot) => (Instruction::UnaryNot, None), + Some(Instruction::CallIntrinsic1 { func }) + if matches!( + func.get(instr.arg), + oparg::IntrinsicFunction1::UnaryPositive + ) => + { + ( + Instruction::CallIntrinsic1 { + func: Arg::marker(), + }, + Some(func.get(instr.arg)), + ) } - } + _ => return false, + }; + let Some(operand_index) = i + .checked_sub(1) + .and_then(|start| get_const_loading_instrs(block, start, 1)) + .and_then(|indices| indices.into_iter().next()) + else { + return false; + }; + let operand = get_const_value(metadata, &block.instructions[operand_index]); + let Some(operand) = operand else { + return false; + }; + let Some(folded_const) = eval_const_unaryop(&operand, op, intrinsic) else { + return false; + }; + nop_out(block, &[operand_index]); + instr_make_load_const(metadata, &mut block.instructions[i], folded_const); + true +} - /// flowgraph.c const_folding_check_complexity - fn const_folding_check_complexity(obj: &ConstantData, mut limit: isize) -> Option { - if let ConstantData::Tuple { elements } = obj { - limit -= isize::try_from(elements.len()).ok()?; - if limit < 0 { +/// flowgraph.c get_const_loading_instrs +fn get_const_loading_instrs(block: &Block, mut start: usize, size: usize) -> Option> { + let mut indices = Vec::with_capacity(size); + loop { + let instr = block.instructions.get(start)?; + if !matches!(instr.instr.real(), Some(Instruction::Nop)) { + if !loads_const(instr) { return None; } - for element in elements { - limit = Self::const_folding_check_complexity(element, limit)?; + indices.push(start); + if indices.len() == size { + break; } } - Some(limit) + start = start.checked_sub(1)?; + } + indices.reverse(); + Some(indices) +} + +/// flowgraph.c nop_out +fn nop_out(block: &mut Block, instrs: &[usize]) { + for &i in instrs { + nop_out_no_location(&mut block.instructions[i]); } +} + +/// flowgraph.c fold_const_binop +fn fold_const_binop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { + use oparg::BinaryOperator as BinOp; + + let Some(Instruction::BinaryOp { .. }) = block.instructions[i].instr.real() else { + return false; + }; + let Some(operand_indices) = i + .checked_sub(1) + .and_then(|start| get_const_loading_instrs(block, start, 2)) + else { + return false; + }; + let op_raw = u32::from(block.instructions[i].arg); + let Ok(op) = BinOp::try_from(op_raw) else { + return false; + }; + let left = get_const_value(metadata, &block.instructions[operand_indices[0]]); + let right = get_const_value(metadata, &block.instructions[operand_indices[1]]); + let (Some(left_val), Some(right_val)) = (left, right) else { + return false; + }; + let Some(result_const) = eval_const_binop(&left_val, &right_val, op) else { + return false; + }; + nop_out(block, &operand_indices); + instr_make_load_const(metadata, &mut block.instructions[i], result_const); + true +} - fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Wtf8Buf { - let mut result = Wtf8Buf::with_capacity(value.len().saturating_mul(n)); - for _ in 0..n { - result.push_wtf8(value); +/// flowgraph.c loads_const +fn loads_const(info: &InstructionInfo) -> bool { + info.instr.has_const() || matches!(info.instr.real(), Some(Instruction::LoadSmallInt { .. })) +} + +/// flowgraph.c get_const_value +fn get_const_value(metadata: &CodeUnitMetadata, info: &InstructionInfo) -> Option { + match info.instr.real() { + Some(Instruction::LoadSmallInt { .. }) => { + let v = u32::from(info.arg) as i32; + Some(ConstantData::Integer { + value: BigInt::from(v), + }) } - result + _ if info.instr.has_const() => { + let idx = u32::from(info.arg) as usize; + metadata.consts.get_index(idx).cloned() + } + _ => None, } +} - fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { - let n = n.to_isize()?; - if item_size != 0 && (n < 0 || n as usize > MAX_STR_SIZE / item_size) { +/// flowgraph.c const_folding_check_complexity +fn const_folding_check_complexity(obj: &ConstantData, mut limit: isize) -> Option { + if let ConstantData::Tuple { elements } = obj { + limit -= isize::try_from(elements.len()).ok()?; + if limit < 0 { return None; } - Some(n.max(0) as usize) + for element in elements { + limit = const_folding_check_complexity(element, limit)?; + } } + Some(limit) +} - /// flowgraph.c const_folding_safe_multiply - fn const_folding_safe_multiply( - left: &ConstantData, - right: &ConstantData, - ) -> Option { - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - if !l.is_zero() && !r.is_zero() && l.bits() + r.bits() > MAX_INT_SIZE { - return None; - } - Some(ConstantData::Integer { value: l * r }) - } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - Some(ConstantData::Float { value: l * r }) - } - (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) => { - let n = Self::checked_repeat_count(n, s.code_points().count())?; - Some(ConstantData::Str { - value: Self::repeat_wtf8(s, n), - }) - } - (ConstantData::Integer { .. }, ConstantData::Str { .. }) => { - Self::const_folding_safe_multiply(right, left) - } - (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) => { - let n = Self::checked_repeat_count(n, b.len())?; - Some(ConstantData::Bytes { value: b.repeat(n) }) - } - (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) => { - Self::const_folding_safe_multiply(right, left) +fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Wtf8Buf { + let mut result = Wtf8Buf::with_capacity(value.len().saturating_mul(n)); + for _ in 0..n { + result.push_wtf8(value); + } + result +} + +fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { + let n = n.to_isize()?; + if item_size != 0 && (n < 0 || n as usize > MAX_STR_SIZE / item_size) { + return None; + } + Some(n.max(0) as usize) +} + +/// flowgraph.c const_folding_safe_multiply +fn const_folding_safe_multiply(left: &ConstantData, right: &ConstantData) -> Option { + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if !l.is_zero() && !r.is_zero() && l.bits() + r.bits() > MAX_INT_SIZE { + return None; } - (ConstantData::Tuple { elements }, ConstantData::Integer { value: n }) => { - let n = n.to_usize()?; - if n != 0 && !elements.is_empty() { - if n > MAX_COLLECTION_SIZE / elements.len() { - return None; - } - Self::const_folding_check_complexity( - &ConstantData::Tuple { - elements: elements.clone(), - }, - MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, - )?; - } - let mut result = Vec::with_capacity(elements.len() * n); - for _ in 0..n { - result.extend(elements.iter().cloned()); + Some(ConstantData::Integer { value: l * r }) + } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + Some(ConstantData::Float { value: l * r }) + } + (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) => { + let n = checked_repeat_count(n, s.code_points().count())?; + Some(ConstantData::Str { + value: repeat_wtf8(s, n), + }) + } + (ConstantData::Integer { .. }, ConstantData::Str { .. }) => { + const_folding_safe_multiply(right, left) + } + (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) => { + let n = checked_repeat_count(n, b.len())?; + Some(ConstantData::Bytes { value: b.repeat(n) }) + } + (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) => { + const_folding_safe_multiply(right, left) + } + (ConstantData::Tuple { elements }, ConstantData::Integer { value: n }) => { + let n = n.to_usize()?; + if n != 0 && !elements.is_empty() { + if n > MAX_COLLECTION_SIZE / elements.len() { + return None; } - Some(ConstantData::Tuple { elements: result }) + const_folding_check_complexity( + &ConstantData::Tuple { + elements: elements.clone(), + }, + MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, + )?; } - (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) => { - Self::const_folding_safe_multiply(right, left) + let mut result = Vec::with_capacity(elements.len() * n); + for _ in 0..n { + result.extend(elements.iter().cloned()); } - _ => None, + Some(ConstantData::Tuple { elements: result }) } + (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) => { + const_folding_safe_multiply(right, left) + } + _ => None, } +} - /// flowgraph.c const_folding_safe_power - fn const_folding_safe_power(left: &ConstantData, right: &ConstantData) -> Option { - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - if r < &BigInt::from(0) { - if l.is_zero() { - return None; - } - let base = l.to_f64()?; - if !base.is_finite() { - return None; - } - let result = if let Some(exp) = r.to_i32() { - base.powi(exp) - } else { - base.powf(r.to_f64()?) - }; - if !result.is_finite() { - return None; - } - return Some(ConstantData::Float { value: result }); +/// flowgraph.c const_folding_safe_power +fn const_folding_safe_power(left: &ConstantData, right: &ConstantData) -> Option { + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if r < &BigInt::from(0) { + if l.is_zero() { + return None; + } + let base = l.to_f64()?; + if !base.is_finite() { + return None; } - let exp: u64 = r.try_into().ok()?; - let exp_usize = usize::try_from(exp).ok()?; - if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE / exp { + let result = if let Some(exp) = r.to_i32() { + base.powi(exp) + } else { + base.powf(r.to_f64()?) + }; + if !result.is_finite() { return None; } - Some(ConstantData::Integer { - value: num_traits::pow::pow(l.clone(), exp_usize), - }) + return Some(ConstantData::Float { value: result }); } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - let result = l.powf(*r); - result - .is_finite() - .then_some(ConstantData::Float { value: result }) + let exp: u64 = r.try_into().ok()?; + let exp_usize = usize::try_from(exp).ok()?; + if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE / exp { + return None; } - _ => None, + Some(ConstantData::Integer { + value: num_traits::pow::pow(l.clone(), exp_usize), + }) } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let result = l.powf(*r); + result + .is_finite() + .then_some(ConstantData::Float { value: result }) + } + _ => None, } +} - /// flowgraph.c const_folding_safe_lshift - fn const_folding_safe_lshift( - left: &ConstantData, - right: &ConstantData, - ) -> Option { - let (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) = - (left, right) - else { - return None; - }; - let shift: u64 = r.try_into().ok()?; - let shift_usize = usize::try_from(shift).ok()?; - if shift > MAX_INT_SIZE || (!l.is_zero() && l.bits() > MAX_INT_SIZE - shift) { - return None; - } - Some(ConstantData::Integer { - value: l << shift_usize, - }) +/// flowgraph.c const_folding_safe_lshift +fn const_folding_safe_lshift(left: &ConstantData, right: &ConstantData) -> Option { + let (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) = (left, right) + else { + return None; + }; + let shift: u64 = r.try_into().ok()?; + let shift_usize = usize::try_from(shift).ok()?; + if shift > MAX_INT_SIZE || (!l.is_zero() && l.bits() > MAX_INT_SIZE - shift) { + return None; } + Some(ConstantData::Integer { + value: l << shift_usize, + }) +} - /// flowgraph.c const_folding_safe_mod - fn const_folding_safe_mod(left: &ConstantData, right: &ConstantData) -> Option { - if matches!(left, ConstantData::Str { .. } | ConstantData::Bytes { .. }) { - return None; - } +/// flowgraph.c const_folding_safe_mod +fn const_folding_safe_mod(left: &ConstantData, right: &ConstantData) -> Option { + if matches!(left, ConstantData::Str { .. } | ConstantData::Bytes { .. }) { + return None; + } - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - if r.is_zero() { - return None; - } - let rem = l.clone() % r.clone(); - let value = if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { - rem + r - } else { - rem - }; - Some(ConstantData::Integer { value }) - } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - let (_, modulo) = Self::float_div_mod(*l, *r)?; - Some(ConstantData::Float { value: modulo }) + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if r.is_zero() { + return None; } - _ => None, + let rem = l.clone() % r.clone(); + let value = if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { + rem + r + } else { + rem + }; + Some(ConstantData::Integer { value }) + } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let (_, modulo) = float_div_mod(*l, *r)?; + Some(ConstantData::Float { value: modulo }) } + _ => None, } +} - fn float_div_mod(left: f64, right: f64) -> Option<(f64, f64)> { - if right == 0.0 { - return None; - } +fn float_div_mod(left: f64, right: f64) -> Option<(f64, f64)> { + if right == 0.0 { + return None; + } - let mut modulo = left % right; - let div = (left - modulo) / right; - let floordiv = if modulo != 0.0 { - let div = if (right < 0.0) != (modulo < 0.0) { - modulo += right; - div - 1.0 - } else { - div - }; - let mut floordiv = div.floor(); - if div - floordiv > 0.5 { - floordiv += 1.0; - } - floordiv + let mut modulo = left % right; + let div = (left - modulo) / right; + let floordiv = if modulo != 0.0 { + let div = if (right < 0.0) != (modulo < 0.0) { + modulo += right; + div - 1.0 } else { - modulo = 0.0f64.copysign(right); - 0.0f64.copysign(left / right) + div }; + let mut floordiv = div.floor(); + if div - floordiv > 0.5 { + floordiv += 1.0; + } + floordiv + } else { + modulo = 0.0f64.copysign(right); + 0.0f64.copysign(left / right) + }; - Some((floordiv, modulo)) - } - - /// flowgraph.c eval_const_binop - fn eval_const_binop( - left: &ConstantData, - right: &ConstantData, - op: oparg::BinaryOperator, - ) -> Option { - use oparg::BinaryOperator as BinOp; - - fn eval_complex_binop( - left: Complex, - right: Complex, - op: BinOp, - ) -> Option { - fn complex_const(value: Complex) -> Option { - (value.re.is_finite() && value.im.is_finite()) - .then_some(ConstantData::Complex { value }) - } - - let value = match op { - BinOp::Add => left + right, - BinOp::Subtract => { - let re = left.re - right.re; - // Preserve CPython's signed-zero behavior for real-zero - // minus zero-complex expressions such as `0 - 0j`. - let im = if left.re == 0.0 - && left.im == 0.0 - && right.re == 0.0 - && right.im == 0.0 - && !right.im.is_sign_negative() - { - -0.0 - } else { - left.im - right.im - }; - Complex::new(re, im) - } - BinOp::Multiply => left * right, - BinOp::TrueDivide => { - if right == Complex::new(0.0, 0.0) { - return None; - } - left / right - } - BinOp::Power => { - if left == Complex::new(0.0, 0.0) { - if right.im != 0.0 || right.re < 0.0 { - return None; - } + Some((floordiv, modulo)) +} - return complex_const(if right.re == 0.0 { - Complex::new(1.0, 0.0) - } else { - Complex::new(0.0, 0.0) - }); - } +/// flowgraph.c eval_const_binop complex result construction +fn eval_const_complex_const(value: Complex) -> Option { + (value.re.is_finite() && value.im.is_finite()).then_some(ConstantData::Complex { value }) +} - if right.im == 0.0 - && right.re.fract() == 0.0 - && right.re >= f64::from(i32::MIN) - && right.re <= f64::from(i32::MAX) - { - left.powi(right.re as i32) - } else { - left.powc(right) - } - } - _ => return None, +/// flowgraph.c eval_const_binop complex operations +fn eval_const_complex_binop( + left: Complex, + right: Complex, + op: oparg::BinaryOperator, +) -> Option { + use oparg::BinaryOperator as BinOp; + + let value = match op { + BinOp::Add => left + right, + BinOp::Subtract => { + let re = left.re - right.re; + // Preserve CPython's signed-zero behavior for real-zero + // minus zero-complex expressions such as `0 - 0j`. + let im = if left.re == 0.0 + && left.im == 0.0 + && right.re == 0.0 + && right.im == 0.0 + && !right.im.is_sign_negative() + { + -0.0 + } else { + left.im - right.im }; - complex_const(value) + Complex::new(re, im) } + BinOp::Multiply => left * right, + BinOp::TrueDivide => { + if right == Complex::new(0.0, 0.0) { + return None; + } + left / right + } + BinOp::Power => { + if left == Complex::new(0.0, 0.0) { + if right.im != 0.0 || right.re < 0.0 { + return None; + } - fn constant_as_index(value: &ConstantData) -> Option { - match value { - ConstantData::Integer { value } => value.to_i64().or_else(|| { - if value < &BigInt::from(0) { - Some(i64::MIN) - } else { - Some(i64::MAX) - } - }), - ConstantData::Boolean { value } => Some(i64::from(*value)), - _ => None, + return eval_const_complex_const(if right.re == 0.0 { + Complex::new(1.0, 0.0) + } else { + Complex::new(0.0, 0.0) + }); + } + + if right.im == 0.0 + && right.re.fract() == 0.0 + && right.re >= f64::from(i32::MIN) + && right.re <= f64::from(i32::MAX) + { + left.powi(right.re as i32) + } else { + left.powc(right) } } + _ => return None, + }; + eval_const_complex_const(value) +} - fn slice_bound(value: &ConstantData) -> Option> { - match value { - ConstantData::None => Some(None), - _ => constant_as_index(value).map(Some), +/// flowgraph.c eval_const_binop subscript index conversion +fn constant_as_index(value: &ConstantData) -> Option { + match value { + ConstantData::Integer { value } => value.to_i64().or_else(|| { + if value < &BigInt::from(0) { + Some(i64::MIN) + } else { + Some(i64::MAX) } + }), + ConstantData::Boolean { value } => Some(i64::from(*value)), + _ => None, + } +} + +/// flowgraph.c eval_const_binop subscript slice bound conversion +fn slice_bound(value: &ConstantData) -> Option> { + match value { + ConstantData::None => Some(None), + _ => constant_as_index(value).map(Some), + } +} + +/// flowgraph.c eval_const_binop subscript slice index adjustment +fn adjusted_slice_indices(len: usize, slice: &[ConstantData; 3]) -> Option> { + let len = i64::try_from(len).ok()?; + let start = slice_bound(&slice[0])?; + let stop = slice_bound(&slice[1])?; + let step = slice_bound(&slice[2])?.unwrap_or(1); + if step == 0 || step == i64::MIN { + return None; + } + + let step_is_negative = step < 0; + let lower = if step_is_negative { -1 } else { 0 }; + let upper = if step_is_negative { len - 1 } else { len }; + let adjust = |value: Option, default: i64| { + let mut value = value.unwrap_or(default); + if value < 0 { + value = value.saturating_add(len); + if value < 0 { + value = lower; + } + } else if value >= len { + value = upper; + } + value + }; + let start = adjust(start, if step_is_negative { upper } else { lower }); + let stop = adjust(stop, if step_is_negative { lower } else { upper }); + + let mut indices = Vec::new(); + let mut index = i128::from(start); + let stop = i128::from(stop); + let step = i128::from(step); + if step > 0 { + while index < stop { + indices.push(usize::try_from(index).ok()?); + index += step; } + } else { + while index > stop { + indices.push(usize::try_from(index).ok()?); + index += step; + } + } + Some(indices) +} + +/// flowgraph.c eval_const_binop subscript index adjustment +fn adjusted_const_index(len: usize, index: &ConstantData) -> Option { + let len = i64::try_from(len).ok()?; + let index = constant_as_index(index)?; + let index = if index < 0 { + index.saturating_add(len) + } else { + index + }; + if index < 0 || index >= len { + return None; + } + usize::try_from(index).ok() +} - fn adjusted_slice_indices(len: usize, slice: &[ConstantData; 3]) -> Option> { - let len = i64::try_from(len).ok()?; - let start = slice_bound(&slice[0])?; - let stop = slice_bound(&slice[1])?; - let step = slice_bound(&slice[2])?.unwrap_or(1); - if step == 0 || step == i64::MIN { +/// flowgraph.c eval_const_binop NB_SUBSCR +fn eval_const_subscript(container: &ConstantData, index: &ConstantData) -> Option { + match (container, index) { + ( + ConstantData::Str { value }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let string = value.to_string(); + if string.contains(char::REPLACEMENT_CHARACTER) { + return None; + } + let chars = string.chars().collect::>(); + let index = adjusted_const_index(chars.len(), index)?; + Some(ConstantData::Str { + value: chars[index].to_string().into(), + }) + } + (ConstantData::Str { value }, ConstantData::Slice { elements }) => { + let string = value.to_string(); + if string.contains(char::REPLACEMENT_CHARACTER) { return None; } + let chars = string.chars().collect::>(); + let mut result = String::new(); + for index in adjusted_slice_indices(chars.len(), elements)? { + result.push(chars[index]); + } + Some(ConstantData::Str { + value: result.into(), + }) + } + ( + ConstantData::Bytes { value }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let index = adjusted_const_index(value.len(), index)?; + Some(ConstantData::Integer { + value: BigInt::from(value[index]), + }) + } + (ConstantData::Bytes { value }, ConstantData::Slice { elements }) => { + let mut result = Vec::new(); + for index in adjusted_slice_indices(value.len(), elements)? { + result.push(value[index]); + } + Some(ConstantData::Bytes { value: result }) + } + ( + ConstantData::Tuple { elements }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let index = adjusted_const_index(elements.len(), index)?; + Some(elements[index].clone()) + } + (ConstantData::Tuple { elements }, ConstantData::Slice { elements: slice }) => { + let elements = adjusted_slice_indices(elements.len(), slice)? + .into_iter() + .map(|index| elements[index].clone()) + .collect(); + Some(ConstantData::Tuple { elements }) + } + _ => None, + } +} - let step_is_negative = step < 0; - let lower = if step_is_negative { -1 } else { 0 }; - let upper = if step_is_negative { len - 1 } else { len }; - let adjust = |value: Option, default: i64| { - let mut value = value.unwrap_or(default); - if value < 0 { - value = value.saturating_add(len); - if value < 0 { - value = lower; - } - } else if value >= len { - value = upper; +/// flowgraph.c eval_const_binop bool/int coercion +fn constant_as_int(value: &ConstantData) -> Option<(BigInt, bool)> { + match value { + ConstantData::Boolean { value } => Some((BigInt::from(u8::from(*value)), true)), + ConstantData::Integer { value } => Some((value.clone(), false)), + _ => None, + } +} + +/// flowgraph.c eval_const_binop +fn eval_const_binop( + left: &ConstantData, + right: &ConstantData, + op: oparg::BinaryOperator, +) -> Option { + use oparg::BinaryOperator as BinOp; + + if matches!(op, BinOp::Subscr) { + return eval_const_subscript(left, right); + } + + if let (Some((left_int, left_is_bool)), Some((right_int, right_is_bool))) = + (constant_as_int(left), constant_as_int(right)) + && (left_is_bool || right_is_bool) + { + if left_is_bool && right_is_bool { + match op { + BinOp::And => { + return Some(ConstantData::Boolean { + value: !left_int.is_zero() & !right_int.is_zero(), + }); } - value - }; - let start = adjust(start, if step_is_negative { upper } else { lower }); - let stop = adjust(stop, if step_is_negative { lower } else { upper }); - - let mut indices = Vec::new(); - let mut index = i128::from(start); - let stop = i128::from(stop); - let step = i128::from(step); - if step > 0 { - while index < stop { - indices.push(usize::try_from(index).ok()?); - index += step; + BinOp::Or => { + return Some(ConstantData::Boolean { + value: !left_int.is_zero() | !right_int.is_zero(), + }); } - } else { - while index > stop { - indices.push(usize::try_from(index).ok()?); - index += step; + BinOp::Xor => { + return Some(ConstantData::Boolean { + value: !left_int.is_zero() ^ !right_int.is_zero(), + }); } + _ => {} } - Some(indices) } - fn adjusted_const_index(len: usize, index: &ConstantData) -> Option { - let len = i64::try_from(len).ok()?; - let index = constant_as_index(index)?; - let index = if index < 0 { - index.saturating_add(len) - } else { - index - }; - if index < 0 || index >= len { - return None; - } - usize::try_from(index).ok() - } + return eval_const_binop( + &ConstantData::Integer { value: left_int }, + &ConstantData::Integer { value: right_int }, + op, + ); + } - fn eval_const_subscript( - container: &ConstantData, - index: &ConstantData, - ) -> Option { - match (container, index) { - ( - ConstantData::Str { value }, - ConstantData::Integer { .. } | ConstantData::Boolean { .. }, - ) => { - let string = value.to_string(); - if string.contains(char::REPLACEMENT_CHARACTER) { + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + let result = match op { + BinOp::Add => l + r, + BinOp::Subtract => l - r, + BinOp::Multiply => { + return const_folding_safe_multiply(left, right); + } + BinOp::TrueDivide => { + if r.is_zero() { + return None; + } + let l_f = l.to_f64()?; + let r_f = r.to_f64()?; + let result = l_f / r_f; + if !result.is_finite() { return None; } - let chars = string.chars().collect::>(); - let index = adjusted_const_index(chars.len(), index)?; - Some(ConstantData::Str { - value: chars[index].to_string().into(), - }) + return Some(ConstantData::Float { value: result }); } - (ConstantData::Str { value }, ConstantData::Slice { elements }) => { - let string = value.to_string(); - if string.contains(char::REPLACEMENT_CHARACTER) { + BinOp::FloorDivide => { + if r.is_zero() { return None; } - let chars = string.chars().collect::>(); - let mut result = String::new(); - for index in adjusted_slice_indices(chars.len(), elements)? { - result.push(chars[index]); + // Python floor division: round towards negative infinity + let (q, rem) = (l.clone() / r.clone(), l.clone() % r.clone()); + if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { + q - 1 + } else { + q } - Some(ConstantData::Str { - value: result.into(), - }) } - ( - ConstantData::Bytes { value }, - ConstantData::Integer { .. } | ConstantData::Boolean { .. }, - ) => { - let index = adjusted_const_index(value.len(), index)?; - Some(ConstantData::Integer { - value: BigInt::from(value[index]), - }) + BinOp::Remainder => return const_folding_safe_mod(left, right), + BinOp::Power => return const_folding_safe_power(left, right), + BinOp::Lshift => return const_folding_safe_lshift(left, right), + BinOp::Rshift => { + let shift: u32 = r.try_into().ok()?; + l >> (shift as usize) } - (ConstantData::Bytes { value }, ConstantData::Slice { elements }) => { - let mut result = Vec::new(); - for index in adjusted_slice_indices(value.len(), elements)? { - result.push(value[index]); + BinOp::And => l & r, + BinOp::Or => l | r, + BinOp::Xor => l ^ r, + _ => return None, + }; + Some(ConstantData::Integer { value: result }) + } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let result = match op { + BinOp::Add => l + r, + BinOp::Subtract => l - r, + BinOp::Multiply => return const_folding_safe_multiply(left, right), + BinOp::TrueDivide => { + if *r == 0.0 { + return None; } - Some(ConstantData::Bytes { value: result }) - } - ( - ConstantData::Tuple { elements }, - ConstantData::Integer { .. } | ConstantData::Boolean { .. }, - ) => { - let index = adjusted_const_index(elements.len(), index)?; - Some(elements[index].clone()) + l / r } - (ConstantData::Tuple { elements }, ConstantData::Slice { elements: slice }) => { - let elements = adjusted_slice_indices(elements.len(), slice)? - .into_iter() - .map(|index| elements[index].clone()) - .collect(); - Some(ConstantData::Tuple { elements }) + BinOp::FloorDivide => { + let (floordiv, _) = float_div_mod(*l, *r)?; + floordiv } - _ => None, + BinOp::Remainder => return const_folding_safe_mod(left, right), + BinOp::Power => return const_folding_safe_power(left, right), + _ => return None, + }; + if matches!(op, BinOp::Power) && !result.is_finite() { + return None; } + Some(ConstantData::Float { value: result }) } - - if matches!(op, BinOp::Subscr) { - return eval_const_subscript(left, right); + // Int op Float or Float op Int → Float + (ConstantData::Integer { value: l }, ConstantData::Float { value: r }) => { + let l_f = l.to_f64()?; + eval_const_binop( + &ConstantData::Float { value: l_f }, + &ConstantData::Float { value: *r }, + op, + ) + } + (ConstantData::Float { value: l }, ConstantData::Integer { value: r }) => { + let r_f = r.to_f64()?; + eval_const_binop( + &ConstantData::Float { value: *l }, + &ConstantData::Float { value: r_f }, + op, + ) + } + (ConstantData::Integer { value: l }, ConstantData::Complex { value: r }) => { + eval_const_complex_binop(Complex::new(l.to_f64()?, 0.0), *r, op) + } + (ConstantData::Complex { value: l }, ConstantData::Integer { value: r }) => { + eval_const_complex_binop(*l, Complex::new(r.to_f64()?, 0.0), op) } + (ConstantData::Float { value: l }, ConstantData::Complex { value: r }) => { + eval_const_complex_binop(Complex::new(*l, 0.0), *r, op) + } + (ConstantData::Complex { value: l }, ConstantData::Float { value: r }) => { + eval_const_complex_binop(*l, Complex::new(*r, 0.0), op) + } + (ConstantData::Complex { value: l }, ConstantData::Complex { value: r }) => { + eval_const_complex_binop(*l, *r, op) + } + // String concatenation and repetition + (ConstantData::Str { value: l }, ConstantData::Str { value: r }) + if matches!(op, BinOp::Add) => + { + let mut result = l.clone(); + result.push_wtf8(r); + Some(ConstantData::Str { value: result }) + } + (ConstantData::Str { .. }, ConstantData::Integer { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Tuple { elements: l }, ConstantData::Tuple { elements: r }) + if matches!(op, BinOp::Add) => + { + let mut result = l.clone(); + result.extend(r.iter().cloned()); + Some(ConstantData::Tuple { elements: result }) + } + (ConstantData::Tuple { .. }, ConstantData::Integer { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Integer { .. }, ConstantData::Str { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Bytes { value: l }, ConstantData::Bytes { value: r }) + if matches!(op, BinOp::Add) => + { + let mut result = l.clone(); + result.extend_from_slice(r); + Some(ConstantData::Bytes { value: result }) + } + (ConstantData::Bytes { .. }, ConstantData::Integer { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + _ => None, + } +} - fn constant_as_int(value: &ConstantData) -> Option<(BigInt, bool)> { - match value { - ConstantData::Boolean { value } => Some((BigInt::from(u8::from(*value)), true)), - ConstantData::Integer { value } => Some((value.clone(), false)), - _ => None, - } +/// flowgraph.c fold_tuple_of_constants +fn fold_tuple_of_constants(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { + let Some(Instruction::BuildTuple { .. }) = block.instructions[i].instr.real() else { + return false; + }; + + let tuple_size = u32::from(block.instructions[i].arg) as usize; + if tuple_size > STACK_USE_GUIDELINE { + return false; + } + + let Some(operand_indices) = (if tuple_size == 0 { + Some(Vec::new()) + } else { + i.checked_sub(1) + .and_then(|start| get_const_loading_instrs(block, start, tuple_size)) + }) else { + return false; + }; + + let mut elements = Vec::with_capacity(tuple_size); + for &j in &operand_indices { + let Some(element) = get_const_value(metadata, &block.instructions[j]) else { + return false; + }; + elements.push(element); + } + + nop_out(block, &operand_indices); + instr_make_load_const( + metadata, + &mut block.instructions[i], + ConstantData::Tuple { elements }, + ); + true +} + +fn fold_constant_intrinsic_list_to_tuple( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, +) -> bool { + let Some(Instruction::CallIntrinsic1 { func }) = block.instructions[i].instr.real() else { + return false; + }; + if func.get(block.instructions[i].arg) != IntrinsicFunction1::ListToTuple { + return false; + } + + let mut consts_found = 0usize; + let mut expect_append = true; + let mut pos = i; + while let Some(prev) = pos.checked_sub(1) { + pos = prev; + let instr = &block.instructions[pos]; + if matches!(instr.instr.real(), Some(Instruction::Nop)) { + continue; } - if let (Some((left_int, left_is_bool)), Some((right_int, right_is_bool))) = - (constant_as_int(left), constant_as_int(right)) - && (left_is_bool || right_is_bool) + if matches!(instr.instr.real(), Some(Instruction::BuildList { .. })) + && u32::from(instr.arg) == 0 { - if left_is_bool && right_is_bool { - match op { - BinOp::And => { - return Some(ConstantData::Boolean { - value: !left_int.is_zero() & !right_int.is_zero(), - }); - } - BinOp::Or => { - return Some(ConstantData::Boolean { - value: !left_int.is_zero() | !right_int.is_zero(), - }); - } - BinOp::Xor => { - return Some(ConstantData::Boolean { - value: !left_int.is_zero() ^ !right_int.is_zero(), - }); - } - _ => {} - } + if !expect_append { + return false; } - return Self::eval_const_binop( - &ConstantData::Integer { value: left_int }, - &ConstantData::Integer { value: right_int }, - op, + let mut elements = vec![None; consts_found]; + let mut remaining = consts_found; + for idx in (pos..i).rev() { + if matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { + continue; + } + if loads_const(&block.instructions[idx]) { + let Some(value) = get_const_value(metadata, &block.instructions[idx]) else { + return false; + }; + debug_assert!(remaining > 0); + remaining -= 1; + elements[remaining] = Some(value); + } + nop_out_no_location(&mut block.instructions[idx]); + } + debug_assert_eq!(remaining, 0); + let elements = elements + .into_iter() + .map(|element| element.expect("constant list item was collected")) + .collect(); + instr_make_load_const( + metadata, + &mut block.instructions[i], + ConstantData::Tuple { elements }, ); + return true; } - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - let result = match op { - BinOp::Add => l + r, - BinOp::Subtract => l - r, - BinOp::Multiply => { - return Self::const_folding_safe_multiply(left, right); - } - BinOp::TrueDivide => { - if r.is_zero() { - return None; - } - let l_f = l.to_f64()?; - let r_f = r.to_f64()?; - let result = l_f / r_f; - if !result.is_finite() { - return None; - } - return Some(ConstantData::Float { value: result }); - } - BinOp::FloorDivide => { - if r.is_zero() { - return None; - } - // Python floor division: round towards negative infinity - let (q, rem) = (l.clone() / r.clone(), l.clone() % r.clone()); - if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { - q - 1 - } else { - q - } - } - BinOp::Remainder => return Self::const_folding_safe_mod(left, right), - BinOp::Power => return Self::const_folding_safe_power(left, right), - BinOp::Lshift => return Self::const_folding_safe_lshift(left, right), - BinOp::Rshift => { - let shift: u32 = r.try_into().ok()?; - l >> (shift as usize) - } - BinOp::And => l & r, - BinOp::Or => l | r, - BinOp::Xor => l ^ r, - _ => return None, - }; - Some(ConstantData::Integer { value: result }) - } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - let result = match op { - BinOp::Add => l + r, - BinOp::Subtract => l - r, - BinOp::Multiply => return Self::const_folding_safe_multiply(left, right), - BinOp::TrueDivide => { - if *r == 0.0 { - return None; - } - l / r - } - BinOp::FloorDivide => { - let (floordiv, _) = Self::float_div_mod(*l, *r)?; - floordiv - } - BinOp::Remainder => return Self::const_folding_safe_mod(left, right), - BinOp::Power => return Self::const_folding_safe_power(left, right), - _ => return None, - }; - if matches!(op, BinOp::Power) && !result.is_finite() { - return None; - } - Some(ConstantData::Float { value: result }) - } - // Int op Float or Float op Int → Float - (ConstantData::Integer { value: l }, ConstantData::Float { value: r }) => { - let l_f = l.to_f64()?; - Self::eval_const_binop( - &ConstantData::Float { value: l_f }, - &ConstantData::Float { value: *r }, - op, - ) - } - (ConstantData::Float { value: l }, ConstantData::Integer { value: r }) => { - let r_f = r.to_f64()?; - Self::eval_const_binop( - &ConstantData::Float { value: *l }, - &ConstantData::Float { value: r_f }, - op, - ) - } - (ConstantData::Integer { value: l }, ConstantData::Complex { value: r }) => { - eval_complex_binop(Complex::new(l.to_f64()?, 0.0), *r, op) - } - (ConstantData::Complex { value: l }, ConstantData::Integer { value: r }) => { - eval_complex_binop(*l, Complex::new(r.to_f64()?, 0.0), op) - } - (ConstantData::Float { value: l }, ConstantData::Complex { value: r }) => { - eval_complex_binop(Complex::new(*l, 0.0), *r, op) - } - (ConstantData::Complex { value: l }, ConstantData::Float { value: r }) => { - eval_complex_binop(*l, Complex::new(*r, 0.0), op) - } - (ConstantData::Complex { value: l }, ConstantData::Complex { value: r }) => { - eval_complex_binop(*l, *r, op) - } - // String concatenation and repetition - (ConstantData::Str { value: l }, ConstantData::Str { value: r }) - if matches!(op, BinOp::Add) => - { - let mut result = l.clone(); - result.push_wtf8(r); - Some(ConstantData::Str { value: result }) - } - (ConstantData::Str { .. }, ConstantData::Integer { .. }) - if matches!(op, BinOp::Multiply) => - { - Self::const_folding_safe_multiply(left, right) - } - (ConstantData::Tuple { elements: l }, ConstantData::Tuple { elements: r }) - if matches!(op, BinOp::Add) => - { - let mut result = l.clone(); - result.extend(r.iter().cloned()); - Some(ConstantData::Tuple { elements: result }) - } - (ConstantData::Tuple { .. }, ConstantData::Integer { .. }) - if matches!(op, BinOp::Multiply) => + if expect_append { + if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) + || u32::from(instr.arg) != 1 { - Self::const_folding_safe_multiply(left, right) - } - (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) - if matches!(op, BinOp::Multiply) => - { - Self::const_folding_safe_multiply(left, right) - } - (ConstantData::Integer { .. }, ConstantData::Str { .. }) - if matches!(op, BinOp::Multiply) => - { - Self::const_folding_safe_multiply(left, right) - } - (ConstantData::Bytes { value: l }, ConstantData::Bytes { value: r }) - if matches!(op, BinOp::Add) => - { - let mut result = l.clone(); - result.extend_from_slice(r); - Some(ConstantData::Bytes { value: result }) - } - (ConstantData::Bytes { .. }, ConstantData::Integer { .. }) - if matches!(op, BinOp::Multiply) => - { - Self::const_folding_safe_multiply(left, right) + return false; } - (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) - if matches!(op, BinOp::Multiply) => - { - Self::const_folding_safe_multiply(left, right) + } else { + if !loads_const(instr) { + return false; } - _ => None, + consts_found += 1; } + expect_append = !expect_append; } - /// flowgraph.c fold_tuple_of_constants - fn fold_tuple_of_constants( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, - ) -> bool { - let Some(Instruction::BuildTuple { .. }) = block.instructions[i].instr.real() else { - return false; - }; + false +} - let tuple_size = u32::from(block.instructions[i].arg) as usize; - if tuple_size > STACK_USE_GUIDELINE { - return false; - } +/// Port of CPython's flowgraph.c optimize_lists_and_sets(). +fn optimize_lists_and_sets( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, + nextop: Option, +) -> bool { + let Some(instr) = block.instructions[i].instr.real() else { + return false; + }; + let is_list = matches!(instr, Instruction::BuildList { .. }); + let is_set = matches!(instr, Instruction::BuildSet { .. }); + if !is_list && !is_set { + return false; + } - let Some(operand_indices) = (if tuple_size == 0 { - Some(Vec::new()) - } else { - i.checked_sub(1) - .and_then(|start| Self::get_const_loading_instrs(block, start, tuple_size)) - }) else { - return false; - }; + let contains_or_iter = matches!( + nextop, + Some(Instruction::GetIter | Instruction::ContainsOp { .. }) + ); + let seq_size = u32::from(block.instructions[i].arg) as usize; + if seq_size > STACK_USE_GUIDELINE || (seq_size < MIN_CONST_SEQUENCE_SIZE && !contains_or_iter) { + return false; + } - let mut elements = Vec::with_capacity(tuple_size); - for &j in &operand_indices { - let Some(element) = Self::get_const_value(metadata, &block.instructions[j]) else { - return false; - }; - elements.push(element); + let Some(operand_indices) = (if seq_size == 0 { + Some(Vec::new()) + } else { + i.checked_sub(1) + .and_then(|start| get_const_loading_instrs(block, start, seq_size)) + }) else { + if contains_or_iter && is_list { + let arg = block.instructions[i].arg; + instr_set_op1(&mut block.instructions[i], Opcode::BuildTuple.into(), arg); + return true; } + return false; + }; - let (const_idx, _) = metadata - .consts - .insert_full(ConstantData::Tuple { elements }); - - Self::nop_out(block, &operand_indices); - - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - true - } - - fn fold_constant_intrinsic_list_to_tuple( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, - ) -> bool { - let Some(Instruction::CallIntrinsic1 { func }) = block.instructions[i].instr.real() else { + let mut elements = Vec::with_capacity(seq_size); + for &j in &operand_indices { + let Some(element) = get_const_value(metadata, &block.instructions[j]) else { return false; }; - if func.get(block.instructions[i].arg) != IntrinsicFunction1::ListToTuple { - return false; - } + elements.push(element); + } - let mut consts_found = 0usize; - let mut expect_append = true; - let mut pos = i; - while let Some(prev) = pos.checked_sub(1) { - pos = prev; - let instr = &block.instructions[pos]; - if matches!(instr.instr.real(), Some(Instruction::Nop)) { - continue; - } + let const_data = if is_list { + ConstantData::Tuple { elements } + } else { + ConstantData::Frozenset { elements } + }; + let const_idx = add_const(metadata, const_data); - if matches!(instr.instr.real(), Some(Instruction::BuildList { .. })) - && u32::from(instr.arg) == 0 - { - if !expect_append { - return false; - } + if !contains_or_iter { + debug_assert!(i >= 2); + let folded_loc = block.instructions[i].location; + let end_loc = block.instructions[i].end_location; - let mut elements = vec![None; consts_found]; - let mut remaining = consts_found; - for idx in (pos..i).rev() { - if matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { - continue; - } - if Self::loads_const(&block.instructions[idx]) { - let Some(value) = Self::get_const_value(metadata, &block.instructions[idx]) - else { - return false; - }; - debug_assert!(remaining > 0); - remaining -= 1; - elements[remaining] = Some(value); - } - nop_out_no_location(&mut block.instructions[idx]); - } - debug_assert_eq!(remaining, 0); - let elements = elements - .into_iter() - .map(|element| element.expect("constant list item was collected")) - .collect(); - let (const_idx, _) = metadata - .consts - .insert_full(ConstantData::Tuple { elements }); - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - return true; - } + nop_out(block, &operand_indices); - if expect_append { - if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) - || u32::from(instr.arg) != 1 - { - return false; - } - } else { - if !Self::loads_const(instr) { - return false; - } - consts_found += 1; + let build_instr = if is_list { + Instruction::BuildList { + count: Arg::marker(), } - expect_append = !expect_append; - } - - false - } - - /// Port of CPython's flowgraph.c optimize_lists_and_sets(). - fn optimize_lists_and_sets( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, - nextop: Option, - ) -> bool { - let Some(instr) = block.instructions[i].instr.real() else { - return false; + .into() + } else { + Instruction::BuildSet { + count: Arg::marker(), + } + .into() }; - let is_list = matches!(instr, Instruction::BuildList { .. }); - let is_set = matches!(instr, Instruction::BuildSet { .. }); - if !is_list && !is_set { - return false; - } + instr_set_op1(&mut block.instructions[i - 2], build_instr, OpArg::new(0)); + block.instructions[i - 2].location = folded_loc; + block.instructions[i - 2].end_location = end_loc; + block.instructions[i - 2].lineno_override = None; - let contains_or_iter = matches!( - nextop, - Some(Instruction::GetIter | Instruction::ContainsOp { .. }) + instr_set_op1( + &mut block.instructions[i - 1], + Instruction::LoadConst { + consti: Arg::marker(), + } + .into(), + OpArg::new(const_idx as u32), ); - let seq_size = u32::from(block.instructions[i].arg) as usize; - if seq_size > STACK_USE_GUIDELINE - || (seq_size < MIN_CONST_SEQUENCE_SIZE && !contains_or_iter) - { - return false; - } - let Some(operand_indices) = (if seq_size == 0 { - Some(Vec::new()) + let extend_instr = if is_list { + Opcode::ListExtend } else { - i.checked_sub(1) - .and_then(|start| Self::get_const_loading_instrs(block, start, seq_size)) - }) else { - if contains_or_iter && is_list { - block.instructions[i].instr = Opcode::BuildTuple.into(); - return true; - } - return false; + Opcode::SetUpdate }; + instr_set_op1( + &mut block.instructions[i], + extend_instr.into(), + OpArg::new(1), + ); + return true; + } - let mut elements = Vec::with_capacity(seq_size); - for &j in &operand_indices { - let Some(element) = Self::get_const_value(metadata, &block.instructions[j]) else { - return false; - }; - elements.push(element); + nop_out(block, &operand_indices); + + instr_set_op1( + &mut block.instructions[i], + Instruction::LoadConst { + consti: Arg::marker(), } + .into(), + OpArg::new(const_idx as u32), + ); + true +} - let const_data = if is_list { - ConstantData::Tuple { elements } - } else { - ConstantData::Frozenset { elements } - }; - let (const_idx, _) = metadata.consts.insert_full(const_data); +const SWAP_OPTIMIZE_VISITED: i32 = -1; - if !contains_or_iter { - debug_assert!(i >= 2); - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; +/// flowgraph.c SWAPPABLE +fn is_swappable(instr: &AnyInstruction) -> bool { + matches!( + (*instr).into(), + AnyOpcode::Real(Opcode::StoreFast | Opcode::PopTop) + | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) + ) +} - Self::nop_out(block, &operand_indices); +/// flowgraph.c STORES_TO +fn stores_to(info: &InstructionInfo) -> i32 { + match info.instr.into() { + AnyOpcode::Real(Opcode::StoreFast) + | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) => { + i32::try_from(u32::from(info.arg)).expect("STORE_FAST oparg fits in int") + } + _ => -1, + } +} - if is_list { - block.instructions[i - 2].instr = Instruction::BuildList { - count: Arg::marker(), - } - .into(); - } else { - block.instructions[i - 2].instr = Instruction::BuildSet { - count: Arg::marker(), - } - .into(); - } - block.instructions[i - 2].arg = OpArg::new(0); - block.instructions[i - 2].location = folded_loc; - block.instructions[i - 2].end_location = end_loc; - block.instructions[i - 2].lineno_override = None; +/// flowgraph.c next_swappable_instruction +fn next_swappable_instruction( + instructions: &[InstructionInfo], + mut i: usize, + lineno: i32, +) -> Option { + loop { + i += 1; + if i >= instructions.len() { + return None; + } + let info = &instructions[i]; + let info_lineno = instruction_lineno(info); + if lineno >= 0 && info_lineno != lineno { + return None; + } + if matches!(info.instr, AnyInstruction::Real(Instruction::Nop)) { + continue; + } + if is_swappable(&info.instr) { + return Some(i); + } + return None; + } +} - block.instructions[i - 1].instr = Instruction::LoadConst { - consti: Arg::marker(), +/// flowgraph.c swaptimize +fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) { + debug_assert!(matches!( + instructions[*ix].instr.real(), + Some(Instruction::Swap { .. }) + )); + let mut depth = u32::from(instructions[*ix].arg) as usize; + let mut len = 1usize; + let mut more = false; + let limit = instructions.len() - *ix; + while len < limit { + match instructions[*ix + len].instr.real() { + Some(Instruction::Swap { .. }) => { + depth = depth.max(u32::from(instructions[*ix + len].arg) as usize); + more = true; + len += 1; } - .into(); - block.instructions[i - 1].arg = OpArg::new(const_idx as u32); - - block.instructions[i].instr = if is_list { - Opcode::ListExtend - } else { - Opcode::SetUpdate + Some(Instruction::Nop) => { + len += 1; } - .into(); - block.instructions[i].arg = OpArg::new(1); - return true; + _ => break, } + } - Self::nop_out(block, &operand_indices); + if !more { + return; + } - block.instructions[i].instr = Opcode::LoadConst.into(); - block.instructions[i].arg = OpArg::new(const_idx as u32); - true + let mut stack = vec![0; depth]; + let mut i = 0; + while i < depth { + stack[i] = i as i32; + i += 1; } - /// apply_static_swaps: eliminate SWAPs by reordering target stores/pops. - /// - /// Ported from CPython Python/flowgraph.c::apply_static_swaps. - /// For each SWAP N, find the 1st and N-th swappable instructions after - /// it. If both are STORE_FAST/POP_TOP and safe to swap, exchange them - /// in the bytecode and replace SWAP with NOP. - /// - /// Safety: abort if the two stores write the same variable, or if any - /// intervening swappable stores to one of the same variables. Do not - /// cross line-number boundaries (user-visible name bindings). - fn apply_static_swaps_block(block: &mut Block) { - const VISITED: i32 = -1; - - /// Instruction classes that are safe to reorder around SWAP. - fn is_swappable(instr: &AnyInstruction) -> bool { - matches!( - (*instr).into(), - AnyOpcode::Real(Opcode::StoreFast | Opcode::PopTop) - | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) - ) + i = 0; + while i < len { + let info = &instructions[*ix + i]; + if matches!(info.instr.real(), Some(Instruction::Swap { .. })) { + let oparg = u32::from(info.arg) as usize; + stack.swap(0, oparg - 1); } + i += 1; + } - /// Variable index that a STORE_FAST writes to, or None. - fn stores_to(info: &InstructionInfo) -> Option { - match info.instr.into() { - AnyOpcode::Real(Opcode::StoreFast) => Some(u32::from(info.arg)), - AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) => Some(u32::from(info.arg)), - _ => None, - } + let mut current = len as isize - 1; + for i in 0..depth { + if stack[i] == SWAP_OPTIMIZE_VISITED || stack[i] == i as i32 { + continue; } - - /// Next swappable index after `i` in `instructions`, skipping NOPs. - /// Returns None if a non-NOP non-swappable instruction blocks, or - /// if `lineno >= 0` and a different lineno is encountered. - fn next_swappable( - instructions: &[InstructionInfo], - mut i: usize, - lineno: i32, - ) -> Option { - loop { - i += 1; - if i >= instructions.len() { - return None; - } - let info = &instructions[i]; - let info_lineno = instruction_lineno(info); - if lineno >= 0 && info_lineno != lineno { - return None; - } - if matches!(info.instr, AnyInstruction::Real(Instruction::Nop)) { - continue; - } - if is_swappable(&info.instr) { - return Some(i); - } - return None; + let mut j = i; + loop { + if j != 0 { + debug_assert!(current >= 0); + let out = &mut instructions[*ix + current as usize]; + out.instr = Opcode::Swap.into(); + out.arg = OpArg::new((j + 1) as u32); + current -= 1; } - } - - fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) { - debug_assert!(matches!( - instructions[*ix].instr.real(), - Some(Instruction::Swap { .. }) - )); - let mut depth = u32::from(instructions[*ix].arg) as usize; - let mut len = 1usize; - let mut more = false; - let limit = instructions.len() - *ix; - while len < limit { - match instructions[*ix + len].instr.real() { - Some(Instruction::Swap { .. }) => { - depth = depth.max(u32::from(instructions[*ix + len].arg) as usize); - more = true; - len += 1; - } - Some(Instruction::Nop) => { - len += 1; - } - _ => break, - } + if stack[j] == SWAP_OPTIMIZE_VISITED { + debug_assert_eq!(j, i); + break; } + let next_j = stack[j] as usize; + stack[j] = SWAP_OPTIMIZE_VISITED; + j = next_j; + } + } - if !more { - return; - } + while current >= 0 { + set_to_nop(&mut instructions[*ix + current as usize]); + current -= 1; + } + *ix += len - 1; +} - let mut stack: Vec = (0..depth as i32).collect(); - for info in &instructions[*ix..*ix + len] { - if matches!(info.instr.real(), Some(Instruction::Swap { .. })) { - let oparg = u32::from(info.arg) as usize; - stack.swap(0, oparg - 1); - } +/// flowgraph.c apply_static_swaps +fn apply_static_swaps(instructions: &mut [InstructionInfo], mut i: isize) { + while i >= 0 { + let idx = i as usize; + let swap_arg = match instructions[idx].instr.real() { + Some(Instruction::Swap { .. }) => u32::from(instructions[idx].arg), + Some(Instruction::Nop | Instruction::PopTop | Instruction::StoreFast { .. }) => { + i -= 1; + continue; } - - let mut current = len as isize - 1; - for i in 0..depth { - if stack[i] == VISITED || stack[i] == i as i32 { - continue; - } - let mut j = i; - loop { - if j != 0 { - debug_assert!(current >= 0); - let out = &mut instructions[*ix + current as usize]; - out.instr = Opcode::Swap.into(); - out.arg = OpArg::new((j + 1) as u32); - current -= 1; - } - if stack[j] == VISITED { - debug_assert_eq!(j, i); - break; - } - let next_j = stack[j] as usize; - stack[j] = VISITED; - j = next_j; - } + _ if matches!( + instructions[idx].instr.pseudo(), + Some(PseudoInstruction::StoreFastMaybeNull { .. }) + ) => + { + i -= 1; + continue; } + _ => return, + }; - while current >= 0 { - set_to_nop(&mut instructions[*ix + current as usize]); - current -= 1; - } - *ix += len - 1; - } - - fn apply_from(instructions: &mut [InstructionInfo], mut i: isize) { - while i >= 0 { - let idx = i as usize; - let swap_arg = match instructions[idx].instr.real() { - Some(Instruction::Swap { .. }) => u32::from(instructions[idx].arg), - Some( - Instruction::Nop | Instruction::PopTop | Instruction::StoreFast { .. }, - ) => { - i -= 1; - continue; - } - _ if matches!( - instructions[idx].instr.pseudo(), - Some(PseudoInstruction::StoreFastMaybeNull { .. }) - ) => - { - i -= 1; - continue; - } - _ => return, - }; + if swap_arg < 2 { + return; + } - if swap_arg < 2 { - return; - } + let Some(j) = next_swappable_instruction(instructions, idx, -1) else { + return; + }; + let lineno = instruction_lineno(&instructions[j]); + let mut k = j; + for _ in 1..swap_arg { + let Some(next) = next_swappable_instruction(instructions, k, lineno) else { + return; + }; + k = next; + } - let Some(j) = next_swappable(instructions, idx, -1) else { + let store_j = stores_to(&instructions[j]); + let store_k = stores_to(&instructions[k]); + if store_j >= 0 || store_k >= 0 { + if store_j == store_k { + return; + } + let mut idx = j + 1; + while idx < k { + let store_idx = stores_to(&instructions[idx]); + if store_idx >= 0 && (store_idx == store_j || store_idx == store_k) { return; - }; - let lineno = instruction_lineno(&instructions[j]); - let mut k = j; - for _ in 1..swap_arg { - let Some(next) = next_swappable(instructions, k, lineno) else { - return; - }; - k = next; - } - - let store_j = stores_to(&instructions[j]); - let store_k = stores_to(&instructions[k]); - if store_j.is_some() || store_k.is_some() { - if store_j == store_k { - return; - } - let conflict = instructions[(j + 1)..k].iter().any(|info| { - if let Some(store_idx) = stores_to(info) { - Some(store_idx) == store_j || Some(store_idx) == store_k - } else { - false - } - }); - if conflict { - return; - } } - - set_to_nop(&mut instructions[idx]); - instructions.swap(j, k); - i -= 1; + idx += 1; } } - let mut i = 0; - while i < block.instructions.len() { - if matches!( - block.instructions[i].instr.real(), - Some(Instruction::Swap { .. }) - ) { - swaptimize(&mut block.instructions, &mut i); - apply_from(&mut block.instructions, i as isize); - } - i += 1; - } + set_to_nop(&mut instructions[idx]); + instructions.swap(j, k); + i -= 1; } +} - /// flowgraph.c maybe_instr_make_load_smallint - fn maybe_instr_make_load_smallint( - instr: &mut InstructionInfo, - constant: &ConstantData, - ) -> bool { - if let ConstantData::Integer { value } = constant - && let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) - { - instr.instr = Opcode::LoadSmallInt.into(); - instr.arg = OpArg::new(small as u32); - return true; +/// flowgraph.c optimize_basic_block swap pass +fn apply_static_swaps_block(block: &mut Block) { + let mut i = 0; + while i < block.instructions.len() { + if matches!( + block.instructions[i].instr.real(), + Some(Instruction::Swap { .. }) + ) { + swaptimize(&mut block.instructions, &mut i); + apply_static_swaps(&mut block.instructions, i as isize); } - false + i += 1; } +} - /// flowgraph.c basicblock_optimize_load_const - fn basicblock_optimize_load_const(metadata: &mut CodeUnitMetadata, block: &mut Block) { - let mut i = 0; - let mut effective_opcode = None; - let mut effective_oparg = OpArg::new(0); - while i < block.instructions.len() { - if matches!( - block.instructions[i].instr.real(), - Some(Instruction::LoadConst { .. }) - ) && let Some(constant) = Self::get_const_value(metadata, &block.instructions[i]) - { - Self::maybe_instr_make_load_smallint(&mut block.instructions[i], &constant); - } - - let curr = block.instructions[i]; - let curr_arg = curr.arg; +/// flowgraph.c maybe_instr_make_load_smallint +fn maybe_instr_make_load_smallint(instr: &mut InstructionInfo, constant: &ConstantData) -> bool { + if let ConstantData::Integer { value } = constant + && let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) + { + instr_set_op1(instr, Opcode::LoadSmallInt.into(), OpArg::new(small as u32)); + return true; + } + false +} - // Only combine if the source is a real instruction. - let Some(curr_instr) = curr.instr.real() else { - i += 1; - continue; - }; +/// flowgraph.c basicblock_optimize_load_const +fn basicblock_optimize_load_const(metadata: &mut CodeUnitMetadata, block: &mut Block) { + let mut i = 0; + let mut effective_opcode = None; + let mut effective_oparg = OpArg::new(0); + while i < block.instructions.len() { + if matches!( + block.instructions[i].instr.real(), + Some(Instruction::LoadConst { .. }) + ) && let Some(constant) = get_const_value(metadata, &block.instructions[i]) + { + maybe_instr_make_load_smallint(&mut block.instructions[i], &constant); + } - let is_copy_of_load_const = matches!( - (effective_opcode, curr_instr), - (Some(Instruction::LoadConst { .. }), Instruction::Copy { i }) if i.get(curr_arg) == 1 - ); - if !is_copy_of_load_const { - effective_opcode = Some(curr_instr); - effective_oparg = curr_arg; - } - let Some(const_instr) = effective_opcode else { - i += 1; - continue; - }; - let const_arg = effective_oparg; + let curr = block.instructions[i]; + let curr_arg = curr.arg; - if i + 1 >= block.instructions.len() { + // Only combine if the source is a real instruction. + let Some(curr_instr) = curr.instr.real() else { + i += 1; + continue; + }; + + let is_copy_of_load_const = matches!( + (effective_opcode, curr_instr), + (Some(Instruction::LoadConst { .. }), Instruction::Copy { i }) if i.get(curr_arg) == 1 + ); + if !is_copy_of_load_const { + effective_opcode = Some(curr_instr); + effective_oparg = curr_arg; + } + let Some(const_instr) = effective_opcode else { + i += 1; + continue; + }; + let const_arg = effective_oparg; + + if i + 1 >= block.instructions.len() { + i += 1; + continue; + } + + let next = block.instructions[i + 1]; + let next_arg = next.arg; + + if let Some(is_true) = load_const_truthiness(const_instr, const_arg, metadata) { + let const_jump = match (next.instr.real(), next.instr.pseudo()) { + (_, Some(PseudoInstruction::JumpIfTrue { .. })) => Some((true, false)), + (_, Some(PseudoInstruction::JumpIfFalse { .. })) => Some((false, false)), + (Some(Instruction::PopJumpIfTrue { .. }), _) => Some((true, true)), + (Some(Instruction::PopJumpIfFalse { .. }), _) => Some((false, true)), + _ => None, + }; + if let Some((jump_if_true, pops_condition)) = const_jump { + if pops_condition { + set_to_nop(&mut block.instructions[i]); + } + if is_true == jump_if_true { + block.instructions[i + 1].instr = PseudoInstruction::Jump { + delta: Arg::marker(), + } + .into(); + } else { + set_to_nop(&mut block.instructions[i + 1]); + } i += 1; continue; } + } - let next = block.instructions[i + 1]; - let next_arg = next.arg; + // The remaining combinations require both instructions to be real. + let Some(next_instr) = next.instr.real() else { + i += 1; + continue; + }; - if let Some(is_true) = Self::load_const_truthiness(const_instr, const_arg, metadata) { - let const_jump = match (next.instr.real(), next.instr.pseudo()) { - (_, Some(PseudoInstruction::JumpIfTrue { .. })) => Some((true, false)), - (_, Some(PseudoInstruction::JumpIfFalse { .. })) => Some((false, false)), - (Some(Instruction::PopJumpIfTrue { .. }), _) => Some((true, true)), - (Some(Instruction::PopJumpIfFalse { .. }), _) => Some((false, true)), - _ => None, - }; - if let Some((jump_if_true, pops_condition)) = const_jump { - if pops_condition { - set_to_nop(&mut block.instructions[i]); - } - if is_true == jump_if_true { - block.instructions[i + 1].instr = PseudoInstruction::Jump { - delta: Arg::marker(), - } - .into(); - if let Some(next_instr) = next.instr.real() - && let Instruction::PopJumpIfTrue { delta } - | Instruction::PopJumpIfFalse { delta } = next_instr - { - block.instructions[i + 1].arg = - OpArg::new(u32::from(delta.get(next.arg))); - } - } else { - set_to_nop(&mut block.instructions[i + 1]); - } + if let Instruction::LoadConst { consti } = const_instr { + let constant = &metadata.consts[consti.get(const_arg).as_usize()]; + if matches!(constant, ConstantData::None) + && let Instruction::IsOp { invert } = next_instr + { + let mut jump_idx = i + 2; + if jump_idx >= block.instructions.len() { i += 1; continue; } - } - // The remaining combinations require both instructions to be real. - let Some(next_instr) = next.instr.real() else { - i += 1; - continue; - }; - - if let Instruction::LoadConst { consti } = const_instr { - let constant = &metadata.consts[consti.get(const_arg).as_usize()]; - if matches!(constant, ConstantData::None) - && let Instruction::IsOp { invert } = next_instr - { - let mut jump_idx = i + 2; + if matches!( + block.instructions[jump_idx].instr.real(), + Some(Instruction::ToBool) + ) { + set_to_nop(&mut block.instructions[jump_idx]); + jump_idx += 1; if jump_idx >= block.instructions.len() { i += 1; continue; } + } - if matches!( - block.instructions[jump_idx].instr.real(), - Some(Instruction::ToBool) - ) { - set_to_nop(&mut block.instructions[jump_idx]); - jump_idx += 1; - if jump_idx >= block.instructions.len() { - i += 1; - continue; - } - } + let Some(jump_instr) = block.instructions[jump_idx].instr.real() else { + i += 1; + continue; + }; - let Some(jump_instr) = block.instructions[jump_idx].instr.real() else { + let mut invert = matches!( + invert.get(next_arg), + rustpython_compiler_core::bytecode::Invert::Yes + ); + match jump_instr { + Instruction::PopJumpIfFalse { .. } => { + invert = !invert; + } + Instruction::PopJumpIfTrue { .. } => {} + _ => { i += 1; continue; - }; - - let mut invert = matches!( - invert.get(next_arg), - rustpython_compiler_core::bytecode::Invert::Yes - ); - let delta = match jump_instr { - Instruction::PopJumpIfFalse { delta } => { - invert = !invert; - delta.get(block.instructions[jump_idx].arg) - } - Instruction::PopJumpIfTrue { delta } => { - delta.get(block.instructions[jump_idx].arg) - } - _ => { - i += 1; - continue; - } - }; - - set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - block.instructions[jump_idx].instr = if invert { - Instruction::PopJumpIfNotNone { - delta: Arg::marker(), - } - } else { - Instruction::PopJumpIfNone { - delta: Arg::marker(), - } } - .into(); - block.instructions[jump_idx].arg = OpArg::new(u32::from(delta)); - i = jump_idx; - continue; - } - } + }; - if matches!( - const_instr, - Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } - ) && matches!(next_instr, Instruction::ToBool) - && let Some(value) = Self::load_const_truthiness(const_instr, const_arg, metadata) - { - let (const_idx, _) = metadata.consts.insert_full(ConstantData::Boolean { value }); set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = Instruction::LoadConst { - consti: Arg::marker(), + set_to_nop(&mut block.instructions[i + 1]); + block.instructions[jump_idx].instr = if invert { + Instruction::PopJumpIfNotNone { + delta: Arg::marker(), + } + } else { + Instruction::PopJumpIfNone { + delta: Arg::marker(), + } } .into(); - block.instructions[i + 1].arg = OpArg::new(const_idx as u32); - i += 1; + i = jump_idx; continue; } + } + if matches!( + const_instr, + Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } + ) && matches!(next_instr, Instruction::ToBool) + && let Some(value) = load_const_truthiness(const_instr, const_arg, metadata) + { + let const_idx = add_const(metadata, ConstantData::Boolean { value }); + set_to_nop(&mut block.instructions[i]); + instr_set_op1( + &mut block.instructions[i + 1], + Instruction::LoadConst { + consti: Arg::marker(), + } + .into(), + OpArg::new(const_idx as u32), + ); i += 1; + continue; } + + i += 1; } +} - /// flowgraph.c optimize_load_const - fn optimize_load_const(&mut self) { - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = self.blocks[block_idx.idx()].next; - { - let metadata = &mut self.metadata; - let block = &mut self.blocks[block_idx]; - Self::basicblock_optimize_load_const(metadata, block); - } - block_idx = next_block; - } +/// flowgraph.c optimize_load_const +fn optimize_load_const(metadata: &mut CodeUnitMetadata, blocks: &mut [Block]) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = blocks[block_idx.idx()].next; + let block = &mut blocks[block_idx]; + basicblock_optimize_load_const(metadata, block); + block_idx = next_block; } +} - /// flowgraph.c optimize_basic_block - fn optimize_basic_block( - blocks: &mut [Block], - metadata: &mut CodeUnitMetadata, - block_idx: BlockIdx, - ) -> crate::InternalResult<()> { - for inst in &blocks[block_idx.idx()].instructions { - if inst.instr.has_target() { - debug_assert!(inst.target != BlockIdx::NULL); - debug_assert!(!blocks[inst.target.idx()].instructions.is_empty()); - debug_assert!( - !blocks[inst.target.idx()].instructions[0] - .instr - .is_assembler() - ); - } - } - { - let block = &mut blocks[block_idx]; - let mut i = 0; - while i < block.instructions.len() { - let inst = block.instructions[i]; - let Some(opcode) = inst.instr.real() else { - i += 1; - continue; - }; - let nextop = block - .instructions - .get(i + 1) - .and_then(|next| next.instr.real()); - - match opcode { - Instruction::BuildTuple { .. } => { - let oparg = u32::from(inst.arg); - if matches!(nextop, Some(Instruction::UnpackSequence { .. })) - && u32::from(block.instructions[i + 1].arg) == oparg - { - match oparg { - 1 => { - set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; - continue; - } - 2 | 3 => { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = - Instruction::Swap { i: Arg::marker() }.into(); - block.instructions[i + 1].arg = OpArg::new(oparg); - i += 1; - continue; - } - _ => {} - } +/// flowgraph.c optimize_basic_block +fn optimize_basic_block( + blocks: &mut [Block], + metadata: &mut CodeUnitMetadata, + block_idx: BlockIdx, +) -> crate::InternalResult<()> { + let bi = block_idx.idx(); + let mut nop = InstructionInfo { + instr: Instruction::Nop.into(), + arg: OpArg::NULL, + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: None, + }; + instr_set_op0(&mut nop, Instruction::Nop.into()); + let mut i = 0; + while i < blocks[bi].instructions.len() { + let inst = blocks[bi].instructions[i]; + debug_assert!(!inst.instr.is_assembler()); + let target = if inst.instr.has_target() { + let target = inst.target; + debug_assert!(target != BlockIdx::NULL); + debug_assert!(!blocks[target.idx()].instructions.is_empty()); + debug_assert!(!blocks[target.idx()].instructions[0].instr.is_assembler()); + blocks[target.idx()].instructions[0] + } else { + nop + }; + + let nextop = blocks[bi] + .instructions + .get(i + 1) + .and_then(|next| next.instr.real()); + + match inst.instr { + AnyInstruction::Real(Instruction::BuildTuple { .. }) => { + let oparg = u32::from(inst.arg); + if matches!(nextop, Some(Instruction::UnpackSequence { .. })) + && u32::from(blocks[bi].instructions[i + 1].arg) == oparg + { + match oparg { + 1 => { + set_to_nop(&mut blocks[bi].instructions[i]); + set_to_nop(&mut blocks[bi].instructions[i + 1]); + i += 1; + continue; } - Self::fold_tuple_of_constants(metadata, block, i); - } - Instruction::BuildList { .. } | Instruction::BuildSet { .. } => { - Self::optimize_lists_and_sets(metadata, block, i, nextop); - } - Instruction::StoreFast { .. } - if matches!(nextop, Some(Instruction::StoreFast { .. })) - && u32::from(inst.arg) == u32::from(block.instructions[i + 1].arg) - && instruction_lineno(&block.instructions[i]) - == instruction_lineno(&block.instructions[i + 1]) => - { - block.instructions[i].instr = Instruction::PopTop.into(); - block.instructions[i].arg = OpArg::NULL; - } - Instruction::Swap { .. } if u32::from(inst.arg) == 1 => { - set_to_nop(&mut block.instructions[i]); - } - Instruction::LoadGlobal { .. } - if matches!(nextop, Some(Instruction::PushNull)) - && (u32::from(inst.arg) & 1) == 0 => - { - block.instructions[i].arg = OpArg::new(u32::from(inst.arg) | 1); - set_to_nop(&mut block.instructions[i + 1]); + 2 | 3 => { + set_to_nop(&mut blocks[bi].instructions[i]); + blocks[bi].instructions[i + 1].instr = + Instruction::Swap { i: Arg::marker() }.into(); + i += 1; + continue; + } + _ => {} } - Instruction::CompareOp { .. } - if matches!(nextop, Some(Instruction::ToBool)) => + } + fold_tuple_of_constants(metadata, &mut blocks[bi], i); + } + AnyInstruction::Real(Instruction::BuildList { .. } | Instruction::BuildSet { .. }) => { + optimize_lists_and_sets(metadata, &mut blocks[bi], i, nextop); + } + AnyInstruction::Real( + Instruction::PopJumpIfNotNone { .. } | Instruction::PopJumpIfNone { .. }, + ) if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) + && jump_thread(blocks, block_idx, i, &target, inst.instr)? => + { + continue; + } + AnyInstruction::Real(Instruction::PopJumpIfFalse { .. }) + if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) + && jump_thread(blocks, block_idx, i, &target, inst.instr)? => + { + continue; + } + AnyInstruction::Real(Instruction::PopJumpIfTrue { .. }) + if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) + && jump_thread(blocks, block_idx, i, &target, inst.instr)? => + { + continue; + } + AnyInstruction::Pseudo( + pseudo @ (PseudoInstruction::JumpIfFalse { .. } + | PseudoInstruction::JumpIfTrue { .. }), + ) => { + let opcode = pseudo.into(); + match target.instr.pseudo().map(Into::into) { + Some(PseudoOpcode::Jump) + if jump_thread(blocks, block_idx, i, &target, opcode)? => { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = Instruction::CompareOp { - opname: Arg::marker(), - } - .into(); - block.instructions[i + 1].arg = - OpArg::new(u32::from(inst.arg) | oparg::COMPARE_OP_BOOL_MASK); - i += 1; continue; } - Instruction::ContainsOp { .. } | Instruction::IsOp { .. } - if matches!(nextop, Some(Instruction::ToBool)) => + Some(PseudoOpcode::JumpIfFalse) + if matches!( + opcode, + AnyInstruction::Pseudo(PseudoInstruction::JumpIfFalse { .. }) + ) && jump_thread(blocks, block_idx, i, &target, opcode)? => { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = opcode.into(); - block.instructions[i + 1].arg = inst.arg; - i += 1; continue; } - Instruction::ContainsOp { .. } | Instruction::IsOp { .. } - if matches!(nextop, Some(Instruction::UnaryNot)) => + Some(PseudoOpcode::JumpIfTrue) + if matches!( + opcode, + AnyInstruction::Pseudo(PseudoInstruction::JumpIfTrue { .. }) + ) && jump_thread(blocks, block_idx, i, &target, opcode)? => { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = opcode.into(); - block.instructions[i + 1].arg = OpArg::new(u32::from(inst.arg) ^ 1); - i += 1; continue; } - Instruction::ToBool if matches!(nextop, Some(Instruction::ToBool)) => { - set_to_nop(&mut block.instructions[i]); - i += 1; + Some(PseudoOpcode::JumpIfFalse | PseudoOpcode::JumpIfTrue) => { + let next = blocks[inst.target.idx()].next; + debug_assert!(next != BlockIdx::NULL); + debug_assert!(next != inst.target); + blocks[bi].instructions[i].target = next; continue; } - Instruction::UnaryNot => { - if matches!(nextop, Some(Instruction::ToBool)) { - set_to_nop(&mut block.instructions[i]); - block.instructions[i + 1].instr = Instruction::UnaryNot.into(); - block.instructions[i + 1].arg = OpArg::new(0); - i += 1; - continue; - } - if matches!(nextop, Some(Instruction::UnaryNot)) { - set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - i += 1; - continue; + _ => {} + } + } + AnyInstruction::Pseudo( + PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. }, + ) => match target.instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::Jump) + if jump_thread(blocks, block_idx, i, &target, PseudoOpcode::Jump.into())? => + { + continue; + } + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) + if jump_thread(blocks, block_idx, i, &target, inst.instr)? => + { + continue; + } + _ => {} + }, + // CPython leaves FOR_ITER jump threading disabled. + AnyInstruction::Real(Instruction::ForIter { .. }) => {} + AnyInstruction::Real(Instruction::StoreFast { .. }) + if matches!(nextop, Some(Instruction::StoreFast { .. })) + && u32::from(inst.arg) == u32::from(blocks[bi].instructions[i + 1].arg) + && instruction_lineno(&blocks[bi].instructions[i]) + == instruction_lineno(&blocks[bi].instructions[i + 1]) => + { + blocks[bi].instructions[i].instr = Instruction::PopTop.into(); + blocks[bi].instructions[i].arg = OpArg::NULL; + } + AnyInstruction::Real(Instruction::Swap { .. }) if u32::from(inst.arg) == 1 => { + set_to_nop(&mut blocks[bi].instructions[i]); + } + AnyInstruction::Real(Instruction::LoadGlobal { .. }) + if matches!(nextop, Some(Instruction::PushNull)) + && (u32::from(inst.arg) & 1) == 0 => + { + instr_set_op1( + &mut blocks[bi].instructions[i], + inst.instr, + OpArg::new(u32::from(inst.arg) | 1), + ); + set_to_nop(&mut blocks[bi].instructions[i + 1]); + } + AnyInstruction::Real(Instruction::CompareOp { .. }) + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut blocks[bi].instructions[i]); + instr_set_op1( + &mut blocks[bi].instructions[i + 1], + inst.instr, + OpArg::new(u32::from(inst.arg) | oparg::COMPARE_OP_BOOL_MASK), + ); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::ContainsOp { .. } | Instruction::IsOp { .. }) + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut blocks[bi].instructions[i]); + instr_set_op1(&mut blocks[bi].instructions[i + 1], inst.instr, inst.arg); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::ContainsOp { .. } | Instruction::IsOp { .. }) + if matches!(nextop, Some(Instruction::UnaryNot)) => + { + set_to_nop(&mut blocks[bi].instructions[i]); + instr_set_op1( + &mut blocks[bi].instructions[i + 1], + inst.instr, + OpArg::new(u32::from(inst.arg) ^ 1), + ); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::ToBool) + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut blocks[bi].instructions[i]); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::UnaryNot) => { + if matches!(nextop, Some(Instruction::ToBool)) { + set_to_nop(&mut blocks[bi].instructions[i]); + instr_set_op0(&mut blocks[bi].instructions[i + 1], inst.instr); + i += 1; + continue; + } + if matches!(nextop, Some(Instruction::UnaryNot)) { + set_to_nop(&mut blocks[bi].instructions[i]); + set_to_nop(&mut blocks[bi].instructions[i + 1]); + i += 1; + continue; + } + fold_const_unaryop(metadata, &mut blocks[bi], i); + } + AnyInstruction::Real(Instruction::UnaryInvert | Instruction::UnaryNegative) => { + fold_const_unaryop(metadata, &mut blocks[bi], i); + } + AnyInstruction::Real(Instruction::CallIntrinsic1 { func }) => { + match func.get(inst.arg) { + IntrinsicFunction1::ListToTuple => { + if matches!(nextop, Some(Instruction::GetIter)) { + set_to_nop(&mut blocks[bi].instructions[i]); + } else { + fold_constant_intrinsic_list_to_tuple(metadata, &mut blocks[bi], i); } - Self::fold_const_unaryop(metadata, block, i); - } - Instruction::UnaryInvert | Instruction::UnaryNegative => { - Self::fold_const_unaryop(metadata, block, i); } - Instruction::CallIntrinsic1 { func } => match func.get(inst.arg) { - IntrinsicFunction1::ListToTuple => { - if matches!(nextop, Some(Instruction::GetIter)) { - set_to_nop(&mut block.instructions[i]); - } else { - Self::fold_constant_intrinsic_list_to_tuple(metadata, block, i); - } - } - IntrinsicFunction1::UnaryPositive => { - Self::fold_const_unaryop(metadata, block, i); - } - _ => {} - }, - Instruction::BinaryOp { .. } => { - Self::fold_const_binop(metadata, block, i); + IntrinsicFunction1::UnaryPositive => { + fold_const_unaryop(metadata, &mut blocks[bi], i); } _ => {} } - - i += 1; } + AnyInstruction::Real(Instruction::BinaryOp { .. }) => { + fold_const_binop(metadata, &mut blocks[bi], i); + } + _ => {} } - jump_threading_block(blocks, block_idx)?; - Self::apply_static_swaps_block(&mut blocks[block_idx]); - Ok(()) + + i += 1; } + apply_static_swaps_block(&mut blocks[block_idx]); + Ok(()) +} - /// flowgraph.c remove_redundant_nops_and_pairs - fn remove_redundant_nops_and_pairs(&mut self) { - let mut done = false; +/// flowgraph.c remove_redundant_nops_and_pairs +#[allow(clippy::if_same_then_else, clippy::useless_let_if_seq)] +fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) { + let mut done = false; - while !done { - done = true; - let mut prev: Option<(BlockIdx, usize)> = None; - let mut block_idx = BlockIdx::new(0); + while !done { + done = true; + let mut instr: Option<(BlockIdx, usize)> = None; + let mut block_idx = BlockIdx::new(0); - while block_idx != BlockIdx::NULL { - basicblock_remove_redundant_nops(&mut self.blocks, block_idx); - if self.blocks[block_idx.idx()].has_cpython_cfg_label() { - prev = None; + while block_idx != BlockIdx::NULL { + basicblock_remove_redundant_nops(blocks, block_idx); + if is_label(blocks[block_idx.idx()].cpython_label) { + instr = None; + } + + let len = blocks[block_idx.idx()].instructions.len(); + for instr_idx in 0..len { + let prev_instr = instr; + instr = Some((block_idx, instr_idx)); + let instr_info = blocks[block_idx.idx()].instructions[instr_idx]; + let mut prev_opcode = None; + let mut prev_oparg = 0; + if let Some((prev_block, prev_instr_idx)) = prev_instr { + let prev_info = blocks[prev_block.idx()].instructions[prev_instr_idx]; + prev_opcode = prev_info.instr.real(); + prev_oparg = match prev_info.instr.real() { + Some(Instruction::Copy { i }) => i.get(prev_info.arg), + _ => u32::from(prev_info.arg), + }; } - - let len = self.blocks[block_idx.idx()].instructions.len(); - for instr_idx in 0..len { - let instr = self.blocks[block_idx.idx()].instructions[instr_idx]; - let is_redundant_pair = - if matches!(instr.instr.real(), Some(Instruction::PopTop)) - && let Some((prev_block, prev_instr)) = prev - { - let prev_info = self.blocks[prev_block.idx()].instructions[prev_instr]; - matches!( - prev_info.instr.real(), - Some( - Instruction::LoadConst { .. } - | Instruction::LoadSmallInt { .. } - ) - ) || matches!( - prev_info.instr.real(), - Some(Instruction::Copy { i }) if i.get(prev_info.arg) == 1 - ) - } else { - false - }; - - if is_redundant_pair { - let (prev_block, prev_instr) = prev.expect("redundant pair has previous"); - set_to_nop(&mut self.blocks[prev_block.idx()].instructions[prev_instr]); - set_to_nop(&mut self.blocks[block_idx.idx()].instructions[instr_idx]); - done = false; + let opcode = instr_info.instr.real(); + let mut is_redundant_pair = false; + if matches!(opcode, Some(Instruction::PopTop)) { + if matches!( + prev_opcode, + Some(Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }) + ) { + is_redundant_pair = true; + } else if matches!(prev_opcode, Some(Instruction::Copy { .. })) + && prev_oparg == 1 + { + is_redundant_pair = true; } - prev = Some((block_idx, instr_idx)); } - let block = &self.blocks[block_idx.idx()]; - if block.instructions.last().is_some_and(is_jump) || !bb_has_fallthrough(block) { - prev = None; + if is_redundant_pair { + let (prev_block, prev_instr_idx) = + prev_instr.expect("redundant pair has previous"); + set_to_nop(&mut blocks[prev_block.idx()].instructions[prev_instr_idx]); + set_to_nop(&mut blocks[block_idx.idx()].instructions[instr_idx]); + done = false; } - block_idx = block.next; } - } - } - - /// Remove constants that are no longer referenced by LOAD_CONST instructions. - /// remove_unused_consts - fn remove_unused_consts(&mut self) { - let nconsts = self.metadata.consts.len(); - if nconsts == 0 { - return; - } - - let mut index_map = vec![-1isize; nconsts]; - // The first constant may be docstring; keep it always. - index_map[0] = 0; - // Mark used consts. - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let block = &self.blocks[block_idx]; - for instr in &block.instructions { - if instr.instr.has_const() { - let index = u32::from(instr.arg) as usize; - debug_assert!(index < nconsts); - index_map[index] = index as isize; - } + let mut instr_is_jump = false; + if let Some((instr_block, instr_idx)) = instr { + instr_is_jump = is_jump(&blocks[instr_block.idx()].instructions[instr_idx]); } - block_idx = block.next; - } - - // Condense consts. - let mut n_used_consts = 0; - for i in 0..nconsts { - if index_map[i] != -1 { - debug_assert_eq!(index_map[i], i as isize); - index_map[n_used_consts] = index_map[i]; - n_used_consts += 1; + let block = &blocks[block_idx.idx()]; + if instr_is_jump || !bb_has_fallthrough(block) { + instr = None; } + block_idx = block.next; } + } +} - if n_used_consts == nconsts { - return; - } - - // Move all used consts to the beginning of the consts list. - let old_consts = core::mem::take(&mut self.metadata.consts.constants); - debug_assert!(n_used_consts < nconsts); - for &old_index in &index_map[..n_used_consts] { - let old_index = old_index as usize; - debug_assert!(old_index < nconsts); - self.metadata - .consts - .constants - .push(old_consts[old_index].clone()); - } - - // Adjust const indices in the bytecode. - let mut reverse_index_map = vec![-1isize; nconsts]; - for (i, &old_index) in index_map[..n_used_consts].iter().enumerate() { - debug_assert!(old_index != -1); - let old_index = old_index as usize; - debug_assert_eq!(reverse_index_map[old_index], -1); - reverse_index_map[old_index] = i as isize; - } - - block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = self.blocks[block_idx.idx()].next; - let block = &mut self.blocks[block_idx]; - for instr in &mut block.instructions { - if instr.instr.has_const() { - let index = u32::from(instr.arg) as usize; - debug_assert!(reverse_index_map[index] >= 0); - debug_assert!(reverse_index_map[index] < n_used_consts as isize); - instr.arg = OpArg::new(reverse_index_map[index] as u32); - } - } - block_idx = next_block; - } +/// flowgraph.c remove_unused_consts +#[allow(clippy::needless_range_loop)] +fn remove_unused_consts(blocks: &mut [Block], consts: &mut ConstantPool) { + let nconsts = consts.len(); + if nconsts == 0 { + return; } - /// flowgraph.c make_super_instruction - fn make_super_instruction( - inst1: &mut InstructionInfo, - inst2: &mut InstructionInfo, - super_op: AnyInstruction, - ) { - let line1 = instruction_lineno(inst1); - let line2 = instruction_lineno(inst2); - if line1 >= 0 && line2 >= 0 && line1 != line2 { - return; - } - let arg1 = u32::from(inst1.arg); - let arg2 = u32::from(inst2.arg); - if arg1 >= 16 || arg2 >= 16 { - return; - } - inst1.instr = super_op; - inst1.arg = OpArg::new((arg1 << 4) | arg2); - set_to_nop(inst2); + let mut index_map = vec![0isize; nconsts]; + for i in 1..nconsts { + index_map[i] = -1; } + // The first constant may be docstring; keep it always. + index_map[0] = 0; - /// flowgraph.c insert_superinstructions - fn insert_superinstructions(&mut self) { - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = self.blocks[block_idx.idx()].next; - let block = &mut self.blocks[block_idx]; - for i in 0..block.instructions.len() { - let nextop = if i + 1 < block.instructions.len() { - block.instructions[i + 1].instr.real() - } else { - None - }; - match block.instructions[i].instr.real() { - Some(Instruction::LoadFast { .. }) => { - if matches!(nextop, Some(Instruction::LoadFast { .. })) { - let (inst1, rest) = block.instructions[i..].split_at_mut(1); - Self::make_super_instruction( - &mut inst1[0], - &mut rest[0], - Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(), - ); - } - } - Some(Instruction::StoreFast { .. }) => match nextop { - Some(Instruction::LoadFast { .. }) => { - let (inst1, rest) = block.instructions[i..].split_at_mut(1); - Self::make_super_instruction( - &mut inst1[0], - &mut rest[0], - Instruction::StoreFastLoadFast { - var_nums: Arg::marker(), - } - .into(), - ); - } - Some(Instruction::StoreFast { .. }) => { - let (inst1, rest) = block.instructions[i..].split_at_mut(1); - Self::make_super_instruction( - &mut inst1[0], - &mut rest[0], - Instruction::StoreFastStoreFast { - var_nums: Arg::marker(), - } - .into(), - ); - } - _ => {} - }, - _ => {} - } + // Mark used consts. + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let block = &blocks[block_idx]; + for i in 0..block.instructions.len() { + let instr = &block.instructions[i]; + if instr.instr.has_const() { + let index = u32::from(instr.arg) as usize; + debug_assert!(index < nconsts); + index_map[index] = index as isize; } - block_idx = next_block; - } - remove_redundant_nops(&mut self.blocks); - #[cfg(debug_assertions)] - { - let no_redundant = no_redundant_nops(&mut self.blocks); - debug_assert!(no_redundant); } + block_idx = block.next; } - fn optimize_load_fast(&mut self) { - // NOT_LOCAL marker: instruction didn't come from a LOAD_FAST - const NOT_LOCAL: usize = usize::MAX; - const DUMMY_INSTR: isize = -1; - const SUPPORT_KILLED: u8 = 1; - const STORED_AS_LOCAL: u8 = 2; - const REF_UNCONSUMED: u8 = 4; - - #[derive(Clone, Copy)] - struct AbstractRef { - instr: isize, - local: usize, - } - - fn push_ref(refs: &mut Vec, instr: isize, local: usize) { - refs.push(AbstractRef { instr, local }); + // Now index_map[i] == i if consts[i] is used, -1 otherwise. + // Condense consts. + let mut n_used_consts = 0; + for i in 0..nconsts { + if index_map[i] != -1 { + debug_assert_eq!(index_map[i], i as isize); + index_map[n_used_consts] = index_map[i]; + n_used_consts += 1; } + } - fn pop_ref(refs: &mut Vec) -> AbstractRef { - refs.pop().expect("ref stack underflow") - } + if n_used_consts == nconsts { + return; + } - fn at_ref(refs: &[AbstractRef], idx: usize) -> AbstractRef { - refs.get(idx).copied().expect("ref stack index in bounds") + // Move all used consts to the beginning of the consts list. + debug_assert!(n_used_consts < nconsts); + for i in 0..n_used_consts { + let old_index = index_map[i] as usize; + debug_assert!(i <= old_index && old_index < nconsts); + if i != old_index { + let value = consts.constants[old_index].clone(); + consts.constants[i] = value; } + } - fn swap_top(refs: &mut [AbstractRef], depth: usize) { - assert!(depth >= 2 && refs.len() >= depth); - let top = refs.len() - 1; - let other = refs.len() - depth; - refs.swap(top, other); - } + // Truncate the consts list at its new size. + consts.constants.truncate(n_used_consts); - fn kill_local(instr_flags: &mut [u8], refs: &[AbstractRef], local: usize) { - for r in refs.iter().copied().filter(|r| r.local == local) { - debug_assert!(r.instr >= 0); - instr_flags[r.instr as usize] |= SUPPORT_KILLED; - } - } + // Adjust const indices in the bytecode. + let mut reverse_index_map = vec![0isize; nconsts]; + for i in 0..nconsts { + reverse_index_map[i] = -1; + } + for i in 0..n_used_consts { + let old_index = index_map[i]; + debug_assert!(old_index != -1); + let old_index = old_index as usize; + debug_assert_eq!(reverse_index_map[old_index], -1); + reverse_index_map[old_index] = i as isize; + } - fn store_local(instr_flags: &mut [u8], refs: &[AbstractRef], local: usize, r: AbstractRef) { - kill_local(instr_flags, refs, local); - if r.instr != DUMMY_INSTR { - instr_flags[r.instr as usize] |= STORED_AS_LOCAL; + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = blocks[block_idx.idx()].next; + let block = &mut blocks[block_idx]; + for i in 0..block.instructions.len() { + let instr = &mut block.instructions[i]; + if instr.instr.has_const() { + let index = u32::from(instr.arg) as usize; + debug_assert!(reverse_index_map[index] >= 0); + debug_assert!(reverse_index_map[index] < n_used_consts as isize); + instr.arg = OpArg::new(reverse_index_map[index] as u32); } } + block_idx = next_block; + } +} - fn decode_packed_fast_locals(arg: OpArg) -> (usize, usize) { - let packed = u32::from(arg); - (((packed >> 4) & 0xF) as usize, (packed & 0xF) as usize) - } +fn optimize_load_fast(blocks: &mut [Block]) { + let mut max_instrs = 0; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + max_instrs = max_instrs.max(blocks[current.idx()].instructions.len()); + current = blocks[current.idx()].next; + } + let mut instr_flags = vec![0u8; max_instrs]; + let mut refs = Vec::new(); + let mut worklist = make_cfg_traversal_stack(blocks); + worklist.push(BlockIdx(0)); + blocks[0].start_depth = 0; + blocks[0].visited = true; + while let Some(block_idx) = worklist.pop() { + let block_i = block_idx.idx(); - fn load_fast_push_block( - worklist: &mut Vec, - blocks: &mut [Block], - target: BlockIdx, - start_depth: usize, - ) { - debug_assert!(target != BlockIdx::NULL); - debug_assert_eq!( - blocks[target.idx()].start_depth.map(|depth| depth as usize), - Some(start_depth) - ); - if !blocks[target.idx()].visited { - blocks[target.idx()].visited = true; - worklist.push(target); - } + let instr_count = blocks[block_i].instructions.len(); + instr_flags[..instr_count].fill(0); + debug_assert!(blocks[block_i].start_depth >= 0); + let start_depth = usize::try_from(blocks[block_i].start_depth) + .expect("visited block has non-negative start depth"); + ref_stack_clear(&mut refs); + refs.reserve(instr_count + start_depth + 2); + for _ in 0..start_depth { + push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL); } - let mut max_instrs = 0; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - max_instrs = max_instrs.max(self.blocks[current.idx()].instructions.len()); - current = self.blocks[current.idx()].next; - } - let mut instr_flags = vec![0u8; max_instrs]; - let mut refs = Vec::new(); - let mut worklist = make_cfg_traversal_stack(&mut self.blocks); - worklist.push(BlockIdx(0)); - self.blocks[0].start_depth = Some(0); - self.blocks[0].visited = true; - while let Some(block_idx) = worklist.pop() { - let block_i = block_idx.idx(); - - let instr_count = self.blocks[block_i].instructions.len(); - instr_flags[..instr_count].fill(0); - let start_depth = self.blocks[block_i].start_depth.unwrap_or(0) as usize; - refs.clear(); - refs.reserve(instr_count + start_depth + 2); - for _ in 0..start_depth { - push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL); - } - - for i in 0..instr_count { - let info = self.blocks[block_i].instructions[i]; - let instr = info.instr; - let arg_u32 = u32::from(info.arg); - - match instr { - AnyInstruction::Real(Instruction::DeleteFast { var_num }) => { - kill_local(&mut instr_flags, &refs, usize::from(var_num.get(info.arg))); - } - AnyInstruction::Real(Instruction::LoadFast { var_num }) => { - push_ref(&mut refs, i as isize, usize::from(var_num.get(info.arg))); - } - AnyInstruction::Real(Instruction::LoadFastAndClear { var_num }) => { - let local = usize::from(var_num.get(info.arg)); - kill_local(&mut instr_flags, &refs, local); - push_ref(&mut refs, i as isize, local); - } - AnyInstruction::Real(Instruction::LoadFastLoadFast { .. }) => { - let (local1, local2) = decode_packed_fast_locals(info.arg); - push_ref(&mut refs, i as isize, local1); - push_ref(&mut refs, i as isize, local2); - } - AnyInstruction::Real(Instruction::StoreFast { var_num }) => { - let r = pop_ref(&mut refs); - store_local( - &mut instr_flags, - &refs, - usize::from(var_num.get(info.arg)), - r, - ); - } - AnyInstruction::Real(Instruction::StoreFastLoadFast { .. }) => { - let (store_local_idx, load_local_idx) = decode_packed_fast_locals(info.arg); - let r = pop_ref(&mut refs); - store_local(&mut instr_flags, &refs, store_local_idx, r); - push_ref(&mut refs, i as isize, load_local_idx); - } - AnyInstruction::Real(Instruction::StoreFastStoreFast { .. }) => { - let (local1, local2) = decode_packed_fast_locals(info.arg); - let r1 = pop_ref(&mut refs); - store_local(&mut instr_flags, &refs, local1, r1); - let r2 = pop_ref(&mut refs); - store_local(&mut instr_flags, &refs, local2, r2); - } - AnyInstruction::Real(Instruction::Copy { i: _ }) => { - let depth = arg_u32 as usize; - assert!(depth > 0); - assert!(refs.len() >= depth); - let r = at_ref(&refs, refs.len() - depth); - push_ref(&mut refs, r.instr, r.local); - } - AnyInstruction::Real(Instruction::Swap { i: _ }) => { - let depth = arg_u32 as usize; - assert!(depth >= 2); - assert!(refs.len() >= depth); - swap_top(&mut refs, depth); - } - AnyInstruction::Real( - Instruction::FormatSimple - | Instruction::GetAnext - | Instruction::GetLen - | Instruction::GetYieldFromIter - | Instruction::ImportFrom { .. } - | Instruction::MatchKeys - | Instruction::MatchMapping - | Instruction::MatchSequence - | Instruction::WithExceptStart, - ) => { - let effect = instr.stack_effect_info(arg_u32); - let net_pushed = effect.pushed() as isize - effect.popped() as isize; - debug_assert!(net_pushed >= 0); - // CPython optimize_load_fast() shadows the outer - // instruction index in this produced-value loop. - for produced in 0..net_pushed { - push_ref(&mut refs, produced, NOT_LOCAL); - } - } - AnyInstruction::Real( - Instruction::DictMerge { .. } - | Instruction::DictUpdate { .. } - | Instruction::ListAppend { .. } - | Instruction::ListExtend { .. } - | Instruction::MapAdd { .. } - | Instruction::Reraise { .. } - | Instruction::SetAdd { .. } - | Instruction::SetUpdate { .. }, - ) => { - let effect = instr.stack_effect_info(arg_u32); - let net_popped = effect.popped() as isize - effect.pushed() as isize; - debug_assert!(net_popped > 0); - for _ in 0..net_popped { - let _ = pop_ref(&mut refs); - } - } - AnyInstruction::Real( - Instruction::EndSend | Instruction::SetFunctionAttribute { .. }, - ) => { - let effect = instr.stack_effect_info(arg_u32); - debug_assert_eq!(effect.popped(), 2); - debug_assert_eq!(effect.pushed(), 1); - let tos = pop_ref(&mut refs); - let _ = pop_ref(&mut refs); - push_ref(&mut refs, tos.instr, tos.local); - } - AnyInstruction::Real(Instruction::CheckExcMatch) => { - let _ = pop_ref(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); - } - AnyInstruction::Real(Instruction::ForIter { .. }) => { - let target = info.target; - debug_assert!(target != BlockIdx::NULL); - load_fast_push_block( - &mut worklist, - &mut self.blocks, - target, - refs.len() + 1, - ); - push_ref(&mut refs, i as isize, NOT_LOCAL); - } - AnyInstruction::Real(Instruction::LoadAttr { namei }) => { - let self_ref = pop_ref(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); - if namei.get(info.arg).is_method() { - push_ref(&mut refs, self_ref.instr, self_ref.local); - } - } - AnyInstruction::Real(Instruction::LoadSuperAttr { namei }) => { - let self_ref = pop_ref(&mut refs); - let _ = pop_ref(&mut refs); - let _ = pop_ref(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); - if namei.get(info.arg).is_load_method() { - push_ref(&mut refs, self_ref.instr, self_ref.local); - } - } - AnyInstruction::Real( - Instruction::LoadSpecial { .. } | Instruction::PushExcInfo, - ) => { - let tos = pop_ref(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); - push_ref(&mut refs, tos.instr, tos.local); - } - AnyInstruction::Real(Instruction::Send { .. }) => { - let target = info.target; - debug_assert!(target != BlockIdx::NULL); - load_fast_push_block(&mut worklist, &mut self.blocks, target, refs.len()); - let _ = pop_ref(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); - } - _ => { - let effect = instr.stack_effect_info(arg_u32); - let num_popped = effect.popped() as usize; - let num_pushed = effect.pushed() as usize; - let target = info.target; - if instr.has_target() { - debug_assert!(target != BlockIdx::NULL); - debug_assert!(refs.len() >= num_popped); - let target_depth = refs.len() - num_popped + num_pushed; - load_fast_push_block( - &mut worklist, - &mut self.blocks, - target, - target_depth, - ); - } - if !is_block_push(&info) { - for _ in 0..num_popped { - let _ = pop_ref(&mut refs); - } - for _ in 0..num_pushed { - push_ref(&mut refs, i as isize, NOT_LOCAL); - } - } - } + for i in 0..instr_count { + let info = blocks[block_i].instructions[i]; + let instr = info.instr; + let arg_u32 = u32::from(info.arg); + + match instr { + AnyInstruction::Real(Instruction::DeleteFast { var_num }) => { + kill_local( + &mut instr_flags, + &refs, + local_as_ref_local(usize::from(var_num.get(info.arg))), + ); } - } - - let fallthrough = self.blocks[block_i].next; - let term = self.blocks[block_i].instructions.last().copied(); - if let Some(term) = term - && fallthrough != BlockIdx::NULL - && !term.instr.is_unconditional_jump() - && !term.instr.is_scope_exit() - { - debug_assert!(bb_has_fallthrough(&self.blocks[block_i])); - load_fast_push_block(&mut worklist, &mut self.blocks, fallthrough, refs.len()); - } - - for r in refs.iter().copied() { - if r.instr != DUMMY_INSTR { - instr_flags[r.instr as usize] |= REF_UNCONSUMED; + AnyInstruction::Real(Instruction::LoadFast { var_num }) => { + push_ref( + &mut refs, + i as isize, + local_as_ref_local(usize::from(var_num.get(info.arg))), + ); + } + AnyInstruction::Real(Instruction::LoadFastAndClear { var_num }) => { + let local = local_as_ref_local(usize::from(var_num.get(info.arg))); + kill_local(&mut instr_flags, &refs, local); + push_ref(&mut refs, i as isize, local); + } + AnyInstruction::Real(Instruction::LoadFastLoadFast { .. }) => { + let (local1, local2) = decode_packed_fast_locals(info.arg); + push_ref(&mut refs, i as isize, local1); + push_ref(&mut refs, i as isize, local2); + } + AnyInstruction::Real(Instruction::StoreFast { var_num }) => { + let r = ref_stack_pop(&mut refs); + store_local( + &mut instr_flags, + &refs, + local_as_ref_local(usize::from(var_num.get(info.arg))), + r, + ); + } + AnyInstruction::Real(Instruction::StoreFastLoadFast { .. }) => { + let (store_local_idx, load_local_idx) = decode_packed_fast_locals(info.arg); + let r = ref_stack_pop(&mut refs); + store_local(&mut instr_flags, &refs, store_local_idx, r); + push_ref(&mut refs, i as isize, load_local_idx); + } + AnyInstruction::Real(Instruction::StoreFastStoreFast { .. }) => { + let (local1, local2) = decode_packed_fast_locals(info.arg); + let r1 = ref_stack_pop(&mut refs); + store_local(&mut instr_flags, &refs, local1, r1); + let r2 = ref_stack_pop(&mut refs); + store_local(&mut instr_flags, &refs, local2, r2); + } + AnyInstruction::Real(Instruction::Copy { i: _ }) => { + let depth = arg_u32 as usize; + assert!(depth > 0); + assert!(refs.len() >= depth); + let r = ref_stack_at(&refs, refs.len() - depth); + push_ref(&mut refs, r.instr, r.local); } - } - - let block = &mut self.blocks[block_idx]; - for (i, info) in block.instructions.iter_mut().enumerate() { - if instr_flags[i] != 0 { - continue; + AnyInstruction::Real(Instruction::Swap { i: _ }) => { + let depth = arg_u32 as usize; + assert!(depth >= 2); + assert!(refs.len() >= depth); + ref_stack_swap_top(&mut refs, depth); } - match info.instr.real() { - Some(Instruction::LoadFast { .. }) => { - info.instr = Instruction::LoadFastBorrow { - var_num: Arg::marker(), - } - .into(); + AnyInstruction::Real( + Instruction::FormatSimple + | Instruction::GetAnext + | Instruction::GetLen + | Instruction::GetYieldFromIter + | Instruction::ImportFrom { .. } + | Instruction::MatchKeys + | Instruction::MatchMapping + | Instruction::MatchSequence + | Instruction::WithExceptStart, + ) => { + let effect = instr.stack_effect_info(arg_u32); + let net_pushed = effect.pushed() as isize - effect.popped() as isize; + debug_assert!(net_pushed >= 0); + // CPython optimize_load_fast() shadows the outer + // instruction index in this produced-value loop. + for produced in 0..net_pushed { + push_ref(&mut refs, produced, NOT_LOCAL); } - Some(Instruction::LoadFastLoadFast { .. }) => { - info.instr = Instruction::LoadFastBorrowLoadFastBorrow { - var_nums: Arg::marker(), - } - .into(); + } + AnyInstruction::Real( + Instruction::DictMerge { .. } + | Instruction::DictUpdate { .. } + | Instruction::ListAppend { .. } + | Instruction::ListExtend { .. } + | Instruction::MapAdd { .. } + | Instruction::Reraise { .. } + | Instruction::SetAdd { .. } + | Instruction::SetUpdate { .. }, + ) => { + let effect = instr.stack_effect_info(arg_u32); + let net_popped = effect.popped() as isize - effect.pushed() as isize; + debug_assert!(net_popped > 0); + for _ in 0..net_popped { + let _ = ref_stack_pop(&mut refs); } - _ => {} } - } - } - } - - fn fast_scan_many_locals(&mut self, nlocals: usize) { - debug_assert!(nlocals > 64); - let mut states = vec![0usize; nlocals - 64]; - let mut blocknum = 0usize; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - blocknum += 1; - for info in &mut self.blocks[current.idx()].instructions { - let arg = u32::from(info.arg) as usize; - if arg < 64 { - continue; + AnyInstruction::Real( + Instruction::EndSend | Instruction::SetFunctionAttribute { .. }, + ) => { + let effect = instr.stack_effect_info(arg_u32); + debug_assert_eq!(effect.popped(), 2); + debug_assert_eq!(effect.pushed(), 1); + let tos = ref_stack_pop(&mut refs); + let _ = ref_stack_pop(&mut refs); + push_ref(&mut refs, tos.instr, tos.local); + } + AnyInstruction::Real(Instruction::CheckExcMatch) => { + let _ = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL); } - match info.instr.real() { - Some(Instruction::DeleteFast { .. } | Instruction::LoadFastAndClear { .. }) => { - debug_assert!(arg < nlocals); - states[arg - 64] = blocknum - 1; + AnyInstruction::Real(Instruction::ForIter { .. }) => { + let target = info.target; + debug_assert!(target != BlockIdx::NULL); + load_fast_push_block(&mut worklist, blocks, target, refs.len() + 1); + push_ref(&mut refs, i as isize, NOT_LOCAL); + } + AnyInstruction::Real(Instruction::LoadAttr { namei }) => { + let self_ref = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL); + if namei.get(info.arg).is_method() { + push_ref(&mut refs, self_ref.instr, self_ref.local); } - None if matches!( - info.instr.pseudo(), - Some(PseudoInstruction::StoreFastMaybeNull { .. }) - ) => - { - debug_assert!(arg < nlocals); - states[arg - 64] = blocknum - 1; + } + AnyInstruction::Real(Instruction::LoadSuperAttr { namei }) => { + let self_ref = ref_stack_pop(&mut refs); + let _ = ref_stack_pop(&mut refs); + let _ = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL); + if namei.get(info.arg).is_load_method() { + push_ref(&mut refs, self_ref.instr, self_ref.local); } - Some(Instruction::StoreFast { .. }) => { - debug_assert!(arg < nlocals); - states[arg - 64] = blocknum; + } + AnyInstruction::Real( + Instruction::LoadSpecial { .. } | Instruction::PushExcInfo, + ) => { + let tos = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL); + push_ref(&mut refs, tos.instr, tos.local); + } + AnyInstruction::Real(Instruction::Send { .. }) => { + let target = info.target; + debug_assert!(target != BlockIdx::NULL); + load_fast_push_block(&mut worklist, blocks, target, refs.len()); + let _ = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL); + } + _ => { + let effect = instr.stack_effect_info(arg_u32); + let num_popped = effect.popped() as usize; + let num_pushed = effect.pushed() as usize; + let target = info.target; + if instr.has_target() { + debug_assert!(target != BlockIdx::NULL); + debug_assert!(refs.len() >= num_popped); + let target_depth = refs.len() - num_popped + num_pushed; + load_fast_push_block(&mut worklist, blocks, target, target_depth); } - Some(Instruction::LoadFast { .. }) => { - debug_assert!(arg < nlocals); - if states[arg - 64] != blocknum { - info.instr = Opcode::LoadFastCheck.into(); + if !is_block_push(&info) { + for _ in 0..num_popped { + let _ = ref_stack_pop(&mut refs); + } + for _ in 0..num_pushed { + push_ref(&mut refs, i as isize, NOT_LOCAL); } - states[arg - 64] = blocknum; } - _ => {} } } - current = self.blocks[current.idx()].next; - } - } - - fn add_checks_for_loads_of_uninitialized_variables(&mut self) { - let mut nlocals = self.metadata.varnames.len(); - if nlocals == 0 { - return; } - let mut nparams = self.metadata.argcount as usize + self.metadata.kwonlyargcount as usize; - if self.flags.contains(CodeFlags::VARARGS) { - nparams += 1; - } - if self.flags.contains(CodeFlags::VARKEYWORDS) { - nparams += 1; - } - nparams = nparams.min(nlocals); - - if nlocals > 64 { - self.fast_scan_many_locals(nlocals); - nlocals = 64; - } - - let mut worklist = make_cfg_traversal_stack(&mut self.blocks); - let mut start_mask = 0u64; - for i in nparams..nlocals { - start_mask |= 1u64 << i; + let fallthrough = blocks[block_i].next; + let term = basicblock_last_instr(&blocks[block_i]).copied(); + if let Some(term) = term + && fallthrough != BlockIdx::NULL + && !term.instr.is_unconditional_jump() + && !term.instr.is_scope_exit() + { + debug_assert!(bb_has_fallthrough(&blocks[block_i])); + load_fast_push_block(&mut worklist, blocks, fallthrough, refs.len()); } - maybe_push_local_block(&mut self.blocks, &mut worklist, BlockIdx(0), start_mask); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - scan_block_for_locals(&mut self.blocks, current, &mut worklist); - current = self.blocks[current.idx()].next; + for i in 0..refs.len() { + let r = ref_stack_at(&refs, i); + if r.instr != DUMMY_INSTR { + instr_flags[r.instr as usize] |= LoadFastInstrFlag::RefUnconsumed as u8; + } } - while let Some(block_idx) = worklist.pop() { - self.blocks[block_idx.idx()].visited = false; - scan_block_for_locals(&mut self.blocks, block_idx, &mut worklist); + let block = &mut blocks[block_idx]; + let iused = block.instructions.len(); + let mut i = 0; + while i < iused { + let info = &mut block.instructions[i]; + if instr_flags[i] != 0 { + i += 1; + continue; + } + match info.instr.real() { + Some(Instruction::LoadFast { .. }) => { + info.instr = Instruction::LoadFastBorrow { + var_num: Arg::marker(), + } + .into(); + } + Some(Instruction::LoadFastLoadFast { .. }) => { + info.instr = Instruction::LoadFastBorrowLoadFastBorrow { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + i += 1; } } +} - /// flowgraph.c calculate_stackdepth - fn calculate_stackdepth(&mut self) -> crate::InternalResult { - let mut maxdepth = 0u32; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - self.blocks[current.idx()].start_depth = None; - current = self.blocks[current.idx()].next; - } - let mut stack = make_cfg_traversal_stack(&mut self.blocks); - stackdepth_push(&mut stack, &mut self.blocks, BlockIdx(0), 0)?; - const DEBUG: bool = false; - while let Some(block_idx) = stack.pop() { - let idx = block_idx.idx(); - let mut depth = self.blocks[idx] - .start_depth - .expect("stackdepth stack contains only started blocks"); - let mut next = self.blocks[idx].next; - if DEBUG { - eprintln!("===BLOCK {}===", block_idx.0); - } - let instr_count = self.blocks[idx].instructions.len(); - for i in 0..instr_count { - let ins = self.blocks[idx].instructions[i]; - let instr = &ins.instr; - let effect = instr.stack_effect(ins.arg.into()); - if DEBUG { - let display_arg = if ins.target == BlockIdx::NULL { - ins.arg - } else { - OpArg::new(ins.target.0) - }; - eprint!("{display_arg:?}: {depth} {effect:+} => "); +/// flowgraph.c calculate_stackdepth +fn calculate_stackdepth(blocks: &mut [Block]) -> crate::InternalResult { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + blocks[current.idx()].start_depth = START_DEPTH_UNSET; + current = blocks[current.idx()].next; + } + let mut stack = make_cfg_traversal_stack(blocks); + let mut maxdepth = 0i32; + stackdepth_push(&mut stack, blocks, BlockIdx(0), 0)?; + while let Some(block_idx) = stack.pop() { + let idx = block_idx.idx(); + let mut depth = blocks[idx].start_depth; + debug_assert!(depth >= 0); + let mut next = blocks[idx].next; + let instr_count = blocks[idx].instructions.len(); + for i in 0..instr_count { + let ins = blocks[idx].instructions[i]; + let instr = &ins.instr; + let effects = get_stack_effects(*instr, ins.arg, 0)?; + let new_depth = depth.checked_add(effects.net).ok_or({ + if effects.net < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow } - let new_depth = depth.checked_add_signed(effect).ok_or({ - if effect < 0 { + })?; + if new_depth < 0 { + return Err(InternalError::StackUnderflow); + } + maxdepth = maxdepth.max(depth); + if instr.has_target() && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) { + debug_assert!(ins.target != BlockIdx::NULL); + let effects = get_stack_effects(*instr, ins.arg, 1)?; + let target_depth = depth.checked_add(effects.net).ok_or({ + if effects.net < 0 { InternalError::StackUnderflow } else { InternalError::StackOverflow } })?; - if DEBUG { - eprintln!("{new_depth}"); - } + debug_assert!(target_depth >= 0); maxdepth = maxdepth.max(depth); - // Process target blocks for branching instructions - if instr.has_target() && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) { - debug_assert!(ins.target != BlockIdx::NULL); - let jump_effect = instr.stack_effect_jump(ins.arg.into()); - let target_depth = depth.checked_add_signed(jump_effect).ok_or({ - if jump_effect < 0 { - InternalError::StackUnderflow - } else { - InternalError::StackOverflow - } - })?; - maxdepth = maxdepth.max(depth); - stackdepth_push(&mut stack, &mut self.blocks, ins.target, target_depth)?; - } - depth = new_depth; - debug_assert!(!instr.is_assembler()); - if instr.is_no_fallthrough() { - next = BlockIdx::NULL; - break; - } + stackdepth_push(&mut stack, blocks, ins.target, target_depth)?; } - if next != BlockIdx::NULL { - debug_assert!(bb_has_fallthrough(&self.blocks[idx])); - stackdepth_push(&mut stack, &mut self.blocks, next, depth)?; + depth = new_depth; + debug_assert!(!instr.is_assembler()); + if instr.is_unconditional_jump() || instr.is_scope_exit() { + next = BlockIdx::NULL; + break; } } - if DEBUG { - eprintln!("DONE: {maxdepth}"); - } - - // Fix up handler stack_depth in ExceptHandlerInfo using b_startdepth - // computed above: depth = start_depth - 1 - preserve_lasti - current = BlockIdx(0); - while current != BlockIdx::NULL { - let next = self.blocks[current.idx()].next; - let instr_count = self.blocks[current.idx()].instructions.len(); - for i in 0..instr_count { - let h_start = self.blocks[current.idx()].instructions[i] - .except_handler - .and_then(|handler| self.blocks[handler.handler_block.idx()].start_depth); - if let (Some(handler), Some(h_start)) = ( - &mut self.blocks[current.idx()].instructions[i].except_handler, - h_start, - ) { - let adjustment = 1 + handler.preserve_lasti as u32; - debug_assert!( - h_start >= adjustment, - "handler start depth {h_start} too shallow for adjustment {adjustment}" - ); - handler.stack_depth = h_start.saturating_sub(adjustment); - } - } - current = next; + if next != BlockIdx::NULL { + debug_assert!(bb_has_fallthrough(&blocks[idx])); + stackdepth_push(&mut stack, blocks, next, depth)?; } - - Ok(maxdepth) } + + let stackdepth = maxdepth; + u32::try_from(stackdepth).map_err(|_| InternalError::StackUnderflow) } #[cfg(test)] @@ -3917,9 +4222,11 @@ impl CodeInfo { block.cold, block.except_handler, block.preserve_lasti, - block - .start_depth - .map_or_else(|| String::from("None"), |depth| depth.to_string()), + if block.start_depth < 0 { + String::from("None") + } else { + block.start_depth.to_string() + }, block_return, ); for info in &block.instructions { @@ -3958,18 +4265,16 @@ impl CodeInfo { "after_cfg_from_instruction_sequence".to_owned(), self.debug_block_dump(), )); - translate_jump_labels_to_targets(&mut self.blocks); - mark_except_handlers(&mut self.blocks); - label_exception_targets(&mut self.blocks); + optimize_code_unit_preprocess(&mut self.blocks); check_cfg(&self.blocks)?; inline_small_or_no_lineno_blocks(&mut self.blocks); trace.push(( "after_inline_small_or_no_lineno_blocks".to_owned(), self.debug_block_dump(), )); - Self::remove_unreachable(&mut self.blocks); - resolve_line_numbers(&mut self.blocks); - self.optimize_load_const(); + remove_unreachable(&mut self.blocks); + resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno); + optimize_load_const(&mut self.metadata, &mut self.blocks); trace.push(( "after_optimize_load_const".to_owned(), self.debug_block_dump(), @@ -3977,30 +4282,33 @@ impl CodeInfo { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next_block = self.blocks[block_idx.idx()].next; - Self::optimize_basic_block(&mut self.blocks, &mut self.metadata, block_idx)?; + optimize_basic_block(&mut self.blocks, &mut self.metadata, block_idx)?; block_idx = next_block; } trace.push(( "after_optimize_basic_block".to_owned(), self.debug_block_dump(), )); - self.remove_redundant_nops_and_pairs(); - Self::remove_unreachable(&mut self.blocks); + remove_redundant_nops_and_pairs(&mut self.blocks); + remove_unreachable(&mut self.blocks); remove_redundant_nops_and_jumps(&mut self.blocks)?; - debug_assert!(no_redundant_jumps(&self.blocks)); - self.remove_unused_consts(); + #[cfg(debug_assertions)] + assert!(no_redundant_jumps(&self.blocks)); + remove_unused_consts(&mut self.blocks, &mut self.metadata.consts); trace.push(( "after_optimize_cfg_cleanup".to_owned(), self.debug_block_dump(), )); - self.add_checks_for_loads_of_uninitialized_variables(); - self.insert_superinstructions(); + let nlocals = self.metadata.varnames.len(); + let nparams = self.nparams; + add_checks_for_loads_of_uninitialized_variables(&mut self.blocks, nlocals, nparams); + insert_superinstructions(&mut self.blocks); push_cold_blocks_to_end(&mut self.blocks)?; trace.push(( "after_push_cold_before_chain_reorder".to_owned(), self.debug_block_dump(), )); - resolve_line_numbers(&mut self.blocks); + resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno); trace.push(( "after_push_cold_resolve_line_numbers".to_owned(), self.debug_block_dump(), @@ -4017,8 +4325,8 @@ impl CodeInfo { self.debug_block_dump(), )); - let _max_stackdepth = self.calculate_stackdepth()?; - let _nlocalsplus = self.prepare_localsplus(); + let _max_stackdepth = calculate_stackdepth(&mut self.blocks)?; + let _nlocalsplus = prepare_localsplus(&self.metadata, &mut self.blocks, self.flags); convert_pseudo_ops(&mut self.blocks)?; trace.push(( "after_convert_pseudo_ops".to_owned(), @@ -4026,9 +4334,10 @@ impl CodeInfo { )); normalize_jumps(&mut self.blocks)?; - debug_assert!(no_redundant_jumps(&self.blocks)); + #[cfg(debug_assertions)] + assert!(no_redundant_jumps(&self.blocks)); trace.push(("after_normalize_jumps".to_owned(), self.debug_block_dump())); - self.optimize_load_fast(); + optimize_load_fast(&mut self.blocks); trace.push(( "after_optimize_load_fast".to_owned(), self.debug_block_dump(), @@ -4069,285 +4378,523 @@ impl InstrDisplayContext for CodeInfo { } } +const NOT_LOCAL: isize = -1; +const DUMMY_INSTR: isize = -1; + +/// flowgraph.c make_super_instruction +fn make_super_instruction( + inst1: &mut InstructionInfo, + inst2: &mut InstructionInfo, + super_op: AnyInstruction, +) { + let line1 = instruction_lineno(inst1); + let line2 = instruction_lineno(inst2); + if line1 >= 0 && line2 >= 0 && line1 != line2 { + return; + } + let arg1 = u32::from(inst1.arg); + let arg2 = u32::from(inst2.arg); + if arg1 >= 16 || arg2 >= 16 { + return; + } + instr_set_op1(inst1, super_op, OpArg::new((arg1 << 4) | arg2)); + set_to_nop(inst2); +} + +/// flowgraph.c insert_superinstructions +fn insert_superinstructions(blocks: &mut [Block]) { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = blocks[block_idx.idx()].next; + let block = &mut blocks[block_idx]; + for i in 0..block.instructions.len() { + let nextop = (i + 1 < block.instructions.len()) + .then(|| block.instructions[i + 1].instr.real()) + .flatten(); + match block.instructions[i].instr.real() { + Some(Instruction::LoadFast { .. }) => { + if matches!(nextop, Some(Instruction::LoadFast { .. })) { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + make_super_instruction( + &mut inst1[0], + &mut rest[0], + Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + } + .into(), + ); + } + } + Some(Instruction::StoreFast { .. }) => match nextop { + Some(Instruction::LoadFast { .. }) => { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + make_super_instruction( + &mut inst1[0], + &mut rest[0], + Instruction::StoreFastLoadFast { + var_nums: Arg::marker(), + } + .into(), + ); + } + Some(Instruction::StoreFast { .. }) => { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + make_super_instruction( + &mut inst1[0], + &mut rest[0], + Instruction::StoreFastStoreFast { + var_nums: Arg::marker(), + } + .into(), + ); + } + _ => {} + }, + _ => {} + } + } + block_idx = next_block; + } + remove_redundant_nops(blocks); + #[cfg(debug_assertions)] + { + let no_redundant = no_redundant_nops(blocks); + debug_assert!(no_redundant); + } +} + +/// flowgraph.c LoadFastInstrFlag +#[repr(u8)] +enum LoadFastInstrFlag { + SupportKilled = 1, + StoredAsLocal = 2, + RefUnconsumed = 4, +} + +/// flowgraph.c ref +#[derive(Clone, Copy)] +struct Ref { + instr: isize, + local: isize, +} + +/// flowgraph.c ref_stack +type RefStack = Vec; + +/// flowgraph.c ref_stack_push +fn ref_stack_push(stack: &mut RefStack, r: Ref) { + stack.push(r); +} + +/// flowgraph.c ref_stack_pop +fn ref_stack_pop(stack: &mut RefStack) -> Ref { + stack.pop().expect("ref stack underflow") +} + +/// flowgraph.c ref_stack_swap_top +fn ref_stack_swap_top(stack: &mut RefStack, off: usize) { + assert!(off >= 2 && stack.len() >= off); + let top = stack.len() - 1; + let other = stack.len() - off; + stack.swap(top, other); +} + +/// flowgraph.c ref_stack_at +fn ref_stack_at(stack: &RefStack, idx: usize) -> Ref { + stack.get(idx).copied().expect("ref stack index in bounds") +} + +/// flowgraph.c ref_stack_clear +fn ref_stack_clear(stack: &mut RefStack) { + stack.clear(); +} + +/// flowgraph.c optimize_load_fast PUSH_REF +fn push_ref(stack: &mut RefStack, instr: isize, local: isize) { + ref_stack_push(stack, Ref { instr, local }); +} + +/// flowgraph.c kill_local +fn kill_local(instr_flags: &mut [u8], refs: &RefStack, local: isize) { + for i in 0..refs.len() { + let r = ref_stack_at(refs, i); + if r.local != local { + continue; + } + debug_assert!(r.instr >= 0); + instr_flags[r.instr as usize] |= LoadFastInstrFlag::SupportKilled as u8; + } +} + +/// flowgraph.c store_local +fn store_local(instr_flags: &mut [u8], refs: &RefStack, local: isize, r: Ref) { + kill_local(instr_flags, refs, local); + if r.instr != DUMMY_INSTR { + instr_flags[r.instr as usize] |= LoadFastInstrFlag::StoredAsLocal as u8; + } +} + +fn decode_packed_fast_locals(arg: OpArg) -> (isize, isize) { + let packed = u32::from(arg); + ( + isize::try_from((packed >> 4) & 0xF).expect("local index fits in isize"), + isize::try_from(packed & 0xF).expect("local index fits in isize"), + ) +} + +fn local_as_ref_local(local: usize) -> isize { + isize::try_from(local).expect("local index fits in isize") +} + +/// flowgraph.c load_fast_push_block +fn load_fast_push_block( + worklist: &mut Vec, + blocks: &mut [Block], + target: BlockIdx, + start_depth: usize, +) { + debug_assert!(target != BlockIdx::NULL); + debug_assert_eq!( + usize::try_from(blocks[target.idx()].start_depth).ok(), + Some(start_depth), + ); + if !blocks[target.idx()].visited { + blocks[target.idx()].visited = true; + worklist.push(target); + } +} + fn stackdepth_push( stack: &mut Vec, blocks: &mut [Block], target: BlockIdx, - depth: u32, + depth: i32, ) -> crate::InternalResult<()> { let idx = target.idx(); let block_depth = &mut blocks[idx].start_depth; - if block_depth.is_some_and(|block_depth| block_depth != depth) { + if !(*block_depth < 0 || *block_depth == depth) { return Err(InternalError::InconsistentStackDepth); } - if block_depth.is_none() { - *block_depth = Some(depth); + if *block_depth < depth && *block_depth < 100 { + debug_assert!(*block_depth < 0); + *block_depth = depth; stack.push(target); } Ok(()) } -/// Generate Python 3.11+ format linetable from source locations -fn generate_linetable( - locations: &[LineTableLocation], - first_line: i32, - debug_ranges: bool, -) -> Box<[u8]> { - if locations.is_empty() { - return Box::new([]); - } +/// flowgraph.c stack_effects +struct StackEffects { + net: i32, +} - let mut linetable = Vec::new(); - // Initialize prev_line to first_line - // The first entry's delta is relative to co_firstlineno - let mut prev_line = first_line; - let mut i = 0; +/// flowgraph.c get_stack_effects +#[allow(clippy::unnecessary_wraps)] +fn get_stack_effects( + instr: AnyInstruction, + oparg: OpArg, + jump: i32, +) -> crate::InternalResult { + let oparg = u32::from(oparg); + let net = if instr.is_block_push() && jump == 0 { + 0 + } else if jump != 0 { + instr.stack_effect_jump(oparg) + } else { + instr.stack_effect(oparg) + }; + Ok(StackEffects { net }) +} - while i < locations.len() { - let loc = &locations[i]; +/// assemble.c write_location_first_byte +fn write_location_first_byte(linetable: &mut Vec, code: u8, length: usize) { + linetable.extend(write_location_entry_start(code, length)); +} - // Count consecutive instructions with the same location - let mut length = 1; - while i + length < locations.len() && locations[i + length] == locations[i] { - length += 1; - } +/// pycore_code.h write_location_entry_start +fn write_location_entry_start(code: u8, length: usize) -> [u8; 1] { + debug_assert!(length > 0 && length <= 8); + debug_assert_eq!(code & 15, code); + [0x80 | (code << 3) | ((length - 1) as u8)] +} + +/// assemble.c write_location_byte +fn write_location_byte(linetable: &mut Vec, value: u8) { + linetable.push(value); +} - // Process in chunks of up to 8 instructions - while length > 0 { - let entry_length = length.min(8); +/// assemble.c write_location_varint +fn write_location_varint(linetable: &mut Vec, value: u32) { + write_varint(linetable, value); +} - // Get line information - let line = loc.line; +/// assemble.c write_location_signed_varint +fn write_location_signed_varint(linetable: &mut Vec, value: i32) { + write_signed_varint(linetable, value); +} - // NO_LOCATION: emit PyCodeLocationInfoKind::None entries (CACHE, etc.) - if line == -1 { - linetable.push( - 0x80 | ((PyCodeLocationInfoKind::None as u8) << 3) | ((entry_length - 1) as u8), - ); - // Do NOT update prev_line - length -= entry_length; - i += entry_length; - continue; - } +/// assemble.c write_location_info_short_form +fn write_location_info_short_form( + linetable: &mut Vec, + length: usize, + column: i32, + end_column: i32, +) { + debug_assert!(length > 0 && length <= 8); + debug_assert!(column < 80); + debug_assert!(end_column >= column); + debug_assert!(end_column - column < 16); + let column_low_bits = column & 7; + let column_group = column >> 3; + let code = PyCodeLocationInfoKind::Short0 as u8 + column_group as u8; + write_location_first_byte(linetable, code, length); + write_location_byte( + linetable, + ((column_low_bits as u8) << 4) | ((end_column - column) as u8), + ); +} - let end_line = loc.end_line; - let line_delta = line - prev_line; - let end_line_delta = end_line - line; +/// assemble.c write_location_info_oneline_form +fn write_location_info_oneline_form( + linetable: &mut Vec, + length: usize, + line_delta: i32, + column: i32, + end_column: i32, +) { + debug_assert!(length > 0 && length <= 8); + debug_assert!((0..3).contains(&line_delta)); + debug_assert!(column < 128); + debug_assert!(end_column < 128); + let code = PyCodeLocationInfoKind::OneLine0 as u8 + line_delta as u8; + write_location_first_byte(linetable, code, length); + write_location_byte(linetable, column as u8); + write_location_byte(linetable, end_column as u8); +} - // When debug_ranges is disabled, only emit line info (NoColumns format) - if !debug_ranges { - // NoColumns format (code 13): line info only, no column data - linetable.push( - 0x80 | ((PyCodeLocationInfoKind::NoColumns as u8) << 3) - | ((entry_length - 1) as u8), - ); - write_signed_varint(&mut linetable, line_delta); +/// assemble.c write_location_info_long_form +fn write_location_info_long_form( + linetable: &mut Vec, + loc: LineTableLocation, + length: usize, + line_delta: i32, +) { + debug_assert!(length > 0 && length <= 8); + write_location_first_byte(linetable, PyCodeLocationInfoKind::Long as u8, length); + write_location_signed_varint(linetable, line_delta); + debug_assert!(loc.end_line >= loc.line); + write_location_varint(linetable, (loc.end_line - loc.line) as u32); + write_location_varint( + linetable, + if loc.col < 0 { 0 } else { (loc.col as u32) + 1 }, + ); + write_location_varint( + linetable, + if loc.end_col < 0 { + 0 + } else { + (loc.end_col as u32) + 1 + }, + ); +} - prev_line = line; - length -= entry_length; - i += entry_length; - continue; - } +/// assemble.c write_location_info_none +fn write_location_info_none(linetable: &mut Vec, length: usize) { + write_location_first_byte(linetable, PyCodeLocationInfoKind::None as u8, length); +} - // Get column information (only when debug_ranges is enabled) - let col = loc.col; - let end_col = loc.end_col; - if (col < 0 || end_col < 0) && end_line == line { - linetable.push( - 0x80 | ((PyCodeLocationInfoKind::NoColumns as u8) << 3) - | ((entry_length - 1) as u8), - ); - write_signed_varint(&mut linetable, line_delta); +/// assemble.c write_location_info_no_column +fn write_location_info_no_column(linetable: &mut Vec, length: usize, line_delta: i32) { + write_location_first_byte(linetable, PyCodeLocationInfoKind::NoColumns as u8, length); + write_location_signed_varint(linetable, line_delta); +} - prev_line = line; - length -= entry_length; - i += entry_length; - continue; - } +/// assemble.c write_location_info_entry +fn write_location_info_entry( + linetable: &mut Vec, + loc: LineTableLocation, + length: usize, + prev_line: &mut i32, + debug_ranges: bool, +) { + if loc.line == NO_LOCATION_OVERRIDE { + write_location_info_none(linetable, length); + return; + } - // Choose the appropriate encoding based on line delta and column info - if line_delta == 0 && end_line_delta == 0 { - if col < 80 && end_col - col < 16 && end_col >= col { - // Short form (codes 0-9) for common cases - let code = (col / 8).min(9) as u8; // Short0 to Short9 - linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); - let col_byte = (((col % 8) as u8) << 4) | ((end_col - col) as u8 & 0xf); - linetable.push(col_byte); - } else if col < 128 && end_col < 128 { - // One-line form (code 10) for same line - linetable.push( - 0x80 | ((PyCodeLocationInfoKind::OneLine0 as u8) << 3) - | ((entry_length - 1) as u8), - ); - linetable.push(col as u8); - linetable.push(end_col as u8); - } else { - // Long form for columns >= 128 - linetable.push( - 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) - | ((entry_length - 1) as u8), - ); - write_signed_varint(&mut linetable, 0); // line_delta = 0 - write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); - write_varint(&mut linetable, (end_col as u32) + 1); - } - } else if line_delta > 0 && line_delta < 3 && end_line_delta == 0 { - // One-line form (codes 11-12) for line deltas 1-2 - if col < 128 && end_col < 128 { - let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); - linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); - linetable.push(col as u8); - linetable.push(end_col as u8); - } else { - // Long form for columns >= 128 - linetable.push( - 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) - | ((entry_length - 1) as u8), - ); - write_signed_varint(&mut linetable, line_delta); - write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); - write_varint(&mut linetable, (end_col as u32) + 1); - } - } else { - // Long form (code 14) for all other cases - // Handles: line_delta < 0, line_delta >= 3, multi-line spans, or columns >= 128 - linetable.push( - 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), - ); - write_signed_varint(&mut linetable, line_delta); - write_varint(&mut linetable, end_line_delta as u32); - write_varint(&mut linetable, if col < 0 { 0 } else { (col as u32) + 1 }); - write_varint( - &mut linetable, - if end_col < 0 { 0 } else { (end_col as u32) + 1 }, - ); - } + let line_delta = loc.line - *prev_line; + let column = loc.col; + let end_column = loc.end_col; + if !debug_ranges + || ((column < 0 || end_column < 0) && (loc.end_line == loc.line || loc.end_line < 0)) + { + write_location_info_no_column(linetable, length, line_delta); + *prev_line = loc.line; + return; + } - prev_line = line; - length -= entry_length; - i += entry_length; + if loc.end_line == loc.line { + if line_delta == 0 && column < 80 && end_column - column < 16 && end_column >= column { + write_location_info_short_form(linetable, length, column, end_column); + return; + } + if (0..3).contains(&line_delta) && column < 128 && end_column < 128 { + write_location_info_oneline_form(linetable, length, line_delta, column, end_column); + *prev_line = loc.line; + return; } } - linetable.into_boxed_slice() + write_location_info_long_form(linetable, loc, length, line_delta); + *prev_line = loc.line; +} + +/// assemble.c assemble_emit_location +fn assemble_emit_location( + linetable: &mut Vec, + loc: LineTableLocation, + mut size: usize, + prev_line: &mut i32, + debug_ranges: bool, +) { + if size == 0 { + return; + } + while size > 8 { + write_location_info_entry(linetable, loc, 8, prev_line, debug_ranges); + size -= 8; + } + write_location_info_entry(linetable, loc, size, prev_line, debug_ranges); } fn no_linetable_location() -> LineTableLocation { LineTableLocation { - line: -1, - end_line: -1, - col: -1, - end_col: -1, + line: NO_LOCATION_OVERRIDE, + end_line: NO_LOCATION_OVERRIDE, + col: NO_LOCATION_OVERRIDE, + end_col: NO_LOCATION_OVERRIDE, + } +} + +fn next_linetable_location() -> LineTableLocation { + LineTableLocation { + line: NEXT_LOCATION_OVERRIDE, + end_line: NEXT_LOCATION_OVERRIDE, + col: NEXT_LOCATION_OVERRIDE, + end_col: NEXT_LOCATION_OVERRIDE, } } -fn instruction_is_terminator(op: Instruction) -> bool { - op.is_terminator() +/// assemble.c assemble_emit_exception_table_item +fn assemble_emit_exception_table_item(table: &mut Vec, value: i32, mut msb: u8) { + debug_assert!((msb | 128) == 128); + debug_assert!((0..(1 << 30)).contains(&value)); + let value = u32::try_from(value).expect("exception table item is non-negative"); + const CONTINUATION_BIT: u8 = 64; + if value >= 1 << 24 { + table.push(((value >> 24) as u8) | CONTINUATION_BIT | msb); + msb = 0; + } + if value >= 1 << 18 { + table.push((((value >> 18) & 0x3f) as u8) | CONTINUATION_BIT | msb); + msb = 0; + } + if value >= 1 << 12 { + table.push((((value >> 12) & 0x3f) as u8) | CONTINUATION_BIT | msb); + msb = 0; + } + if value >= 1 << 6 { + table.push((((value >> 6) & 0x3f) as u8) | CONTINUATION_BIT | msb); + msb = 0; + } + table.push(((value & 0x3f) as u8) | msb); } -fn resolve_next_locations(instructions: &[CodeUnit], locations: &mut [LineTableLocation]) { - debug_assert_eq!(instructions.len(), locations.len()); - let mut next_location = no_linetable_location(); - for (instruction, location) in instructions.iter().zip(locations.iter_mut()).rev() { - if location.line == NEXT_LOCATION_OVERRIDE { - *location = if instruction_is_terminator(instruction.op) { - no_linetable_location() - } else { - next_location - }; - } - next_location = *location; - } +/// assemble.c assemble_emit_exception_table_entry +fn assemble_emit_exception_table_entry( + table: &mut Vec, + start: i32, + end: i32, + handler_offset: i32, + handler: InstructionSequenceExceptHandlerInfo, +) { + let size = end - start; + debug_assert!(end > start); + let target = handler_offset; + let mut depth = handler + .start_depth + .checked_sub(1) + .expect("exception handler start depth includes exception item"); + if handler.preserve_lasti > 0 { + depth = depth + .checked_sub(1) + .expect("preserve_lasti handler start depth includes lasti"); + } + debug_assert!(depth >= 0); + let depth_lasti = (depth << 1) | handler.preserve_lasti; + assemble_emit_exception_table_item(table, start, 1 << 7); + assemble_emit_exception_table_item(table, size, 0); + assemble_emit_exception_table_item(table, target, 0); + assemble_emit_exception_table_item(table, depth_lasti, 0); } /// assemble.c assemble_exception_table -fn generate_exception_table( - instrs: &[InstructionSequenceEntry], - instruction_offsets: &[u32], - end_offset: u32, -) -> Box<[u8]> { - let mut entries: Vec = Vec::new(); - let mut current_entry: Option<(InstructionSequenceExceptHandlerInfo, u32)> = None; - let same_handler = |left: InstructionSequenceExceptHandlerInfo, - right: InstructionSequenceExceptHandlerInfo| { - // CPython assemble_exception_table() starts a new table entry only - // when h_label changes. h_startdepth and h_preserve_lasti come from - // the active handler for that label. - left.target_offset == right.target_offset +fn assemble_exception_table(instrs: &[InstructionSequenceEntry]) -> Box<[u8]> { + let mut table = Vec::new(); + let mut handler = InstructionSequenceExceptHandlerInfo { + h_label: NO_EXCEPTION_HANDLER_LABEL, + start_depth: -1, + preserve_lasti: -1, }; - - for (idx, instr) in instrs.iter().enumerate() { - let instr_offset = instruction_offsets[idx]; - match (¤t_entry, instr.except_handler) { - // No current entry, no handler - nothing to do - (None, None) => {} - - // No current entry, handler starts - begin new entry - (None, Some(handler)) => { - current_entry = Some((handler, instr_offset)); - } - - // Current entry exists, same handler - continue - (Some((curr_handler, _)), Some(handler)) if same_handler(*curr_handler, handler) => {} - - // Current entry exists, different handler - finish current, start new - (Some((curr_handler, start)), Some(handler)) => { - let target_offset = instruction_offsets - [curr_handler.target_offset.expect("missing handler target")]; - entries.push(ExceptionTableEntry::new( - *start, - instr_offset, - target_offset, - curr_handler.stack_depth as u16, - curr_handler.preserve_lasti, - )); - current_entry = Some((handler, instr_offset)); - } - - // Current entry exists, no handler - finish current entry - (Some((curr_handler, start)), None) => { - let target_offset = instruction_offsets - [curr_handler.target_offset.expect("missing handler target")]; - entries.push(ExceptionTableEntry::new( - *start, - instr_offset, - target_offset, - curr_handler.stack_depth as u16, - curr_handler.preserve_lasti, - )); - current_entry = None; + let mut start = -1; + let mut ioffset = 0i32; + + for i in 0..instrs.len() { + let instr = &instrs[i]; + if instr.except_handler.h_label != handler.h_label { + if handler.h_label >= 0 { + let handler_label = + usize::try_from(handler.h_label).expect("handler label is non-negative"); + let handler_offset = instrs[handler_label].i_offset; + assemble_emit_exception_table_entry( + &mut table, + start, + ioffset, + handler_offset, + handler, + ); } + start = ioffset; + handler = instr.except_handler; } + ioffset += instr_size(&instr.info) + .to_i32() + .expect("assembled instruction offset fits in i32"); } - // Finish any remaining entry - if let Some((curr_handler, start)) = current_entry { - let target_offset = - instruction_offsets[curr_handler.target_offset.expect("missing handler target")]; - entries.push(ExceptionTableEntry::new( - start, - end_offset, - target_offset, - curr_handler.stack_depth as u16, - curr_handler.preserve_lasti, - )); + if handler.h_label >= 0 { + let handler_label = + usize::try_from(handler.h_label).expect("handler label is non-negative"); + let handler_offset = instrs[handler_label].i_offset; + assemble_emit_exception_table_entry(&mut table, start, ioffset, handler_offset, handler); } - encode_exception_table(&entries) + table.into_boxed_slice() } /// Mark exception handler target blocks. /// flowgraph.c mark_except_handlers pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - debug_assert!(!blocks[block_idx.idx()].except_handler); - block_idx = blocks[block_idx.idx()].next; + #[cfg(debug_assertions)] + { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + assert!(!blocks[block_idx.idx()].except_handler); + block_idx = blocks[block_idx.idx()].next; + } } - block_idx = BlockIdx(0); + let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; let instr_count = blocks[block_idx.idx()].instructions.len(); @@ -4391,8 +4938,8 @@ fn mark_warm(blocks: &mut [Block]) { let next = blocks[idx].next; if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) && !blocks[next.idx()].visited { - blocks[next.idx()].visited = true; stack.push(next); + blocks[next.idx()].visited = true; } let instr_count = blocks[idx].instructions.len(); @@ -4402,8 +4949,8 @@ fn mark_warm(blocks: &mut [Block]) { let target = instr.target; debug_assert!(target != BlockIdx::NULL); if !blocks[target.idx()].visited { - blocks[target.idx()].visited = true; stack.push(target); + blocks[target.idx()].visited = true; } } } @@ -4441,8 +4988,8 @@ fn mark_cold(blocks: &mut [Block]) { if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { let next_idx = next.idx(); if !blocks[next_idx].warm && !blocks[next_idx].visited { - blocks[next_idx].visited = true; cold_stack.push(next); + blocks[next_idx].visited = true; } } @@ -4453,10 +5000,9 @@ fn mark_cold(blocks: &mut [Block]) { debug_assert_eq!(i, instr_count - 1); let target = instr.target; debug_assert!(target != BlockIdx::NULL); - let target_idx = target.idx(); - if !blocks[target_idx].warm && !blocks[target_idx].visited { - blocks[target_idx].visited = true; + if !blocks[target.idx()].warm && !blocks[target.idx()].visited { cold_stack.push(target); + blocks[target.idx()].visited = true; } } } @@ -4465,12 +5011,12 @@ fn mark_cold(blocks: &mut [Block]) { /// flowgraph.c push_cold_blocks_to_end fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> { - if blocks.is_empty() || blocks[0].next == BlockIdx::NULL { + if blocks[0].next == BlockIdx::NULL { return Ok(()); } mark_cold(blocks); - let mut next_label = next_cpython_cfg_label(blocks); + let mut next_label = (get_max_label(blocks) + 1) as usize; // If a cold block falls through to a warm block, add an explicit jump let mut block_idx = BlockIdx(0); @@ -4481,41 +5027,45 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> && next != BlockIdx::NULL && blocks[next.idx()].warm { - if blocks[next.idx()].cpython_label_id.is_none() { - blocks[next.idx()].cpython_label_id = Some(InstructionSequenceLabel(next_label)); + let explicit_jump = blocks_new_block(blocks); + if !is_label(blocks[next.idx()].cpython_label) { + blocks[next.idx()].cpython_label = InstructionSequenceLabel::from_index(next_label); next_label += 1; } - let jump_arg = cpython_target_label_arg(blocks, next); - let jump_block_idx = BlockIdx(blocks.len() as u32); - let mut jump_block = Block { - cold: true, - predecessors: 1, - ..Block::default() - }; - basicblock_add_jump_op( - &mut jump_block, + let jump_label = blocks[next.idx()].cpython_label; + debug_assert!(is_label(jump_label)); + basicblock_addop( + &mut blocks[explicit_jump.idx()], InstructionInfo { instr: PseudoOpcode::JumpNoInterrupt.into(), - arg: jump_arg, + arg: OpArg::new( + jump_label + .idx() + .to_u32() + .expect("too many CPython CFG labels"), + ), target: BlockIdx::NULL, location: SourceLocation::default(), end_location: SourceLocation::default(), except_handler: None, - lineno_override: Some(-1), - cache_entries: 0, + lineno_override: Some(NO_LOCATION_OVERRIDE), }, - next, - )?; - jump_block.next = next; - blocks[block_idx.idx()].next = jump_block_idx; - blocks.push(jump_block); + ); + blocks[explicit_jump.idx()].cold = true; + blocks[explicit_jump.idx()].next = next; + blocks[explicit_jump.idx()].predecessors = 1; + blocks[block_idx.idx()].next = explicit_jump; + let target = blocks[explicit_jump.idx()].next; + let last = basicblock_last_instr_mut(&mut blocks[explicit_jump.idx()]) + .expect("missing explicit jump"); + last.target = target; } block_idx = blocks[block_idx.idx()].next; } + assert!(!blocks[0].cold); let mut cold_blocks: BlockIdx = BlockIdx::NULL; let mut cold_blocks_tail: BlockIdx = BlockIdx::NULL; - assert!(!blocks[0].cold); let mut block_idx = BlockIdx(0); while blocks[block_idx.idx()].next != BlockIdx::NULL { @@ -4529,6 +5079,9 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> break; } + debug_assert!(!blocks[block_idx.idx()].cold); + debug_assert!(blocks[blocks[block_idx.idx()].next.idx()].cold); + let mut block_end = blocks[block_idx.idx()].next; while blocks[block_end.idx()].next != BlockIdx::NULL && blocks[blocks[block_end.idx()].next.idx()].cold @@ -4536,8 +5089,6 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> block_end = blocks[block_end.idx()].next; } - debug_assert!(!blocks[block_idx.idx()].cold); - debug_assert!(blocks[blocks[block_idx.idx()].next.idx()].cold); debug_assert!(blocks[block_end.idx()].cold); debug_assert!( blocks[block_end.idx()].next == BlockIdx::NULL @@ -4564,14 +5115,18 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> } /// flowgraph.c check_cfg +#[allow(clippy::collapsible_if)] fn check_cfg(blocks: &[Block]) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let block = &blocks[block_idx.idx()]; - for (i, ins) in block.instructions.iter().enumerate() { - debug_assert!(!ins.instr.is_assembler()); - if ins.instr.is_terminator() && i + 1 != block.instructions.len() { - return Err(InternalError::MalformedControlFlowGraph); + for i in 0..block.instructions.len() { + let opcode = block.instructions[i].instr; + debug_assert!(!opcode.is_assembler()); + if opcode.is_terminator() { + if i != block.instructions.len() - 1 { + return Err(InternalError::MalformedControlFlowGraph); + } } } block_idx = block.next; @@ -4579,120 +5134,25 @@ fn check_cfg(blocks: &[Block]) -> crate::InternalResult<()> { Ok(()) } -#[derive(Clone, Copy, PartialEq, Eq)] -enum JumpThreadKind { - Plain, - NoInterrupt, -} - -fn jump_thread_kind(instr: AnyInstruction) -> Option { - Some(match instr.into() { - AnyOpcode::Pseudo(PseudoOpcode::Jump) - | AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward) => JumpThreadKind::Plain, - AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) - | AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => JumpThreadKind::NoInterrupt, - _ => return None, - }) -} - -fn threaded_jump_instr(source: AnyInstruction, target: AnyInstruction) -> Option { - let target_kind = jump_thread_kind(target)?; - let source_kind = jump_thread_kind(source)?; - let result_kind = if source_kind == JumpThreadKind::NoInterrupt - && target_kind == JumpThreadKind::NoInterrupt - { - JumpThreadKind::NoInterrupt - } else { - JumpThreadKind::Plain - }; - - Some(match (source.into(), result_kind) { - (AnyOpcode::Pseudo(_), JumpThreadKind::Plain) => PseudoOpcode::Jump.into(), - (AnyOpcode::Pseudo(_), JumpThreadKind::NoInterrupt) => PseudoOpcode::JumpNoInterrupt.into(), - (AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt), JumpThreadKind::Plain) => { - Opcode::JumpBackward.into() - } - (AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt), JumpThreadKind::NoInterrupt) => source, - (AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward), JumpThreadKind::Plain) => { - source - } - ( - AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward), - JumpThreadKind::NoInterrupt, - ) => PseudoOpcode::JumpNoInterrupt.into(), - _ => return None, - }) -} - -/// flowgraph.c optimize_basic_block + jump_thread -fn jump_threading_block(blocks: &mut [Block], block_idx: BlockIdx) -> crate::InternalResult<()> { +/// flowgraph.c jump_thread +fn jump_thread( + blocks: &mut [Block], + block_idx: BlockIdx, + instr_idx: usize, + target: &InstructionInfo, + opcode: AnyInstruction, +) -> crate::InternalResult { let bi = block_idx.idx(); - while let Some(last_idx) = blocks[bi].instructions.len().checked_sub(1) { - let ins = blocks[bi].instructions[last_idx]; - if !(ins.instr.is_unconditional_jump() || is_cfg_conditional_jump(&ins.instr)) { - return Ok(()); - } - let target = ins.target; - debug_assert!(target != BlockIdx::NULL); - debug_assert!(!blocks[target.idx()].instructions.is_empty()); - let target_ins = blocks[target.idx()].instructions[0]; - match ( - ins.instr.pseudo().map(Into::into), - target_ins.instr.pseudo().map(Into::into), - ) { - ( - Some(source @ (PseudoOpcode::JumpIfFalse | PseudoOpcode::JumpIfTrue)), - Some(PseudoOpcode::Jump), - ) - | (Some(source @ PseudoOpcode::JumpIfFalse), Some(PseudoOpcode::JumpIfFalse)) - | (Some(source @ PseudoOpcode::JumpIfTrue), Some(PseudoOpcode::JumpIfTrue)) => { - let final_target = target_ins.target; - debug_assert!(final_target != BlockIdx::NULL); - if final_target == ins.target { - return Ok(()); - } - set_to_nop(&mut blocks[bi].instructions[last_idx]); - basicblock_add_jump(blocks, block_idx, source.into(), final_target, target_ins)?; - continue; - } - (Some(PseudoOpcode::JumpIfFalse), Some(PseudoOpcode::JumpIfTrue)) - | (Some(PseudoOpcode::JumpIfTrue), Some(PseudoOpcode::JumpIfFalse)) => { - let next = blocks[target.idx()].next; - debug_assert!(next != BlockIdx::NULL); - debug_assert!(next != target); - blocks[bi].instructions[last_idx].target = next; - continue; - } - _ => {} - } - if !target_ins.instr.is_unconditional_jump() || target_ins.target == target { - return Ok(()); - } - debug_assert!(target_ins.target != BlockIdx::NULL); - if is_cfg_conditional_jump(&ins.instr) { - if jump_thread_kind(target_ins.instr) != Some(JumpThreadKind::Plain) { - return Ok(()); - } - let final_target = target_ins.target; - if ins.target == final_target { - return Ok(()); - } - set_to_nop(&mut blocks[bi].instructions[last_idx]); - basicblock_add_jump(blocks, block_idx, ins.instr, final_target, target_ins)?; - continue; - } - let final_target = target_ins.target; - let Some(threaded_instr) = threaded_jump_instr(ins.instr, target_ins.instr) else { - return Ok(()); - }; - if ins.target == final_target { - return Ok(()); - } - set_to_nop(&mut blocks[bi].instructions[last_idx]); - basicblock_add_jump(blocks, block_idx, threaded_instr, final_target, target_ins)?; - continue; - } - Ok(()) + debug_assert!(is_jump(&blocks[bi].instructions[instr_idx])); + debug_assert!(is_jump(target)); + debug_assert_eq!(instr_idx + 1, blocks[bi].instructions.len()); + debug_assert!(target.target != BlockIdx::NULL); + if blocks[bi].instructions[instr_idx].target != target.target { + set_to_nop(&mut blocks[bi].instructions[instr_idx]); + basicblock_add_jump(blocks, block_idx, opcode, target.target, target)?; + return Ok(true); + } + Ok(false) } /// flowgraph.c basicblock_add_jump @@ -4701,12 +5161,20 @@ fn basicblock_add_jump( block_idx: BlockIdx, instr: AnyInstruction, target: BlockIdx, - loc_source: InstructionInfo, + loc_source: &InstructionInfo, ) -> crate::InternalResult<()> { let bi = block_idx.idx(); - let arg = cpython_target_label_arg(blocks, target); - basicblock_add_jump_op( - &mut blocks[bi], + let last = basicblock_last_instr(&blocks[bi]); + if last.is_some_and(is_jump) { + return Err(InternalError::MalformedControlFlowGraph); + } + debug_assert!(target != BlockIdx::NULL); + let label = blocks[target.idx()].cpython_label; + debug_assert!(is_label(label)); + let arg = OpArg::new(label.idx().to_u32().expect("too many CPython CFG labels")); + let block = &mut blocks[bi]; + basicblock_addop( + block, InstructionInfo { instr, arg, @@ -4715,10 +5183,18 @@ fn basicblock_add_jump( end_location: loc_source.end_location, except_handler: None, lineno_override: loc_source.lineno_override, - cache_entries: 0, }, - target, - ) + ); + let last = basicblock_last_instr_mut(block).expect("missing jump"); + debug_assert!(match (last.instr, instr) { + (AnyInstruction::Real(last), AnyInstruction::Real(opcode)) => + last.as_opcode() == opcode.as_opcode(), + (AnyInstruction::Pseudo(last), AnyInstruction::Pseudo(opcode)) => + last.as_opcode() == opcode.as_opcode(), + _ => false, + }); + last.target = target; + Ok(()) } /// pycore_opcode_utils.h IS_CONDITIONAL_JUMP_OPCODE @@ -4734,16 +5210,6 @@ fn is_conditional_jump_opcode(instr: &AnyInstruction) -> bool { ) } -/// flowgraph.c optimize_basic_block handles pseudo `JUMP_IF_*` before -/// convert_pseudo_conditional_jumps lowers them to POP_JUMP_IF_*. -fn is_cfg_conditional_jump(instr: &AnyInstruction) -> bool { - is_conditional_jump_opcode(instr) - || matches!( - instr.pseudo(), - Some(PseudoInstruction::JumpIfFalse { .. } | PseudoInstruction::JumpIfTrue { .. }) - ) -} - /// flowgraph.c convert_pseudo_conditional_jumps fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { let mut block_idx = BlockIdx(0); @@ -4752,62 +5218,60 @@ fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { let block = &mut blocks[block_idx.idx()]; let mut i = 0; while i < block.instructions.len() { - let Some(pseudo) = block.instructions[i].instr.pseudo() else { - i += 1; - continue; - }; - let jump = match pseudo { - PseudoInstruction::JumpIfFalse { .. } => { - debug_assert_eq!(i, block.instructions.len() - 1); - Instruction::PopJumpIfFalse { - delta: Arg::marker(), - } - } - PseudoInstruction::JumpIfTrue { .. } => { - debug_assert_eq!(i, block.instructions.len() - 1); - Instruction::PopJumpIfTrue { - delta: Arg::marker(), - } - } - _ => { - i += 1; - continue; - } - }; - - let original = block.instructions[i]; - block.instructions[i].instr = jump.into(); - - let mut copy = original; - copy.instr = Instruction::Copy { i: Arg::marker() }.into(); - copy.arg = OpArg::new(1); - copy.target = BlockIdx::NULL; + let instr = block.instructions[i]; + let opcode = instr.instr; + if matches!( + opcode.pseudo(), + Some(PseudoInstruction::JumpIfFalse { .. } | PseudoInstruction::JumpIfTrue { .. }) + ) { + debug_assert_eq!(i, block.instructions.len() - 1); + block.instructions[i].instr = + if matches!(opcode.pseudo(), Some(PseudoInstruction::JumpIfFalse { .. })) { + Instruction::PopJumpIfFalse { + delta: Arg::marker(), + } + .into() + } else { + Instruction::PopJumpIfTrue { + delta: Arg::marker(), + } + .into() + }; - let mut to_bool = original; - to_bool.instr = Instruction::ToBool.into(); - to_bool.arg = OpArg::new(0); - to_bool.target = BlockIdx::NULL; + let location = instr.location; + let end_location = instr.end_location; + let except_handler = instr.except_handler; + let lineno_override = instr.lineno_override; + let copy = InstructionInfo { + instr: Instruction::Copy { i: Arg::marker() }.into(), + arg: OpArg::new(1), + target: BlockIdx::NULL, + location, + end_location, + except_handler, + lineno_override, + }; + basicblock_insert_instruction(block, i, copy); + i += 1; - basicblock_insert_instruction(block, i, copy); + let to_bool = InstructionInfo { + instr: Instruction::ToBool.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location, + end_location, + except_handler, + lineno_override, + }; + basicblock_insert_instruction(block, i, to_bool); + i += 1; + } i += 1; - basicblock_insert_instruction(block, i, to_bool); - i += 2; } block_idx = next; } } -/// Invert a conditional jump opcode. -fn reversed_conditional(instr: &AnyInstruction) -> Option { - Some(match AnyOpcode::from(*instr).real()? { - Opcode::PopJumpIfFalse => Opcode::PopJumpIfTrue.into(), - Opcode::PopJumpIfTrue => Opcode::PopJumpIfFalse.into(), - Opcode::PopJumpIfNone => Opcode::PopJumpIfNotNone.into(), - Opcode::PopJumpIfNotNone => Opcode::PopJumpIfNone.into(), - _ => return None, - }) -} - /// flowgraph.c normalize_jumps_in_block fn normalize_jumps_in_block( blocks: &mut Vec, @@ -4822,9 +5286,8 @@ fn normalize_jumps_in_block( } debug_assert!(!last_ins.instr.is_assembler()); - let target = last_ins.target; - debug_assert!(target != BlockIdx::NULL); - let is_forward = !blocks[target.idx()].visited; + debug_assert!(last_ins.target != BlockIdx::NULL); + let is_forward = !blocks[last_ins.target.idx()].visited; if is_forward { // Insert NOT_TAKEN after forward conditional jump. @@ -4836,34 +5299,28 @@ fn normalize_jumps_in_block( end_location: last_ins.end_location, except_handler: None, lineno_override: last_ins.lineno_override, - cache_entries: 0, }; basicblock_addop(&mut blocks[idx], not_taken); return Ok(()); } - // Backward conditional jump: invert and create new block - // Transform: `cond_jump T` (backward) - // Into: `reversed_cond_jump b_next` + new block [NOT_TAKEN, JUMP T] + let reversed_opcode = match AnyOpcode::from(last_ins.instr).real() { + Some(Opcode::PopJumpIfNotNone) => Opcode::PopJumpIfNone.into(), + Some(Opcode::PopJumpIfNone) => Opcode::PopJumpIfNotNone.into(), + Some(Opcode::PopJumpIfFalse) => Opcode::PopJumpIfTrue.into(), + Some(Opcode::PopJumpIfTrue) => Opcode::PopJumpIfFalse.into(), + _ => unreachable!("conditional jump has reverse opcode"), + }; + + // Transform 'conditional jump T' to 'reversed_jump b_next' followed by + // 'jump_backwards T'. let loc = last_ins.location; let end_loc = last_ins.end_location; - let reversed = - reversed_conditional(&last_ins.instr).expect("conditional jump has reverse opcode"); - let old_next = blocks[idx].next; - debug_assert!(old_next != BlockIdx::NULL); - let is_cold = blocks[idx].cold; - let target_arg = cpython_target_label_arg(blocks, target); - - // Create new block with NOT_TAKEN + JUMP to original backward target - let new_block_idx = BlockIdx(blocks.len() as u32); - let mut new_block = Block { - cold: is_cold, - start_depth: blocks[target.idx()].start_depth, - ..Block::default() - }; + let target = last_ins.target; + let backwards_jump_idx = blocks_new_block(blocks); basicblock_addop( - &mut new_block, + &mut blocks[backwards_jump_idx.idx()], InstructionInfo { instr: Opcode::NotTaken.into(), arg: OpArg::new(0), @@ -4872,33 +5329,27 @@ fn normalize_jumps_in_block( end_location: end_loc, except_handler: None, lineno_override: last_ins.lineno_override, - cache_entries: 0, }, ); - basicblock_add_jump_op( - &mut new_block, - InstructionInfo { - instr: PseudoOpcode::Jump.into(), - arg: target_arg, - target: BlockIdx::NULL, - location: loc, - end_location: end_loc, - except_handler: None, - lineno_override: last_ins.lineno_override, - cache_entries: 0, - }, + basicblock_add_jump( + blocks, + backwards_jump_idx, + PseudoOpcode::Jump.into(), target, + &last_ins, )?; - new_block.next = old_next; + blocks[backwards_jump_idx.idx()].start_depth = blocks[target.idx()].start_depth; + + let old_next = blocks[idx].next; + debug_assert!(old_next != BlockIdx::NULL); - // Update the conditional jump: invert opcode, target = old next block - let last_mut = blocks[idx].instructions.last_mut().unwrap(); - last_mut.instr = reversed; + let last_mut = basicblock_last_instr_mut(&mut blocks[idx]).unwrap(); + last_mut.instr = reversed_opcode; last_mut.target = old_next; - // Splice new block between current and old next - blocks[idx].next = new_block_idx; - blocks.push(new_block); + blocks[backwards_jump_idx.idx()].cold = blocks[idx].cold; + blocks[backwards_jump_idx.idx()].next = old_next; + blocks[idx].next = backwards_jump_idx; Ok(()) } @@ -4922,8 +5373,6 @@ fn normalize_jumps(blocks: &mut Vec) -> crate::InternalResult<()> { /// flowgraph.c basicblock_inline_small_or_no_lineno_blocks fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: BlockIdx) -> bool { - const MAX_COPY_SIZE: usize = 4; - let Some(last) = basicblock_last_instr(&blocks[block_idx.idx()]).copied() else { return false; }; @@ -4937,29 +5386,29 @@ fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: && blocks[target.idx()].instructions.len() <= MAX_COPY_SIZE; let no_lineno_no_fallthrough = basicblock_has_no_lineno(&blocks[target.idx()]) && !bb_has_fallthrough(&blocks[target.idx()]); - if !small_exit_block && !no_lineno_no_fallthrough { - return false; - } - - let removed_jump_was_jump = matches!(last.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)); - let target_instructions = blocks[target.idx()].instructions.clone(); - if let Some(last_instr) = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]) { - set_to_nop(last_instr); - } - basicblock_append_instructions(&mut blocks[block_idx.idx()], &target_instructions); - if no_lineno_no_fallthrough - && removed_jump_was_jump - && let Some(last) = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]) - && jump_thread_kind(last.instr) == Some(JumpThreadKind::NoInterrupt) - { - last.instr = match last.instr.into() { - AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => PseudoOpcode::Jump.into(), - AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => Opcode::JumpBackward.into(), - _ => last.instr, - }; + if small_exit_block || no_lineno_no_fallthrough { + debug_assert!(is_jump(&last)); + let removed_jump_opcode = last.instr; + let last = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]) + .expect("non-empty block has last instruction"); + set_to_nop(last); + let target_instructions = blocks[target.idx()].instructions.clone(); + basicblock_append_instructions(&mut blocks[block_idx.idx()], &target_instructions); + if no_lineno_no_fallthrough { + let last = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]).unwrap(); + if last.instr.is_unconditional_jump() + && matches!( + removed_jump_opcode.into(), + AnyOpcode::Pseudo(PseudoOpcode::Jump) + ) + { + last.instr = PseudoOpcode::Jump.into(); + } + } + blocks[target.idx()].predecessors -= 1; + return true; } - blocks[target.idx()].predecessors -= 1; - true + false } /// flowgraph.c inline_small_or_no_lineno_blocks @@ -4969,11 +5418,13 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { let mut current = BlockIdx(0); while current != BlockIdx::NULL { let next = blocks[current.idx()].next; - changes |= basicblock_inline_small_or_no_lineno_blocks(blocks, current); + let res = basicblock_inline_small_or_no_lineno_blocks(blocks, current); + if res { + changes = true; + } current = next; } - if !changes { break; } @@ -4983,13 +5434,12 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { /// flowgraph.c basicblock_remove_redundant_nops fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) -> usize { let bi = block_idx.idx(); - let mut instructions = core::mem::take(&mut blocks[bi].instructions); - let spare_instr_slots = core::mem::take(&mut blocks[bi].cpython_spare_instr_slots); let mut dest = 0; let mut prev_lineno = -1i32; + let instr_count = blocks[bi].instructions.len(); - for src in 0..instructions.len() { - let instr = instructions[src]; + for src in 0..instr_count { + let instr = blocks[bi].instructions[src]; let lineno = instruction_lineno(&instr); if matches!(instr.instr.real(), Some(Instruction::Nop)) { @@ -4999,31 +5449,37 @@ fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) - if prev_lineno == lineno { continue; } - if src < instructions.len() - 1 { - let next_lineno = instruction_lineno(&instructions[src + 1]); + if src < instr_count - 1 { + let next_lineno = instruction_lineno(&blocks[bi].instructions[src + 1]); if next_lineno == lineno { continue; - } else if next_lineno < 0 { - copy_instruction_location(instr, &mut instructions[src + 1]); + } + if next_lineno < 0 { + instr_set_loc( + &mut blocks[bi].instructions[src + 1], + instr.location, + instr.end_location, + instr.lineno_override, + ); continue; } } else { let next = next_nonempty_block(blocks, blocks[bi].next); if next != BlockIdx::NULL { - let mut next_location = None; - for next_instr in &blocks[next.idx()].instructions { - let next_lineno = instruction_lineno(next_instr); - if matches!(next_instr.instr.real(), Some(Instruction::Nop)) - && next_lineno < 0 + let mut next_loc = no_linetable_location(); + let mut next_i = 0; + while next_i < blocks[next.idx()].instructions.len() { + let instr = blocks[next.idx()].instructions[next_i]; + if matches!(instr.instr.real(), Some(Instruction::Nop)) + && instruction_lineno(&instr) < 0 { + next_i += 1; continue; } - next_location = Some(next_lineno); + next_loc = instruction_linetable_location(&instr); break; } - if let Some(next_lineno) = next_location - && next_lineno == lineno - { + if lineno == next_loc.line { continue; } } @@ -5031,21 +5487,22 @@ fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) - } if dest != src { - instructions[dest] = instructions[src]; + blocks[bi].instructions[dest] = blocks[bi].instructions[src]; } dest += 1; prev_lineno = lineno; } - let num_removed = instructions.len() - dest; - blocks[bi] - .cpython_spare_instr_slots - .extend(instructions[dest..].iter().copied()); - blocks[bi] - .cpython_spare_instr_slots - .extend(spare_instr_slots); - instructions.truncate(dest); - blocks[bi].instructions = instructions; + debug_assert!(dest <= instr_count); + let num_removed = instr_count - dest; + let mut src = instr_count; + while src > dest { + src -= 1; + blocks[bi] + .cpython_spare_instr_slots + .push(blocks[bi].instructions[src]); + } + blocks[bi].instructions.truncate(dest); num_removed } @@ -5063,7 +5520,10 @@ fn remove_redundant_nops(blocks: &mut [Block]) -> usize { /// flowgraph.c no_redundant_nops #[cfg(debug_assertions)] fn no_redundant_nops(blocks: &mut [Block]) -> bool { - remove_redundant_nops(blocks) == 0 + if remove_redundant_nops(blocks) != 0 { + return false; + } + true } /// flowgraph.c remove_redundant_jumps @@ -5071,48 +5531,47 @@ fn remove_redundant_jumps(blocks: &mut [Block]) -> crate::InternalResult let mut changes = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { - let idx = current.idx(); - let Some(last_instr) = basicblock_last_instr(&blocks[idx]).copied() else { - current = blocks[idx].next; + let block_idx = current.idx(); + let Some(last) = basicblock_last_instr(&blocks[block_idx]).copied() else { + current = blocks[block_idx].next; continue; }; - debug_assert!(!last_instr.instr.is_assembler()); - if last_instr.instr.is_unconditional_jump() { - let jump_target = next_nonempty_block(blocks, last_instr.target); + debug_assert!(!last.instr.is_assembler()); + if last.instr.is_unconditional_jump() { + let jump_target = next_nonempty_block(blocks, last.target); if jump_target == BlockIdx::NULL { return Err(InternalError::MalformedControlFlowGraph); } - let next = next_nonempty_block(blocks, blocks[idx].next); + let next = next_nonempty_block(blocks, blocks[block_idx].next); if jump_target == next { - let last_instr = basicblock_last_instr_mut(&mut blocks[idx]).unwrap(); - set_to_nop(last_instr); changes += 1; + let last = basicblock_last_instr_mut(&mut blocks[block_idx]).unwrap(); + set_to_nop(last); } } - current = blocks[idx].next; + current = blocks[block_idx].next; } Ok(changes) } /// flowgraph.c no_redundant_jumps +#[cfg(debug_assertions)] +#[allow(clippy::collapsible_if)] fn no_redundant_jumps(blocks: &[Block]) -> bool { let mut current = BlockIdx(0); while current != BlockIdx::NULL { let block = &blocks[current.idx()]; - let Some(last) = basicblock_last_instr(block) else { - current = block.next; - continue; - }; - if last.instr.is_unconditional_jump() { - let next = next_nonempty_block(blocks, block.next); - let jump_target = next_nonempty_block(blocks, last.target); - if jump_target == next { - debug_assert!(next != BlockIdx::NULL); - if instruction_lineno(last) - == instruction_lineno(&blocks[next.idx()].instructions[0]) - { - debug_assert!(false, "redundant jump has same line as fallthrough target"); - return false; + if let Some(last) = basicblock_last_instr(block) { + if last.instr.is_unconditional_jump() { + let next = next_nonempty_block(blocks, block.next); + let jump_target = next_nonempty_block(blocks, last.target); + if jump_target == next { + assert!(next != BlockIdx::NULL); + if instruction_lineno(last) + == instruction_lineno(&blocks[next.idx()].instructions[0]) + { + panic!("redundant jump has same line as fallthrough target"); + } } } } @@ -5122,113 +5581,184 @@ fn no_redundant_jumps(blocks: &[Block]) -> bool { } fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResult<()> { - let mut removed_nops; - let mut removed_jumps; loop { - removed_nops = remove_redundant_nops(blocks); - removed_jumps = remove_redundant_jumps(blocks)?; - if removed_nops + removed_jumps > 0 { - continue; + // Convergence is guaranteed because the number of redundant jumps and + // nops only decreases. + let removed_nops = remove_redundant_nops(blocks); + let removed_jumps = remove_redundant_jumps(blocks)?; + if removed_nops + removed_jumps == 0 { + break; } - break; } Ok(()) } /// flowgraph.c make_cfg_traversal_stack fn make_cfg_traversal_stack(blocks: &mut [Block]) -> Vec { + debug_assert!(!blocks.is_empty()); let mut nblocks = 0; - if !blocks.is_empty() { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - blocks[current.idx()].visited = false; - nblocks += 1; - current = blocks[current.idx()].next; - } + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + blocks[current.idx()].visited = false; + nblocks += 1; + current = blocks[current.idx()].next; } Vec::with_capacity(nblocks) } +fn blocks_new_block(blocks: &mut Vec) -> BlockIdx { + let block_idx = BlockIdx(blocks.len() as u32); + blocks.push(Block::default()); + block_idx +} + /// flowgraph.c struct _PyCfgBuilder struct CfgBuilder { blocks: Vec, + entry: BlockIdx, + block_list: BlockIdx, current: BlockIdx, - current_label: Option, + current_label: InstructionSequenceLabel, } -impl CfgBuilder { - /// flowgraph.c init_cfg_builder - fn new_with_capacity(capacity: usize) -> Self { - let mut blocks = Vec::with_capacity(capacity.max(1)); - blocks.push(Block::default()); - Self { - blocks, - current: BlockIdx(0), - current_label: None, - } - } +/// flowgraph.c cfg_builder_new_block +fn cfg_builder_new_block(g: &mut CfgBuilder) -> BlockIdx { + let block = blocks_new_block(&mut g.blocks); + g.blocks[block.idx()].allocation_next = g.block_list; + g.blocks[block.idx()].cpython_label = InstructionSequenceLabel::NO_LABEL; + g.block_list = block; + block +} - /// flowgraph.c cfg_builder_current_block_is_terminated - fn current_block_is_terminated(&mut self) -> bool { - let block = &mut self.blocks[self.current.idx()]; - if block - .instructions - .last() - .is_some_and(|instr| instr.instr.is_terminator()) - { +/// flowgraph.c cfg_builder_use_next_block +fn cfg_builder_use_next_block(g: &mut CfgBuilder, block: BlockIdx) -> BlockIdx { + debug_assert!(block != BlockIdx::NULL); + g.blocks[g.current.idx()].next = block; + g.current = block; + block +} + +/// flowgraph.c init_cfg_builder +fn init_cfg_builder(g: &mut CfgBuilder) { + g.block_list = BlockIdx::NULL; + let block = cfg_builder_new_block(g); + g.entry = block; + g.current = block; + g.current_label = InstructionSequenceLabel::NO_LABEL; +} + +/// flowgraph.c _PyCfgBuilder_New +fn cfg_builder_new() -> CfgBuilder { + let mut builder = CfgBuilder { + blocks: Vec::new(), + entry: BlockIdx::NULL, + block_list: BlockIdx::NULL, + current: BlockIdx::NULL, + current_label: InstructionSequenceLabel::NO_LABEL, + }; + init_cfg_builder(&mut builder); + builder +} + +/// flowgraph.c cfg_builder_current_block_is_terminated +fn cfg_builder_current_block_is_terminated(g: &mut CfgBuilder) -> bool { + let block = &mut g.blocks[g.current.idx()]; + let last = basicblock_last_instr(block).copied(); + if last.is_some_and(|last| last.instr.is_terminator()) { + return true; + } + if is_label(g.current_label) { + if last.is_some() || is_label(block.cpython_label) { return true; } - if let Some(label_id) = self.current_label { - if !block.instructions.is_empty() || block.has_cpython_cfg_label() { - return true; - } - block.cpython_label_id = Some(label_id); - self.current_label = None; - } - false + block.cpython_label = g.current_label; + g.current_label = InstructionSequenceLabel::NO_LABEL; } + false +} - /// flowgraph.c cfg_builder_maybe_start_new_block - fn maybe_start_new_block(&mut self) { - if self.current_block_is_terminated() { - let next = BlockIdx(self.blocks.len() as u32); - let block = Block { - cpython_label_id: self.current_label.take(), - ..Default::default() - }; - self.blocks[self.current.idx()].next = next; - self.blocks.push(block); - self.current = next; - } +/// flowgraph.c cfg_builder_maybe_start_new_block +fn cfg_builder_maybe_start_new_block(g: &mut CfgBuilder) { + if cfg_builder_current_block_is_terminated(g) { + let block = cfg_builder_new_block(g); + g.blocks[block.idx()].cpython_label = g.current_label; + g.current_label = InstructionSequenceLabel::NO_LABEL; + cfg_builder_use_next_block(g, block); } +} - /// flowgraph.c _PyCfgBuilder_UseLabel - fn use_label(&mut self, label_id: InstructionSequenceLabel) { - self.current_label = Some(label_id); - self.maybe_start_new_block(); - } +/// flowgraph.c _PyCfgBuilder_UseLabel +fn cfg_builder_use_label(g: &mut CfgBuilder, label_id: InstructionSequenceLabel) { + g.current_label = label_id; + cfg_builder_maybe_start_new_block(g); +} - /// flowgraph.c _PyCfgBuilder_Addop - fn addop(&mut self, info: InstructionInfo) { - self.maybe_start_new_block(); - basicblock_addop(&mut self.blocks[self.current.idx()], info); - } +/// flowgraph.c _PyCfgBuilder_Addop +fn cfg_builder_addop(g: &mut CfgBuilder, info: InstructionInfo) { + cfg_builder_maybe_start_new_block(g); + basicblock_addop(&mut g.blocks[g.current.idx()], info); +} - fn into_blocks(self) -> Vec { - self.blocks +/// flowgraph.c cfg_builder_check +fn cfg_builder_check(g: &CfgBuilder) -> bool { + debug_assert!(g.entry != BlockIdx::NULL); + debug_assert!(!g.blocks[g.entry.idx()].instructions.is_empty()); + let mut seen = vec![false; g.blocks.len()]; + let mut block = g.block_list; + while block != BlockIdx::NULL { + debug_assert!(block.idx() < g.blocks.len()); + debug_assert!(!seen[block.idx()]); + seen[block.idx()] = true; + let block_ref = &g.blocks[block.idx()]; + let has_instr_array = + !block_ref.instructions.is_empty() || !block_ref.cpython_spare_instr_slots.is_empty(); + if has_instr_array { + debug_assert!(block_ref.instruction_allocation > 0); + debug_assert!(block_ref.instruction_allocation >= block_ref.instructions.len()); + debug_assert!( + block_ref.instruction_allocation + >= block_ref.instructions.len() + block_ref.cpython_spare_instr_slots.len() + ); + } else { + debug_assert!(block_ref.instructions.is_empty()); + debug_assert_eq!(block_ref.instruction_allocation, 0); + } + block = block_ref.allocation_next; } + debug_assert!(seen.into_iter().all(core::convert::identity)); + true +} + +/// flowgraph.c _PyCfgBuilder_CheckSize +fn cfg_builder_check_size(g: &CfgBuilder) -> crate::InternalResult<()> { + debug_assert!(g.entry != BlockIdx::NULL); + debug_assert!(g.block_list != BlockIdx::NULL); + debug_assert!(g.current != BlockIdx::NULL); + let mut nblocks = 0usize; + let mut block = g.block_list; + while block != BlockIdx::NULL { + debug_assert!(block.idx() < g.blocks.len()); + nblocks += 1; + block = g.blocks[block.idx()].allocation_next; + } + debug_assert_eq!(nblocks, g.blocks.len()); + nblocks + .checked_mul(core::mem::size_of::()) + .ok_or(InternalError::MalformedControlFlowGraph)?; + Ok(()) } /// flowgraph.c translate_jump_labels_to_targets fn translate_jump_labels_to_targets(blocks: &mut [Block]) { - let max_label = get_max_cpython_cfg_label(blocks); + let max_label = get_max_label(blocks); let mapsize = (max_label + 1) as usize; let mut label_to_block = vec![BlockIdx::NULL; mapsize]; let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let block = &blocks[block_idx.idx()]; - if let Some(label_id) = block.cpython_label_id { + if is_label(block.cpython_label) { + let label_id = block.cpython_label; debug_assert!(label_id.idx() <= max_label as usize); label_to_block[label_id.idx()] = block_idx; } @@ -5238,14 +5768,19 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) { block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; - for info in &mut blocks[block_idx.idx()].instructions { + for i in 0..blocks[block_idx.idx()].instructions.len() { + let info = &mut blocks[block_idx.idx()].instructions[i]; + debug_assert_eq!(info.target, BlockIdx::NULL); if info.instr.has_target() { - let label_id = InstructionSequenceLabel(u32::from(info.arg) as usize); + let lbl = u32::from(info.arg) as usize; debug_assert!(max_label >= 0); - debug_assert!(label_id.idx() <= max_label as usize); - let target = label_to_block[label_id.idx()]; + debug_assert!(lbl <= max_label as usize); + let target = label_to_block[lbl]; debug_assert!(target != BlockIdx::NULL); - debug_assert_eq!(blocks[target.idx()].cpython_label_id, Some(label_id)); + debug_assert_eq!( + blocks[target.idx()].cpython_label, + InstructionSequenceLabel::from_index(lbl) + ); info.target = target; } } @@ -5253,90 +5788,100 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) { } } +/// flowgraph.c _PyCfg_OptimizeCodeUnit preprocessing +fn optimize_code_unit_preprocess(blocks: &mut [Block]) { + translate_jump_labels_to_targets(blocks); + mark_except_handlers(blocks); + label_exception_targets(blocks); +} + /// flowgraph.c _PyCfg_FromInstructionSequence fn cfg_from_instruction_sequence( mut instr_sequence: InstructionSequence, ) -> crate::InternalResult> { - instr_sequence.apply_label_map()?; - instr_sequence.mark_targets(); + instruction_sequence_apply_label_map(&mut instr_sequence)?; + for i in 0..instr_sequence.instrs.len() { + instr_sequence.instrs[i].i_target = 0; + } + for i in 0..instr_sequence.instrs.len() { + if instr_sequence.instrs[i].info.instr.has_target() { + let target_offset = usize::try_from(u32::from(instr_sequence.instrs[i].info.arg)) + .expect("instruction-sequence target index fits in usize"); + debug_assert!(target_offset < instr_sequence.instrs.len()); + instr_sequence.instrs[target_offset].i_target = 1; + } + } let InstructionSequence { instrs, label_map, annotations_code, + .. } = instr_sequence; - debug_assert!(matches!( - label_map, - InstructionSequenceLabelOffsets::Applied - )); + debug_assert!(label_map.is_none()); - let final_capacity = instrs.len() + annotations_code.as_ref().map_or(0, |seq| seq.instrs.len()); - let mut sequence = Vec::with_capacity(final_capacity); - let mut builder = CfgBuilder::new_with_capacity(final_capacity); - let mut offset_delta = 0isize; + let mut builder = cfg_builder_new(); + let mut offset = 0isize; - for mut entry in instrs { + let mut i = 0; + while i < instrs.len() { + let mut entry = instrs[i]; if matches!( entry.info.instr.pseudo(), Some(PseudoInstruction::AnnotationsPlaceholder) ) { if let Some(annotations_code) = &annotations_code { - debug_assert!(matches!( - annotations_code.label_map, - InstructionSequenceLabelOffsets::Applied - )); + debug_assert!(annotations_code.label_map.is_none()); debug_assert!(annotations_code.annotations_code.is_none()); - for ann_entry in annotations_code.instrs.iter().copied() { + for j in 0..annotations_code.instrs.len() { + let ann_entry = annotations_code.instrs[j]; debug_assert!(!ann_entry.info.instr.has_target()); let mut info = ann_entry.info; info.target = BlockIdx::NULL; - builder.addop(info); - sequence.push(ann_entry); + cfg_builder_addop(&mut builder, info); } - offset_delta += annotations_code.instrs.len() as isize - 1; + offset += annotations_code.instrs.len() as isize - 1; } else { - offset_delta -= 1; + offset -= 1; } + i += 1; continue; } - if entry.is_target { - let label = InstructionSequenceLabel(sequence.len()); - builder.use_label(label); - } - - if let Some(target_offset) = entry.target_offset { - entry.target_offset = Some( - target_offset - .checked_add_signed(offset_delta) + if entry.i_target != 0 { + let label = InstructionSequenceLabel::from_index( + i.checked_add_signed(offset) .ok_or(InternalError::MalformedControlFlowGraph)?, ); + cfg_builder_use_label(&mut builder, label); } - let mut info = entry.info; - if let Some(target_offset) = entry.target_offset { - info.arg = OpArg::new( + + let opcode = entry.info.instr; + let mut oparg = entry.info.arg; + if opcode.has_target() { + let target_offset = usize::try_from(u32::from(oparg)) + .map_err(|_| InternalError::MalformedControlFlowGraph)? + .checked_add_signed(offset) + .ok_or(InternalError::MalformedControlFlowGraph)?; + oparg = OpArg::new( target_offset .to_u32() .ok_or(InternalError::MalformedControlFlowGraph)?, ); } - info.target = BlockIdx::NULL; - builder.addop(info); - sequence.push(entry); - } - - for entry in &sequence { - if entry - .target_offset - .is_some_and(|target_offset| target_offset >= sequence.len()) - { - return Err(InternalError::MalformedControlFlowGraph); - } + entry.info.instr = opcode; + entry.info.arg = oparg; + entry.info.target = BlockIdx::NULL; + cfg_builder_addop(&mut builder, entry.info); + i += 1; } - Ok(builder.into_blocks()) + cfg_builder_check_size(&builder)?; + debug_assert!(cfg_builder_check(&builder)); + Ok(builder.blocks) } -fn maybe_push_local_block( +/// flowgraph.c maybe_push +fn maybe_push( blocks: &mut [Block], worklist: &mut Vec, block: BlockIdx, @@ -5355,6 +5900,8 @@ fn maybe_push_local_block( } } +/// flowgraph.c scan_block_for_locals +#[allow(clippy::collapsible_if)] fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &mut Vec) { let idx = block_idx.idx(); let mut unsafe_mask = blocks[idx].unsafe_locals_mask; @@ -5369,13 +5916,14 @@ fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &m info.except_handler.map(|eh| eh.handler_block), ) }; + debug_assert!(!matches!(instr.real(), Some(Instruction::ExtendedArg))); if let Some(handler_block) = except_handler { - maybe_push_local_block(blocks, worklist, handler_block, unsafe_mask); + maybe_push(blocks, worklist, handler_block, unsafe_mask); } let oparg = u32::from(arg) as usize; - if oparg >= 64 { + if oparg >= LOCAL_UNSAFE_MASK_BITS { continue; } @@ -5387,9 +5935,11 @@ fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &m | AnyInstruction::Pseudo(PseudoInstruction::StoreFastMaybeNull { .. }) => { unsafe_mask |= bit; } - AnyInstruction::Real( - Instruction::StoreFast { .. } | Instruction::LoadFastCheck { .. }, - ) => { + AnyInstruction::Real(Instruction::StoreFast { .. }) => { + unsafe_mask &= !bit; + } + AnyInstruction::Real(Instruction::LoadFastCheck { .. }) => { + // If this doesn't raise, then the local is defined. unsafe_mask &= !bit; } AnyInstruction::Real(Instruction::LoadFast { .. }) => { @@ -5403,17 +5953,92 @@ fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &m } let next = blocks[idx].next; - if blocks[idx].next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { - maybe_push_local_block(blocks, worklist, next, unsafe_mask); + if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { + maybe_push(blocks, worklist, next, unsafe_mask); + } + + let last = basicblock_last_instr(&blocks[idx]).copied(); + if let Some(last) = last { + if is_jump(&last) { + let target = last.target; + debug_assert!(target != BlockIdx::NULL); + maybe_push(blocks, worklist, target, unsafe_mask); + } + } +} + +/// flowgraph.c fast_scan_many_locals +fn fast_scan_many_locals(blocks: &mut [Block], nlocals: usize) { + debug_assert!(nlocals > LOCAL_UNSAFE_MASK_BITS); + let mut states = vec![0usize; nlocals - LOCAL_UNSAFE_MASK_BITS]; + let mut blocknum = 0usize; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + blocknum += 1; + for i in 0..blocks[current.idx()].instructions.len() { + let info = &mut blocks[current.idx()].instructions[i]; + debug_assert!(!matches!(info.instr.real(), Some(Instruction::ExtendedArg))); + let arg = u32::from(info.arg) as usize; + if arg < LOCAL_UNSAFE_MASK_BITS { + continue; + } + match info.instr { + AnyInstruction::Real( + Instruction::DeleteFast { .. } | Instruction::LoadFastAndClear { .. }, + ) + | AnyInstruction::Pseudo(PseudoInstruction::StoreFastMaybeNull { .. }) => { + debug_assert!(arg < nlocals); + states[arg - LOCAL_UNSAFE_MASK_BITS] = blocknum - 1; + } + AnyInstruction::Real(Instruction::StoreFast { .. }) => { + debug_assert!(arg < nlocals); + states[arg - LOCAL_UNSAFE_MASK_BITS] = blocknum; + } + AnyInstruction::Real(Instruction::LoadFast { .. }) => { + debug_assert!(arg < nlocals); + if states[arg - LOCAL_UNSAFE_MASK_BITS] != blocknum { + info.instr = Opcode::LoadFastCheck.into(); + } + states[arg - LOCAL_UNSAFE_MASK_BITS] = blocknum; + } + _ => {} + } + } + current = blocks[current.idx()].next; + } +} + +/// flowgraph.c add_checks_for_loads_of_uninitialized_variables +fn add_checks_for_loads_of_uninitialized_variables( + blocks: &mut [Block], + mut nlocals: usize, + nparams: usize, +) { + if nlocals == 0 { + return; + } + + if nlocals > LOCAL_UNSAFE_MASK_BITS { + fast_scan_many_locals(blocks, nlocals); + nlocals = LOCAL_UNSAFE_MASK_BITS; } - let jump_target = blocks[idx] - .instructions - .last() - .and_then(|last| is_jump(last).then_some(last.target)); - if let Some(target) = jump_target { - debug_assert!(target != BlockIdx::NULL); - maybe_push_local_block(blocks, worklist, target, unsafe_mask); + let mut worklist = make_cfg_traversal_stack(blocks); + let mut start_mask = 0u64; + for i in nparams..nlocals { + start_mask |= 1u64 << i; + } + maybe_push(blocks, &mut worklist, BlockIdx(0), start_mask); + + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + scan_block_for_locals(blocks, current, &mut worklist); + current = blocks[current.idx()].next; + } + + while let Some(block_idx) = worklist.pop() { + blocks[block_idx.idx()].visited = false; + scan_block_for_locals(blocks, block_idx, &mut worklist); } } @@ -5432,30 +6057,32 @@ fn instruction_lineno(instr: &InstructionInfo) -> i32 { } } -fn instruction_has_lineno(instr: &InstructionInfo) -> bool { - instruction_lineno(instr) >= 0 -} - fn instruction_is_no_location(instr: &InstructionInfo) -> bool { - instruction_lineno(instr) == -1 + instruction_lineno(instr) == NO_LOCATION_OVERRIDE } -fn copy_instruction_location(source: InstructionInfo, target: &mut InstructionInfo) { - target.location = source.location; - target.end_location = source.end_location; - target.lineno_override = source.lineno_override; +/// flowgraph.c basicblock_nofallthrough +fn basicblock_nofallthrough(block: &Block) -> bool { + let last = basicblock_last_instr(block); + last.is_some_and(|last| last.instr.is_scope_exit() || last.instr.is_unconditional_jump()) } -/// flowgraph.c basicblock_nofallthrough / BB_NO_FALLTHROUGH -fn basicblock_nofallthrough(block: &Block) -> bool { - basicblock_last_instr(block).is_some_and(|ins| ins.instr.is_no_fallthrough()) +/// flowgraph.c BB_NO_FALLTHROUGH +fn bb_no_fallthrough(block: &Block) -> bool { + basicblock_nofallthrough(block) } /// flowgraph.c BB_HAS_FALLTHROUGH fn bb_has_fallthrough(block: &Block) -> bool { - !basicblock_nofallthrough(block) + !bb_no_fallthrough(block) } +/// flowgraph.c add_checks_for_loads_of_uninitialized_variables uses uint64_t masks. +const LOCAL_UNSAFE_MASK_BITS: usize = 64; + +/// flowgraph.c MAX_COPY_SIZE +const MAX_COPY_SIZE: usize = 4; + /// flowgraph.c is_jump fn is_jump(instr: &InstructionInfo) -> bool { instr.instr.has_jump() @@ -5470,18 +6097,22 @@ fn is_block_push(instr: &InstructionInfo) -> bool { /// flowgraph.c basicblock_returns #[cfg(test)] fn basicblock_returns(block: &Block) -> bool { - basicblock_last_instr(block) - .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::ReturnValue))) + let last = basicblock_last_instr(block); + if let Some(last) = last { + matches!(last.instr.real(), Some(Instruction::ReturnValue)) + } else { + false + } } /// flowgraph.c basicblock_exits_scope fn basicblock_exits_scope(block: &Block) -> bool { - basicblock_last_instr(block).is_some_and(|instr| instr.instr.is_scope_exit()) + let last = basicblock_last_instr(block); + last.is_some_and(|last| last.instr.is_scope_exit()) } /// flowgraph.c is_exit_or_eval_check_without_lineno -fn is_exit_or_eval_check_without_lineno(blocks: &[Block], block_idx: BlockIdx) -> bool { - let block = &blocks[block_idx.idx()]; +fn is_exit_or_eval_check_without_lineno(block: &Block) -> bool { if basicblock_exits_scope(block) || basicblock_has_eval_break(block) { basicblock_has_no_lineno(block) } else { @@ -5491,50 +6122,31 @@ fn is_exit_or_eval_check_without_lineno(blocks: &[Block], block_idx: BlockIdx) - /// flowgraph.c basicblock_has_eval_break fn basicblock_has_eval_break(block: &Block) -> bool { - block - .instructions - .iter() - .any(|info| info.instr.has_eval_break()) + let mut i = 0; + while i < block.instructions.len() { + if block.instructions[i].instr.has_eval_break() { + return true; + } + i += 1; + } + false } /// flowgraph.c basicblock_has_no_lineno fn basicblock_has_no_lineno(block: &Block) -> bool { - block - .instructions - .iter() - .all(|ins| !instruction_has_lineno(ins)) -} - -fn maybe_propagate_location( - instr: &mut InstructionInfo, - location: SourceLocation, - end_location: SourceLocation, - lineno_override: Option, -) { - if instruction_is_no_location(instr) { - instr.location = location; - instr.end_location = end_location; - instr.lineno_override = lineno_override; + let mut i = 0; + while i < block.instructions.len() { + if instruction_lineno(&block.instructions[i]) >= 0 { + return false; + } + i += 1; } -} - -fn overwrite_location( - instr: &mut InstructionInfo, - location: SourceLocation, - end_location: SourceLocation, - lineno_override: Option, -) { - instr.location = location; - instr.end_location = end_location; - instr.lineno_override = lineno_override; + true } /// flowgraph.c remove_unreachable computes `b_predecessors` by traversing only /// blocks reachable from entry. fn compute_reachable_predecessors(blocks: &mut [Block]) { - if blocks.is_empty() { - return; - } let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { blocks[block_idx.idx()].predecessors = 0; @@ -5543,15 +6155,16 @@ fn compute_reachable_predecessors(blocks: &mut [Block]) { let mut stack = make_cfg_traversal_stack(blocks); blocks[0].predecessors = 1; - blocks[0].visited = true; stack.push(BlockIdx(0)); + blocks[0].visited = true; while let Some(current) = stack.pop() { let idx = current.idx(); let next = blocks[idx].next; if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { if !blocks[next.idx()].visited { - blocks[next.idx()].visited = true; + debug_assert_eq!(blocks[next.idx()].predecessors, 0); stack.push(next); + blocks[next.idx()].visited = true; } blocks[next.idx()].predecessors += 1; } @@ -5562,218 +6175,199 @@ fn compute_reachable_predecessors(blocks: &mut [Block]) { if is_jump(&instr) || is_block_push(&instr) { let target = instr.target; debug_assert!(target != BlockIdx::NULL); - if !blocks[target.idx()].visited { - blocks[target.idx()].visited = true; + let target_idx = target.idx(); + if !blocks[target_idx].visited { stack.push(target); + blocks[target_idx].visited = true; } - blocks[target.idx()].predecessors += 1; + blocks[target_idx].predecessors += 1; } } } } /// flowgraph.c copy_basicblock -fn copy_basicblock(blocks: &[Block], block_idx: BlockIdx) -> Block { - let block = &blocks[block_idx.idx()]; - debug_assert!(!bb_has_fallthrough(block)); - let mut result = Block::default(); - basicblock_append_instructions(&mut result, &block.instructions); +fn copy_basicblock(blocks: &mut Vec, block_idx: BlockIdx) -> BlockIdx { + debug_assert!(bb_no_fallthrough(&blocks[block_idx.idx()])); + let result = blocks_new_block(blocks); + let result_idx = result.idx(); + let (blocks_before_result, result_block) = blocks.split_at_mut(result_idx); + let block = &blocks_before_result[block_idx.idx()]; + basicblock_append_instructions(&mut result_block[0], &block.instructions); result } /// flowgraph.c get_max_label -fn get_max_cpython_cfg_label(blocks: &[Block]) -> isize { - let mut label = -1; +fn get_max_label(blocks: &[Block]) -> i32 { + let mut lbl = -1; let mut current = BlockIdx(0); while current != BlockIdx::NULL { - if let Some(cpython_label) = blocks[current.idx()].cpython_label_id { - label = label.max(cpython_label.idx() as isize); - } + let cpython_label = blocks[current.idx()].cpython_label; + lbl = lbl.max(cpython_label.0); current = blocks[current.idx()].next; } - label -} - -fn next_cpython_cfg_label(blocks: &[Block]) -> usize { - (get_max_cpython_cfg_label(blocks) + 1) as usize + lbl } +#[allow(clippy::collapsible_if)] fn duplicate_exits_without_lineno(blocks: &mut Vec) { - let mut next_label = next_cpython_cfg_label(blocks); - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let block = &blocks[current.idx()]; - let last = match block.instructions.last() { - Some(ins) if is_jump(ins) => { - debug_assert!(ins.target != BlockIdx::NULL); - ins - } - _ => { - current = blocks[current.idx()].next; - continue; - } - }; + let mut next_lbl = get_max_label(blocks) + 1; - let target = next_nonempty_block(blocks, last.target); - debug_assert!(target != BlockIdx::NULL); - if !is_exit_or_eval_check_without_lineno(blocks, target) - || blocks[target.idx()].predecessors <= 1 - { - current = blocks[current.idx()].next; + let entryblock = BlockIdx(0); + let mut b = entryblock; + while b != BlockIdx::NULL { + let Some(last) = basicblock_last_instr(&blocks[b.idx()]).copied() else { + b = blocks[b.idx()].next; continue; + }; + if is_jump(&last) { + debug_assert!(last.target != BlockIdx::NULL); + let target = next_nonempty_block(blocks, last.target); + debug_assert!(target != BlockIdx::NULL); + if is_exit_or_eval_check_without_lineno(&blocks[target.idx()]) + && blocks[target.idx()].predecessors > 1 + { + let new_target = copy_basicblock(blocks, target); + instr_set_location( + &mut blocks[new_target.idx()].instructions[0], + instr_location(&last), + ); + let last_mut = basicblock_last_instr_mut(&mut blocks[b.idx()]).unwrap(); + last_mut.target = new_target; + blocks[target.idx()].predecessors -= 1; + blocks[new_target.idx()].predecessors = 1; + blocks[new_target.idx()].next = blocks[target.idx()].next; + blocks[new_target.idx()].cpython_label = InstructionSequenceLabel(next_lbl); + next_lbl += 1; + blocks[target.idx()].next = new_target; + } } - - let new_idx = BlockIdx(blocks.len() as u32); - let mut new_block = copy_basicblock(blocks, target); - overwrite_location( - &mut new_block.instructions[0], - last.location, - last.end_location, - last.lineno_override, - ); - let old_next = blocks[target.idx()].next; - new_block.next = old_next; - new_block.predecessors = 1; - new_block.cpython_label_id = Some(InstructionSequenceLabel(next_label)); - next_label += 1; - blocks.push(new_block); - blocks[target.idx()].next = new_idx; - - let last_mut = blocks[current.idx()].instructions.last_mut().unwrap(); - last_mut.target = new_idx; - blocks[target.idx()].predecessors -= 1; - current = blocks[current.idx()].next; + b = blocks[b.idx()].next; } - current = BlockIdx(0); - while current != BlockIdx::NULL { - let block = &blocks[current.idx()]; - if bb_has_fallthrough(block) - && block.next != BlockIdx::NULL - && !block.instructions.is_empty() + b = entryblock; + while b != BlockIdx::NULL { + let next = blocks[b.idx()].next; + if bb_has_fallthrough(&blocks[b.idx()]) + && next != BlockIdx::NULL + && !blocks[b.idx()].instructions.is_empty() { - let target = block.next; - let last = *block.instructions.last().expect("block has instructions"); - if is_exit_or_eval_check_without_lineno(blocks, target) { - overwrite_location( - &mut blocks[target.idx()].instructions[0], - last.location, - last.end_location, - last.lineno_override, + if is_exit_or_eval_check_without_lineno(&blocks[next.idx()]) { + let last = + *basicblock_last_instr(&blocks[b.idx()]).expect("block has instructions"); + instr_set_location( + &mut blocks[next.idx()].instructions[0], + instr_location(&last), ); } } - current = blocks[current.idx()].next; + b = blocks[b.idx()].next; } } +#[allow(clippy::collapsible_if)] fn propagate_line_numbers(blocks: &mut [Block]) { let mut current = BlockIdx(0); while current != BlockIdx::NULL { let idx = current.idx(); - let Some(last) = blocks[idx].instructions.last().copied() else { + let Some(last) = basicblock_last_instr(&blocks[idx]).copied() else { current = blocks[idx].next; continue; }; - let mut prev_location = None; - for instr in &mut blocks[idx].instructions { - if let Some((location, end_location, lineno_override)) = prev_location { - maybe_propagate_location(instr, location, end_location, lineno_override); - } - if !instruction_is_no_location(instr) { - prev_location = Some((instr.location, instr.end_location, instr.lineno_override)); + let mut prev_location = no_instruction_location(); + for i in 0..blocks[idx].instructions.len() { + if instruction_is_no_location(&blocks[idx].instructions[i]) { + instr_set_location(&mut blocks[idx].instructions[i], prev_location); + } else { + prev_location = instr_location(&blocks[idx].instructions[i]); } } let next = blocks[idx].next; if bb_has_fallthrough(&blocks[idx]) { debug_assert!(next != BlockIdx::NULL); - if blocks[next.idx()].predecessors == 1 - && !blocks[next.idx()].instructions.is_empty() - && let Some((location, end_location, lineno_override)) = prev_location - { - maybe_propagate_location( - &mut blocks[next.idx()].instructions[0], - location, - end_location, - lineno_override, - ); + if next != BlockIdx::NULL && blocks[next.idx()].predecessors == 1 { + if !blocks[next.idx()].instructions.is_empty() { + if instruction_is_no_location(&blocks[next.idx()].instructions[0]) { + instr_set_location(&mut blocks[next.idx()].instructions[0], prev_location); + } + } } } if is_jump(&last) { let target = last.target; debug_assert!(target != BlockIdx::NULL); - if blocks[target.idx()].predecessors == 1 - && let Some((location, end_location, lineno_override)) = prev_location - { - maybe_propagate_location( - basicblock_raw_first_instr_mut(&mut blocks[target.idx()]), - location, - end_location, - lineno_override, - ); + if blocks[target.idx()].predecessors == 1 { + let instr = basicblock_raw_first_instr_mut(&mut blocks[target.idx()]); + if instruction_is_no_location(instr) { + instr_set_location(instr, prev_location); + } } } current = blocks[current.idx()].next; } } -fn resolve_line_numbers(blocks: &mut Vec) { +fn resolve_line_numbers(blocks: &mut Vec, _firstlineno: OneIndexed) { duplicate_exits_without_lineno(blocks); propagate_line_numbers(blocks); } -pub(crate) fn label_exception_targets(blocks: &mut [Block]) { - fn except_stack_top(stack: &[BlockIdx], blocks: &[Block]) -> Option { - let handler_block = stack.last().copied()?; - if handler_block == BlockIdx::NULL { - return None; - } - Some(ExceptHandlerInfo { - handler_block, - stack_depth: 0, - preserve_lasti: blocks[handler_block.idx()].preserve_lasti, - }) - } +/// flowgraph.c make_except_stack +fn make_except_stack() -> Vec { + Vec::new() +} - fn push_except_block( - stack: &mut Vec, - setup: InstructionInfo, - blocks: &mut [Block], - ) -> Option { - debug_assert!(is_block_push(&setup)); - let instr = setup.instr; - let target = setup.target; - debug_assert!(target != BlockIdx::NULL); - if matches!( - instr.pseudo(), - Some(PseudoInstruction::SetupWith { .. } | PseudoInstruction::SetupCleanup { .. }) - ) { - blocks[target.idx()].preserve_lasti = true; - } - stack.push(target); - except_stack_top(stack, blocks) - } +/// flowgraph.c copy_except_stack +fn copy_except_stack(stack: &[BlockIdx]) -> Vec { + stack.to_vec() +} - fn pop_except_block(stack: &mut Vec, blocks: &[Block]) -> Option { - debug_assert!(!stack.is_empty()); - stack.pop().expect("non-empty exception stack"); - except_stack_top(stack, blocks) - } +/// flowgraph.c except_stack_top +fn except_stack_top(stack: &[BlockIdx], blocks: &[Block]) -> Option { + let handler_block = stack.last().copied()?; + Some(ExceptHandlerInfo { + handler_block, + preserve_lasti: blocks[handler_block.idx()].preserve_lasti, + }) +} - let num_blocks = blocks.len(); - if num_blocks == 0 { - return; +/// flowgraph.c push_except_block +fn push_except_block( + stack: &mut Vec, + setup: InstructionInfo, + blocks: &mut [Block], +) -> Option { + debug_assert!(is_block_push(&setup)); + let instr = setup.instr; + let target = setup.target; + debug_assert!(target != BlockIdx::NULL); + if matches!( + instr.pseudo(), + Some(PseudoInstruction::SetupWith { .. } | PseudoInstruction::SetupCleanup { .. }) + ) { + blocks[target.idx()].preserve_lasti = true; } + debug_assert!(stack.len() <= CO_MAXBLOCKS); + stack.push(target); + except_stack_top(stack, blocks) +} + +/// flowgraph.c pop_except_block +fn pop_except_block(stack: &mut Vec, blocks: &[Block]) -> Option { + debug_assert!(!stack.is_empty()); + stack.pop().expect("non-empty exception stack"); + except_stack_top(stack, blocks) +} +pub(crate) fn label_exception_targets(blocks: &mut [Block]) { let mut todo = make_cfg_traversal_stack(blocks); - // Entry block - blocks[0].visited = true; - blocks[0].except_stack = Some(Vec::new()); todo.push(BlockIdx(0)); + blocks[0].visited = true; + blocks[0].except_stack = Some(make_except_stack()); while let Some(block_idx) = todo.pop() { let bi = block_idx.idx(); @@ -5795,9 +6389,9 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { if is_block_push(&info) { debug_assert!(target != BlockIdx::NULL); if !blocks[target.idx()].visited { - blocks[target.idx()].visited = true; - blocks[target.idx()].except_stack = Some(stack.clone()); + blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)); todo.push(target); + blocks[target.idx()].visited = true; } handler = push_except_block(&mut stack, info, blocks); } else if instr.is_pop_block() { @@ -5812,13 +6406,13 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { // to the jump target. debug_assert!(target != BlockIdx::NULL); if !blocks[target.idx()].visited { - blocks[target.idx()].visited = true; - blocks[target.idx()].except_stack = Some(if bb_has_fallthrough(&blocks[bi]) { - stack.clone() + if bb_has_fallthrough(&blocks[bi]) { + blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)); } else { - core::mem::take(&mut stack) - }); + blocks[target.idx()].except_stack = Some(core::mem::take(&mut stack)); + } todo.push(target); + blocks[target.idx()].visited = true; } } else if matches!(instr.real(), Some(Instruction::YieldValue { .. })) { blocks[bi].instructions[i].except_handler = handler; @@ -5843,9 +6437,9 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { if bb_has_fallthrough(&blocks[bi]) { debug_assert!(next != BlockIdx::NULL); if !blocks[next.idx()].visited { - blocks[next.idx()].visited = true; blocks[next.idx()].except_stack = Some(stack); todo.push(next); + blocks[next.idx()].visited = true; } } } @@ -5867,35 +6461,31 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block]) -> crate::InternalResult< while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; - for info in &mut block.instructions { - let Some(pseudo) = info.instr.pseudo() else { - continue; - }; - match pseudo { - // Block push pseudo ops → NOP - PseudoInstruction::SetupCleanup { .. } - | PseudoInstruction::SetupFinally { .. } - | PseudoInstruction::SetupWith { .. } => { - set_to_nop(info); - } - PseudoInstruction::LoadClosure { .. } => { - info.instr = Opcode::LoadFast.into(); - } - // Jump pseudo ops are resolved during block linearization - PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. } => {} - PseudoInstruction::StoreFastMaybeNull { .. } => { - info.instr = Instruction::StoreFast { - var_num: Arg::marker(), - } - .into(); - } - // These should have been resolved earlier - PseudoInstruction::PopBlock - | PseudoInstruction::AnnotationsPlaceholder - | PseudoInstruction::JumpIfFalse { .. } - | PseudoInstruction::JumpIfTrue { .. } => { - unreachable!("Unexpected pseudo instruction in convert_pseudo_ops: {pseudo:?}") + for i in 0..block.instructions.len() { + let info = &mut block.instructions[i]; + if is_block_push(info) { + set_to_nop(info); + } else if matches!( + info.instr.pseudo(), + Some(PseudoInstruction::LoadClosure { .. }) + ) { + debug_assert!(is_pseudo_target( + PseudoOpcode::LoadClosure, + Opcode::LoadFast + )); + info.instr = Opcode::LoadFast.into(); + } else if matches!( + info.instr.pseudo(), + Some(PseudoInstruction::StoreFastMaybeNull { .. }) + ) { + debug_assert!(is_pseudo_target( + PseudoOpcode::StoreFastMaybeNull, + Opcode::StoreFast + )); + info.instr = Instruction::StoreFast { + var_num: Arg::marker(), } + .into(); } } block_idx = next; @@ -5906,68 +6496,92 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block]) -> crate::InternalResult< } /// flowgraph.c build_cellfixedoffsets -pub(crate) fn build_cellfixedoffsets( - varnames: &IndexSet, - cellvars: &IndexSet, - freevars: &IndexSet, -) -> Vec { - let nlocals = varnames.len(); - let ncells = cellvars.len(); - let nfrees = freevars.len(); - let mut fixed = (0..ncells + nfrees) - .map(|i| (nlocals + i).to_u32().expect("too many localsplus slots")) - .collect::>(); - for (oldindex, cellvar) in cellvars.iter().enumerate() { - if let Some(local_idx) = varnames.get_index_of(cellvar) { - fixed[oldindex] = local_idx.to_u32().expect("too many localsplus slots"); +#[allow(clippy::needless_range_loop)] +pub(crate) fn build_cellfixedoffsets(metadata: &CodeUnitMetadata) -> Vec { + let nlocals = metadata.varnames.len(); + let ncellvars = metadata.cellvars.len(); + let nfreevars = metadata.freevars.len(); + let noffsets = ncellvars + nfreevars; + let mut fixed = vec![0; noffsets]; + for i in 0..noffsets { + fixed[i] = (nlocals + i) + .to_i32() + .expect("localsplus offset fits in int"); + } + for oldindex in 0..ncellvars { + let varname = metadata + .cellvars + .get_index(oldindex) + .expect("cellvar index is in range"); + if let Some(varindex) = metadata.varnames.get_index_of(varname) { + let argoffset = varindex.to_i32().expect("localsplus offset fits in int"); + fixed[oldindex] = argoffset; } } fixed } -/// First half of flowgraph.c fix_cell_offsets. -fn fix_cellfixedoffsets(nlocals: usize, fixedmap: &mut [u32]) -> usize { +/// flowgraph.c fix_cell_offsets +#[allow(clippy::needless_range_loop)] +pub(crate) fn fix_cell_offsets( + metadata: &CodeUnitMetadata, + blocks: &mut [Block], + cellfixedoffsets: &mut [i32], +) -> usize { + let nlocals = metadata.varnames.len(); + let ncellvars = metadata.cellvars.len(); + let nfreevars = metadata.freevars.len(); + let noffsets = ncellvars + nfreevars; + debug_assert_eq!(cellfixedoffsets.len(), noffsets); + let mut numdropped = 0usize; - for (i, fixed) in fixedmap.iter_mut().enumerate() { - if usize::try_from(*fixed).expect("localsplus index overflow") == i + nlocals { - *fixed -= numdropped.to_u32().expect("too many dropped cell vars"); + for i in 0..noffsets { + if cellfixedoffsets[i] + == (i + nlocals) + .to_i32() + .expect("localsplus index fits in int") + { + cellfixedoffsets[i] -= numdropped.to_i32().expect("too many dropped cell vars"); } else { numdropped += 1; } } - numdropped -} -/// flowgraph.c fix_cell_offsets -pub(crate) fn fix_cell_offsets( - blocks: &mut [Block], - nlocals: usize, - cellfixedoffsets: &mut [u32], -) -> usize { - let numdropped = fix_cellfixedoffsets(nlocals, cellfixedoffsets); let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; - for info in &mut block.instructions { + for i in 0..block.instructions.len() { + let inst = &mut block.instructions[i]; debug_assert!( - !matches!(info.instr.real(), Some(Instruction::ExtendedArg)), + !matches!(inst.instr.real(), Some(Instruction::ExtendedArg)), "fix_cell_offsets is called before extended args are generated" ); - let needs_fixup = matches!( - info.instr, + let oldoffset = i32::try_from(u32::from(inst.arg)).expect("oparg fits in int"); + match inst.instr { AnyInstruction::Real( - Instruction::LoadDeref { .. } - | Instruction::StoreDeref { .. } - | Instruction::DeleteDeref { .. } - | Instruction::LoadFromDictOrDeref { .. } - | Instruction::MakeCell { .. } - ) | AnyInstruction::Pseudo(PseudoInstruction::LoadClosure { .. }) - ); - if needs_fixup { - let cell_relative = u32::from(info.arg) as usize; - debug_assert!(cell_relative < cellfixedoffsets.len()); - info.arg = OpArg::new(cellfixedoffsets[cell_relative]); + Instruction::MakeCell { .. } + | Instruction::LoadDeref { .. } + | Instruction::StoreDeref { .. } + | Instruction::DeleteDeref { .. } + | Instruction::LoadFromDictOrDeref { .. }, + ) + | AnyInstruction::Pseudo(PseudoInstruction::LoadClosure { .. }) => { + debug_assert!(oldoffset >= 0); + debug_assert!( + oldoffset < noffsets.to_i32().expect("localsplus offset fits in int") + ); + let oldoffset = + usize::try_from(oldoffset).expect("localsplus offset is non-negative"); + let fixed_offset = cellfixedoffsets[oldoffset]; + debug_assert!(fixed_offset >= 0); + inst.arg = OpArg::new( + fixed_offset + .to_u32() + .expect("localsplus offset is non-negative"), + ); + } + _ => {} } } block_idx = next; @@ -5995,7 +6609,6 @@ mod tests { end_location: test_location(line), except_handler: None, lineno_override: None, - cache_entries: 0, } } @@ -6020,7 +6633,7 @@ mod tests { private: None, blocks: vec![block], current_block: BlockIdx::new(0), - instr_sequence: InstructionSequence::new(), + instr_sequence: instruction_sequence_new(), instr_sequence_label_map: InstructionSequenceLabelMap::new(), annotations_instr_sequence: None, metadata: CodeUnitMetadata { @@ -6042,6 +6655,7 @@ mod tests { in_inlined_comp: false, fblock: Vec::new(), symbol_table_index: 0, + nparams: 0, in_conditional_block: 0, next_conditional_annotation_index: 0, } @@ -6049,36 +6663,110 @@ mod tests { #[test] fn instruction_sequence_label_shadow_preserves_cpython_offset_aliases() { + let mut seq = instruction_sequence_new(); let mut labels = InstructionSequenceLabelMap::new(); - labels.push_unmapped_label(); - labels.push_unmapped_label(); + instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq); + instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq); let first = BlockIdx::new(1); let second = BlockIdx::new(2); assert_ne!( - labels.label_for_block(first), - labels.label_for_block(second) + instruction_sequence_label_map_label_for_block(&labels, first), + instruction_sequence_label_map_label_for_block(&labels, second) ); // CPython `_PyInstructionSequence_UseLabel()` can map consecutive // labels to the same instruction offset. The codegen CFG shadow must // resolve the later block label to the block owning that shared offset. - labels.use_label_at_block(second, first); - assert_eq!(labels.resolve_label(first), first); - assert_eq!(labels.resolve_label(second), first); + instruction_sequence_label_map_use_label_at_block(&mut labels, &mut seq, second, first); + assert_eq!( + instruction_sequence_label_map_resolve_label(&labels, first), + first + ); + assert_eq!( + instruction_sequence_label_map_resolve_label(&labels, second), + first + ); + } + + #[test] + fn instruction_sequence_insert_preserves_cpython_slot_metadata() { + let handler = InstructionSequenceExceptHandlerInfo { + h_label: 7, + start_depth: 3, + preserve_lasti: 1, + }; + let mut seq = instruction_sequence_new(); + let entry = instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 11)); + entry.except_handler = handler; + entry.i_target = 1; + entry.i_offset = 42; + + instruction_sequence_insert_instruction(&mut seq, 0, test_instr(Instruction::PopTop, 12)); + + // CPython `_PyInstructionSequence_InsertInstruction()` shifts the + // backing instruction slots, then overwrites only opcode/oparg/loc. + let inserted = &seq.instrs[0]; + assert!(matches!( + inserted.info.instr.real(), + Some(Instruction::PopTop) + )); + assert_eq!(inserted.except_handler.h_label, handler.h_label); + assert_eq!(inserted.except_handler.start_depth, handler.start_depth); + assert_eq!( + inserted.except_handler.preserve_lasti, + handler.preserve_lasti + ); + assert_eq!(inserted.i_target, 1); + assert_eq!(inserted.i_offset, 42); + } + + #[test] + fn instruction_sequence_tracks_cpython_c_array_allocation() { + let mut seq = instruction_sequence_new(); + for i in 0..99 { + instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 10 + i)); + } + assert_eq!(seq.instr_allocation, INITIAL_INSTR_SEQUENCE_SIZE); + + // CPython calls `_Py_CArray_EnsureCapacity(s_used + 1)`, so the 100th + // instruction expands a 100-slot array to 200 before returning offset 99. + instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 109)); + assert_eq!(seq.instr_allocation, INITIAL_INSTR_SEQUENCE_SIZE * 2); + } + + #[test] + fn instruction_sequence_label_map_tracks_cpython_c_array_allocation() { + let mut seq = instruction_sequence_new(); + instruction_sequence_use_label(&mut seq, InstructionSequenceLabel::from_index(1)); + assert_eq!( + seq.label_map_allocation, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE + ); + assert_eq!( + seq.label_map.as_ref().expect("label map allocated").len(), + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE + ); + + // CPython passes the label id itself to `_Py_CArray_EnsureCapacity()`. + // Label 10 therefore expands the initial 10-slot map to 20. + instruction_sequence_use_label(&mut seq, InstructionSequenceLabel::from_index(10)); + assert_eq!( + seq.label_map_allocation, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE * 2 + ); } #[test] fn basicblock_addop_reuses_cpython_spare_except_handler_slot() { let handler = ExceptHandlerInfo { handler_block: BlockIdx::new(7), - stack_depth: 3, preserve_lasti: true, }; let mut block = Block::default(); let mut stale = test_instr(Instruction::Nop, 11); stale.except_handler = Some(handler); - block.cpython_spare_instr_slots.push_back(stale); + block.cpython_spare_instr_slots.push(stale); basicblock_addop(&mut block, test_instr(Instruction::PopTop, 12)); @@ -6090,18 +6778,31 @@ mod tests { assert_eq!(block.instructions[0].target, BlockIdx::NULL); } + #[test] + fn basicblock_next_instr_tracks_cpython_c_array_allocation() { + let mut block = Block::default(); + for i in 0..15 { + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 10 + i)); + } + assert_eq!(block.instruction_allocation, DEFAULT_BLOCK_SIZE); + + // CPython calls `_Py_CArray_EnsureCapacity(b_iused + 1)`, so the 16th + // instruction expands a 16-slot array to 32 before returning offset 15. + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 25)); + assert_eq!(block.instruction_allocation, DEFAULT_BLOCK_SIZE * 2); + } + #[test] fn basicblock_insert_instruction_consumes_spare_without_inheriting_except_handler() { let handler = ExceptHandlerInfo { handler_block: BlockIdx::new(9), - stack_depth: 1, preserve_lasti: false, }; let mut block = Block::default(); block.instructions.push(test_instr(Instruction::Nop, 21)); let mut stale = test_instr(Instruction::Nop, 22); stale.except_handler = Some(handler); - block.cpython_spare_instr_slots.push_back(stale); + block.cpython_spare_instr_slots.push(stale); basicblock_insert_instruction(&mut block, 0, test_instr(Instruction::PopTop, 23)); @@ -6117,7 +6818,6 @@ mod tests { fn basicblock_clear_preserves_cpython_spare_slots() { let handler = ExceptHandlerInfo { handler_block: BlockIdx::new(3), - stack_depth: 2, preserve_lasti: true, }; let mut block = Block::default(); @@ -6136,17 +6836,50 @@ mod tests { assert_eq!(block.instructions[0].except_handler, Some(handler)); } + #[test] + fn basicblock_clear_reuses_cpython_spare_slots_in_offset_order() { + let mut block = Block::default(); + for i in 0..3 { + let mut stale = test_instr(Instruction::Nop, 35 + i); + stale.except_handler = Some(ExceptHandlerInfo { + handler_block: BlockIdx::new(i as u32 + 1), + preserve_lasti: false, + }); + block.instructions.push(stale); + } + + basicblock_clear(&mut block); + for i in 0..3 { + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 38 + i)); + } + + let handlers = block + .instructions + .iter() + .map(|instr| { + instr + .except_handler + .expect("reused CPython slot") + .handler_block + }) + .collect::>(); + assert_eq!( + handlers, + [BlockIdx::new(1), BlockIdx::new(2), BlockIdx::new(3)] + ); + assert_eq!(block.cpython_spare_instr_slots.len(), 0); + } + #[test] fn basicblock_append_instructions_overwrites_cpython_spare_slot() { let handler = ExceptHandlerInfo { handler_block: BlockIdx::new(5), - stack_depth: 4, preserve_lasti: false, }; let mut to = Block::default(); let mut stale = test_instr(Instruction::Nop, 41); stale.except_handler = Some(handler); - to.cpython_spare_instr_slots.push_back(stale); + to.cpython_spare_instr_slots.push(stale); let from = [test_instr(Instruction::PopTop, 42)]; basicblock_append_instructions(&mut to, &from); @@ -6170,7 +6903,8 @@ mod tests { blocks[0].instructions.push(info); blocks[0].next = BlockIdx::new(1); - cfg_to_instruction_sequence(&mut blocks) + let mut instr_sequence = instruction_sequence_new(); + cfg_to_instruction_sequence(&mut blocks, &mut instr_sequence) .expect("non-target NOP should ignore stale CPython i_target"); } @@ -6182,7 +6916,8 @@ mod tests { block.instructions.push(test_jump(BlockIdx::NULL, 51)); let mut blocks = vec![block]; - let _ = cfg_to_instruction_sequence(&mut blocks); + let mut instr_sequence = instruction_sequence_new(); + let _ = cfg_to_instruction_sequence(&mut blocks, &mut instr_sequence); } #[test] @@ -6198,10 +6933,10 @@ mod tests { ); store.arg = OpArg::new(0); let mut pop = test_instr(Instruction::PopTop, 60); - pop.lineno_override = Some(-1); + pop.lineno_override = Some(NO_LOCATION_OVERRIDE); block.instructions.extend([swap, store, pop]); - CodeInfo::apply_static_swaps_block(&mut block); + apply_static_swaps_block(&mut block); // CPython `next_swappable_instruction()` compares `i_loc.lineno` // directly, so a following NO_LOCATION swaperand does not match the @@ -6229,11 +6964,11 @@ mod tests { 70, ); store.arg = OpArg::new(0); - store.lineno_override = Some(-1); + store.lineno_override = Some(NO_LOCATION_OVERRIDE); let pop = test_instr(Instruction::PopTop, 71); block.instructions.extend([swap, store, pop]); - CodeInfo::apply_static_swaps_block(&mut block); + apply_static_swaps_block(&mut block); // Conversely, when the first swaperand has NO_LOCATION, CPython passes // `-1` as the line filter and does not enforce a boundary. @@ -6273,7 +7008,7 @@ mod tests { }); code.blocks[0].instructions[0].arg = OpArg::new(const_idx as u32); - code.optimize_load_const(); + optimize_load_const(&mut code.metadata, &mut code.blocks); // CPython `basicblock_optimize_load_const()` keeps the previous // LOAD_CONST as the effective opcode for a following `COPY 1`, so the @@ -6313,7 +7048,7 @@ mod tests { block.instructions.push(test_instr(Instruction::PopTop, 10)); let mut code = test_code_info(block); - code.optimize_load_fast(); + optimize_load_fast(&mut code.blocks); // CPython `optimize_load_fast()` shadows the outer instruction index in // the produced-value loop for GET_LEN, so the produced ref is recorded @@ -6365,11 +7100,7 @@ mod tests { let mut block = Block::default(); block.instructions.extend([immortal, mortal, build]); - assert!(CodeInfo::fold_tuple_of_constants( - &mut metadata, - &mut block, - 2 - )); + assert!(fold_tuple_of_constants(&mut metadata, &mut block, 2)); // CPython `loads_const()` accepts every `OPCODE_HAS_CONST` opcode, not // just canonical LOAD_CONST, so LOAD_CONST_IMMORTAL/MORTAL participate @@ -6397,9 +7128,9 @@ mod tests { fn resolve_line_numbers_duplicates_exit_blocks_like_cpython() { let exit = BlockIdx::new(2); let mut blocks = vec![Block::default(), Block::default(), Block::default()]; - blocks[0].cpython_label_id = Some(InstructionSequenceLabel(0)); - blocks[1].cpython_label_id = Some(InstructionSequenceLabel(1)); - blocks[2].cpython_label_id = Some(InstructionSequenceLabel(2)); + blocks[0].cpython_label = InstructionSequenceLabel::from_index(0); + blocks[1].cpython_label = InstructionSequenceLabel::from_index(1); + blocks[2].cpython_label = InstructionSequenceLabel::from_index(2); blocks[0].next = BlockIdx::new(1); blocks[0].instructions.push(test_cond_jump(exit, 10)); blocks[1].next = exit; @@ -6407,18 +7138,18 @@ mod tests { blocks[2] .instructions .push(test_instr(Instruction::ReturnValue, 30)); - blocks[2].instructions[0].lineno_override = Some(-1); + blocks[2].instructions[0].lineno_override = Some(NO_LOCATION_OVERRIDE); compute_reachable_predecessors(&mut blocks); - resolve_line_numbers(&mut blocks); + resolve_line_numbers(&mut blocks, OneIndexed::MIN); // CPython `duplicate_exits_without_lineno()` copies a shared exit block // reached by jumps so each copy can inherit its sole predecessor's line. let duplicate = blocks[0].instructions[0].target; assert_ne!(duplicate, exit); assert_eq!( - blocks[duplicate.idx()].cpython_label_id, - Some(InstructionSequenceLabel(3)) + blocks[duplicate.idx()].cpython_label, + InstructionSequenceLabel::from_index(3) ); assert_eq!( instruction_lineno(&blocks[duplicate.idx()].instructions[0]), @@ -6437,14 +7168,14 @@ mod tests { block .instructions .push(test_instr(Instruction::ReturnValue, 30)); - block.instructions[2].lineno_override = Some(-1); + block.instructions[2].lineno_override = Some(NO_LOCATION_OVERRIDE); let mut blocks = vec![block]; compute_reachable_predecessors(&mut blocks); propagate_line_numbers(&mut blocks); // CPython `propagate_line_numbers()` only copies over NO_LOCATION - // (`lineno == -1`). `NEXT_LOCATION` (`lineno == -2`) becomes the + // (`lineno == NO_LOCATION`). `NEXT_LOCATION` (`lineno == -2`) becomes the // current previous location and is copied to following NO_LOCATION // instructions for assemble.c to resolve later. assert_eq!( @@ -6466,8 +7197,8 @@ mod tests { .push(test_cond_jump(BlockIdx::new(1), 10)); blocks[1] .cpython_spare_instr_slots - .push_back(test_instr(Instruction::Nop, 20)); - blocks[1].cpython_spare_instr_slots[0].lineno_override = Some(-1); + .push(test_instr(Instruction::Nop, 20)); + blocks[1].cpython_spare_instr_slots[0].lineno_override = Some(NO_LOCATION_OVERRIDE); blocks[2] .instructions .push(test_instr(Instruction::ReturnValue, 30)); @@ -6508,7 +7239,7 @@ mod tests { Block::default(), ]; for (i, block) in blocks.iter_mut().enumerate() { - block.cpython_label_id = Some(InstructionSequenceLabel(i)); + block.cpython_label = InstructionSequenceLabel::from_index(i); } blocks[0].next = BlockIdx::new(1); blocks[1].next = BlockIdx::new(2); @@ -6520,7 +7251,9 @@ mod tests { .instructions .push(test_instr(Instruction::ReturnValue, 40)); - jump_threading_block(&mut blocks, BlockIdx::new(0)).expect("valid jump chain"); + let mut metadata = test_code_info(Block::default()).metadata; + optimize_basic_block(&mut blocks, &mut metadata, BlockIdx::new(0)) + .expect("valid jump chain"); // CPython `optimize_basic_block()` continues after `jump_thread()`, so // the appended jump is immediately checked against the next jump target. diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index 7a9e34efcf2..5872c0ffbd2 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -415,6 +415,10 @@ impl IndexMut for [T] { } /// Per-slot kind flags for localsplus (co_localspluskinds). +pub const CO_FAST_ARG_POS: u8 = 0x02; +pub const CO_FAST_ARG_KW: u8 = 0x04; +pub const CO_FAST_ARG_VAR: u8 = 0x08; +pub const CO_FAST_ARG: u8 = CO_FAST_ARG_POS | CO_FAST_ARG_KW | CO_FAST_ARG_VAR; pub const CO_FAST_HIDDEN: u8 = 0x10; pub const CO_FAST_LOCAL: u8 = 0x20; pub const CO_FAST_CELL: u8 = 0x40; @@ -443,7 +447,8 @@ pub struct CodeObject { pub varnames: Box<[C::Name]>, pub cellvars: Box<[C::Name]>, pub freevars: Box<[C::Name]>, - /// Per-slot kind flags: CO_FAST_LOCAL, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN. + /// Per-slot kind flags: CO_FAST_ARG_*, CO_FAST_LOCAL, CO_FAST_CELL, + /// CO_FAST_FREE, CO_FAST_HIDDEN. /// Length = nlocalsplus (nlocals + ncells + nfrees). pub localspluskinds: Box<[u8]>, /// Line number table (CPython 3.11+ format) From 8b5773378389f70ae176043d94dd1e958f486eb4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 10:45:37 +0900 Subject: [PATCH 012/131] Match CPython CFG annotation offset arithmetic --- crates/codegen/src/ir.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8118b971713..92a0c2a7722 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5820,7 +5820,7 @@ fn cfg_from_instruction_sequence( debug_assert!(label_map.is_none()); let mut builder = cfg_builder_new(); - let mut offset = 0isize; + let mut offset = 0i32; let mut i = 0; while i < instrs.len() { @@ -5839,7 +5839,12 @@ fn cfg_from_instruction_sequence( info.target = BlockIdx::NULL; cfg_builder_addop(&mut builder, info); } - offset += annotations_code.instrs.len() as isize - 1; + offset += annotations_code + .instrs + .len() + .to_i32() + .ok_or(InternalError::MalformedControlFlowGraph)? + - 1; } else { offset -= 1; } @@ -5848,19 +5853,21 @@ fn cfg_from_instruction_sequence( } if entry.i_target != 0 { - let label = InstructionSequenceLabel::from_index( - i.checked_add_signed(offset) - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + let label_id = i + .to_i32() + .ok_or(InternalError::MalformedControlFlowGraph)? + .checked_add(offset) + .ok_or(InternalError::MalformedControlFlowGraph)?; + let label = InstructionSequenceLabel(label_id); cfg_builder_use_label(&mut builder, label); } let opcode = entry.info.instr; let mut oparg = entry.info.arg; if opcode.has_target() { - let target_offset = usize::try_from(u32::from(oparg)) + let target_offset = i32::try_from(u32::from(oparg)) .map_err(|_| InternalError::MalformedControlFlowGraph)? - .checked_add_signed(offset) + .checked_add(offset) .ok_or(InternalError::MalformedControlFlowGraph)?; oparg = OpArg::new( target_offset From 11d9f92a7d946fe1ed457eae4391638fcb2710b6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 10:49:26 +0900 Subject: [PATCH 013/131] Propagate CPython CFG label translation errors --- crates/codegen/src/ir.rs | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 92a0c2a7722..18752ced47c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1702,7 +1702,7 @@ fn optimize_code_unit( ) -> crate::InternalResult<()> { // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) *blocks = cfg_from_instruction_sequence(instr_sequence)?; - optimize_code_unit_preprocess(blocks); + optimize_code_unit_preprocess(blocks)?; optimize_cfg(metadata, blocks, metadata.firstlineno)?; remove_unused_consts(blocks, &mut metadata.consts); add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams); @@ -4265,7 +4265,7 @@ impl CodeInfo { "after_cfg_from_instruction_sequence".to_owned(), self.debug_block_dump(), )); - optimize_code_unit_preprocess(&mut self.blocks); + optimize_code_unit_preprocess(&mut self.blocks)?; check_cfg(&self.blocks)?; inline_small_or_no_lineno_blocks(&mut self.blocks); trace.push(( @@ -5749,17 +5749,23 @@ fn cfg_builder_check_size(g: &CfgBuilder) -> crate::InternalResult<()> { } /// flowgraph.c translate_jump_labels_to_targets -fn translate_jump_labels_to_targets(blocks: &mut [Block]) { +fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResult<()> { let max_label = get_max_label(blocks); - let mapsize = (max_label + 1) as usize; - let mut label_to_block = vec![BlockIdx::NULL; mapsize]; + let label_count = max_label + .checked_add(1) + .and_then(|count| count.to_usize()) + .ok_or(InternalError::MalformedControlFlowGraph)?; + label_count + .checked_mul(core::mem::size_of::()) + .ok_or(InternalError::MalformedControlFlowGraph)?; + let mut label_to_block = vec![BlockIdx::NULL; label_count]; let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let block = &blocks[block_idx.idx()]; if is_label(block.cpython_label) { let label_id = block.cpython_label; - debug_assert!(label_id.idx() <= max_label as usize); + debug_assert!(label_id.0 <= max_label); label_to_block[label_id.idx()] = block_idx; } block_idx = block.next; @@ -5772,27 +5778,32 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) { let info = &mut blocks[block_idx.idx()].instructions[i]; debug_assert_eq!(info.target, BlockIdx::NULL); if info.instr.has_target() { - let lbl = u32::from(info.arg) as usize; - debug_assert!(max_label >= 0); - debug_assert!(lbl <= max_label as usize); - let target = label_to_block[lbl]; + let lbl = i32::try_from(u32::from(info.arg)) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + if lbl < 0 || lbl > max_label { + return Err(InternalError::MalformedControlFlowGraph); + } + let target = label_to_block + [usize::try_from(lbl).map_err(|_| InternalError::MalformedControlFlowGraph)?]; debug_assert!(target != BlockIdx::NULL); debug_assert_eq!( blocks[target.idx()].cpython_label, - InstructionSequenceLabel::from_index(lbl) + InstructionSequenceLabel(lbl) ); info.target = target; } } block_idx = next; } + Ok(()) } /// flowgraph.c _PyCfg_OptimizeCodeUnit preprocessing -fn optimize_code_unit_preprocess(blocks: &mut [Block]) { - translate_jump_labels_to_targets(blocks); +fn optimize_code_unit_preprocess(blocks: &mut [Block]) -> crate::InternalResult<()> { + translate_jump_labels_to_targets(blocks)?; mark_except_handlers(blocks); label_exception_targets(blocks); + Ok(()) } /// flowgraph.c _PyCfg_FromInstructionSequence From b6cd3985f8d4431e11d65edc092bcaa6156d3a45 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 10:53:54 +0900 Subject: [PATCH 014/131] Align CPython exception target labeling flow --- crates/codegen/src/ir.rs | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 18752ced47c..48e44ab5a9a 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4884,7 +4884,8 @@ fn assemble_exception_table(instrs: &[InstructionSequenceEntry]) -> Box<[u8]> { /// Mark exception handler target blocks. /// flowgraph.c mark_except_handlers -pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn mark_except_handlers(blocks: &mut [Block]) -> crate::InternalResult<()> { #[cfg(debug_assertions)] { let mut block_idx = BlockIdx(0); @@ -4907,6 +4908,7 @@ pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { } block_idx = next; } + Ok(()) } /// flowgraph.c mark_cold (two-pass to match CPython). @@ -5801,8 +5803,8 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResu /// flowgraph.c _PyCfg_OptimizeCodeUnit preprocessing fn optimize_code_unit_preprocess(blocks: &mut [Block]) -> crate::InternalResult<()> { translate_jump_labels_to_targets(blocks)?; - mark_except_handlers(blocks); - label_exception_targets(blocks); + mark_except_handlers(blocks)?; + label_exception_targets(blocks)?; Ok(()) } @@ -6334,13 +6336,21 @@ fn resolve_line_numbers(blocks: &mut Vec, _firstlineno: OneIndexed) { } /// flowgraph.c make_except_stack -fn make_except_stack() -> Vec { - Vec::new() +fn make_except_stack() -> crate::InternalResult> { + let mut stack = Vec::new(); + stack + .try_reserve_exact(CO_MAXBLOCKS + 2) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + Ok(stack) } /// flowgraph.c copy_except_stack -fn copy_except_stack(stack: &[BlockIdx]) -> Vec { - stack.to_vec() +fn copy_except_stack(stack: &[BlockIdx]) -> crate::InternalResult> { + let mut copy = Vec::new(); + copy.try_reserve_exact(CO_MAXBLOCKS + 2) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + copy.extend_from_slice(stack); + Ok(copy) } /// flowgraph.c except_stack_top @@ -6380,12 +6390,12 @@ fn pop_except_block(stack: &mut Vec, blocks: &[Block]) -> Option crate::InternalResult<()> { let mut todo = make_cfg_traversal_stack(blocks); todo.push(BlockIdx(0)); blocks[0].visited = true; - blocks[0].except_stack = Some(make_except_stack()); + blocks[0].except_stack = Some(make_except_stack()?); while let Some(block_idx) = todo.pop() { let bi = block_idx.idx(); @@ -6407,7 +6417,7 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { if is_block_push(&info) { debug_assert!(target != BlockIdx::NULL); if !blocks[target.idx()].visited { - blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)); + blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)?); todo.push(target); blocks[target.idx()].visited = true; } @@ -6425,7 +6435,7 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { debug_assert!(target != BlockIdx::NULL); if !blocks[target.idx()].visited { if bb_has_fallthrough(&blocks[bi]) { - blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)); + blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)?); } else { blocks[target.idx()].except_stack = Some(core::mem::take(&mut stack)); } @@ -6470,6 +6480,7 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { block_idx = block.next; } } + Ok(()) } /// Convert remaining pseudo ops to real instructions or NOP. From cda874c233621298525e416196c2d7685970b7d5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 10:58:11 +0900 Subject: [PATCH 015/131] Propagate CPython CFG traversal stack allocation errors --- crates/codegen/src/ir.rs | 83 ++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 48e44ab5a9a..5f0dcdf5eb4 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1705,7 +1705,7 @@ fn optimize_code_unit( optimize_code_unit_preprocess(blocks)?; optimize_cfg(metadata, blocks, metadata.firstlineno)?; remove_unused_consts(blocks, &mut metadata.consts); - add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams); + add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams)?; // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before // later jump normalization / block reordering can create adjacencies // that never exist at this stage in flowgraph.c. @@ -1730,7 +1730,7 @@ fn optimize_cfg( // CPython does not re-run instruction-sequence label-map/CFG conversion // after this point. Unreferenced label blocks left by jump inlining // remain block boundaries and can preserve line-marker NOPs. - remove_unreachable(blocks); + remove_unreachable(blocks)?; // CPython optimize_cfg resolves line numbers before local checks and // superinstruction insertion, so fusion decisions see propagated // source locations. @@ -1748,7 +1748,7 @@ fn optimize_cfg( // CPython optimize_cfg() removes newly-unreachable blocks and // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes // unused constants. - remove_unreachable(blocks); + remove_unreachable(blocks)?; remove_redundant_nops_and_jumps(blocks)?; #[cfg(debug_assertions)] assert!(no_redundant_jumps(blocks)); @@ -1772,7 +1772,7 @@ fn optimized_cfg_to_instruction_sequence( #[cfg(debug_assertions)] assert!(no_redundant_jumps(blocks)); // optimize_load_fast: after normalize_jumps - optimize_load_fast(blocks); + optimize_load_fast(blocks)?; let mut instr_sequence = instruction_sequence_new(); cfg_to_instruction_sequence(blocks, &mut instr_sequence)?; @@ -2011,8 +2011,8 @@ fn prepare_localsplus( } /// flowgraph.c remove_unreachable -fn remove_unreachable(blocks: &mut [Block]) { - compute_reachable_predecessors(blocks); +fn remove_unreachable(blocks: &mut [Block]) -> crate::InternalResult<()> { + compute_reachable_predecessors(blocks)?; let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { @@ -2025,6 +2025,7 @@ fn remove_unreachable(blocks: &mut [Block]) { } block_idx = next; } + Ok(()) } /// flowgraph.c eval_const_unaryop @@ -3886,7 +3887,7 @@ fn remove_unused_consts(blocks: &mut [Block], consts: &mut ConstantPool) { } } -fn optimize_load_fast(blocks: &mut [Block]) { +fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut max_instrs = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { @@ -3895,7 +3896,7 @@ fn optimize_load_fast(blocks: &mut [Block]) { } let mut instr_flags = vec![0u8; max_instrs]; let mut refs = Vec::new(); - let mut worklist = make_cfg_traversal_stack(blocks); + let mut worklist = make_cfg_traversal_stack(blocks)?; worklist.push(BlockIdx(0)); blocks[0].start_depth = 0; blocks[0].visited = true; @@ -4133,6 +4134,7 @@ fn optimize_load_fast(blocks: &mut [Block]) { i += 1; } } + Ok(()) } /// flowgraph.c calculate_stackdepth @@ -4142,7 +4144,7 @@ fn calculate_stackdepth(blocks: &mut [Block]) -> crate::InternalResult { blocks[current.idx()].start_depth = START_DEPTH_UNSET; current = blocks[current.idx()].next; } - let mut stack = make_cfg_traversal_stack(blocks); + let mut stack = make_cfg_traversal_stack(blocks)?; let mut maxdepth = 0i32; stackdepth_push(&mut stack, blocks, BlockIdx(0), 0)?; while let Some(block_idx) = stack.pop() { @@ -4272,7 +4274,7 @@ impl CodeInfo { "after_inline_small_or_no_lineno_blocks".to_owned(), self.debug_block_dump(), )); - remove_unreachable(&mut self.blocks); + remove_unreachable(&mut self.blocks)?; resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno); optimize_load_const(&mut self.metadata, &mut self.blocks); trace.push(( @@ -4290,7 +4292,7 @@ impl CodeInfo { self.debug_block_dump(), )); remove_redundant_nops_and_pairs(&mut self.blocks); - remove_unreachable(&mut self.blocks); + remove_unreachable(&mut self.blocks)?; remove_redundant_nops_and_jumps(&mut self.blocks)?; #[cfg(debug_assertions)] assert!(no_redundant_jumps(&self.blocks)); @@ -4301,7 +4303,7 @@ impl CodeInfo { )); let nlocals = self.metadata.varnames.len(); let nparams = self.nparams; - add_checks_for_loads_of_uninitialized_variables(&mut self.blocks, nlocals, nparams); + add_checks_for_loads_of_uninitialized_variables(&mut self.blocks, nlocals, nparams)?; insert_superinstructions(&mut self.blocks); push_cold_blocks_to_end(&mut self.blocks)?; trace.push(( @@ -4337,7 +4339,7 @@ impl CodeInfo { #[cfg(debug_assertions)] assert!(no_redundant_jumps(&self.blocks)); trace.push(("after_normalize_jumps".to_owned(), self.debug_block_dump())); - optimize_load_fast(&mut self.blocks); + optimize_load_fast(&mut self.blocks)?; trace.push(( "after_optimize_load_fast".to_owned(), self.debug_block_dump(), @@ -4928,8 +4930,8 @@ pub(crate) fn mark_except_handlers(blocks: &mut [Block]) -> crate::InternalResul /// optimize_cfg). This matches CPython's behavior and is necessary for /// optimize_load_fast to terminate fall-through at those placeholders. /// flowgraph.c mark_warm -fn mark_warm(blocks: &mut [Block]) { - let mut stack = make_cfg_traversal_stack(blocks); +fn mark_warm(blocks: &mut [Block]) -> crate::InternalResult<()> { + let mut stack = make_cfg_traversal_stack(blocks)?; stack.push(BlockIdx(0)); blocks[0].visited = true; while let Some(block_idx) = stack.pop() { @@ -4957,9 +4959,10 @@ fn mark_warm(blocks: &mut [Block]) { } } } + Ok(()) } -fn mark_cold(blocks: &mut [Block]) { +fn mark_cold(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let block = &mut blocks[block_idx.idx()]; @@ -4968,9 +4971,9 @@ fn mark_cold(blocks: &mut [Block]) { block_idx = block.next; } - mark_warm(blocks); + mark_warm(blocks)?; - let mut cold_stack = make_cfg_traversal_stack(blocks); + let mut cold_stack = make_cfg_traversal_stack(blocks)?; block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let i = block_idx.idx(); @@ -5009,6 +5012,7 @@ fn mark_cold(blocks: &mut [Block]) { } } } + Ok(()) } /// flowgraph.c push_cold_blocks_to_end @@ -5017,7 +5021,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> return Ok(()); } - mark_cold(blocks); + mark_cold(blocks)?; let mut next_label = (get_max_label(blocks) + 1) as usize; // If a cold block falls through to a warm block, add an explicit jump @@ -5596,7 +5600,7 @@ fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResul } /// flowgraph.c make_cfg_traversal_stack -fn make_cfg_traversal_stack(blocks: &mut [Block]) -> Vec { +fn make_cfg_traversal_stack(blocks: &mut [Block]) -> crate::InternalResult> { debug_assert!(!blocks.is_empty()); let mut nblocks = 0; let mut current = BlockIdx(0); @@ -5605,7 +5609,11 @@ fn make_cfg_traversal_stack(blocks: &mut [Block]) -> Vec { nblocks += 1; current = blocks[current.idx()].next; } - Vec::with_capacity(nblocks) + let mut stack = Vec::new(); + stack + .try_reserve_exact(nblocks) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + Ok(stack) } fn blocks_new_block(blocks: &mut Vec) -> BlockIdx { @@ -5988,9 +5996,13 @@ fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &m } /// flowgraph.c fast_scan_many_locals -fn fast_scan_many_locals(blocks: &mut [Block], nlocals: usize) { +fn fast_scan_many_locals(blocks: &mut [Block], nlocals: usize) -> crate::InternalResult<()> { debug_assert!(nlocals > LOCAL_UNSAFE_MASK_BITS); - let mut states = vec![0usize; nlocals - LOCAL_UNSAFE_MASK_BITS]; + let mut states = Vec::new(); + states + .try_reserve_exact(nlocals - LOCAL_UNSAFE_MASK_BITS) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + states.resize(nlocals - LOCAL_UNSAFE_MASK_BITS, 0usize); let mut blocknum = 0usize; let mut current = BlockIdx(0); while current != BlockIdx::NULL { @@ -6026,6 +6038,7 @@ fn fast_scan_many_locals(blocks: &mut [Block], nlocals: usize) { } current = blocks[current.idx()].next; } + Ok(()) } /// flowgraph.c add_checks_for_loads_of_uninitialized_variables @@ -6033,17 +6046,17 @@ fn add_checks_for_loads_of_uninitialized_variables( blocks: &mut [Block], mut nlocals: usize, nparams: usize, -) { +) -> crate::InternalResult<()> { if nlocals == 0 { - return; + return Ok(()); } if nlocals > LOCAL_UNSAFE_MASK_BITS { - fast_scan_many_locals(blocks, nlocals); + fast_scan_many_locals(blocks, nlocals)?; nlocals = LOCAL_UNSAFE_MASK_BITS; } - let mut worklist = make_cfg_traversal_stack(blocks); + let mut worklist = make_cfg_traversal_stack(blocks)?; let mut start_mask = 0u64; for i in nparams..nlocals { start_mask |= 1u64 << i; @@ -6060,6 +6073,7 @@ fn add_checks_for_loads_of_uninitialized_variables( blocks[block_idx.idx()].visited = false; scan_block_for_locals(blocks, block_idx, &mut worklist); } + Ok(()) } /// Follow chain of empty blocks to find first non-empty block. @@ -6166,14 +6180,14 @@ fn basicblock_has_no_lineno(block: &Block) -> bool { /// flowgraph.c remove_unreachable computes `b_predecessors` by traversing only /// blocks reachable from entry. -fn compute_reachable_predecessors(blocks: &mut [Block]) { +fn compute_reachable_predecessors(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { blocks[block_idx.idx()].predecessors = 0; block_idx = blocks[block_idx.idx()].next; } - let mut stack = make_cfg_traversal_stack(blocks); + let mut stack = make_cfg_traversal_stack(blocks)?; blocks[0].predecessors = 1; stack.push(BlockIdx(0)); blocks[0].visited = true; @@ -6204,6 +6218,7 @@ fn compute_reachable_predecessors(blocks: &mut [Block]) { } } } + Ok(()) } /// flowgraph.c copy_basicblock @@ -6391,7 +6406,7 @@ fn pop_except_block(stack: &mut Vec, blocks: &[Block]) -> Option crate::InternalResult<()> { - let mut todo = make_cfg_traversal_stack(blocks); + let mut todo = make_cfg_traversal_stack(blocks)?; todo.push(BlockIdx(0)); blocks[0].visited = true; @@ -7077,7 +7092,7 @@ mod tests { block.instructions.push(test_instr(Instruction::PopTop, 10)); let mut code = test_code_info(block); - optimize_load_fast(&mut code.blocks); + optimize_load_fast(&mut code.blocks).expect("optimize_load_fast succeeds"); // CPython `optimize_load_fast()` shadows the outer instruction index in // the produced-value loop for GET_LEN, so the produced ref is recorded @@ -7169,7 +7184,7 @@ mod tests { .push(test_instr(Instruction::ReturnValue, 30)); blocks[2].instructions[0].lineno_override = Some(NO_LOCATION_OVERRIDE); - compute_reachable_predecessors(&mut blocks); + compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); resolve_line_numbers(&mut blocks, OneIndexed::MIN); // CPython `duplicate_exits_without_lineno()` copies a shared exit block @@ -7200,7 +7215,7 @@ mod tests { block.instructions[2].lineno_override = Some(NO_LOCATION_OVERRIDE); let mut blocks = vec![block]; - compute_reachable_predecessors(&mut blocks); + compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); propagate_line_numbers(&mut blocks); // CPython `propagate_line_numbers()` only copies over NO_LOCATION @@ -7232,7 +7247,7 @@ mod tests { .instructions .push(test_instr(Instruction::ReturnValue, 30)); - compute_reachable_predecessors(&mut blocks); + compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); propagate_line_numbers(&mut blocks); // CPython `propagate_line_numbers()` directly reads `target->b_instr[0]` From e2fd26d9f8c67d63d03b8afb63b534e8e22d31d1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 10:59:58 +0900 Subject: [PATCH 016/131] Match CPython optimize_load_fast allocation flow --- crates/codegen/src/ir.rs | 57 +++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5f0dcdf5eb4..9e7578541b3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3894,7 +3894,11 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { max_instrs = max_instrs.max(blocks[current.idx()].instructions.len()); current = blocks[current.idx()].next; } - let mut instr_flags = vec![0u8; max_instrs]; + let mut instr_flags = Vec::new(); + instr_flags + .try_reserve_exact(max_instrs) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + instr_flags.resize(max_instrs, 0u8); let mut refs = Vec::new(); let mut worklist = make_cfg_traversal_stack(blocks)?; worklist.push(BlockIdx(0)); @@ -3909,9 +3913,10 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let start_depth = usize::try_from(blocks[block_i].start_depth) .expect("visited block has non-negative start depth"); ref_stack_clear(&mut refs); - refs.reserve(instr_count + start_depth + 2); + refs.try_reserve(instr_count + start_depth + 2) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; for _ in 0..start_depth { - push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL); + push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL)?; } for i in 0..instr_count { @@ -3932,17 +3937,17 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { &mut refs, i as isize, local_as_ref_local(usize::from(var_num.get(info.arg))), - ); + )?; } AnyInstruction::Real(Instruction::LoadFastAndClear { var_num }) => { let local = local_as_ref_local(usize::from(var_num.get(info.arg))); kill_local(&mut instr_flags, &refs, local); - push_ref(&mut refs, i as isize, local); + push_ref(&mut refs, i as isize, local)?; } AnyInstruction::Real(Instruction::LoadFastLoadFast { .. }) => { let (local1, local2) = decode_packed_fast_locals(info.arg); - push_ref(&mut refs, i as isize, local1); - push_ref(&mut refs, i as isize, local2); + push_ref(&mut refs, i as isize, local1)?; + push_ref(&mut refs, i as isize, local2)?; } AnyInstruction::Real(Instruction::StoreFast { var_num }) => { let r = ref_stack_pop(&mut refs); @@ -3957,7 +3962,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let (store_local_idx, load_local_idx) = decode_packed_fast_locals(info.arg); let r = ref_stack_pop(&mut refs); store_local(&mut instr_flags, &refs, store_local_idx, r); - push_ref(&mut refs, i as isize, load_local_idx); + push_ref(&mut refs, i as isize, load_local_idx)?; } AnyInstruction::Real(Instruction::StoreFastStoreFast { .. }) => { let (local1, local2) = decode_packed_fast_locals(info.arg); @@ -3971,7 +3976,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { assert!(depth > 0); assert!(refs.len() >= depth); let r = ref_stack_at(&refs, refs.len() - depth); - push_ref(&mut refs, r.instr, r.local); + push_ref(&mut refs, r.instr, r.local)?; } AnyInstruction::Real(Instruction::Swap { i: _ }) => { let depth = arg_u32 as usize; @@ -3996,7 +4001,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { // CPython optimize_load_fast() shadows the outer // instruction index in this produced-value loop. for produced in 0..net_pushed { - push_ref(&mut refs, produced, NOT_LOCAL); + push_ref(&mut refs, produced, NOT_LOCAL)?; } } AnyInstruction::Real( @@ -4024,47 +4029,47 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { debug_assert_eq!(effect.pushed(), 1); let tos = ref_stack_pop(&mut refs); let _ = ref_stack_pop(&mut refs); - push_ref(&mut refs, tos.instr, tos.local); + push_ref(&mut refs, tos.instr, tos.local)?; } AnyInstruction::Real(Instruction::CheckExcMatch) => { let _ = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; } AnyInstruction::Real(Instruction::ForIter { .. }) => { let target = info.target; debug_assert!(target != BlockIdx::NULL); load_fast_push_block(&mut worklist, blocks, target, refs.len() + 1); - push_ref(&mut refs, i as isize, NOT_LOCAL); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; } AnyInstruction::Real(Instruction::LoadAttr { namei }) => { let self_ref = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; if namei.get(info.arg).is_method() { - push_ref(&mut refs, self_ref.instr, self_ref.local); + push_ref(&mut refs, self_ref.instr, self_ref.local)?; } } AnyInstruction::Real(Instruction::LoadSuperAttr { namei }) => { let self_ref = ref_stack_pop(&mut refs); let _ = ref_stack_pop(&mut refs); let _ = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; if namei.get(info.arg).is_load_method() { - push_ref(&mut refs, self_ref.instr, self_ref.local); + push_ref(&mut refs, self_ref.instr, self_ref.local)?; } } AnyInstruction::Real( Instruction::LoadSpecial { .. } | Instruction::PushExcInfo, ) => { let tos = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); - push_ref(&mut refs, tos.instr, tos.local); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; + push_ref(&mut refs, tos.instr, tos.local)?; } AnyInstruction::Real(Instruction::Send { .. }) => { let target = info.target; debug_assert!(target != BlockIdx::NULL); load_fast_push_block(&mut worklist, blocks, target, refs.len()); let _ = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; } _ => { let effect = instr.stack_effect_info(arg_u32); @@ -4082,7 +4087,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let _ = ref_stack_pop(&mut refs); } for _ in 0..num_pushed { - push_ref(&mut refs, i as isize, NOT_LOCAL); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; } } } @@ -4484,8 +4489,12 @@ struct Ref { type RefStack = Vec; /// flowgraph.c ref_stack_push -fn ref_stack_push(stack: &mut RefStack, r: Ref) { +fn ref_stack_push(stack: &mut RefStack, r: Ref) -> crate::InternalResult<()> { + stack + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; stack.push(r); + Ok(()) } /// flowgraph.c ref_stack_pop @@ -4512,8 +4521,8 @@ fn ref_stack_clear(stack: &mut RefStack) { } /// flowgraph.c optimize_load_fast PUSH_REF -fn push_ref(stack: &mut RefStack, instr: isize, local: isize) { - ref_stack_push(stack, Ref { instr, local }); +fn push_ref(stack: &mut RefStack, instr: isize, local: isize) -> crate::InternalResult<()> { + ref_stack_push(stack, Ref { instr, local }) } /// flowgraph.c kill_local From 9500d4ed29b8ef5ca5454c581b2bc46d5b18e964 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:09:30 +0900 Subject: [PATCH 017/131] Propagate CPython basicblock allocation errors --- crates/codegen/src/ir.rs | 212 +++++++++++++++++++++++---------------- 1 file changed, 127 insertions(+), 85 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 9e7578541b3..6e5d04b499c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -319,18 +319,21 @@ fn c_array_ensure_capacity( } /// flowgraph.c basicblock_next_instr -fn basicblock_next_instr(block: &mut Block) -> usize { +fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { let off = block.instructions.len(); let new_allocation = c_array_ensure_capacity(block.instruction_allocation, off + 1, DEFAULT_BLOCK_SIZE); if new_allocation > block.instruction_allocation { block.instruction_allocation = new_allocation; if new_allocation > block.instructions.capacity() + block.cpython_spare_instr_slots.len() { - block.instructions.reserve_exact( - new_allocation - - block.instructions.capacity() - - block.cpython_spare_instr_slots.len(), - ); + block + .instructions + .try_reserve_exact( + new_allocation + - block.instructions.capacity() + - block.cpython_spare_instr_slots.len(), + ) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; } } let slot = block @@ -338,7 +341,7 @@ fn basicblock_next_instr(block: &mut Block) -> usize { .pop() .unwrap_or_else(empty_instruction_info); block.instructions.push(slot); - off + Ok(off) } /// flowgraph.c basicblock_last_instr @@ -352,7 +355,7 @@ fn basicblock_last_instr_mut(block: &mut Block) -> Option<&mut InstructionInfo> } /// flowgraph.c basicblock_addop -fn basicblock_addop(block: &mut Block, mut info: InstructionInfo) { +fn basicblock_addop(block: &mut Block, mut info: InstructionInfo) -> crate::InternalResult<()> { debug_assert!(!info.instr.is_assembler()); debug_assert!( info.instr.has_arg() || info.instr.has_target() || u32::from(info.arg) == 0, @@ -362,30 +365,40 @@ fn basicblock_addop(block: &mut Block, mut info: InstructionInfo) { u32::from(info.arg) < (1 << 30), "CPython basicblock_addop requires 0 <= oparg < (1 << 30)" ); - let off = basicblock_next_instr(block); + let off = basicblock_next_instr(block)?; let except_handler = block.instructions[off].except_handler; info.target = BlockIdx::NULL; info.except_handler = except_handler; block.instructions[off] = info; + Ok(()) } /// flowgraph.c basicblock_insert_instruction -fn basicblock_insert_instruction(block: &mut Block, pos: usize, info: InstructionInfo) { +fn basicblock_insert_instruction( + block: &mut Block, + pos: usize, + info: InstructionInfo, +) -> crate::InternalResult<()> { let old_len = block.instructions.len(); debug_assert!(pos <= old_len); - basicblock_next_instr(block); + basicblock_next_instr(block)?; for i in (pos + 1..=old_len).rev() { block.instructions[i] = block.instructions[i - 1]; } block.instructions[pos] = info; + Ok(()) } /// flowgraph.c basicblock_append_instructions -fn basicblock_append_instructions(to: &mut Block, from: &[InstructionInfo]) { +fn basicblock_append_instructions( + to: &mut Block, + from: &[InstructionInfo], +) -> crate::InternalResult<()> { for info in from { - let off = basicblock_next_instr(to); + let off = basicblock_next_instr(to)?; to.instructions[off] = *info; } + Ok(()) } /// flowgraph.c direct `b_iused = 0` @@ -1712,7 +1725,7 @@ fn optimize_code_unit( insert_superinstructions(blocks); push_cold_blocks_to_end(blocks)?; // CPython resolves line numbers again after cold-block extraction. - resolve_line_numbers(blocks, metadata.firstlineno); + resolve_line_numbers(blocks, metadata.firstlineno)?; Ok(()) } @@ -1726,7 +1739,7 @@ fn optimize_cfg( // SystemError if a jump or scope exit is not the last instruction in // its block. check_cfg(blocks)?; - inline_small_or_no_lineno_blocks(blocks); + inline_small_or_no_lineno_blocks(blocks)?; // CPython does not re-run instruction-sequence label-map/CFG conversion // after this point. Unreferenced label blocks left by jump inlining // remain block boundaries and can preserve line-marker NOPs. @@ -1734,7 +1747,7 @@ fn optimize_cfg( // CPython optimize_cfg resolves line numbers before local checks and // superinstruction insertion, so fusion decisions see propagated // source locations. - resolve_line_numbers(blocks, firstlineno); + resolve_line_numbers(blocks, firstlineno)?; // CPython optimize_cfg() runs optimize_load_const() and then // optimize_basic_block() after line numbers are resolved. optimize_load_const(metadata, blocks); @@ -1761,10 +1774,10 @@ fn optimized_cfg_to_instruction_sequence( blocks: &mut Vec, ) -> crate::InternalResult<(u32, usize, InstructionSequence)> { // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) - convert_pseudo_conditional_jumps(blocks); + convert_pseudo_conditional_jumps(blocks)?; let max_stackdepth = calculate_stackdepth(blocks)?; debug_assert!(!is_generator(flags) || max_stackdepth != 0); - let nlocalsplus = prepare_localsplus(metadata, blocks, flags); + let nlocalsplus = prepare_localsplus(metadata, blocks, flags)?; // Match CPython order: pseudo ops are lowered after stackdepth and // localsplus preparation, before normalize_jumps. convert_pseudo_ops(blocks)?; @@ -1886,7 +1899,7 @@ fn insert_prefix_instructions( cellfixedoffsets: &[i32], nfreevars: usize, flags: CodeFlags, -) { +) -> crate::InternalResult<()> { debug_assert!(!blocks.is_empty()); let entry = &mut blocks[0]; let ncellvars = metadata.cellvars.len(); @@ -1910,7 +1923,7 @@ fn insert_prefix_instructions( except_handler: None, lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), }, - ); + )?; basicblock_insert_instruction( entry, 1, @@ -1923,7 +1936,7 @@ fn insert_prefix_instructions( except_handler: None, lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), }, - ); + )?; } if ncellvars > 0 { @@ -1953,7 +1966,7 @@ fn insert_prefix_instructions( except_handler: None, lineno_override: Some(NO_LOCATION_OVERRIDE), }, - ); + )?; ncellsused += 1; } } @@ -1971,8 +1984,9 @@ fn insert_prefix_instructions( except_handler: None, lineno_override: Some(NO_LOCATION_OVERRIDE), }, - ); + )?; } + Ok(()) } /// flowgraph.c prepare_localsplus @@ -1980,7 +1994,7 @@ fn prepare_localsplus( metadata: &CodeUnitMetadata, blocks: &mut [Block], flags: CodeFlags, -) -> usize { +) -> crate::InternalResult { let nlocals = metadata.varnames.len(); let ncellvars = metadata.cellvars.len(); let nfreevars = metadata.freevars.len(); @@ -2003,11 +2017,11 @@ fn prepare_localsplus( let mut cellfixedoffsets = build_cellfixedoffsets(metadata); // This must be called before fix_cell_offsets(). - insert_prefix_instructions(metadata, blocks, &cellfixedoffsets, nfreevars, flags); + insert_prefix_instructions(metadata, blocks, &cellfixedoffsets, nfreevars, flags)?; let numdropped = fix_cell_offsets(metadata, blocks, &mut cellfixedoffsets); nlocalsplus -= numdropped; - nlocalsplus + Ok(nlocalsplus) } /// flowgraph.c remove_unreachable @@ -4274,13 +4288,13 @@ impl CodeInfo { )); optimize_code_unit_preprocess(&mut self.blocks)?; check_cfg(&self.blocks)?; - inline_small_or_no_lineno_blocks(&mut self.blocks); + inline_small_or_no_lineno_blocks(&mut self.blocks)?; trace.push(( "after_inline_small_or_no_lineno_blocks".to_owned(), self.debug_block_dump(), )); remove_unreachable(&mut self.blocks)?; - resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno); + resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno)?; optimize_load_const(&mut self.metadata, &mut self.blocks); trace.push(( "after_optimize_load_const".to_owned(), @@ -4315,7 +4329,7 @@ impl CodeInfo { "after_push_cold_before_chain_reorder".to_owned(), self.debug_block_dump(), )); - resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno); + resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno)?; trace.push(( "after_push_cold_resolve_line_numbers".to_owned(), self.debug_block_dump(), @@ -4326,14 +4340,14 @@ impl CodeInfo { self.debug_block_dump(), )); - convert_pseudo_conditional_jumps(&mut self.blocks); + convert_pseudo_conditional_jumps(&mut self.blocks)?; trace.push(( "after_convert_pseudo_conditional_jumps".to_owned(), self.debug_block_dump(), )); let _max_stackdepth = calculate_stackdepth(&mut self.blocks)?; - let _nlocalsplus = prepare_localsplus(&self.metadata, &mut self.blocks, self.flags); + let _nlocalsplus = prepare_localsplus(&self.metadata, &mut self.blocks, self.flags)?; convert_pseudo_ops(&mut self.blocks)?; trace.push(( "after_convert_pseudo_ops".to_owned(), @@ -5042,7 +5056,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> && next != BlockIdx::NULL && blocks[next.idx()].warm { - let explicit_jump = blocks_new_block(blocks); + let explicit_jump = blocks_new_block(blocks)?; if !is_label(blocks[next.idx()].cpython_label) { blocks[next.idx()].cpython_label = InstructionSequenceLabel::from_index(next_label); next_label += 1; @@ -5065,7 +5079,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> except_handler: None, lineno_override: Some(NO_LOCATION_OVERRIDE), }, - ); + )?; blocks[explicit_jump.idx()].cold = true; blocks[explicit_jump.idx()].next = next; blocks[explicit_jump.idx()].predecessors = 1; @@ -5199,7 +5213,7 @@ fn basicblock_add_jump( except_handler: None, lineno_override: loc_source.lineno_override, }, - ); + )?; let last = basicblock_last_instr_mut(block).expect("missing jump"); debug_assert!(match (last.instr, instr) { (AnyInstruction::Real(last), AnyInstruction::Real(opcode)) => @@ -5226,7 +5240,7 @@ fn is_conditional_jump_opcode(instr: &AnyInstruction) -> bool { } /// flowgraph.c convert_pseudo_conditional_jumps -fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { +fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; @@ -5266,7 +5280,7 @@ fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { except_handler, lineno_override, }; - basicblock_insert_instruction(block, i, copy); + basicblock_insert_instruction(block, i, copy)?; i += 1; let to_bool = InstructionInfo { @@ -5278,13 +5292,14 @@ fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) { except_handler, lineno_override, }; - basicblock_insert_instruction(block, i, to_bool); + basicblock_insert_instruction(block, i, to_bool)?; i += 1; } i += 1; } block_idx = next; } + Ok(()) } /// flowgraph.c normalize_jumps_in_block @@ -5315,7 +5330,7 @@ fn normalize_jumps_in_block( except_handler: None, lineno_override: last_ins.lineno_override, }; - basicblock_addop(&mut blocks[idx], not_taken); + basicblock_addop(&mut blocks[idx], not_taken)?; return Ok(()); } @@ -5333,7 +5348,7 @@ fn normalize_jumps_in_block( let end_loc = last_ins.end_location; let target = last_ins.target; - let backwards_jump_idx = blocks_new_block(blocks); + let backwards_jump_idx = blocks_new_block(blocks)?; basicblock_addop( &mut blocks[backwards_jump_idx.idx()], InstructionInfo { @@ -5345,7 +5360,7 @@ fn normalize_jumps_in_block( except_handler: None, lineno_override: last_ins.lineno_override, }, - ); + )?; basicblock_add_jump( blocks, backwards_jump_idx, @@ -5387,12 +5402,15 @@ fn normalize_jumps(blocks: &mut Vec) -> crate::InternalResult<()> { } /// flowgraph.c basicblock_inline_small_or_no_lineno_blocks -fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: BlockIdx) -> bool { +fn basicblock_inline_small_or_no_lineno_blocks( + blocks: &mut [Block], + block_idx: BlockIdx, +) -> crate::InternalResult { let Some(last) = basicblock_last_instr(&blocks[block_idx.idx()]).copied() else { - return false; + return Ok(false); }; if !last.instr.is_unconditional_jump() { - return false; + return Ok(false); } let target = last.target; @@ -5408,7 +5426,7 @@ fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: .expect("non-empty block has last instruction"); set_to_nop(last); let target_instructions = blocks[target.idx()].instructions.clone(); - basicblock_append_instructions(&mut blocks[block_idx.idx()], &target_instructions); + basicblock_append_instructions(&mut blocks[block_idx.idx()], &target_instructions)?; if no_lineno_no_fallthrough { let last = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]).unwrap(); if last.instr.is_unconditional_jump() @@ -5421,19 +5439,19 @@ fn basicblock_inline_small_or_no_lineno_blocks(blocks: &mut [Block], block_idx: } } blocks[target.idx()].predecessors -= 1; - return true; + return Ok(true); } - false + Ok(false) } /// flowgraph.c inline_small_or_no_lineno_blocks -fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { +fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) -> crate::InternalResult<()> { loop { let mut changes = false; let mut current = BlockIdx(0); while current != BlockIdx::NULL { let next = blocks[current.idx()].next; - let res = basicblock_inline_small_or_no_lineno_blocks(blocks, current); + let res = basicblock_inline_small_or_no_lineno_blocks(blocks, current)?; if res { changes = true; } @@ -5444,6 +5462,7 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) { break; } } + Ok(()) } /// flowgraph.c basicblock_remove_redundant_nops @@ -5625,10 +5644,13 @@ fn make_cfg_traversal_stack(blocks: &mut [Block]) -> crate::InternalResult) -> BlockIdx { +fn blocks_new_block(blocks: &mut Vec) -> crate::InternalResult { + blocks + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; let block_idx = BlockIdx(blocks.len() as u32); blocks.push(Block::default()); - block_idx + Ok(block_idx) } /// flowgraph.c struct _PyCfgBuilder @@ -5641,12 +5663,12 @@ struct CfgBuilder { } /// flowgraph.c cfg_builder_new_block -fn cfg_builder_new_block(g: &mut CfgBuilder) -> BlockIdx { - let block = blocks_new_block(&mut g.blocks); +fn cfg_builder_new_block(g: &mut CfgBuilder) -> crate::InternalResult { + let block = blocks_new_block(&mut g.blocks)?; g.blocks[block.idx()].allocation_next = g.block_list; g.blocks[block.idx()].cpython_label = InstructionSequenceLabel::NO_LABEL; g.block_list = block; - block + Ok(block) } /// flowgraph.c cfg_builder_use_next_block @@ -5658,16 +5680,17 @@ fn cfg_builder_use_next_block(g: &mut CfgBuilder, block: BlockIdx) -> BlockIdx { } /// flowgraph.c init_cfg_builder -fn init_cfg_builder(g: &mut CfgBuilder) { +fn init_cfg_builder(g: &mut CfgBuilder) -> crate::InternalResult<()> { g.block_list = BlockIdx::NULL; - let block = cfg_builder_new_block(g); + let block = cfg_builder_new_block(g)?; g.entry = block; g.current = block; g.current_label = InstructionSequenceLabel::NO_LABEL; + Ok(()) } /// flowgraph.c _PyCfgBuilder_New -fn cfg_builder_new() -> CfgBuilder { +fn cfg_builder_new() -> crate::InternalResult { let mut builder = CfgBuilder { blocks: Vec::new(), entry: BlockIdx::NULL, @@ -5675,8 +5698,8 @@ fn cfg_builder_new() -> CfgBuilder { current: BlockIdx::NULL, current_label: InstructionSequenceLabel::NO_LABEL, }; - init_cfg_builder(&mut builder); - builder + init_cfg_builder(&mut builder)?; + Ok(builder) } /// flowgraph.c cfg_builder_current_block_is_terminated @@ -5697,25 +5720,29 @@ fn cfg_builder_current_block_is_terminated(g: &mut CfgBuilder) -> bool { } /// flowgraph.c cfg_builder_maybe_start_new_block -fn cfg_builder_maybe_start_new_block(g: &mut CfgBuilder) { +fn cfg_builder_maybe_start_new_block(g: &mut CfgBuilder) -> crate::InternalResult<()> { if cfg_builder_current_block_is_terminated(g) { - let block = cfg_builder_new_block(g); + let block = cfg_builder_new_block(g)?; g.blocks[block.idx()].cpython_label = g.current_label; g.current_label = InstructionSequenceLabel::NO_LABEL; cfg_builder_use_next_block(g, block); } + Ok(()) } /// flowgraph.c _PyCfgBuilder_UseLabel -fn cfg_builder_use_label(g: &mut CfgBuilder, label_id: InstructionSequenceLabel) { +fn cfg_builder_use_label( + g: &mut CfgBuilder, + label_id: InstructionSequenceLabel, +) -> crate::InternalResult<()> { g.current_label = label_id; - cfg_builder_maybe_start_new_block(g); + cfg_builder_maybe_start_new_block(g) } /// flowgraph.c _PyCfgBuilder_Addop -fn cfg_builder_addop(g: &mut CfgBuilder, info: InstructionInfo) { - cfg_builder_maybe_start_new_block(g); - basicblock_addop(&mut g.blocks[g.current.idx()], info); +fn cfg_builder_addop(g: &mut CfgBuilder, info: InstructionInfo) -> crate::InternalResult<()> { + cfg_builder_maybe_start_new_block(g)?; + basicblock_addop(&mut g.blocks[g.current.idx()], info) } /// flowgraph.c cfg_builder_check @@ -5849,7 +5876,7 @@ fn cfg_from_instruction_sequence( } = instr_sequence; debug_assert!(label_map.is_none()); - let mut builder = cfg_builder_new(); + let mut builder = cfg_builder_new()?; let mut offset = 0i32; let mut i = 0; @@ -5867,7 +5894,7 @@ fn cfg_from_instruction_sequence( debug_assert!(!ann_entry.info.instr.has_target()); let mut info = ann_entry.info; info.target = BlockIdx::NULL; - cfg_builder_addop(&mut builder, info); + cfg_builder_addop(&mut builder, info)?; } offset += annotations_code .instrs @@ -5889,7 +5916,7 @@ fn cfg_from_instruction_sequence( .checked_add(offset) .ok_or(InternalError::MalformedControlFlowGraph)?; let label = InstructionSequenceLabel(label_id); - cfg_builder_use_label(&mut builder, label); + cfg_builder_use_label(&mut builder, label)?; } let opcode = entry.info.instr; @@ -5908,7 +5935,7 @@ fn cfg_from_instruction_sequence( entry.info.instr = opcode; entry.info.arg = oparg; entry.info.target = BlockIdx::NULL; - cfg_builder_addop(&mut builder, entry.info); + cfg_builder_addop(&mut builder, entry.info)?; i += 1; } @@ -6231,14 +6258,17 @@ fn compute_reachable_predecessors(blocks: &mut [Block]) -> crate::InternalResult } /// flowgraph.c copy_basicblock -fn copy_basicblock(blocks: &mut Vec, block_idx: BlockIdx) -> BlockIdx { +fn copy_basicblock( + blocks: &mut Vec, + block_idx: BlockIdx, +) -> crate::InternalResult { debug_assert!(bb_no_fallthrough(&blocks[block_idx.idx()])); - let result = blocks_new_block(blocks); + let result = blocks_new_block(blocks)?; let result_idx = result.idx(); let (blocks_before_result, result_block) = blocks.split_at_mut(result_idx); let block = &blocks_before_result[block_idx.idx()]; - basicblock_append_instructions(&mut result_block[0], &block.instructions); - result + basicblock_append_instructions(&mut result_block[0], &block.instructions)?; + Ok(result) } /// flowgraph.c get_max_label @@ -6254,7 +6284,7 @@ fn get_max_label(blocks: &[Block]) -> i32 { } #[allow(clippy::collapsible_if)] -fn duplicate_exits_without_lineno(blocks: &mut Vec) { +fn duplicate_exits_without_lineno(blocks: &mut Vec) -> crate::InternalResult<()> { let mut next_lbl = get_max_label(blocks) + 1; let entryblock = BlockIdx(0); @@ -6271,7 +6301,7 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec) { if is_exit_or_eval_check_without_lineno(&blocks[target.idx()]) && blocks[target.idx()].predecessors > 1 { - let new_target = copy_basicblock(blocks, target); + let new_target = copy_basicblock(blocks, target)?; instr_set_location( &mut blocks[new_target.idx()].instructions[0], instr_location(&last), @@ -6307,6 +6337,7 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec) { } b = blocks[b.idx()].next; } + Ok(()) } #[allow(clippy::collapsible_if)] @@ -6354,9 +6385,13 @@ fn propagate_line_numbers(blocks: &mut [Block]) { } } -fn resolve_line_numbers(blocks: &mut Vec, _firstlineno: OneIndexed) { - duplicate_exits_without_lineno(blocks); +fn resolve_line_numbers( + blocks: &mut Vec, + _firstlineno: OneIndexed, +) -> crate::InternalResult<()> { + duplicate_exits_without_lineno(blocks)?; propagate_line_numbers(blocks); + Ok(()) } /// flowgraph.c make_except_stack @@ -6821,7 +6856,8 @@ mod tests { stale.except_handler = Some(handler); block.cpython_spare_instr_slots.push(stale); - basicblock_addop(&mut block, test_instr(Instruction::PopTop, 12)); + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 12)) + .expect("basicblock_addop succeeds"); // CPython `basicblock_addop()` writes opcode/oparg/target/location into // the reused `b_instr[b_iused]` slot, but does not clear `i_except`. @@ -6835,13 +6871,15 @@ mod tests { fn basicblock_next_instr_tracks_cpython_c_array_allocation() { let mut block = Block::default(); for i in 0..15 { - basicblock_addop(&mut block, test_instr(Instruction::PopTop, 10 + i)); + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 10 + i)) + .expect("basicblock_addop succeeds"); } assert_eq!(block.instruction_allocation, DEFAULT_BLOCK_SIZE); // CPython calls `_Py_CArray_EnsureCapacity(b_iused + 1)`, so the 16th // instruction expands a 16-slot array to 32 before returning offset 15. - basicblock_addop(&mut block, test_instr(Instruction::PopTop, 25)); + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 25)) + .expect("basicblock_addop succeeds"); assert_eq!(block.instruction_allocation, DEFAULT_BLOCK_SIZE * 2); } @@ -6857,7 +6895,8 @@ mod tests { stale.except_handler = Some(handler); block.cpython_spare_instr_slots.push(stale); - basicblock_insert_instruction(&mut block, 0, test_instr(Instruction::PopTop, 23)); + basicblock_insert_instruction(&mut block, 0, test_instr(Instruction::PopTop, 23)) + .expect("basicblock_insert_instruction succeeds"); // CPython `basicblock_insert_instruction()` also obtains a slot with // `basicblock_next_instr()`, then overwrites the inserted position with @@ -6879,7 +6918,8 @@ mod tests { block.instructions.push(stale); basicblock_clear(&mut block); - basicblock_addop(&mut block, test_instr(Instruction::Nop, 32)); + basicblock_addop(&mut block, test_instr(Instruction::Nop, 32)) + .expect("basicblock_addop succeeds"); // CPython `remove_unreachable()` sets `b_iused = 0` without clearing the // backing `b_instr` slot. A later `basicblock_addop()` reuses that slot @@ -6903,7 +6943,8 @@ mod tests { basicblock_clear(&mut block); for i in 0..3 { - basicblock_addop(&mut block, test_instr(Instruction::PopTop, 38 + i)); + basicblock_addop(&mut block, test_instr(Instruction::PopTop, 38 + i)) + .expect("basicblock_addop succeeds"); } let handlers = block @@ -6935,7 +6976,8 @@ mod tests { to.cpython_spare_instr_slots.push(stale); let from = [test_instr(Instruction::PopTop, 42)]; - basicblock_append_instructions(&mut to, &from); + basicblock_append_instructions(&mut to, &from) + .expect("basicblock_append_instructions succeeds"); // CPython `basicblock_append_instructions()` obtains a slot with // `basicblock_next_instr()`, then overwrites it with the copied @@ -7194,7 +7236,7 @@ mod tests { blocks[2].instructions[0].lineno_override = Some(NO_LOCATION_OVERRIDE); compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); - resolve_line_numbers(&mut blocks, OneIndexed::MIN); + resolve_line_numbers(&mut blocks, OneIndexed::MIN).expect("resolve_line_numbers succeeds"); // CPython `duplicate_exits_without_lineno()` copies a shared exit block // reached by jumps so each copy can inherit its sole predecessor's line. From c45852be08d792cf85889dca3b8122b7175dcd56 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:12:21 +0900 Subject: [PATCH 018/131] Propagate CPython redundant NOP cleanup errors --- crates/codegen/src/ir.rs | 47 +++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 6e5d04b499c..70f18f0284c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1722,7 +1722,7 @@ fn optimize_code_unit( // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before // later jump normalization / block reordering can create adjacencies // that never exist at this stage in flowgraph.c. - insert_superinstructions(blocks); + insert_superinstructions(blocks)?; push_cold_blocks_to_end(blocks)?; // CPython resolves line numbers again after cold-block extraction. resolve_line_numbers(blocks, metadata.firstlineno)?; @@ -1757,7 +1757,7 @@ fn optimize_cfg( optimize_basic_block(blocks, metadata, block_idx)?; block_idx = next_block; } - remove_redundant_nops_and_pairs(blocks); + remove_redundant_nops_and_pairs(blocks)?; // CPython optimize_cfg() removes newly-unreachable blocks and // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes // unused constants. @@ -3746,7 +3746,7 @@ fn optimize_basic_block( /// flowgraph.c remove_redundant_nops_and_pairs #[allow(clippy::if_same_then_else, clippy::useless_let_if_seq)] -fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) { +fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut done = false; while !done { @@ -3755,7 +3755,7 @@ fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) { let mut block_idx = BlockIdx::new(0); while block_idx != BlockIdx::NULL { - basicblock_remove_redundant_nops(blocks, block_idx); + basicblock_remove_redundant_nops(blocks, block_idx)?; if is_label(blocks[block_idx.idx()].cpython_label) { instr = None; } @@ -3810,6 +3810,7 @@ fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) { block_idx = block.next; } } + Ok(()) } /// flowgraph.c remove_unused_consts @@ -4310,7 +4311,7 @@ impl CodeInfo { "after_optimize_basic_block".to_owned(), self.debug_block_dump(), )); - remove_redundant_nops_and_pairs(&mut self.blocks); + remove_redundant_nops_and_pairs(&mut self.blocks)?; remove_unreachable(&mut self.blocks)?; remove_redundant_nops_and_jumps(&mut self.blocks)?; #[cfg(debug_assertions)] @@ -4323,7 +4324,7 @@ impl CodeInfo { let nlocals = self.metadata.varnames.len(); let nparams = self.nparams; add_checks_for_loads_of_uninitialized_variables(&mut self.blocks, nlocals, nparams)?; - insert_superinstructions(&mut self.blocks); + insert_superinstructions(&mut self.blocks)?; push_cold_blocks_to_end(&mut self.blocks)?; trace.push(( "after_push_cold_before_chain_reorder".to_owned(), @@ -4423,7 +4424,7 @@ fn make_super_instruction( } /// flowgraph.c insert_superinstructions -fn insert_superinstructions(blocks: &mut [Block]) { +fn insert_superinstructions(blocks: &mut [Block]) -> crate::InternalResult { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next_block = blocks[block_idx.idx()].next; @@ -4476,12 +4477,13 @@ fn insert_superinstructions(blocks: &mut [Block]) { } block_idx = next_block; } - remove_redundant_nops(blocks); + let res = remove_redundant_nops(blocks)?; #[cfg(debug_assertions)] { - let no_redundant = no_redundant_nops(blocks); + let no_redundant = no_redundant_nops(blocks)?; debug_assert!(no_redundant); } + Ok(res) } /// flowgraph.c LoadFastInstrFlag @@ -5466,7 +5468,10 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) -> crate::InternalResu } /// flowgraph.c basicblock_remove_redundant_nops -fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) -> usize { +fn basicblock_remove_redundant_nops( + blocks: &mut [Block], + block_idx: BlockIdx, +) -> crate::InternalResult { let bi = block_idx.idx(); let mut dest = 0; let mut prev_lineno = -1i32; @@ -5529,6 +5534,10 @@ fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) - debug_assert!(dest <= instr_count); let num_removed = instr_count - dest; + blocks[bi] + .cpython_spare_instr_slots + .try_reserve(num_removed) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; let mut src = instr_count; while src > dest { src -= 1; @@ -5537,27 +5546,25 @@ fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) - .push(blocks[bi].instructions[src]); } blocks[bi].instructions.truncate(dest); - num_removed + Ok(num_removed) } /// flowgraph.c remove_redundant_nops -fn remove_redundant_nops(blocks: &mut [Block]) -> usize { +fn remove_redundant_nops(blocks: &mut [Block]) -> crate::InternalResult { let mut changes = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { let next = blocks[current.idx()].next; - changes += basicblock_remove_redundant_nops(blocks, current); + changes += basicblock_remove_redundant_nops(blocks, current)?; current = next; } - changes + Ok(changes) } + /// flowgraph.c no_redundant_nops #[cfg(debug_assertions)] -fn no_redundant_nops(blocks: &mut [Block]) -> bool { - if remove_redundant_nops(blocks) != 0 { - return false; - } - true +fn no_redundant_nops(blocks: &mut [Block]) -> crate::InternalResult { + Ok(remove_redundant_nops(blocks)? == 0) } /// flowgraph.c remove_redundant_jumps @@ -5618,7 +5625,7 @@ fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResul loop { // Convergence is guaranteed because the number of redundant jumps and // nops only decreases. - let removed_nops = remove_redundant_nops(blocks); + let removed_nops = remove_redundant_nops(blocks)?; let removed_jumps = remove_redundant_jumps(blocks)?; if removed_nops + removed_jumps == 0 { break; From a85d8f4b369f7f63285c789ee9d603ec8c57f846 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:13:22 +0900 Subject: [PATCH 019/131] Propagate CPython unused const cleanup errors --- crates/codegen/src/ir.rs | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 70f18f0284c..38b26b67bdc 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1717,7 +1717,7 @@ fn optimize_code_unit( *blocks = cfg_from_instruction_sequence(instr_sequence)?; optimize_code_unit_preprocess(blocks)?; optimize_cfg(metadata, blocks, metadata.firstlineno)?; - remove_unused_consts(blocks, &mut metadata.consts); + remove_unused_consts(blocks, &mut metadata.consts)?; add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams)?; // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before // later jump normalization / block reordering can create adjacencies @@ -3815,13 +3815,20 @@ fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) -> crate::InternalResul /// flowgraph.c remove_unused_consts #[allow(clippy::needless_range_loop)] -fn remove_unused_consts(blocks: &mut [Block], consts: &mut ConstantPool) { +fn remove_unused_consts( + blocks: &mut [Block], + consts: &mut ConstantPool, +) -> crate::InternalResult<()> { let nconsts = consts.len(); if nconsts == 0 { - return; + return Ok(()); } - let mut index_map = vec![0isize; nconsts]; + let mut index_map = Vec::new(); + index_map + .try_reserve_exact(nconsts) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + index_map.resize(nconsts, 0isize); for i in 1..nconsts { index_map[i] = -1; } @@ -3855,7 +3862,7 @@ fn remove_unused_consts(blocks: &mut [Block], consts: &mut ConstantPool) { } if n_used_consts == nconsts { - return; + return Ok(()); } // Move all used consts to the beginning of the consts list. @@ -3873,7 +3880,11 @@ fn remove_unused_consts(blocks: &mut [Block], consts: &mut ConstantPool) { consts.constants.truncate(n_used_consts); // Adjust const indices in the bytecode. - let mut reverse_index_map = vec![0isize; nconsts]; + let mut reverse_index_map = Vec::new(); + reverse_index_map + .try_reserve_exact(nconsts) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + reverse_index_map.resize(nconsts, 0isize); for i in 0..nconsts { reverse_index_map[i] = -1; } @@ -3900,6 +3911,7 @@ fn remove_unused_consts(blocks: &mut [Block], consts: &mut ConstantPool) { } block_idx = next_block; } + Ok(()) } fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { @@ -4316,7 +4328,7 @@ impl CodeInfo { remove_redundant_nops_and_jumps(&mut self.blocks)?; #[cfg(debug_assertions)] assert!(no_redundant_jumps(&self.blocks)); - remove_unused_consts(&mut self.blocks, &mut self.metadata.consts); + remove_unused_consts(&mut self.blocks, &mut self.metadata.consts)?; trace.push(( "after_optimize_cfg_cleanup".to_owned(), self.debug_block_dump(), From 71eb1d7ec4f711b8362156c25a210bad2c92dd74 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:19:07 +0900 Subject: [PATCH 020/131] Propagate CPython const folding errors --- crates/codegen/src/ir.rs | 241 +++++++++++++++++++++++++-------------- 1 file changed, 157 insertions(+), 84 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 38b26b67bdc..402f0112371 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -81,6 +81,25 @@ impl ConstantPool { (idx, true) } + fn try_insert_full(&mut self, constant: ConstantData) -> crate::InternalResult<(usize, bool)> { + // CPython's _PyCode_ConstantKey() keeps NaN-bearing constants distinct + // because Python-level NaN keys do not compare equal. + if !Self::constant_contains_nan(&constant) + && let Some(idx) = self + .constants + .iter() + .position(|existing| existing == &constant) + { + return Ok((idx, false)); + } + self.constants + .try_reserve_exact(1) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + let idx = self.constants.len(); + self.constants.push(constant); + Ok((idx, true)) + } + pub fn insert(&mut self, constant: ConstantData) -> bool { self.insert_full(constant).1 } @@ -1750,7 +1769,7 @@ fn optimize_cfg( resolve_line_numbers(blocks, firstlineno)?; // CPython optimize_cfg() runs optimize_load_const() and then // optimize_basic_block() after line numbers are resolved. - optimize_load_const(metadata, blocks); + optimize_load_const(metadata, blocks)?; let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next_block = blocks[block_idx.idx()].next; @@ -2130,20 +2149,23 @@ fn load_const_truthiness( } /// flowgraph.c add_const -fn add_const(metadata: &mut CodeUnitMetadata, constant: ConstantData) -> usize { - metadata.consts.insert_full(constant).0 +fn add_const( + metadata: &mut CodeUnitMetadata, + constant: ConstantData, +) -> crate::InternalResult { + Ok(metadata.consts.try_insert_full(constant)?.0) } fn instr_make_load_const( metadata: &mut CodeUnitMetadata, instr: &mut InstructionInfo, constant: ConstantData, -) { +) -> crate::InternalResult<()> { if maybe_instr_make_load_smallint(instr, &constant) { - return; + return Ok(()); } - let const_idx = add_const(metadata, constant); + let const_idx = add_const(metadata, constant)?; instr_set_op1( instr, Instruction::LoadConst { @@ -2152,10 +2174,15 @@ fn instr_make_load_const( .into(), OpArg::new(const_idx as u32), ); + Ok(()) } /// flowgraph.c fold_const_unaryop -fn fold_const_unaryop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { +fn fold_const_unaryop( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, +) -> crate::InternalResult { let instr = &block.instructions[i]; let (op, intrinsic) = match instr.instr.real() { Some(Instruction::UnaryNegative) => (Instruction::UnaryNegative, None), @@ -2174,45 +2201,58 @@ fn fold_const_unaryop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usi Some(func.get(instr.arg)), ) } - _ => return false, + _ => return Ok(false), }; - let Some(operand_index) = i - .checked_sub(1) - .and_then(|start| get_const_loading_instrs(block, start, 1)) - .and_then(|indices| indices.into_iter().next()) - else { - return false; + let Some(operand_index) = (if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, 1)? + } else { + None + }) + .and_then(|indices| indices.into_iter().next()) else { + return Ok(false); }; let operand = get_const_value(metadata, &block.instructions[operand_index]); let Some(operand) = operand else { - return false; + return Ok(false); }; let Some(folded_const) = eval_const_unaryop(&operand, op, intrinsic) else { - return false; + return Ok(false); }; nop_out(block, &[operand_index]); - instr_make_load_const(metadata, &mut block.instructions[i], folded_const); - true + instr_make_load_const(metadata, &mut block.instructions[i], folded_const)?; + Ok(true) } /// flowgraph.c get_const_loading_instrs -fn get_const_loading_instrs(block: &Block, mut start: usize, size: usize) -> Option> { - let mut indices = Vec::with_capacity(size); +fn get_const_loading_instrs( + block: &Block, + mut start: usize, + size: usize, +) -> crate::InternalResult>> { + let mut indices = Vec::new(); + indices + .try_reserve_exact(size) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; loop { - let instr = block.instructions.get(start)?; + let Some(instr) = block.instructions.get(start) else { + return Ok(None); + }; if !matches!(instr.instr.real(), Some(Instruction::Nop)) { if !loads_const(instr) { - return None; + return Ok(None); } indices.push(start); if indices.len() == size { break; } } - start = start.checked_sub(1)?; + let Some(prev) = start.checked_sub(1) else { + return Ok(None); + }; + start = prev; } indices.reverse(); - Some(indices) + Ok(Some(indices)) } /// flowgraph.c nop_out @@ -2223,33 +2263,38 @@ fn nop_out(block: &mut Block, instrs: &[usize]) { } /// flowgraph.c fold_const_binop -fn fold_const_binop(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { +fn fold_const_binop( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, +) -> crate::InternalResult { use oparg::BinaryOperator as BinOp; let Some(Instruction::BinaryOp { .. }) = block.instructions[i].instr.real() else { - return false; + return Ok(false); }; - let Some(operand_indices) = i - .checked_sub(1) - .and_then(|start| get_const_loading_instrs(block, start, 2)) - else { - return false; + let Some(operand_indices) = (if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, 2)? + } else { + None + }) else { + return Ok(false); }; let op_raw = u32::from(block.instructions[i].arg); let Ok(op) = BinOp::try_from(op_raw) else { - return false; + return Ok(false); }; let left = get_const_value(metadata, &block.instructions[operand_indices[0]]); let right = get_const_value(metadata, &block.instructions[operand_indices[1]]); let (Some(left_val), Some(right_val)) = (left, right) else { - return false; + return Ok(false); }; let Some(result_const) = eval_const_binop(&left_val, &right_val, op) else { - return false; + return Ok(false); }; nop_out(block, &operand_indices); - instr_make_load_const(metadata, &mut block.instructions[i], result_const); - true + instr_make_load_const(metadata, &mut block.instructions[i], result_const)?; + Ok(true) } /// flowgraph.c loads_const @@ -2896,29 +2941,37 @@ fn eval_const_binop( } /// flowgraph.c fold_tuple_of_constants -fn fold_tuple_of_constants(metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize) -> bool { +fn fold_tuple_of_constants( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, +) -> crate::InternalResult { let Some(Instruction::BuildTuple { .. }) = block.instructions[i].instr.real() else { - return false; + return Ok(false); }; let tuple_size = u32::from(block.instructions[i].arg) as usize; if tuple_size > STACK_USE_GUIDELINE { - return false; + return Ok(false); } let Some(operand_indices) = (if tuple_size == 0 { Some(Vec::new()) + } else if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, tuple_size)? } else { - i.checked_sub(1) - .and_then(|start| get_const_loading_instrs(block, start, tuple_size)) + None }) else { - return false; + return Ok(false); }; - let mut elements = Vec::with_capacity(tuple_size); + let mut elements = Vec::new(); + elements + .try_reserve_exact(tuple_size) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; for &j in &operand_indices { let Some(element) = get_const_value(metadata, &block.instructions[j]) else { - return false; + return Ok(false); }; elements.push(element); } @@ -2928,20 +2981,20 @@ fn fold_tuple_of_constants(metadata: &mut CodeUnitMetadata, block: &mut Block, i metadata, &mut block.instructions[i], ConstantData::Tuple { elements }, - ); - true + )?; + Ok(true) } fn fold_constant_intrinsic_list_to_tuple( metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize, -) -> bool { +) -> crate::InternalResult { let Some(Instruction::CallIntrinsic1 { func }) = block.instructions[i].instr.real() else { - return false; + return Ok(false); }; if func.get(block.instructions[i].arg) != IntrinsicFunction1::ListToTuple { - return false; + return Ok(false); } let mut consts_found = 0usize; @@ -2958,10 +3011,14 @@ fn fold_constant_intrinsic_list_to_tuple( && u32::from(instr.arg) == 0 { if !expect_append { - return false; + return Ok(false); } - let mut elements = vec![None; consts_found]; + let mut elements = Vec::new(); + elements + .try_reserve_exact(consts_found) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + elements.resize_with(consts_found, || None); let mut remaining = consts_found; for idx in (pos..i).rev() { if matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { @@ -2969,7 +3026,7 @@ fn fold_constant_intrinsic_list_to_tuple( } if loads_const(&block.instructions[idx]) { let Some(value) = get_const_value(metadata, &block.instructions[idx]) else { - return false; + return Ok(false); }; debug_assert!(remaining > 0); remaining -= 1; @@ -2986,26 +3043,26 @@ fn fold_constant_intrinsic_list_to_tuple( metadata, &mut block.instructions[i], ConstantData::Tuple { elements }, - ); - return true; + )?; + return Ok(true); } if expect_append { if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) || u32::from(instr.arg) != 1 { - return false; + return Ok(false); } } else { if !loads_const(instr) { - return false; + return Ok(false); } consts_found += 1; } expect_append = !expect_append; } - false + Ok(false) } /// Port of CPython's flowgraph.c optimize_lists_and_sets(). @@ -3014,14 +3071,14 @@ fn optimize_lists_and_sets( block: &mut Block, i: usize, nextop: Option, -) -> bool { +) -> crate::InternalResult { let Some(instr) = block.instructions[i].instr.real() else { - return false; + return Ok(false); }; let is_list = matches!(instr, Instruction::BuildList { .. }); let is_set = matches!(instr, Instruction::BuildSet { .. }); if !is_list && !is_set { - return false; + return Ok(false); } let contains_or_iter = matches!( @@ -3030,27 +3087,31 @@ fn optimize_lists_and_sets( ); let seq_size = u32::from(block.instructions[i].arg) as usize; if seq_size > STACK_USE_GUIDELINE || (seq_size < MIN_CONST_SEQUENCE_SIZE && !contains_or_iter) { - return false; + return Ok(false); } let Some(operand_indices) = (if seq_size == 0 { Some(Vec::new()) + } else if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, seq_size)? } else { - i.checked_sub(1) - .and_then(|start| get_const_loading_instrs(block, start, seq_size)) + None }) else { if contains_or_iter && is_list { let arg = block.instructions[i].arg; instr_set_op1(&mut block.instructions[i], Opcode::BuildTuple.into(), arg); - return true; + return Ok(true); } - return false; + return Ok(false); }; - let mut elements = Vec::with_capacity(seq_size); + let mut elements = Vec::new(); + elements + .try_reserve_exact(seq_size) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; for &j in &operand_indices { let Some(element) = get_const_value(metadata, &block.instructions[j]) else { - return false; + return Ok(false); }; elements.push(element); } @@ -3060,7 +3121,7 @@ fn optimize_lists_and_sets( } else { ConstantData::Frozenset { elements } }; - let const_idx = add_const(metadata, const_data); + let const_idx = add_const(metadata, const_data)?; if !contains_or_iter { debug_assert!(i >= 2); @@ -3104,7 +3165,7 @@ fn optimize_lists_and_sets( extend_instr.into(), OpArg::new(1), ); - return true; + return Ok(true); } nop_out(block, &operand_indices); @@ -3117,7 +3178,7 @@ fn optimize_lists_and_sets( .into(), OpArg::new(const_idx as u32), ); - true + Ok(true) } const SWAP_OPTIMIZE_VISITED: i32 = -1; @@ -3330,7 +3391,10 @@ fn maybe_instr_make_load_smallint(instr: &mut InstructionInfo, constant: &Consta } /// flowgraph.c basicblock_optimize_load_const -fn basicblock_optimize_load_const(metadata: &mut CodeUnitMetadata, block: &mut Block) { +fn basicblock_optimize_load_const( + metadata: &mut CodeUnitMetadata, + block: &mut Block, +) -> crate::InternalResult<()> { let mut i = 0; let mut effective_opcode = None; let mut effective_oparg = OpArg::new(0); @@ -3471,7 +3535,7 @@ fn basicblock_optimize_load_const(metadata: &mut CodeUnitMetadata, block: &mut B ) && matches!(next_instr, Instruction::ToBool) && let Some(value) = load_const_truthiness(const_instr, const_arg, metadata) { - let const_idx = add_const(metadata, ConstantData::Boolean { value }); + let const_idx = add_const(metadata, ConstantData::Boolean { value })?; set_to_nop(&mut block.instructions[i]); instr_set_op1( &mut block.instructions[i + 1], @@ -3487,17 +3551,22 @@ fn basicblock_optimize_load_const(metadata: &mut CodeUnitMetadata, block: &mut B i += 1; } + Ok(()) } /// flowgraph.c optimize_load_const -fn optimize_load_const(metadata: &mut CodeUnitMetadata, blocks: &mut [Block]) { +fn optimize_load_const( + metadata: &mut CodeUnitMetadata, + blocks: &mut [Block], +) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next_block = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx]; - basicblock_optimize_load_const(metadata, block); + basicblock_optimize_load_const(metadata, block)?; block_idx = next_block; } + Ok(()) } /// flowgraph.c optimize_basic_block @@ -3559,10 +3628,10 @@ fn optimize_basic_block( _ => {} } } - fold_tuple_of_constants(metadata, &mut blocks[bi], i); + fold_tuple_of_constants(metadata, &mut blocks[bi], i)?; } AnyInstruction::Real(Instruction::BuildList { .. } | Instruction::BuildSet { .. }) => { - optimize_lists_and_sets(metadata, &mut blocks[bi], i, nextop); + optimize_lists_and_sets(metadata, &mut blocks[bi], i, nextop)?; } AnyInstruction::Real( Instruction::PopJumpIfNotNone { .. } | Instruction::PopJumpIfNone { .. }, @@ -3712,10 +3781,10 @@ fn optimize_basic_block( i += 1; continue; } - fold_const_unaryop(metadata, &mut blocks[bi], i); + fold_const_unaryop(metadata, &mut blocks[bi], i)?; } AnyInstruction::Real(Instruction::UnaryInvert | Instruction::UnaryNegative) => { - fold_const_unaryop(metadata, &mut blocks[bi], i); + fold_const_unaryop(metadata, &mut blocks[bi], i)?; } AnyInstruction::Real(Instruction::CallIntrinsic1 { func }) => { match func.get(inst.arg) { @@ -3723,17 +3792,17 @@ fn optimize_basic_block( if matches!(nextop, Some(Instruction::GetIter)) { set_to_nop(&mut blocks[bi].instructions[i]); } else { - fold_constant_intrinsic_list_to_tuple(metadata, &mut blocks[bi], i); + fold_constant_intrinsic_list_to_tuple(metadata, &mut blocks[bi], i)?; } } IntrinsicFunction1::UnaryPositive => { - fold_const_unaryop(metadata, &mut blocks[bi], i); + fold_const_unaryop(metadata, &mut blocks[bi], i)?; } _ => {} } } AnyInstruction::Real(Instruction::BinaryOp { .. }) => { - fold_const_binop(metadata, &mut blocks[bi], i); + fold_const_binop(metadata, &mut blocks[bi], i)?; } _ => {} } @@ -4308,7 +4377,7 @@ impl CodeInfo { )); remove_unreachable(&mut self.blocks)?; resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno)?; - optimize_load_const(&mut self.metadata, &mut self.blocks); + optimize_load_const(&mut self.metadata, &mut self.blocks)?; trace.push(( "after_optimize_load_const".to_owned(), self.debug_block_dump(), @@ -7122,7 +7191,8 @@ mod tests { }); code.blocks[0].instructions[0].arg = OpArg::new(const_idx as u32); - optimize_load_const(&mut code.metadata, &mut code.blocks); + optimize_load_const(&mut code.metadata, &mut code.blocks) + .expect("optimize_load_const succeeds"); // CPython `basicblock_optimize_load_const()` keeps the previous // LOAD_CONST as the effective opcode for a following `COPY 1`, so the @@ -7214,7 +7284,10 @@ mod tests { let mut block = Block::default(); block.instructions.extend([immortal, mortal, build]); - assert!(fold_tuple_of_constants(&mut metadata, &mut block, 2)); + assert!( + fold_tuple_of_constants(&mut metadata, &mut block, 2) + .expect("fold_tuple_of_constants succeeds") + ); // CPython `loads_const()` accepts every `OPCODE_HAS_CONST` opcode, not // just canonical LOAD_CONST, so LOAD_CONST_IMMORTAL/MORTAL participate From 13f4ef579a42494fc833bacb414d6e6c6dbe7c44 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:21:40 +0900 Subject: [PATCH 021/131] Propagate CPython swaptimize allocation errors --- crates/codegen/src/ir.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 402f0112371..f1d2e1e2fe0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3230,7 +3230,7 @@ fn next_swappable_instruction( } /// flowgraph.c swaptimize -fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) { +fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::InternalResult<()> { debug_assert!(matches!( instructions[*ix].instr.real(), Some(Instruction::Swap { .. }) @@ -3254,13 +3254,16 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) { } if !more { - return; + return Ok(()); } - let mut stack = vec![0; depth]; + let mut stack = Vec::new(); + stack + .try_reserve_exact(depth) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; let mut i = 0; while i < depth { - stack[i] = i as i32; + stack.push(i as i32); i += 1; } @@ -3303,6 +3306,7 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) { current -= 1; } *ix += len - 1; + Ok(()) } /// flowgraph.c apply_static_swaps @@ -3365,18 +3369,19 @@ fn apply_static_swaps(instructions: &mut [InstructionInfo], mut i: isize) { } /// flowgraph.c optimize_basic_block swap pass -fn apply_static_swaps_block(block: &mut Block) { +fn apply_static_swaps_block(block: &mut Block) -> crate::InternalResult<()> { let mut i = 0; while i < block.instructions.len() { if matches!( block.instructions[i].instr.real(), Some(Instruction::Swap { .. }) ) { - swaptimize(&mut block.instructions, &mut i); + swaptimize(&mut block.instructions, &mut i)?; apply_static_swaps(&mut block.instructions, i as isize); } i += 1; } + Ok(()) } /// flowgraph.c maybe_instr_make_load_smallint @@ -3809,7 +3814,7 @@ fn optimize_basic_block( i += 1; } - apply_static_swaps_block(&mut blocks[block_idx]); + apply_static_swaps_block(&mut blocks[block_idx])?; Ok(()) } @@ -7119,7 +7124,7 @@ mod tests { pop.lineno_override = Some(NO_LOCATION_OVERRIDE); block.instructions.extend([swap, store, pop]); - apply_static_swaps_block(&mut block); + apply_static_swaps_block(&mut block).expect("apply_static_swaps_block succeeds"); // CPython `next_swappable_instruction()` compares `i_loc.lineno` // directly, so a following NO_LOCATION swaperand does not match the @@ -7151,7 +7156,7 @@ mod tests { let pop = test_instr(Instruction::PopTop, 71); block.instructions.extend([swap, store, pop]); - apply_static_swaps_block(&mut block); + apply_static_swaps_block(&mut block).expect("apply_static_swaps_block succeeds"); // Conversely, when the first swaperand has NO_LOCATION, CPython passes // `-1` as the line filter and does not enforce a boundary. From 71f9bef6f0db331dae0dbd33c03dd02db0997fe5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:23:36 +0900 Subject: [PATCH 022/131] Match CPython list-to-tuple fold allocation --- crates/codegen/src/ir.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f1d2e1e2fe0..1845932f633 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3018,8 +3018,6 @@ fn fold_constant_intrinsic_list_to_tuple( elements .try_reserve_exact(consts_found) .map_err(|_| InternalError::MalformedControlFlowGraph)?; - elements.resize_with(consts_found, || None); - let mut remaining = consts_found; for idx in (pos..i).rev() { if matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { continue; @@ -3028,17 +3026,12 @@ fn fold_constant_intrinsic_list_to_tuple( let Some(value) = get_const_value(metadata, &block.instructions[idx]) else { return Ok(false); }; - debug_assert!(remaining > 0); - remaining -= 1; - elements[remaining] = Some(value); + elements.push(value); } nop_out_no_location(&mut block.instructions[idx]); } - debug_assert_eq!(remaining, 0); - let elements = elements - .into_iter() - .map(|element| element.expect("constant list item was collected")) - .collect(); + debug_assert_eq!(elements.len(), consts_found); + elements.reverse(); instr_make_load_const( metadata, &mut block.instructions[i], From ed0ad7405633971a4a9512abde88a36f7302de95 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:24:53 +0900 Subject: [PATCH 023/131] Skip const folding on CPython allocation failures --- crates/codegen/src/ir.rs | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 1845932f633..bd20433724d 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -2333,12 +2333,13 @@ fn const_folding_check_complexity(obj: &ConstantData, mut limit: isize) -> Optio Some(limit) } -fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Wtf8Buf { - let mut result = Wtf8Buf::with_capacity(value.len().saturating_mul(n)); +fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Option { + let mut result = Wtf8Buf::new(); + result.try_reserve_exact(value.len().checked_mul(n)?).ok()?; for _ in 0..n { result.push_wtf8(value); } - result + Some(result) } fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { @@ -2364,7 +2365,7 @@ fn const_folding_safe_multiply(left: &ConstantData, right: &ConstantData) -> Opt (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) => { let n = checked_repeat_count(n, s.code_points().count())?; Some(ConstantData::Str { - value: repeat_wtf8(s, n), + value: repeat_wtf8(s, n)?, }) } (ConstantData::Integer { .. }, ConstantData::Str { .. }) => { @@ -2372,7 +2373,12 @@ fn const_folding_safe_multiply(left: &ConstantData, right: &ConstantData) -> Opt } (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) => { let n = checked_repeat_count(n, b.len())?; - Some(ConstantData::Bytes { value: b.repeat(n) }) + let mut value = Vec::new(); + value.try_reserve_exact(b.len().checked_mul(n)?).ok()?; + for _ in 0..n { + value.extend_from_slice(b); + } + Some(ConstantData::Bytes { value }) } (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) => { const_folding_safe_multiply(right, left) @@ -2390,7 +2396,10 @@ fn const_folding_safe_multiply(left: &ConstantData, right: &ConstantData) -> Opt MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, )?; } - let mut result = Vec::with_capacity(elements.len() * n); + let mut result = Vec::new(); + result + .try_reserve_exact(elements.len().checked_mul(n)?) + .ok()?; for _ in 0..n { result.extend(elements.iter().cloned()); } @@ -2888,7 +2897,11 @@ fn eval_const_binop( (ConstantData::Str { value: l }, ConstantData::Str { value: r }) if matches!(op, BinOp::Add) => { - let mut result = l.clone(); + let mut result = Wtf8Buf::new(); + result + .try_reserve_exact(l.len().checked_add(r.len())?) + .ok()?; + result.push_wtf8(l); result.push_wtf8(r); Some(ConstantData::Str { value: result }) } @@ -2900,7 +2913,11 @@ fn eval_const_binop( (ConstantData::Tuple { elements: l }, ConstantData::Tuple { elements: r }) if matches!(op, BinOp::Add) => { - let mut result = l.clone(); + let mut result = Vec::new(); + result + .try_reserve_exact(l.len().checked_add(r.len())?) + .ok()?; + result.extend(l.iter().cloned()); result.extend(r.iter().cloned()); Some(ConstantData::Tuple { elements: result }) } @@ -2922,7 +2939,11 @@ fn eval_const_binop( (ConstantData::Bytes { value: l }, ConstantData::Bytes { value: r }) if matches!(op, BinOp::Add) => { - let mut result = l.clone(); + let mut result = Vec::new(); + result + .try_reserve_exact(l.len().checked_add(r.len())?) + .ok()?; + result.extend_from_slice(l); result.extend_from_slice(r); Some(ConstantData::Bytes { value: result }) } From 1e0d4f96046d9286f9941460b3ed6aead3c39e64 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:26:27 +0900 Subject: [PATCH 024/131] Skip subscript folding on CPython allocation failures --- crates/codegen/src/ir.rs | 45 +++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index bd20433724d..d677d2a1b96 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -2640,10 +2640,22 @@ fn adjusted_slice_indices(len: usize, slice: &[ConstantData; 3]) -> Option 0 { + if index < stop { + usize::try_from((stop - index - 1) / step + 1).ok()? + } else { + 0 + } + } else if index > stop { + usize::try_from((index - stop - 1) / -step + 1).ok()? + } else { + 0 + }; + let mut indices = Vec::new(); + indices.try_reserve_exact(slice_len).ok()?; if step > 0 { while index < stop { indices.push(usize::try_from(index).ok()?); @@ -2684,7 +2696,9 @@ fn eval_const_subscript(container: &ConstantData, index: &ConstantData) -> Optio if string.contains(char::REPLACEMENT_CHARACTER) { return None; } - let chars = string.chars().collect::>(); + let mut chars = Vec::new(); + chars.try_reserve_exact(string.chars().count()).ok()?; + chars.extend(string.chars()); let index = adjusted_const_index(chars.len(), index)?; Some(ConstantData::Str { value: chars[index].to_string().into(), @@ -2695,9 +2709,16 @@ fn eval_const_subscript(container: &ConstantData, index: &ConstantData) -> Optio if string.contains(char::REPLACEMENT_CHARACTER) { return None; } - let chars = string.chars().collect::>(); + let mut chars = Vec::new(); + chars.try_reserve_exact(string.chars().count()).ok()?; + chars.extend(string.chars()); + let indices = adjusted_slice_indices(chars.len(), elements)?; + let capacity = indices.iter().try_fold(0usize, |capacity, &index| { + capacity.checked_add(chars[index].len_utf8()) + })?; let mut result = String::new(); - for index in adjusted_slice_indices(chars.len(), elements)? { + result.try_reserve_exact(capacity).ok()?; + for index in indices { result.push(chars[index]); } Some(ConstantData::Str { @@ -2714,8 +2735,10 @@ fn eval_const_subscript(container: &ConstantData, index: &ConstantData) -> Optio }) } (ConstantData::Bytes { value }, ConstantData::Slice { elements }) => { + let indices = adjusted_slice_indices(value.len(), elements)?; let mut result = Vec::new(); - for index in adjusted_slice_indices(value.len(), elements)? { + result.try_reserve_exact(indices.len()).ok()?; + for index in indices { result.push(value[index]); } Some(ConstantData::Bytes { value: result }) @@ -2728,11 +2751,13 @@ fn eval_const_subscript(container: &ConstantData, index: &ConstantData) -> Optio Some(elements[index].clone()) } (ConstantData::Tuple { elements }, ConstantData::Slice { elements: slice }) => { - let elements = adjusted_slice_indices(elements.len(), slice)? - .into_iter() - .map(|index| elements[index].clone()) - .collect(); - Some(ConstantData::Tuple { elements }) + let indices = adjusted_slice_indices(elements.len(), slice)?; + let mut result = Vec::new(); + result.try_reserve_exact(indices.len()).ok()?; + for index in indices { + result.push(elements[index].clone()); + } + Some(ConstantData::Tuple { elements: result }) } _ => None, } From af868ab07d4ca78d77cca46f33510a9afd21e422 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:29:43 +0900 Subject: [PATCH 025/131] Propagate CPython assembler allocation errors --- crates/codegen/src/ir.rs | 85 +++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d677d2a1b96..fbd0cbf4dcc 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1036,7 +1036,7 @@ fn assemble_location_info( instr_sequence: &mut InstructionSequence, first_line: i32, debug_ranges: bool, -) -> Box<[u8]> { +) -> crate::InternalResult> { for i in (0..instr_sequence.instrs.len()).rev() { let loc = instruction_linetable_location(&instr_sequence.instrs[i].info); if same_location(loc, next_linetable_location()) { @@ -1068,14 +1068,14 @@ fn assemble_location_info( let entry = &instr_sequence.instrs[i]; let instr_loc = instruction_linetable_location(&entry.info); if !same_location(loc, instr_loc) { - assemble_emit_location(&mut linetable, loc, size, &mut prev_line, debug_ranges); + assemble_emit_location(&mut linetable, loc, size, &mut prev_line, debug_ranges)?; loc = instr_loc; size = 0; } size += instr_size(&entry.info); } - assemble_emit_location(&mut linetable, loc, size, &mut prev_line, debug_ranges); - linetable.into_boxed_slice() + assemble_emit_location(&mut linetable, loc, size, &mut prev_line, debug_ranges)?; + Ok(linetable.into_boxed_slice()) } /// assemble.c assemble_emit @@ -1088,16 +1088,16 @@ fn assemble_emit( let num_instructions = usize::try_from(end_offset).map_err(|_| InternalError::MalformedControlFlowGraph)?; let mut instructions = Vec::new(); - instructions.reserve_exact(num_instructions); + vec_try_reserve_exact(&mut instructions, num_instructions)?; for i in 0..instr_sequence.instrs.len() { let instr = &mut instr_sequence.instrs[i].info; assemble_emit_instr(&mut instructions, instr); } - let linetable = assemble_location_info(instr_sequence, first_line, debug_ranges); + let linetable = assemble_location_info(instr_sequence, first_line, debug_ranges)?; - let exceptiontable = assemble_exception_table(&instr_sequence.instrs); + let exceptiontable = assemble_exception_table(&instr_sequence.instrs)?; Ok(AssembledCode { instructions, @@ -4758,6 +4758,11 @@ fn get_stack_effects( Ok(StackEffects { net }) } +fn vec_try_reserve_exact(vec: &mut Vec, additional: usize) -> crate::InternalResult<()> { + vec.try_reserve_exact(additional) + .map_err(|_| InternalError::MalformedControlFlowGraph) +} + /// assemble.c write_location_first_byte fn write_location_first_byte(linetable: &mut Vec, code: u8, length: usize) { linetable.extend(write_location_entry_start(code, length)); @@ -4791,11 +4796,12 @@ fn write_location_info_short_form( length: usize, column: i32, end_column: i32, -) { +) -> crate::InternalResult<()> { debug_assert!(length > 0 && length <= 8); debug_assert!(column < 80); debug_assert!(end_column >= column); debug_assert!(end_column - column < 16); + vec_try_reserve_exact(linetable, 2)?; let column_low_bits = column & 7; let column_group = column >> 3; let code = PyCodeLocationInfoKind::Short0 as u8 + column_group as u8; @@ -4804,6 +4810,7 @@ fn write_location_info_short_form( linetable, ((column_low_bits as u8) << 4) | ((end_column - column) as u8), ); + Ok(()) } /// assemble.c write_location_info_oneline_form @@ -4813,15 +4820,17 @@ fn write_location_info_oneline_form( line_delta: i32, column: i32, end_column: i32, -) { +) -> crate::InternalResult<()> { debug_assert!(length > 0 && length <= 8); debug_assert!((0..3).contains(&line_delta)); debug_assert!(column < 128); debug_assert!(end_column < 128); + vec_try_reserve_exact(linetable, 3)?; let code = PyCodeLocationInfoKind::OneLine0 as u8 + line_delta as u8; write_location_first_byte(linetable, code, length); write_location_byte(linetable, column as u8); write_location_byte(linetable, end_column as u8); + Ok(()) } /// assemble.c write_location_info_long_form @@ -4830,8 +4839,9 @@ fn write_location_info_long_form( loc: LineTableLocation, length: usize, line_delta: i32, -) { +) -> crate::InternalResult<()> { debug_assert!(length > 0 && length <= 8); + vec_try_reserve_exact(linetable, 25)?; write_location_first_byte(linetable, PyCodeLocationInfoKind::Long as u8, length); write_location_signed_varint(linetable, line_delta); debug_assert!(loc.end_line >= loc.line); @@ -4848,17 +4858,26 @@ fn write_location_info_long_form( (loc.end_col as u32) + 1 }, ); + Ok(()) } /// assemble.c write_location_info_none -fn write_location_info_none(linetable: &mut Vec, length: usize) { +fn write_location_info_none(linetable: &mut Vec, length: usize) -> crate::InternalResult<()> { + vec_try_reserve_exact(linetable, 1)?; write_location_first_byte(linetable, PyCodeLocationInfoKind::None as u8, length); + Ok(()) } /// assemble.c write_location_info_no_column -fn write_location_info_no_column(linetable: &mut Vec, length: usize, line_delta: i32) { +fn write_location_info_no_column( + linetable: &mut Vec, + length: usize, + line_delta: i32, +) -> crate::InternalResult<()> { + vec_try_reserve_exact(linetable, 7)?; write_location_first_byte(linetable, PyCodeLocationInfoKind::NoColumns as u8, length); write_location_signed_varint(linetable, line_delta); + Ok(()) } /// assemble.c write_location_info_entry @@ -4868,10 +4887,9 @@ fn write_location_info_entry( length: usize, prev_line: &mut i32, debug_ranges: bool, -) { +) -> crate::InternalResult<()> { if loc.line == NO_LOCATION_OVERRIDE { - write_location_info_none(linetable, length); - return; + return write_location_info_none(linetable, length); } let line_delta = loc.line - *prev_line; @@ -4880,25 +4898,25 @@ fn write_location_info_entry( if !debug_ranges || ((column < 0 || end_column < 0) && (loc.end_line == loc.line || loc.end_line < 0)) { - write_location_info_no_column(linetable, length, line_delta); + write_location_info_no_column(linetable, length, line_delta)?; *prev_line = loc.line; - return; + return Ok(()); } if loc.end_line == loc.line { if line_delta == 0 && column < 80 && end_column - column < 16 && end_column >= column { - write_location_info_short_form(linetable, length, column, end_column); - return; + return write_location_info_short_form(linetable, length, column, end_column); } if (0..3).contains(&line_delta) && column < 128 && end_column < 128 { - write_location_info_oneline_form(linetable, length, line_delta, column, end_column); + write_location_info_oneline_form(linetable, length, line_delta, column, end_column)?; *prev_line = loc.line; - return; + return Ok(()); } } - write_location_info_long_form(linetable, loc, length, line_delta); + write_location_info_long_form(linetable, loc, length, line_delta)?; *prev_line = loc.line; + Ok(()) } /// assemble.c assemble_emit_location @@ -4908,15 +4926,15 @@ fn assemble_emit_location( mut size: usize, prev_line: &mut i32, debug_ranges: bool, -) { +) -> crate::InternalResult<()> { if size == 0 { - return; + return Ok(()); } while size > 8 { - write_location_info_entry(linetable, loc, 8, prev_line, debug_ranges); + write_location_info_entry(linetable, loc, 8, prev_line, debug_ranges)?; size -= 8; } - write_location_info_entry(linetable, loc, size, prev_line, debug_ranges); + write_location_info_entry(linetable, loc, size, prev_line, debug_ranges) } fn no_linetable_location() -> LineTableLocation { @@ -4969,7 +4987,9 @@ fn assemble_emit_exception_table_entry( end: i32, handler_offset: i32, handler: InstructionSequenceExceptHandlerInfo, -) { +) -> crate::InternalResult<()> { + const MAX_SIZE_OF_ENTRY: usize = 20; + vec_try_reserve_exact(table, MAX_SIZE_OF_ENTRY)?; let size = end - start; debug_assert!(end > start); let target = handler_offset; @@ -4988,10 +5008,13 @@ fn assemble_emit_exception_table_entry( assemble_emit_exception_table_item(table, size, 0); assemble_emit_exception_table_item(table, target, 0); assemble_emit_exception_table_item(table, depth_lasti, 0); + Ok(()) } /// assemble.c assemble_exception_table -fn assemble_exception_table(instrs: &[InstructionSequenceEntry]) -> Box<[u8]> { +fn assemble_exception_table( + instrs: &[InstructionSequenceEntry], +) -> crate::InternalResult> { let mut table = Vec::new(); let mut handler = InstructionSequenceExceptHandlerInfo { h_label: NO_EXCEPTION_HANDLER_LABEL, @@ -5014,7 +5037,7 @@ fn assemble_exception_table(instrs: &[InstructionSequenceEntry]) -> Box<[u8]> { ioffset, handler_offset, handler, - ); + )?; } start = ioffset; handler = instr.except_handler; @@ -5028,10 +5051,10 @@ fn assemble_exception_table(instrs: &[InstructionSequenceEntry]) -> Box<[u8]> { let handler_label = usize::try_from(handler.h_label).expect("handler label is non-negative"); let handler_offset = instrs[handler_label].i_offset; - assemble_emit_exception_table_entry(&mut table, start, ioffset, handler_offset, handler); + assemble_emit_exception_table_entry(&mut table, start, ioffset, handler_offset, handler)?; } - table.into_boxed_slice() + Ok(table.into_boxed_slice()) } /// Mark exception handler target blocks. From 7571fbd9b29dc7195f65a1d637d9d432d71c3cfa Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:31:32 +0900 Subject: [PATCH 026/131] Propagate CPython localsplus allocation errors --- crates/codegen/src/ir.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index fbd0cbf4dcc..97d715dd9ca 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1111,12 +1111,15 @@ fn compute_localsplus_info( umd: &CodeUnitMetadata, nlocalsplus: usize, flags: CodeFlags, -) -> LocalsPlusInfo { +) -> crate::InternalResult { let nlocals = umd.varnames.len(); let ncells = umd.cellvars.len(); let nfrees = umd.freevars.len(); - let mut localspluskinds = vec![0u8; nlocalsplus]; - let mut cellvars = Vec::with_capacity(ncells); + let mut localspluskinds = Vec::new(); + vec_try_reserve_exact(&mut localspluskinds, nlocalsplus)?; + localspluskinds.resize(nlocalsplus, 0); + let mut cellvars = Vec::new(); + vec_try_reserve_exact(&mut cellvars, ncells)?; let argvarkinds = [ (umd.posonlyargcount as usize, CO_FAST_ARG_POS), @@ -1196,10 +1199,10 @@ fn compute_localsplus_info( "CPython prepare_localsplus() result must match assemble.c localsplus sizing" ); debug_assert_eq!(cellvars.len(), ncells); - LocalsPlusInfo { + Ok(LocalsPlusInfo { cellvars: cellvars.into_boxed_slice(), kinds: localspluskinds.into_boxed_slice(), - } + }) } #[derive(Debug, Clone)] @@ -1829,7 +1832,7 @@ impl CodeInfo { )?; let (max_stackdepth, nlocalsplus, mut instr_sequence) = optimized_cfg_to_instruction_sequence(&self.metadata, self.flags, &mut self.blocks)?; - let localsplusinfo = compute_localsplus_info(&self.metadata, nlocalsplus, self.flags); + let localsplusinfo = compute_localsplus_info(&self.metadata, nlocalsplus, self.flags)?; let Self { flags, From bf31c4ab04fa91ab583342acf99d39f14fa5794f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:33:13 +0900 Subject: [PATCH 027/131] Propagate CPython localsplus setup allocation errors --- crates/codegen/src/ir.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 97d715dd9ca..fe947dd16f8 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1963,7 +1963,9 @@ fn insert_prefix_instructions( if ncellvars > 0 { let nvars = metadata.varnames.len() + ncellvars; - let mut sorted = vec![0i32; nvars]; + let mut sorted = Vec::new(); + vec_try_reserve_exact(&mut sorted, nvars)?; + sorted.resize(nvars, 0i32); for i in 0..ncellvars { sorted[usize::try_from(cellfixedoffsets[i]) .expect("localsplus offset is non-negative")] = i as i32 + 1; @@ -2036,7 +2038,7 @@ fn prepare_localsplus( .is_some_and(|sum| sum < int_max) ); let mut nlocalsplus = nlocals + ncellvars + nfreevars; - let mut cellfixedoffsets = build_cellfixedoffsets(metadata); + let mut cellfixedoffsets = build_cellfixedoffsets(metadata)?; // This must be called before fix_cell_offsets(). insert_prefix_instructions(metadata, blocks, &cellfixedoffsets, nfreevars, flags)?; @@ -6743,12 +6745,16 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block]) -> crate::InternalResult< /// flowgraph.c build_cellfixedoffsets #[allow(clippy::needless_range_loop)] -pub(crate) fn build_cellfixedoffsets(metadata: &CodeUnitMetadata) -> Vec { +pub(crate) fn build_cellfixedoffsets( + metadata: &CodeUnitMetadata, +) -> crate::InternalResult> { let nlocals = metadata.varnames.len(); let ncellvars = metadata.cellvars.len(); let nfreevars = metadata.freevars.len(); let noffsets = ncellvars + nfreevars; - let mut fixed = vec![0; noffsets]; + let mut fixed = Vec::new(); + vec_try_reserve_exact(&mut fixed, noffsets)?; + fixed.resize(noffsets, 0); for i in 0..noffsets { fixed[i] = (nlocals + i) .to_i32() @@ -6764,7 +6770,7 @@ pub(crate) fn build_cellfixedoffsets(metadata: &CodeUnitMetadata) -> Vec { fixed[oldindex] = argoffset; } } - fixed + Ok(fixed) } /// flowgraph.c fix_cell_offsets From df9e93ffa134b23711f92b8bb7bd220d3c3e3de3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:34:10 +0900 Subject: [PATCH 028/131] Propagate CPython jump label map allocation errors --- crates/codegen/src/ir.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index fe947dd16f8..21eb7a7ec82 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5964,7 +5964,9 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResu label_count .checked_mul(core::mem::size_of::()) .ok_or(InternalError::MalformedControlFlowGraph)?; - let mut label_to_block = vec![BlockIdx::NULL; label_count]; + let mut label_to_block = Vec::new(); + vec_try_reserve_exact(&mut label_to_block, label_count)?; + label_to_block.resize(label_count, BlockIdx::NULL); let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { From 63cdf7e7320a91f2300f9b4e73c5253d0d8019e1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:38:24 +0900 Subject: [PATCH 029/131] Propagate CPython instruction sequence allocation errors --- crates/codegen/src/compile.rs | 3 ++- crates/codegen/src/ir.rs | 40 ++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 5987183bb37..0a44967d515 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -10224,7 +10224,8 @@ impl Compiler { "CPython inserts StopIteration cleanup before CFG prefix instructions" ); - code.insert_start_setup_cleanup(handler_block); + let result = code.insert_start_setup_cleanup(handler_block); + unwrap_internal(self, result); } fn emit_no_arg>(&mut self, ins: I) { diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 21eb7a7ec82..4fff005686e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -538,7 +538,7 @@ fn instruction_sequence_new() -> InstructionSequence { } /// instruction_sequence.c instr_sequence_next_inst -fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> usize { +fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::InternalResult { let idx = seq.instrs.len(); let new_allocation = c_array_ensure_capacity(seq.instr_allocation, idx + 1, INITIAL_INSTR_SEQUENCE_SIZE); @@ -546,7 +546,8 @@ fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> usize { seq.instr_allocation = new_allocation; if new_allocation > seq.instrs.capacity() { seq.instrs - .reserve_exact(new_allocation - seq.instrs.capacity()); + .try_reserve_exact(new_allocation - seq.instrs.capacity()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; } } seq.instrs.push(InstructionSequenceEntry::new( @@ -561,7 +562,7 @@ fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> usize { }, ZERO_EXCEPTION_HANDLER_INFO, )); - idx + Ok(idx) } /// instruction_sequence.c _PyInstructionSequence_NewLabel @@ -622,12 +623,12 @@ fn instruction_sequence_use_label(seq: &mut InstructionSequence, label: Instruct fn instruction_sequence_addop( seq: &mut InstructionSequence, info: InstructionInfo, -) -> &mut InstructionSequenceEntry { +) -> crate::InternalResult<&mut InstructionSequenceEntry> { instruction_sequence_debug_check_addop(&info); - let idx = instruction_sequence_next_inst(seq); + let idx = instruction_sequence_next_inst(seq)?; let entry = &mut seq.instrs[idx]; entry.info = info; - entry + Ok(entry) } fn instruction_sequence_last_info_mut( @@ -642,10 +643,10 @@ fn instruction_sequence_insert_instruction( seq: &mut InstructionSequence, pos: usize, info: InstructionInfo, -) { +) -> crate::InternalResult<()> { debug_assert!(pos <= seq.instrs.len()); instruction_sequence_debug_check_addop(&info); - let last_idx = instruction_sequence_next_inst(seq); + let last_idx = instruction_sequence_next_inst(seq)?; for i in (pos..last_idx).rev() { seq.instrs[i + 1] = seq.instrs[i]; } @@ -658,6 +659,7 @@ fn instruction_sequence_insert_instruction( } } } + Ok(()) } /// instruction_sequence.c _PyInstructionSequence_ApplyLabelMap @@ -734,7 +736,7 @@ fn cfg_to_instruction_sequence( let mut info = blocks[block_idx.idx()].instructions[i]; info.target = BlockIdx::NULL; let except_handler = info.except_handler.take(); - let entry = instruction_sequence_addop(instr_sequence, info); + let entry = instruction_sequence_addop(instr_sequence, info)?; let hi = &mut entry.except_handler; if let Some(handler) = except_handler { debug_assert!(handler.handler_block != BlockIdx::NULL); @@ -1559,7 +1561,7 @@ impl CodeInfo { ); info.target = BlockIdx::NULL; } - instruction_sequence_addop(&mut self.instr_sequence, info); + instruction_sequence_addop(&mut self.instr_sequence, info)?; Ok(()) } @@ -1578,7 +1580,7 @@ impl CodeInfo { .ok_or(InternalError::MalformedControlFlowGraph)?, ); info.target = BlockIdx::NULL; - instruction_sequence_addop(&mut self.instr_sequence, info); + instruction_sequence_addop(&mut self.instr_sequence, info)?; Ok(()) } @@ -1649,7 +1651,10 @@ impl CodeInfo { } } - pub(crate) fn insert_start_setup_cleanup(&mut self, handler_block: BlockIdx) { + pub(crate) fn insert_start_setup_cleanup( + &mut self, + handler_block: BlockIdx, + ) -> crate::InternalResult<()> { let handler_label = instruction_sequence_label_map_ensure_label_for_block( &mut self.instr_sequence_label_map, &mut self.instr_sequence, @@ -1670,7 +1675,7 @@ impl CodeInfo { except_handler: None, lineno_override: Some(NO_LOCATION_OVERRIDE), }, - ); + ) } pub(crate) fn push_unmapped_instr_sequence_label(&mut self) { @@ -6951,12 +6956,13 @@ mod tests { preserve_lasti: 1, }; let mut seq = instruction_sequence_new(); - let entry = instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 11)); + let entry = instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 11)).unwrap(); entry.except_handler = handler; entry.i_target = 1; entry.i_offset = 42; - instruction_sequence_insert_instruction(&mut seq, 0, test_instr(Instruction::PopTop, 12)); + instruction_sequence_insert_instruction(&mut seq, 0, test_instr(Instruction::PopTop, 12)) + .unwrap(); // CPython `_PyInstructionSequence_InsertInstruction()` shifts the // backing instruction slots, then overwrites only opcode/oparg/loc. @@ -6979,13 +6985,13 @@ mod tests { fn instruction_sequence_tracks_cpython_c_array_allocation() { let mut seq = instruction_sequence_new(); for i in 0..99 { - instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 10 + i)); + instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 10 + i)).unwrap(); } assert_eq!(seq.instr_allocation, INITIAL_INSTR_SEQUENCE_SIZE); // CPython calls `_Py_CArray_EnsureCapacity(s_used + 1)`, so the 100th // instruction expands a 100-slot array to 200 before returning offset 99. - instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 109)); + instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 109)).unwrap(); assert_eq!(seq.instr_allocation, INITIAL_INSTR_SEQUENCE_SIZE * 2); } From b6393209dc82d0133485fd31fd37f2cebee20bf8 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:41:19 +0900 Subject: [PATCH 030/131] Propagate CPython instruction label allocation errors --- crates/codegen/src/compile.rs | 32 +++++++++++++++++++++++--------- crates/codegen/src/ir.rs | 30 +++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 0a44967d515..61417657307 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -4676,8 +4676,10 @@ impl Compiler { if let Some(not_set_block) = not_set_block { self.use_cpython_label_block(not_set_block); } else if let Some(not_set_label) = not_set_label { - self.current_code_info() + let result = self + .current_code_info() .use_raw_instr_sequence_label(not_set_label); + unwrap_internal(self, result); } } @@ -5913,8 +5915,10 @@ impl Compiler { let saved_range = self.current_source_range; self.set_source_range(target.range()); emit!(self, Instruction::Nop); - self.current_code_info() + let result = self + .current_code_info() .use_raw_instr_sequence_label(body_label.expect("sync for must have body label")); + unwrap_internal(self, result); self.compile_store(target)?; self.set_source_range(saved_range); }; @@ -8788,13 +8792,17 @@ impl Compiler { // `skip_optimization` for every sync name(genexpr) shape after // loading the function, even when the name is not all/any/tuple. let skip_optimization = self.current_code_info().new_instr_sequence_label(); - self.current_code_info() + let result = self + .current_code_info() .use_raw_instr_sequence_label(skip_optimization); + unwrap_internal(self, result); } emit!(self, Instruction::PushNull); self.codegen_call_helper(0, args, call_range, None)?; - self.current_code_info() + let result = self + .current_code_info() .use_raw_instr_sequence_label(skip_normal_call); + unwrap_internal(self, result); } Ok(()) } @@ -11012,9 +11020,13 @@ impl Compiler { /// instruction-sequence label map, but it does not become a CFG /// `basicblock.b_label` unless some emitted instruction targets it. fn use_cpython_function_start_label(&mut self) -> ir::InstructionSequenceLabel { - let code = self.current_code_info(); - let label = code.new_instr_sequence_label(); - code.use_raw_instr_sequence_label(label); + let (label, result) = { + let code = self.current_code_info(); + let label = code.new_instr_sequence_label(); + let result = code.use_raw_instr_sequence_label(label); + (label, result) + }; + unwrap_internal(self, result); label } @@ -11025,8 +11037,9 @@ impl Compiler { /// CFG. Reuse an empty current block even if it already carries a label so /// codegen CFG path preserves that aliasing. fn use_cpython_label_block(&mut self, block: BlockIdx) { + let result = self.current_code_info().use_instr_sequence_label(block); + unwrap_internal(self, result); let code = self.current_code_info(); - code.use_instr_sequence_label(block); let block = code.resolve_instr_sequence_label(block); let cur = code.current_block; let can_reuse_current = cur != block @@ -11084,8 +11097,9 @@ impl Compiler { } fn switch_to_block(&mut self, block: BlockIdx) { + let result = self.current_code_info().use_instr_sequence_label(block); + unwrap_internal(self, result); let code = self.current_code_info(); - code.use_instr_sequence_label(block); let block = code.resolve_instr_sequence_label(block); let prev = code.current_block; assert_ne!(prev, block, "recursive switching {prev:?} -> {block:?}"); diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4fff005686e..e986b9f2ae0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -595,7 +595,10 @@ fn instruction_sequence_set_annotations_code( /// instruction_sequence.c _PyInstructionSequence_UseLabel #[allow(clippy::needless_range_loop)] -fn instruction_sequence_use_label(seq: &mut InstructionSequence, label: InstructionSequenceLabel) { +fn instruction_sequence_use_label( + seq: &mut InstructionSequence, + label: InstructionSequenceLabel, +) -> crate::InternalResult<()> { let label_map = seq.label_map.get_or_insert_with(Vec::new); let old_len = label_map.len(); let new_allocation = c_array_ensure_capacity( @@ -606,7 +609,9 @@ fn instruction_sequence_use_label(seq: &mut InstructionSequence, label: Instruct if new_allocation > seq.label_map_allocation { seq.label_map_allocation = new_allocation; if new_allocation > label_map.capacity() { - label_map.reserve_exact(new_allocation - label_map.capacity()); + label_map + .try_reserve_exact(new_allocation - label_map.capacity()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; } } if label_map.len() < seq.label_map_allocation { @@ -617,6 +622,7 @@ fn instruction_sequence_use_label(seq: &mut InstructionSequence, label: Instruct } label_map[label.idx()] = i32::try_from(seq.instrs.len()).expect("instruction offset fits in int"); + Ok(()) } /// instruction_sequence.c _PyInstructionSequence_Addop @@ -717,7 +723,7 @@ fn cfg_to_instruction_sequence( while block_idx != BlockIdx::NULL { let block_label = blocks[block_idx.idx()].cpython_label; debug_assert!(is_label(block_label)); - instruction_sequence_use_label(instr_sequence, block_label); + instruction_sequence_use_label(instr_sequence, block_label)?; let instr_count = blocks[block_idx.idx()].instructions.len(); for i in 0..instr_count { @@ -1590,21 +1596,27 @@ impl CodeInfo { } } - pub(crate) fn use_instr_sequence_label(&mut self, block: BlockIdx) { + pub(crate) fn use_instr_sequence_label( + &mut self, + block: BlockIdx, + ) -> crate::InternalResult<()> { let label = instruction_sequence_label_map_ensure_label_for_block( &mut self.instr_sequence_label_map, &mut self.instr_sequence, block, ); - instruction_sequence_use_label(&mut self.instr_sequence, label); + instruction_sequence_use_label(&mut self.instr_sequence, label) } pub(crate) fn new_instr_sequence_label(&mut self) -> InstructionSequenceLabel { instruction_sequence_new_label(&mut self.instr_sequence) } - pub(crate) fn use_raw_instr_sequence_label(&mut self, label: InstructionSequenceLabel) { - instruction_sequence_use_label(&mut self.instr_sequence, label); + pub(crate) fn use_raw_instr_sequence_label( + &mut self, + label: InstructionSequenceLabel, + ) -> crate::InternalResult<()> { + instruction_sequence_use_label(&mut self.instr_sequence, label) } pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) { @@ -6998,7 +7010,7 @@ mod tests { #[test] fn instruction_sequence_label_map_tracks_cpython_c_array_allocation() { let mut seq = instruction_sequence_new(); - instruction_sequence_use_label(&mut seq, InstructionSequenceLabel::from_index(1)); + instruction_sequence_use_label(&mut seq, InstructionSequenceLabel::from_index(1)).unwrap(); assert_eq!( seq.label_map_allocation, INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE @@ -7010,7 +7022,7 @@ mod tests { // CPython passes the label id itself to `_Py_CArray_EnsureCapacity()`. // Label 10 therefore expands the initial 10-slot map to 20. - instruction_sequence_use_label(&mut seq, InstructionSequenceLabel::from_index(10)); + instruction_sequence_use_label(&mut seq, InstructionSequenceLabel::from_index(10)).unwrap(); assert_eq!( seq.label_map_allocation, INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE * 2 From 67b695861c6de5b49c3268b80f3781389690d6cd Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:42:31 +0900 Subject: [PATCH 031/131] Guard CPython codegen block allocation --- crates/codegen/src/compile.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 61417657307..19d8b19ccd0 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -11068,6 +11068,12 @@ impl Compiler { } fn new_block(&mut self) -> BlockIdx { + let result = self + .current_code_info() + .blocks + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph); + unwrap_internal(self, result); let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); @@ -11076,6 +11082,12 @@ impl Compiler { } fn new_unlabeled_block(&mut self) -> BlockIdx { + let result = self + .current_code_info() + .blocks + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph); + unwrap_internal(self, result); let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); From bb464429dd7a498ae4219fa6d857e3c4f538c1f2 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:45:09 +0900 Subject: [PATCH 032/131] Guard CPython label shadow allocation --- crates/codegen/src/compile.rs | 6 ++++-- crates/codegen/src/ir.rs | 31 +++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 19d8b19ccd0..c6a5abb42a3 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -11077,7 +11077,8 @@ impl Compiler { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); - code.push_unmapped_instr_sequence_label(); + let result = code.push_unmapped_instr_sequence_label(); + unwrap_internal(self, result); idx } @@ -11091,7 +11092,8 @@ impl Compiler { let code = self.current_code_info(); let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); - code.push_unlabeled_instr_sequence_block(); + let result = code.push_unlabeled_instr_sequence_block(); + unwrap_internal(self, result); idx } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e986b9f2ae0..8f74cb11466 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1392,15 +1392,26 @@ fn instruction_sequence_label_map_use_label_at_block( map.cpython_block_by_label[from_label.idx()] = to_block; } -fn instruction_sequence_label_map_push_unlabeled_block(map: &mut InstructionSequenceLabelMap) { +fn instruction_sequence_label_map_push_unlabeled_block( + map: &mut InstructionSequenceLabelMap, +) -> crate::InternalResult<()> { + map.block_labels + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; map.block_labels.push(InstructionSequenceLabel::NO_LABEL); + Ok(()) } fn instruction_sequence_label_map_push_unmapped_label( map: &mut InstructionSequenceLabelMap, seq: &mut InstructionSequence, -) { +) -> crate::InternalResult<()> { let label = instruction_sequence_new_label(seq); + if map.cpython_block_by_label.len() <= label.idx() { + map.cpython_block_by_label + .try_reserve(label.idx() + 1 - map.cpython_block_by_label.len()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + } instruction_sequence_label_map_register_label(map, label); let block = BlockIdx( map.block_labels @@ -1409,7 +1420,11 @@ fn instruction_sequence_label_map_push_unmapped_label( .expect("too many codegen CFG blocks"), ); map.cpython_block_by_label[label.idx()] = block; + map.block_labels + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; map.block_labels.push(label); + Ok(()) } impl InstructionSequenceLabelMap { @@ -1690,15 +1705,15 @@ impl CodeInfo { ) } - pub(crate) fn push_unmapped_instr_sequence_label(&mut self) { + pub(crate) fn push_unmapped_instr_sequence_label(&mut self) -> crate::InternalResult<()> { instruction_sequence_label_map_push_unmapped_label( &mut self.instr_sequence_label_map, &mut self.instr_sequence, - ); + ) } - pub(crate) fn push_unlabeled_instr_sequence_block(&mut self) { - instruction_sequence_label_map_push_unlabeled_block(&mut self.instr_sequence_label_map); + pub(crate) fn push_unlabeled_instr_sequence_block(&mut self) -> crate::InternalResult<()> { + instruction_sequence_label_map_push_unlabeled_block(&mut self.instr_sequence_label_map) } fn take_recorded_instr_sequence(&mut self) -> crate::InternalResult { @@ -6936,8 +6951,8 @@ mod tests { fn instruction_sequence_label_shadow_preserves_cpython_offset_aliases() { let mut seq = instruction_sequence_new(); let mut labels = InstructionSequenceLabelMap::new(); - instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq); - instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq); + instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq).unwrap(); + instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq).unwrap(); let first = BlockIdx::new(1); let second = BlockIdx::new(2); From 9b5b6a1d4cf35c59e32627ad798bc88e37fbd94e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:50:45 +0900 Subject: [PATCH 033/131] Propagate CPython label shadow allocation errors --- crates/codegen/src/compile.rs | 98 ++++++++++++----------------------- crates/codegen/src/ir.rs | 56 +++++++++++--------- 2 files changed, 64 insertions(+), 90 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c6a5abb42a3..d83cf7116d7 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3550,13 +3550,8 @@ impl Compiler { } ); self.use_cpython_label_block(body_block); - let (body_label, finally_except_label) = { - let code = self.current_code_info(); - ( - code.instr_sequence_label_for_block(body_block), - code.instr_sequence_label_for_block(finally_except_block), - ) - }; + let body_label = self.instr_sequence_label_for_block(body_block); + let finally_except_label = self.instr_sequence_label_for_block(finally_except_block); self.push_fblock_labels( FBlockType::FinallyTry, body_label, @@ -3641,9 +3636,7 @@ impl Compiler { } ); self.use_cpython_label_block(body_block); - let body_label = self - .current_code_info() - .instr_sequence_label_for_block(body_block); + let body_label = self.instr_sequence_label_for_block(body_block); self.push_fblock_labels( FBlockType::TryExcept, body_label, @@ -3714,9 +3707,7 @@ impl Compiler { emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup_end }); self.use_cpython_label_block(cleanup_body); - let cleanup_body_label = self - .current_code_info() - .instr_sequence_label_for_block(cleanup_body); + let cleanup_body_label = self.instr_sequence_label_for_block(cleanup_body); self.push_fblock_labels( FBlockType::HandlerCleanup, cleanup_body_label, @@ -3751,9 +3742,7 @@ impl Compiler { emit!(self, Instruction::PopTop); self.use_cpython_label_block(cleanup_body); - let cleanup_body_label = self - .current_code_info() - .instr_sequence_label_for_block(cleanup_body); + let cleanup_body_label = self.instr_sequence_label_for_block(cleanup_body); self.push_fblock_labels( FBlockType::HandlerCleanup, cleanup_body_label, @@ -3820,13 +3809,8 @@ impl Compiler { } ); self.use_cpython_label_block(body_block); - let (body_label, finally_except_label) = { - let code = self.current_code_info(); - ( - code.instr_sequence_label_for_block(body_block), - code.instr_sequence_label_for_block(finally_except_block), - ) - }; + let body_label = self.instr_sequence_label_for_block(body_block); + let finally_except_label = self.instr_sequence_label_for_block(finally_except_block); self.push_fblock_labels( FBlockType::FinallyTry, body_label, @@ -3916,9 +3900,7 @@ impl Compiler { } ); self.use_cpython_label_block(body_block); - let body_label = self - .current_code_info() - .instr_sequence_label_for_block(body_block); + let body_label = self.instr_sequence_label_for_block(body_block); self.push_fblock_labels( FBlockType::TryExcept, body_label, @@ -4030,9 +4012,7 @@ impl Compiler { } ); self.use_cpython_label_block(cleanup_body_block); - let cleanup_body_label = self - .current_code_info() - .instr_sequence_label_for_block(cleanup_body_block); + let cleanup_body_label = self.instr_sequence_label_for_block(cleanup_body_block); self.push_fblock_labels( FBlockType::HandlerCleanup, cleanup_body_label, @@ -5599,13 +5579,8 @@ impl Compiler { let loop_block = self.new_block(); let end_block = self.new_block(); let anchor_block = self.new_block(); - let (loop_label, end_label) = { - let code = self.current_code_info(); - ( - code.instr_sequence_label_for_block(loop_block), - code.instr_sequence_label_for_block(end_block), - ) - }; + let loop_label = self.instr_sequence_label_for_block(loop_block); + let end_label = self.instr_sequence_label_for_block(end_block); self.use_cpython_label_block(loop_block); self.push_fblock_labels( FBlockType::WhileLoop, @@ -5737,13 +5712,8 @@ impl Compiler { } else { FBlockType::With }; - let (body_label, exc_handler_label) = { - let code = self.current_code_info(); - ( - code.instr_sequence_label_for_block(body_block), - code.instr_sequence_label_for_block(exc_handler_block), - ) - }; + let body_label = self.instr_sequence_label_for_block(body_block); + let exc_handler_label = self.instr_sequence_label_for_block(exc_handler_block); self.push_fblock_labels( fblock_type, body_label, @@ -5850,13 +5820,8 @@ impl Compiler { }; let else_block = self.new_block(); let after_block = self.new_block(); - let (for_label, after_label) = { - let code = self.current_code_info(); - ( - code.instr_sequence_label_for_block(for_block), - code.instr_sequence_label_for_block(after_block), - ) - }; + let for_label = self.instr_sequence_label_for_block(for_block); + let after_label = self.instr_sequence_label_for_block(after_block); let mut end_async_for_target = BlockIdx::NULL; if !is_async { @@ -9497,9 +9462,7 @@ impl Compiler { self.use_cpython_label_block(loop_block); let mut end_async_for_target = BlockIdx::NULL; if generator.is_async { - let loop_label = self - .current_code_info() - .instr_sequence_label_for_block(loop_block); + let loop_label = self.instr_sequence_label_for_block(loop_block); self.push_fblock_labels( FBlockType::AsyncComprehensionGenerator, loop_label, @@ -9565,9 +9528,7 @@ impl Compiler { emit!(self, PseudoInstruction::Jump { delta: loop_block }); if is_async { - let loop_label = self - .current_code_info() - .instr_sequence_label_for_block(loop_block); + let loop_label = self.instr_sequence_label_for_block(loop_block); self.pop_fblock_label(FBlockType::AsyncComprehensionGenerator, loop_label); } @@ -9877,9 +9838,7 @@ impl Compiler { let mut end_async_for_target = BlockIdx::NULL; if generator.is_async { - let loop_label = self - .current_code_info() - .instr_sequence_label_for_block(loop_block); + let loop_label = self.instr_sequence_label_for_block(loop_block); self.push_fblock_labels( FBlockType::AsyncComprehensionGenerator, loop_label, @@ -9943,9 +9902,7 @@ impl Compiler { emit!(self, PseudoInstruction::Jump { delta: loop_block }); if is_async { - let loop_label = self - .current_code_info() - .instr_sequence_label_for_block(loop_block); + let loop_label = self.instr_sequence_label_for_block(loop_block); self.pop_fblock_label( FBlockType::AsyncComprehensionGenerator, loop_label, @@ -11030,6 +10987,13 @@ impl Compiler { label } + fn instr_sequence_label_for_block(&mut self, block: BlockIdx) -> ir::InstructionSequenceLabel { + let result = self + .current_code_info() + .instr_sequence_label_for_block(block); + unwrap_internal(self, result) + } + /// Switch to a block as CPython instruction-sequence labels would resolve. /// /// Consecutive `USE_LABEL()` calls can map multiple labels to the same @@ -11049,7 +11013,8 @@ impl Compiler { && code.blocks[block.idx()].next == BlockIdx::NULL; if !can_reuse_current { - code.mark_cpython_cfg_label(block); + let result = code.mark_cpython_cfg_label(block); + unwrap_internal(self, result); self.switch_to_block(block); return; } @@ -11061,10 +11026,13 @@ impl Compiler { debug_assert_eq!(target.start_depth, ir::START_DEPTH_UNSET); debug_assert!(!target.cold); } - code.mark_cpython_cfg_label(cur); + let result = code.mark_cpython_cfg_label(cur); + unwrap_internal(self, result); - self.current_code_info() + let result = self + .current_code_info() .use_instr_sequence_label_at_block(block, cur); + unwrap_internal(self, result); } fn new_block(&mut self) -> BlockIdx { diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 8f74cb11466..7db6e7e31a8 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1287,28 +1287,32 @@ pub(crate) struct InstructionSequenceLabelMap { fn instruction_sequence_label_map_register_label( map: &mut InstructionSequenceLabelMap, label: InstructionSequenceLabel, -) { +) -> crate::InternalResult<()> { if map.cpython_block_by_label.len() <= label.idx() { + map.cpython_block_by_label + .try_reserve(label.idx() + 1 - map.cpython_block_by_label.len()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; map.cpython_block_by_label .resize(label.idx() + 1, BlockIdx::NULL); } + Ok(()) } fn instruction_sequence_label_map_ensure_label_for_block( map: &mut InstructionSequenceLabelMap, seq: &mut InstructionSequence, block: BlockIdx, -) -> InstructionSequenceLabel { +) -> crate::InternalResult { debug_assert_ne!(block, BlockIdx::NULL); let block_label = map.block_labels[block.idx()]; if is_label(block_label) { - return block_label; + return Ok(block_label); } let label = instruction_sequence_new_label(seq); - instruction_sequence_label_map_register_label(map, label); + instruction_sequence_label_map_register_label(map, label)?; map.cpython_block_by_label[label.idx()] = block; map.block_labels[block.idx()] = label; - label + Ok(label) } fn instruction_sequence_label_map_label_for_block( @@ -1376,20 +1380,21 @@ fn instruction_sequence_label_map_use_label_at_block( seq: &mut InstructionSequence, from: BlockIdx, to: BlockIdx, -) { +) -> crate::InternalResult<()> { if from == BlockIdx::NULL || from == to { - return; + return Ok(()); } - let from_label = instruction_sequence_label_map_ensure_label_for_block(map, seq, from); + let from_label = instruction_sequence_label_map_ensure_label_for_block(map, seq, from)?; let to_block = instruction_sequence_label_map_resolve_label(map, to); if to_block == BlockIdx::NULL { debug_assert!( false, "CPython label target must map to a codegen CFG block" ); - return; + return Ok(()); } map.cpython_block_by_label[from_label.idx()] = to_block; + Ok(()) } fn instruction_sequence_label_map_push_unlabeled_block( @@ -1407,12 +1412,7 @@ fn instruction_sequence_label_map_push_unmapped_label( seq: &mut InstructionSequence, ) -> crate::InternalResult<()> { let label = instruction_sequence_new_label(seq); - if map.cpython_block_by_label.len() <= label.idx() { - map.cpython_block_by_label - .try_reserve(label.idx() + 1 - map.cpython_block_by_label.len()) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - } - instruction_sequence_label_map_register_label(map, label); + instruction_sequence_label_map_register_label(map, label)?; let block = BlockIdx( map.block_labels .len() @@ -1573,7 +1573,7 @@ impl CodeInfo { &mut self.instr_sequence_label_map, &mut self.instr_sequence, info.target, - ); + )?; info.arg = OpArg::new( label .idx() @@ -1619,7 +1619,7 @@ impl CodeInfo { &mut self.instr_sequence_label_map, &mut self.instr_sequence, block, - ); + )?; instruction_sequence_use_label(&mut self.instr_sequence, label) } @@ -1634,13 +1634,14 @@ impl CodeInfo { instruction_sequence_use_label(&mut self.instr_sequence, label) } - pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) { + pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) -> crate::InternalResult<()> { let label = instruction_sequence_label_map_ensure_label_for_block( &mut self.instr_sequence_label_map, &mut self.instr_sequence, block, - ); + )?; self.blocks[block.idx()].cpython_label = label; + Ok(()) } pub(crate) fn resolve_instr_sequence_label(&self, block: BlockIdx) -> BlockIdx { @@ -1654,21 +1655,25 @@ impl CodeInfo { instruction_sequence_label_map_resolve_label_to_block(&self.instr_sequence_label_map, label) } - pub(crate) fn use_instr_sequence_label_at_block(&mut self, from: BlockIdx, to: BlockIdx) { + pub(crate) fn use_instr_sequence_label_at_block( + &mut self, + from: BlockIdx, + to: BlockIdx, + ) -> crate::InternalResult<()> { instruction_sequence_label_map_use_label_at_block( &mut self.instr_sequence_label_map, &mut self.instr_sequence, from, to, - ); + ) } pub(crate) fn instr_sequence_label_for_block( &mut self, block: BlockIdx, - ) -> InstructionSequenceLabel { + ) -> crate::InternalResult { if block == BlockIdx::NULL { - InstructionSequenceLabel::NO_LABEL + Ok(InstructionSequenceLabel::NO_LABEL) } else { instruction_sequence_label_map_ensure_label_for_block( &mut self.instr_sequence_label_map, @@ -1686,7 +1691,7 @@ impl CodeInfo { &mut self.instr_sequence_label_map, &mut self.instr_sequence, handler_block, - ); + )?; instruction_sequence_insert_instruction( &mut self.instr_sequence, 0, @@ -6964,7 +6969,8 @@ mod tests { // CPython `_PyInstructionSequence_UseLabel()` can map consecutive // labels to the same instruction offset. The codegen CFG shadow must // resolve the later block label to the block owning that shared offset. - instruction_sequence_label_map_use_label_at_block(&mut labels, &mut seq, second, first); + instruction_sequence_label_map_use_label_at_block(&mut labels, &mut seq, second, first) + .unwrap(); assert_eq!( instruction_sequence_label_map_resolve_label(&labels, first), first From f39ff54a899427e2d7fdcc7f5711182ac7176c57 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:53:11 +0900 Subject: [PATCH 034/131] Align CPython c-array allocation updates --- crates/codegen/src/ir.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7db6e7e31a8..4147d3a108b 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -343,7 +343,6 @@ fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { let new_allocation = c_array_ensure_capacity(block.instruction_allocation, off + 1, DEFAULT_BLOCK_SIZE); if new_allocation > block.instruction_allocation { - block.instruction_allocation = new_allocation; if new_allocation > block.instructions.capacity() + block.cpython_spare_instr_slots.len() { block .instructions @@ -354,6 +353,7 @@ fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { ) .map_err(|_| InternalError::MalformedControlFlowGraph)?; } + block.instruction_allocation = new_allocation; } let slot = block .cpython_spare_instr_slots @@ -543,12 +543,12 @@ fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::Inter let new_allocation = c_array_ensure_capacity(seq.instr_allocation, idx + 1, INITIAL_INSTR_SEQUENCE_SIZE); if new_allocation > seq.instr_allocation { - seq.instr_allocation = new_allocation; if new_allocation > seq.instrs.capacity() { seq.instrs .try_reserve_exact(new_allocation - seq.instrs.capacity()) .map_err(|_| InternalError::MalformedControlFlowGraph)?; } + seq.instr_allocation = new_allocation; } seq.instrs.push(InstructionSequenceEntry::new( InstructionInfo { @@ -599,21 +599,32 @@ fn instruction_sequence_use_label( seq: &mut InstructionSequence, label: InstructionSequenceLabel, ) -> crate::InternalResult<()> { - let label_map = seq.label_map.get_or_insert_with(Vec::new); - let old_len = label_map.len(); + let old_len = seq.label_map.as_ref().map_or(0, Vec::len); let new_allocation = c_array_ensure_capacity( seq.label_map_allocation, label.idx(), INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, ); if new_allocation > seq.label_map_allocation { - seq.label_map_allocation = new_allocation; - if new_allocation > label_map.capacity() { + if let Some(label_map) = &mut seq.label_map { + if new_allocation > label_map.capacity() { + label_map + .try_reserve_exact(new_allocation - label_map.capacity()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + } + } else { + let mut label_map = Vec::new(); label_map - .try_reserve_exact(new_allocation - label_map.capacity()) + .try_reserve_exact(new_allocation) .map_err(|_| InternalError::MalformedControlFlowGraph)?; + seq.label_map = Some(label_map); } + seq.label_map_allocation = new_allocation; } + let label_map = seq + .label_map + .as_mut() + .ok_or(InternalError::MalformedControlFlowGraph)?; if label_map.len() < seq.label_map_allocation { label_map.resize(seq.label_map_allocation, INSTRUCTION_SEQUENCE_UNSET_LABEL); } From d093fa63d2c015785ca09f6a32f1769fdf046177 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:54:54 +0900 Subject: [PATCH 035/131] Align CPython ref stack growth --- crates/codegen/src/ir.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4147d3a108b..a138a6a9565 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4106,8 +4106,6 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let start_depth = usize::try_from(blocks[block_i].start_depth) .expect("visited block has non-negative start depth"); ref_stack_clear(&mut refs); - refs.try_reserve(instr_count + start_depth + 2) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; for _ in 0..start_depth { push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL)?; } @@ -4684,9 +4682,16 @@ type RefStack = Vec; /// flowgraph.c ref_stack_push fn ref_stack_push(stack: &mut RefStack, r: Ref) -> crate::InternalResult<()> { - stack - .try_reserve(1) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; + if stack.len() == stack.capacity() { + let doubled = stack + .capacity() + .checked_mul(2) + .ok_or(InternalError::MalformedControlFlowGraph)?; + let new_cap = 32.max(doubled); + stack + .try_reserve_exact(new_cap - stack.capacity()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + } stack.push(r); Ok(()) } From cc0fc01cd9dd201c69322c561e34fb46b8b9d7b4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:57:50 +0900 Subject: [PATCH 036/131] Match CPython CFG builder debug check --- crates/codegen/src/ir.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a138a6a9565..08ad2364d91 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5962,12 +5962,9 @@ fn cfg_builder_addop(g: &mut CfgBuilder, info: InstructionInfo) -> crate::Intern fn cfg_builder_check(g: &CfgBuilder) -> bool { debug_assert!(g.entry != BlockIdx::NULL); debug_assert!(!g.blocks[g.entry.idx()].instructions.is_empty()); - let mut seen = vec![false; g.blocks.len()]; let mut block = g.block_list; while block != BlockIdx::NULL { debug_assert!(block.idx() < g.blocks.len()); - debug_assert!(!seen[block.idx()]); - seen[block.idx()] = true; let block_ref = &g.blocks[block.idx()]; let has_instr_array = !block_ref.instructions.is_empty() || !block_ref.cpython_spare_instr_slots.is_empty(); @@ -5984,7 +5981,6 @@ fn cfg_builder_check(g: &CfgBuilder) -> bool { } block = block_ref.allocation_next; } - debug_assert!(seen.into_iter().all(core::convert::identity)); true } From 3e0faecf5146fa4483c16f196e130daf4449a100 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 11:59:21 +0900 Subject: [PATCH 037/131] Avoid Rust-only CFG append clone --- crates/codegen/src/ir.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 08ad2364d91..e599093af19 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -420,6 +420,22 @@ fn basicblock_append_instructions( Ok(()) } +/// flowgraph.c basicblock_append_instructions +fn basicblock_append_block_instructions( + blocks: &mut [Block], + to: BlockIdx, + from: BlockIdx, +) -> crate::InternalResult<()> { + debug_assert_ne!(to, from); + let from_len = blocks[from.idx()].instructions.len(); + for i in 0..from_len { + let info = blocks[from.idx()].instructions[i]; + let off = basicblock_next_instr(&mut blocks[to.idx()])?; + blocks[to.idx()].instructions[off] = info; + } + Ok(()) +} + /// flowgraph.c direct `b_iused = 0` fn basicblock_clear(block: &mut Block) { let instructions = core::mem::take(&mut block.instructions); @@ -5633,8 +5649,7 @@ fn basicblock_inline_small_or_no_lineno_blocks( let last = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]) .expect("non-empty block has last instruction"); set_to_nop(last); - let target_instructions = blocks[target.idx()].instructions.clone(); - basicblock_append_instructions(&mut blocks[block_idx.idx()], &target_instructions)?; + basicblock_append_block_instructions(blocks, block_idx, target)?; if no_lineno_no_fallthrough { let last = basicblock_last_instr_mut(&mut blocks[block_idx.idx()]).unwrap(); if last.instr.is_unconditional_jump() From 657bc7df31770ad3bad940e910c7d63bb13ed960 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:00:37 +0900 Subject: [PATCH 038/131] Reuse CPython cleared block slots --- crates/codegen/src/ir.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e599093af19..d77eecd393a 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -438,11 +438,14 @@ fn basicblock_append_block_instructions( /// flowgraph.c direct `b_iused = 0` fn basicblock_clear(block: &mut Block) { - let instructions = core::mem::take(&mut block.instructions); + let mut instructions = core::mem::take(&mut block.instructions); if !instructions.is_empty() { - block - .cpython_spare_instr_slots - .extend(instructions.into_iter().rev()); + instructions.reverse(); + if block.cpython_spare_instr_slots.is_empty() { + block.cpython_spare_instr_slots = instructions; + } else { + block.cpython_spare_instr_slots.extend(instructions); + } } } From fb20c2a7d060c92f9754fd26754a8ec222931937 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:34:43 +0900 Subject: [PATCH 039/131] Use CPython block append in copy_basicblock --- crates/codegen/src/ir.rs | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d77eecd393a..293012ca578 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -408,18 +408,6 @@ fn basicblock_insert_instruction( Ok(()) } -/// flowgraph.c basicblock_append_instructions -fn basicblock_append_instructions( - to: &mut Block, - from: &[InstructionInfo], -) -> crate::InternalResult<()> { - for info in from { - let off = basicblock_next_instr(to)?; - to.instructions[off] = *info; - } - Ok(()) -} - /// flowgraph.c basicblock_append_instructions fn basicblock_append_block_instructions( blocks: &mut [Block], @@ -6493,10 +6481,7 @@ fn copy_basicblock( ) -> crate::InternalResult { debug_assert!(bb_no_fallthrough(&blocks[block_idx.idx()])); let result = blocks_new_block(blocks)?; - let result_idx = result.idx(); - let (blocks_before_result, result_block) = blocks.split_at_mut(result_idx); - let block = &blocks_before_result[block_idx.idx()]; - basicblock_append_instructions(&mut result_block[0], &block.instructions)?; + basicblock_append_block_instructions(blocks, result, block_idx)?; Ok(result) } @@ -7205,21 +7190,23 @@ mod tests { handler_block: BlockIdx::new(5), preserve_lasti: false, }; - let mut to = Block::default(); + let mut blocks = vec![Block::default(), Block::default()]; let mut stale = test_instr(Instruction::Nop, 41); stale.except_handler = Some(handler); - to.cpython_spare_instr_slots.push(stale); + blocks[0].cpython_spare_instr_slots.push(stale); - let from = [test_instr(Instruction::PopTop, 42)]; - basicblock_append_instructions(&mut to, &from) - .expect("basicblock_append_instructions succeeds"); + blocks[1] + .instructions + .push(test_instr(Instruction::PopTop, 42)); + basicblock_append_block_instructions(&mut blocks, BlockIdx::new(0), BlockIdx::new(1)) + .expect("basicblock_append_block_instructions succeeds"); // CPython `basicblock_append_instructions()` obtains a slot with // `basicblock_next_instr()`, then overwrites it with the copied // instruction, including `i_except`. - assert_eq!(to.cpython_spare_instr_slots.len(), 0); - assert_eq!(to.instructions.len(), 1); - assert_eq!(to.instructions[0].except_handler, None); + assert_eq!(blocks[0].cpython_spare_instr_slots.len(), 0); + assert_eq!(blocks[0].instructions.len(), 1); + assert_eq!(blocks[0].instructions[0].except_handler, None); } #[test] From 8a03dc3fb01d5b57fdd8bfaf74b80ce9b76d5ce5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:36:51 +0900 Subject: [PATCH 040/131] Match CPython cfg builder creation order --- crates/codegen/src/ir.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 293012ca578..f385d603803 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6074,6 +6074,8 @@ fn cfg_from_instruction_sequence( mut instr_sequence: InstructionSequence, ) -> crate::InternalResult> { instruction_sequence_apply_label_map(&mut instr_sequence)?; + let mut builder = cfg_builder_new()?; + for i in 0..instr_sequence.instrs.len() { instr_sequence.instrs[i].i_target = 0; } @@ -6093,7 +6095,6 @@ fn cfg_from_instruction_sequence( } = instr_sequence; debug_assert!(label_map.is_none()); - let mut builder = cfg_builder_new()?; let mut offset = 0i32; let mut i = 0; From 1c3de545e29baab8dc6f9524171a40f35823825a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:38:02 +0900 Subject: [PATCH 041/131] Clear label map after CPython apply pass --- crates/codegen/src/ir.rs | 57 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f385d603803..7d905c302df 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -691,36 +691,39 @@ fn instruction_sequence_insert_instruction( fn instruction_sequence_apply_label_map( instrs: &mut InstructionSequence, ) -> crate::InternalResult<()> { - let Some(label_map) = instrs.label_map.take() else { - return Ok(()); - }; - instrs.label_map_allocation = 0; - for i in 0..instrs.instrs.len() { - let entry = &mut instrs.instrs[i]; - if entry.info.instr.has_target() { - let label = usize::try_from(u32::from(entry.info.arg)) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - let target = *label_map - .get(label) - .ok_or(InternalError::MalformedControlFlowGraph)?; - if target < 0 { - return Err(InternalError::MalformedControlFlowGraph); + { + let Some(label_map) = instrs.label_map.as_ref() else { + return Ok(()); + }; + for i in 0..instrs.instrs.len() { + let entry = &mut instrs.instrs[i]; + if entry.info.instr.has_target() { + let label = usize::try_from(u32::from(entry.info.arg)) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + let target = *label_map + .get(label) + .ok_or(InternalError::MalformedControlFlowGraph)?; + if target < 0 { + return Err(InternalError::MalformedControlFlowGraph); + } + entry.info.arg = OpArg::new( + target + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); + } + let handler = &mut entry.except_handler; + if handler.h_label >= 0 { + let label = usize::try_from(handler.h_label) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + handler.h_label = *label_map + .get(label) + .ok_or(InternalError::MalformedControlFlowGraph)?; } - entry.info.arg = OpArg::new( - target - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); - } - let handler = &mut entry.except_handler; - if handler.h_label >= 0 { - let label = usize::try_from(handler.h_label) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - handler.h_label = *label_map - .get(label) - .ok_or(InternalError::MalformedControlFlowGraph)?; } } + instrs.label_map = None; + instrs.label_map_allocation = 0; Ok(()) } From f08bf9bd1c170d6f70aff6bb2babeb42d0deb09b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:41:42 +0900 Subject: [PATCH 042/131] Match CPython cfg builder allocation check --- crates/codegen/src/ir.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7d905c302df..bc7dba0c0e7 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5980,10 +5980,6 @@ fn cfg_builder_check(g: &CfgBuilder) -> bool { if has_instr_array { debug_assert!(block_ref.instruction_allocation > 0); debug_assert!(block_ref.instruction_allocation >= block_ref.instructions.len()); - debug_assert!( - block_ref.instruction_allocation - >= block_ref.instructions.len() + block_ref.cpython_spare_instr_slots.len() - ); } else { debug_assert!(block_ref.instructions.is_empty()); debug_assert_eq!(block_ref.instruction_allocation, 0); From a859ce8c175e3b493f1b6dd4371e01c60bb4b12c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:44:32 +0900 Subject: [PATCH 043/131] Propagate CPython c-array size errors --- crates/codegen/src/ir.rs | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index bc7dba0c0e7..9b67bcf7d27 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -316,24 +316,28 @@ fn c_array_ensure_capacity( allocated_entries: usize, idx: usize, initial_num_entries: usize, -) -> usize { +) -> crate::InternalResult { if allocated_entries == 0 { - if idx >= initial_num_entries { - idx + initial_num_entries + let new_alloc = if idx >= initial_num_entries { + idx.checked_add(initial_num_entries) + .ok_or(InternalError::MalformedControlFlowGraph)? } else { initial_num_entries - } + }; + Ok(new_alloc) } else if idx >= allocated_entries { let doubled = allocated_entries .checked_mul(2) - .expect("CPython C array allocation size overflow"); - if idx >= doubled { - idx + initial_num_entries + .ok_or(InternalError::MalformedControlFlowGraph)?; + let new_alloc = if idx >= doubled { + idx.checked_add(initial_num_entries) + .ok_or(InternalError::MalformedControlFlowGraph)? } else { doubled - } + }; + Ok(new_alloc) } else { - allocated_entries + Ok(allocated_entries) } } @@ -341,7 +345,7 @@ fn c_array_ensure_capacity( fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { let off = block.instructions.len(); let new_allocation = - c_array_ensure_capacity(block.instruction_allocation, off + 1, DEFAULT_BLOCK_SIZE); + c_array_ensure_capacity(block.instruction_allocation, off + 1, DEFAULT_BLOCK_SIZE)?; if new_allocation > block.instruction_allocation { if new_allocation > block.instructions.capacity() + block.cpython_spare_instr_slots.len() { block @@ -548,7 +552,7 @@ fn instruction_sequence_new() -> InstructionSequence { fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::InternalResult { let idx = seq.instrs.len(); let new_allocation = - c_array_ensure_capacity(seq.instr_allocation, idx + 1, INITIAL_INSTR_SEQUENCE_SIZE); + c_array_ensure_capacity(seq.instr_allocation, idx + 1, INITIAL_INSTR_SEQUENCE_SIZE)?; if new_allocation > seq.instr_allocation { if new_allocation > seq.instrs.capacity() { seq.instrs @@ -611,7 +615,7 @@ fn instruction_sequence_use_label( seq.label_map_allocation, label.idx(), INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, - ); + )?; if new_allocation > seq.label_map_allocation { if let Some(label_map) = &mut seq.label_map { if new_allocation > label_map.capacity() { From fbdf35613dddc307cc1aba26913c0fb6248103d0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:46:09 +0900 Subject: [PATCH 044/131] Drop Rust-only label uniqueness check --- crates/codegen/src/ir.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 9b67bcf7d27..fad06759de8 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1467,7 +1467,6 @@ impl InstructionSequenceLabelMap { "every codegen CFG block must have an instruction-sequence label slot" ); debug_assert!(self.cpython_block_by_label.len() <= next_free_label + 1); - let mut seen_labels = vec![false; next_free_label + 1]; for &label in &self.block_labels { if !is_label(label) { continue; @@ -1476,11 +1475,6 @@ impl InstructionSequenceLabelMap { label.idx() <= next_free_label, "codegen CFG block labels must come from _PyInstructionSequence_NewLabel()" ); - debug_assert!( - !seen_labels[label.idx()], - "codegen CFG blocks must not share a CPython instruction-sequence label" - ); - seen_labels[label.idx()] = true; } for &block in &self.cpython_block_by_label { if block != BlockIdx::NULL { From fa41c61c0a046741c0911f2c36d50edad727012d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:48:27 +0900 Subject: [PATCH 045/131] Drop Rust-only label shadow debug checks --- crates/codegen/src/ir.rs | 88 ---------------------------------------- 1 file changed, 88 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index fad06759de8..7397778ba12 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1459,90 +1459,6 @@ impl InstructionSequenceLabelMap { cpython_block_by_label: Vec::new(), } } - - fn debug_check_for_blocks(&self, blocks_len: usize, next_free_label: usize) { - debug_assert_eq!( - self.block_labels.len(), - blocks_len, - "every codegen CFG block must have an instruction-sequence label slot" - ); - debug_assert!(self.cpython_block_by_label.len() <= next_free_label + 1); - for &label in &self.block_labels { - if !is_label(label) { - continue; - } - debug_assert!( - label.idx() <= next_free_label, - "codegen CFG block labels must come from _PyInstructionSequence_NewLabel()" - ); - } - for &block in &self.cpython_block_by_label { - if block != BlockIdx::NULL { - debug_assert!( - block.idx() < blocks_len, - "CPython label must map to an existing codegen CFG block" - ); - } - } - for &label in &self.block_labels { - if !is_label(label) { - continue; - } - debug_assert!( - instruction_sequence_label_map_block_for_label(self, label) - .is_some_and(|block| block.idx() < blocks_len), - "codegen CFG block label must map to a codegen CFG block" - ); - } - } - - fn debug_check_label_blocks_match_instruction_sequence( - &self, - instr_sequence: &InstructionSequence, - ) { - let Some(label_map) = &instr_sequence.label_map else { - debug_assert!( - self.cpython_block_by_label - .iter() - .all(|&block| block == BlockIdx::NULL), - "codegen CFG labels require a CPython instruction-sequence label map" - ); - return; - }; - for (label_idx, block) in self.cpython_block_by_label.iter().copied().enumerate() { - if block == BlockIdx::NULL { - continue; - } - let Some(&label_offset) = label_map.get(label_idx) else { - continue; - }; - if label_offset < 0 { - continue; - } - let block_label = self - .block_labels - .get(block.idx()) - .copied() - .unwrap_or(InstructionSequenceLabel::NO_LABEL); - if !is_label(block_label) { - debug_assert!( - false, - "CPython label must map to an existing codegen CFG block" - ); - continue; - } - let Some(&block_offset) = label_map.get(block_label.idx()) else { - continue; - }; - if block_offset < 0 { - continue; - } - debug_assert!( - label_offset == block_offset, - "codegen CFG labels may share a block only when CPython maps them to the same instruction offset" - ); - } - } } pub struct CodeInfo { @@ -1759,10 +1675,6 @@ impl CodeInfo { debug_assert!(!self.blocks[0].instructions.is_empty()); debug_assert!(!self.instr_sequence.instrs.is_empty()); debug_assert!(self.current_block.idx() < self.blocks.len()); - self.instr_sequence_label_map - .debug_check_for_blocks(self.blocks.len(), self.instr_sequence.next_free_label); - self.instr_sequence_label_map - .debug_check_label_blocks_match_instruction_sequence(&self.instr_sequence); for block in &self.blocks { if block.next != BlockIdx::NULL { debug_assert!(block.next.idx() < self.blocks.len()); From 0a993324d2e56f94cf20b0b8c20b74018f8eeaa6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:50:50 +0900 Subject: [PATCH 046/131] Propagate CFG block index overflow --- crates/codegen/src/ir.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7397778ba12..f4b8f1c0e15 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1442,7 +1442,7 @@ fn instruction_sequence_label_map_push_unmapped_label( map.block_labels .len() .to_u32() - .expect("too many codegen CFG blocks"), + .ok_or(InternalError::MalformedControlFlowGraph)?, ); map.cpython_block_by_label[label.idx()] = block; map.block_labels @@ -5780,7 +5780,12 @@ fn blocks_new_block(blocks: &mut Vec) -> crate::InternalResult blocks .try_reserve(1) .map_err(|_| InternalError::MalformedControlFlowGraph)?; - let block_idx = BlockIdx(blocks.len() as u32); + let block_idx = BlockIdx( + blocks + .len() + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + ); blocks.push(Block::default()); Ok(block_idx) } From 2076342367df0ce0373d11187ff26f03757862ba Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 12:52:20 +0900 Subject: [PATCH 047/131] Propagate CFG label oparg overflow --- crates/codegen/src/ir.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f4b8f1c0e15..233e62a94ce 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1400,6 +1400,18 @@ fn instruction_sequence_label_map_resolve_label_to_block( }) } +fn instruction_sequence_label_oparg( + label: InstructionSequenceLabel, +) -> crate::InternalResult { + debug_assert!(is_label(label)); + Ok(OpArg::new( + label + .idx() + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, + )) +} + fn instruction_sequence_label_map_use_label_at_block( map: &mut InstructionSequenceLabelMap, seq: &mut InstructionSequence, @@ -1635,7 +1647,7 @@ impl CodeInfo { delta: Arg::marker(), } .into(), - arg: OpArg::new(handler_label.idx().to_u32().expect("too many labels")), + arg: instruction_sequence_label_oparg(handler_label)?, target: BlockIdx::NULL, location: SourceLocation::default(), end_location: SourceLocation::default(), @@ -5195,12 +5207,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> &mut blocks[explicit_jump.idx()], InstructionInfo { instr: PseudoOpcode::JumpNoInterrupt.into(), - arg: OpArg::new( - jump_label - .idx() - .to_u32() - .expect("too many CPython CFG labels"), - ), + arg: instruction_sequence_label_oparg(jump_label)?, target: BlockIdx::NULL, location: SourceLocation::default(), end_location: SourceLocation::default(), @@ -5328,7 +5335,7 @@ fn basicblock_add_jump( debug_assert!(target != BlockIdx::NULL); let label = blocks[target.idx()].cpython_label; debug_assert!(is_label(label)); - let arg = OpArg::new(label.idx().to_u32().expect("too many CPython CFG labels")); + let arg = instruction_sequence_label_oparg(label)?; let block = &mut blocks[bi]; basicblock_addop( block, From 58d468574ddbc838a5fb2a480254d7b71c25eda1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:04:03 +0900 Subject: [PATCH 048/131] Model CPython basicblock instruction storage --- crates/codegen/src/compile.rs | 23 +- crates/codegen/src/ir.rs | 398 +++++++++++++++++----------------- 2 files changed, 208 insertions(+), 213 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index d83cf7116d7..5de5084beab 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -10037,7 +10037,9 @@ impl Compiler { self.current_code_info() .addop_to_instr_sequence(info) .expect("malformed instruction sequence emission"); - self.current_block().instructions.push(info); + self.current_code_info() + .addop_to_current_block(info) + .expect("malformed CFG emission"); } fn push_emitted_instruction_with_target_label( @@ -10048,11 +10050,13 @@ impl Compiler { self.current_code_info() .addop_to_instr_sequence_with_target_label(info, target_label) .expect("malformed instruction sequence emission"); - self.current_block().instructions.push(info); + self.current_code_info() + .addop_to_current_block(info) + .expect("malformed CFG emission"); } fn last_emitted_instruction_mut(&mut self) -> Option<&mut ir::InstructionInfo> { - self.current_block().instructions.last_mut() + self.current_code_info().last_current_block_instr_mut() } fn set_last_emitted_lineno_override(&mut self, lineno_override: i32) { @@ -10166,7 +10170,7 @@ impl Compiler { .expect("code unit must have an entry block"); debug_assert!( entry - .instructions + .used_instructions() .first() .is_some_and(|info| match info.instr.real() { Some(Instruction::Resume { context }) => matches!( @@ -10178,7 +10182,7 @@ impl Compiler { "scope entry must start with a function-start RESUME" ); debug_assert!( - !entry.instructions.iter().any(|info| matches!( + !entry.used_instructions().iter().any(|info| matches!( info.instr.real(), Some( Instruction::ReturnGenerator @@ -10947,11 +10951,6 @@ impl Compiler { Ok(()) } - fn current_block(&mut self) -> &mut ir::Block { - let info = self.current_code_info(); - &mut info.blocks[info.current_block] - } - /// CPython `_PyCfgBuilder_Addop()` calls /// `cfg_builder_maybe_start_new_block()` before appending each instruction. /// That keeps any `IS_TERMINATOR_OPCODE` as the final instruction in its @@ -11007,9 +11006,9 @@ impl Compiler { let block = code.resolve_instr_sequence_label(block); let cur = code.current_block; let can_reuse_current = cur != block - && code.blocks[cur.idx()].instructions.is_empty() + && code.blocks[cur.idx()].is_empty() && code.blocks[cur.idx()].next == BlockIdx::NULL - && code.blocks[block.idx()].instructions.is_empty() + && code.blocks[block.idx()].is_empty() && code.blocks[block.idx()].next == BlockIdx::NULL; if !can_reuse_current { diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 233e62a94ce..1dd05ec61e7 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -343,38 +343,41 @@ fn c_array_ensure_capacity( /// flowgraph.c basicblock_next_instr fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { - let off = block.instructions.len(); + let off = block.instruction_used; let new_allocation = c_array_ensure_capacity(block.instruction_allocation, off + 1, DEFAULT_BLOCK_SIZE)?; if new_allocation > block.instruction_allocation { - if new_allocation > block.instructions.capacity() + block.cpython_spare_instr_slots.len() { + if new_allocation > block.instructions.len() { block .instructions - .try_reserve_exact( - new_allocation - - block.instructions.capacity() - - block.cpython_spare_instr_slots.len(), - ) + .try_reserve_exact(new_allocation - block.instructions.len()) .map_err(|_| InternalError::MalformedControlFlowGraph)?; + block + .instructions + .resize_with(new_allocation, empty_instruction_info); } block.instruction_allocation = new_allocation; } - let slot = block - .cpython_spare_instr_slots - .pop() - .unwrap_or_else(empty_instruction_info); - block.instructions.push(slot); + block.instruction_used += 1; Ok(off) } /// flowgraph.c basicblock_last_instr fn basicblock_last_instr(block: &Block) -> Option<&InstructionInfo> { - block.instructions.last() + if block.instruction_used > 0 { + Some(&block.instructions[block.instruction_used - 1]) + } else { + None + } } /// flowgraph.c basicblock_last_instr fn basicblock_last_instr_mut(block: &mut Block) -> Option<&mut InstructionInfo> { - block.instructions.last_mut() + if block.instruction_used > 0 { + Some(&mut block.instructions[block.instruction_used - 1]) + } else { + None + } } /// flowgraph.c basicblock_addop @@ -402,7 +405,7 @@ fn basicblock_insert_instruction( pos: usize, info: InstructionInfo, ) -> crate::InternalResult<()> { - let old_len = block.instructions.len(); + let old_len = block.instruction_used; debug_assert!(pos <= old_len); basicblock_next_instr(block)?; for i in (pos + 1..=old_len).rev() { @@ -419,7 +422,7 @@ fn basicblock_append_block_instructions( from: BlockIdx, ) -> crate::InternalResult<()> { debug_assert_ne!(to, from); - let from_len = blocks[from.idx()].instructions.len(); + let from_len = blocks[from.idx()].instruction_used; for i in 0..from_len { let info = blocks[from.idx()].instructions[i]; let off = basicblock_next_instr(&mut blocks[to.idx()])?; @@ -430,29 +433,15 @@ fn basicblock_append_block_instructions( /// flowgraph.c direct `b_iused = 0` fn basicblock_clear(block: &mut Block) { - let mut instructions = core::mem::take(&mut block.instructions); - if !instructions.is_empty() { - instructions.reverse(); - if block.cpython_spare_instr_slots.is_empty() { - block.cpython_spare_instr_slots = instructions; - } else { - block.cpython_spare_instr_slots.extend(instructions); - } - } + block.instruction_used = 0; } /// CPython direct `b_instr[0]` access. Some passes set `b_iused = 0` /// without clearing the backing array, so an empty basic block can still have /// a first raw instruction slot. fn basicblock_raw_first_instr_mut(block: &mut Block) -> &mut InstructionInfo { - if !block.instructions.is_empty() { - &mut block.instructions[0] - } else { - block - .cpython_spare_instr_slots - .last_mut() - .expect("CPython basicblock has a backing instruction slot") - } + debug_assert!(block.instruction_allocation > 0); + &mut block.instructions[0] } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -750,7 +739,7 @@ fn cfg_to_instruction_sequence( debug_assert!(is_label(block_label)); instruction_sequence_use_label(instr_sequence, block_label)?; - let instr_count = blocks[block_idx.idx()].instructions.len(); + let instr_count = blocks[block_idx.idx()].instruction_used; for i in 0..instr_count { if blocks[block_idx.idx()].instructions[i].instr.has_target() { let target_block = blocks[block_idx.idx()].instructions[i].target; @@ -1248,12 +1237,11 @@ pub struct Block { instruction_allocation: usize, /// Exception stack at start of block, used by label_exception_targets (b_exceptstack) except_stack: Option>, + /// CPython `basicblock.b_instr`, including allocated slots beyond `b_iused`. pub instructions: Vec, pub next: BlockIdx, - /// CPython keeps `b_instr` allocated beyond `b_iused`. Instructions removed - /// by compaction remain in those spare slots until `basicblock_next_instr()` - /// reuses them. The next CPython slot is stored at the end so reuse is O(1). - cpython_spare_instr_slots: Vec, + /// CPython `basicblock.b_iused`, the number of used entries in `b_instr`. + instruction_used: usize, /// Potentially uninitialized locals mask for local-check analysis (b_unsafe_locals_mask) unsafe_locals_mask: u64, /// Number of incoming CFG edges from reachable blocks (b_predecessors) @@ -1281,7 +1269,7 @@ impl Default for Block { except_stack: None, instructions: Vec::new(), next: BlockIdx::NULL, - cpython_spare_instr_slots: Vec::new(), + instruction_used: 0, unsafe_locals_mask: 0, predecessors: 0, start_depth: START_DEPTH_UNSET, @@ -1294,6 +1282,16 @@ impl Default for Block { } } +impl Block { + pub(crate) fn used_instructions(&self) -> &[InstructionInfo] { + &self.instructions[..self.instruction_used] + } + + pub(crate) fn is_empty(&self) -> bool { + self.instruction_used == 0 + } +} + pub(crate) const START_DEPTH_UNSET: i32 = i32::MIN; const CO_MAXBLOCKS: usize = 20; @@ -1552,6 +1550,17 @@ impl CodeInfo { Ok(()) } + pub(crate) fn addop_to_current_block( + &mut self, + info: InstructionInfo, + ) -> crate::InternalResult<()> { + basicblock_addop(&mut self.blocks[self.current_block.idx()], info) + } + + pub(crate) fn last_current_block_instr_mut(&mut self) -> Option<&mut InstructionInfo> { + basicblock_last_instr_mut(&mut self.blocks[self.current_block.idx()]) + } + pub(crate) fn set_last_instr_sequence_lineno_override(&mut self, lineno_override: i32) { if let Some(last) = instruction_sequence_last_info_mut(&mut self.instr_sequence) { last.lineno_override = Some(lineno_override); @@ -1684,14 +1693,14 @@ impl CodeInfo { /// flowgraph.c cfg_builder_check fn debug_check_recorded_cfg_builder(&self) { debug_assert!(!self.blocks.is_empty()); - debug_assert!(!self.blocks[0].instructions.is_empty()); + debug_assert!(self.blocks[0].instruction_used != 0); debug_assert!(!self.instr_sequence.instrs.is_empty()); debug_assert!(self.current_block.idx() < self.blocks.len()); for block in &self.blocks { if block.next != BlockIdx::NULL { debug_assert!(block.next.idx() < self.blocks.len()); } - for instr in &block.instructions { + for instr in &block.instructions[..block.instruction_used] { debug_assert!(!instr.instr.is_assembler()); if instr.target != BlockIdx::NULL { debug_assert!(instr.target.idx() < self.blocks.len()); @@ -1724,7 +1733,7 @@ fn optimize_code_unit( // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before // later jump normalization / block reordering can create adjacencies // that never exist at this stage in flowgraph.c. - insert_superinstructions(blocks)?; + insert_superinstructions(blocks); push_cold_blocks_to_end(blocks)?; // CPython resolves line numbers again after cold-block extraction. resolve_line_numbers(blocks, metadata.firstlineno)?; @@ -1759,7 +1768,7 @@ fn optimize_cfg( optimize_basic_block(blocks, metadata, block_idx)?; block_idx = next_block; } - remove_redundant_nops_and_pairs(blocks)?; + remove_redundant_nops_and_pairs(blocks); // CPython optimize_cfg() removes newly-unreachable blocks and // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes // unused constants. @@ -2219,9 +2228,10 @@ fn get_const_loading_instrs( .try_reserve_exact(size) .map_err(|_| InternalError::MalformedControlFlowGraph)?; loop { - let Some(instr) = block.instructions.get(start) else { + if start >= block.instruction_used { return Ok(None); - }; + } + let instr = &block.instructions[start]; if !matches!(instr.instr.real(), Some(Instruction::Nop)) { if !loads_const(instr) { return Ok(None); @@ -3395,13 +3405,14 @@ fn apply_static_swaps(instructions: &mut [InstructionInfo], mut i: isize) { /// flowgraph.c optimize_basic_block swap pass fn apply_static_swaps_block(block: &mut Block) -> crate::InternalResult<()> { let mut i = 0; - while i < block.instructions.len() { + while i < block.instruction_used { if matches!( block.instructions[i].instr.real(), Some(Instruction::Swap { .. }) ) { - swaptimize(&mut block.instructions, &mut i)?; - apply_static_swaps(&mut block.instructions, i as isize); + let instructions = &mut block.instructions[..block.instruction_used]; + swaptimize(instructions, &mut i)?; + apply_static_swaps(instructions, i as isize); } i += 1; } @@ -3427,7 +3438,7 @@ fn basicblock_optimize_load_const( let mut i = 0; let mut effective_opcode = None; let mut effective_oparg = OpArg::new(0); - while i < block.instructions.len() { + while i < block.instruction_used { if matches!( block.instructions[i].instr.real(), Some(Instruction::LoadConst { .. }) @@ -3459,7 +3470,7 @@ fn basicblock_optimize_load_const( }; let const_arg = effective_oparg; - if i + 1 >= block.instructions.len() { + if i + 1 >= block.instruction_used { i += 1; continue; } @@ -3504,7 +3515,7 @@ fn basicblock_optimize_load_const( && let Instruction::IsOp { invert } = next_instr { let mut jump_idx = i + 2; - if jump_idx >= block.instructions.len() { + if jump_idx >= block.instruction_used { i += 1; continue; } @@ -3515,7 +3526,7 @@ fn basicblock_optimize_load_const( ) { set_to_nop(&mut block.instructions[jump_idx]); jump_idx += 1; - if jump_idx >= block.instructions.len() { + if jump_idx >= block.instruction_used { i += 1; continue; } @@ -3616,13 +3627,13 @@ fn optimize_basic_block( }; instr_set_op0(&mut nop, Instruction::Nop.into()); let mut i = 0; - while i < blocks[bi].instructions.len() { + while i < blocks[bi].instruction_used { let inst = blocks[bi].instructions[i]; debug_assert!(!inst.instr.is_assembler()); let target = if inst.instr.has_target() { let target = inst.target; debug_assert!(target != BlockIdx::NULL); - debug_assert!(!blocks[target.idx()].instructions.is_empty()); + debug_assert!(blocks[target.idx()].instruction_used != 0); debug_assert!(!blocks[target.idx()].instructions[0].instr.is_assembler()); blocks[target.idx()].instructions[0] } else { @@ -3844,7 +3855,7 @@ fn optimize_basic_block( /// flowgraph.c remove_redundant_nops_and_pairs #[allow(clippy::if_same_then_else, clippy::useless_let_if_seq)] -fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) -> crate::InternalResult<()> { +fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) { let mut done = false; while !done { @@ -3853,12 +3864,12 @@ fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) -> crate::InternalResul let mut block_idx = BlockIdx::new(0); while block_idx != BlockIdx::NULL { - basicblock_remove_redundant_nops(blocks, block_idx)?; + basicblock_remove_redundant_nops(blocks, block_idx); if is_label(blocks[block_idx.idx()].cpython_label) { instr = None; } - let len = blocks[block_idx.idx()].instructions.len(); + let len = blocks[block_idx.idx()].instruction_used; for instr_idx in 0..len { let prev_instr = instr; instr = Some((block_idx, instr_idx)); @@ -3908,7 +3919,6 @@ fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) -> crate::InternalResul block_idx = block.next; } } - Ok(()) } /// flowgraph.c remove_unused_consts @@ -3937,7 +3947,7 @@ fn remove_unused_consts( let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let block = &blocks[block_idx]; - for i in 0..block.instructions.len() { + for i in 0..block.instruction_used { let instr = &block.instructions[i]; if instr.instr.has_const() { let index = u32::from(instr.arg) as usize; @@ -3998,7 +4008,7 @@ fn remove_unused_consts( while block_idx != BlockIdx::NULL { let next_block = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx]; - for i in 0..block.instructions.len() { + for i in 0..block.instruction_used { let instr = &mut block.instructions[i]; if instr.instr.has_const() { let index = u32::from(instr.arg) as usize; @@ -4016,7 +4026,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut max_instrs = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { - max_instrs = max_instrs.max(blocks[current.idx()].instructions.len()); + max_instrs = max_instrs.max(blocks[current.idx()].instruction_used); current = blocks[current.idx()].next; } let mut instr_flags = Vec::new(); @@ -4032,7 +4042,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { while let Some(block_idx) = worklist.pop() { let block_i = block_idx.idx(); - let instr_count = blocks[block_i].instructions.len(); + let instr_count = blocks[block_i].instruction_used; instr_flags[..instr_count].fill(0); debug_assert!(blocks[block_i].start_depth >= 0); let start_depth = usize::try_from(blocks[block_i].start_depth) @@ -4236,7 +4246,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { } let block = &mut blocks[block_idx]; - let iused = block.instructions.len(); + let iused = block.instruction_used; let mut i = 0; while i < iused { let info = &mut block.instructions[i]; @@ -4280,7 +4290,7 @@ fn calculate_stackdepth(blocks: &mut [Block]) -> crate::InternalResult { let mut depth = blocks[idx].start_depth; debug_assert!(depth >= 0); let mut next = blocks[idx].next; - let instr_count = blocks[idx].instructions.len(); + let instr_count = blocks[idx].instruction_used; for i in 0..instr_count { let ins = blocks[idx].instructions[i]; let instr = &ins.instr; @@ -4359,7 +4369,7 @@ impl CodeInfo { }, block_return, ); - for info in &block.instructions { + for info in &block.instructions[..block.instruction_used] { let lineno = instruction_lineno(info); let _ = writeln!( out, @@ -4419,7 +4429,7 @@ impl CodeInfo { "after_optimize_basic_block".to_owned(), self.debug_block_dump(), )); - remove_redundant_nops_and_pairs(&mut self.blocks)?; + remove_redundant_nops_and_pairs(&mut self.blocks); remove_unreachable(&mut self.blocks)?; remove_redundant_nops_and_jumps(&mut self.blocks)?; #[cfg(debug_assertions)] @@ -4432,7 +4442,7 @@ impl CodeInfo { let nlocals = self.metadata.varnames.len(); let nparams = self.nparams; add_checks_for_loads_of_uninitialized_variables(&mut self.blocks, nlocals, nparams)?; - insert_superinstructions(&mut self.blocks)?; + insert_superinstructions(&mut self.blocks); push_cold_blocks_to_end(&mut self.blocks)?; trace.push(( "after_push_cold_before_chain_reorder".to_owned(), @@ -4532,13 +4542,13 @@ fn make_super_instruction( } /// flowgraph.c insert_superinstructions -fn insert_superinstructions(blocks: &mut [Block]) -> crate::InternalResult { +fn insert_superinstructions(blocks: &mut [Block]) -> usize { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next_block = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx]; - for i in 0..block.instructions.len() { - let nextop = (i + 1 < block.instructions.len()) + for i in 0..block.instruction_used { + let nextop = (i + 1 < block.instruction_used) .then(|| block.instructions[i + 1].instr.real()) .flatten(); match block.instructions[i].instr.real() { @@ -4585,13 +4595,13 @@ fn insert_superinstructions(blocks: &mut [Block]) -> crate::InternalResult crate::InternalResul let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; - let instr_count = blocks[block_idx.idx()].instructions.len(); + let instr_count = blocks[block_idx.idx()].instruction_used; for i in 0..instr_count { let instr = blocks[block_idx.idx()].instructions[i]; if is_block_push(&instr) { @@ -5109,7 +5119,7 @@ fn mark_warm(blocks: &mut [Block]) -> crate::InternalResult<()> { blocks[next.idx()].visited = true; } - let instr_count = blocks[idx].instructions.len(); + let instr_count = blocks[idx].instruction_used; for i in 0..instr_count { let instr = blocks[idx].instructions[i]; if is_jump(&instr) { @@ -5161,7 +5171,7 @@ fn mark_cold(blocks: &mut [Block]) -> crate::InternalResult<()> { } } - let instr_count = blocks[idx].instructions.len(); + let instr_count = blocks[idx].instruction_used; for i in 0..instr_count { let instr = blocks[idx].instructions[i]; if is_jump(&instr) { @@ -5284,11 +5294,11 @@ fn check_cfg(blocks: &[Block]) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let block = &blocks[block_idx.idx()]; - for i in 0..block.instructions.len() { + for i in 0..block.instruction_used { let opcode = block.instructions[i].instr; debug_assert!(!opcode.is_assembler()); if opcode.is_terminator() { - if i != block.instructions.len() - 1 { + if i != block.instruction_used - 1 { return Err(InternalError::MalformedControlFlowGraph); } } @@ -5309,7 +5319,7 @@ fn jump_thread( let bi = block_idx.idx(); debug_assert!(is_jump(&blocks[bi].instructions[instr_idx])); debug_assert!(is_jump(target)); - debug_assert_eq!(instr_idx + 1, blocks[bi].instructions.len()); + debug_assert_eq!(instr_idx + 1, blocks[bi].instruction_used); debug_assert!(target.target != BlockIdx::NULL); if blocks[bi].instructions[instr_idx].target != target.target { set_to_nop(&mut blocks[bi].instructions[instr_idx]); @@ -5381,14 +5391,14 @@ fn convert_pseudo_conditional_jumps(blocks: &mut [Block]) -> crate::InternalResu let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; let mut i = 0; - while i < block.instructions.len() { + while i < block.instruction_used { let instr = block.instructions[i]; let opcode = instr.instr; if matches!( opcode.pseudo(), Some(PseudoInstruction::JumpIfFalse { .. } | PseudoInstruction::JumpIfTrue { .. }) ) { - debug_assert_eq!(i, block.instructions.len() - 1); + debug_assert_eq!(i, block.instruction_used - 1); block.instructions[i].instr = if matches!(opcode.pseudo(), Some(PseudoInstruction::JumpIfFalse { .. })) { Instruction::PopJumpIfFalse { @@ -5551,7 +5561,7 @@ fn basicblock_inline_small_or_no_lineno_blocks( let target = last.target; debug_assert!(target != BlockIdx::NULL); let small_exit_block = basicblock_exits_scope(&blocks[target.idx()]) - && blocks[target.idx()].instructions.len() <= MAX_COPY_SIZE; + && blocks[target.idx()].instruction_used <= MAX_COPY_SIZE; let no_lineno_no_fallthrough = basicblock_has_no_lineno(&blocks[target.idx()]) && !bb_has_fallthrough(&blocks[target.idx()]); if small_exit_block || no_lineno_no_fallthrough { @@ -5600,14 +5610,11 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) -> crate::InternalResu } /// flowgraph.c basicblock_remove_redundant_nops -fn basicblock_remove_redundant_nops( - blocks: &mut [Block], - block_idx: BlockIdx, -) -> crate::InternalResult { +fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) -> usize { let bi = block_idx.idx(); let mut dest = 0; let mut prev_lineno = -1i32; - let instr_count = blocks[bi].instructions.len(); + let instr_count = blocks[bi].instruction_used; for src in 0..instr_count { let instr = blocks[bi].instructions[src]; @@ -5639,7 +5646,7 @@ fn basicblock_remove_redundant_nops( if next != BlockIdx::NULL { let mut next_loc = no_linetable_location(); let mut next_i = 0; - while next_i < blocks[next.idx()].instructions.len() { + while next_i < blocks[next.idx()].instruction_used { let instr = blocks[next.idx()].instructions[next_i]; if matches!(instr.instr.real(), Some(Instruction::Nop)) && instruction_lineno(&instr) < 0 @@ -5666,37 +5673,26 @@ fn basicblock_remove_redundant_nops( debug_assert!(dest <= instr_count); let num_removed = instr_count - dest; - blocks[bi] - .cpython_spare_instr_slots - .try_reserve(num_removed) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - let mut src = instr_count; - while src > dest { - src -= 1; - blocks[bi] - .cpython_spare_instr_slots - .push(blocks[bi].instructions[src]); - } - blocks[bi].instructions.truncate(dest); - Ok(num_removed) + blocks[bi].instruction_used = dest; + num_removed } /// flowgraph.c remove_redundant_nops -fn remove_redundant_nops(blocks: &mut [Block]) -> crate::InternalResult { +fn remove_redundant_nops(blocks: &mut [Block]) -> usize { let mut changes = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { let next = blocks[current.idx()].next; - changes += basicblock_remove_redundant_nops(blocks, current)?; + changes += basicblock_remove_redundant_nops(blocks, current); current = next; } - Ok(changes) + changes } /// flowgraph.c no_redundant_nops #[cfg(debug_assertions)] -fn no_redundant_nops(blocks: &mut [Block]) -> crate::InternalResult { - Ok(remove_redundant_nops(blocks)? == 0) +fn no_redundant_nops(blocks: &mut [Block]) -> bool { + remove_redundant_nops(blocks) == 0 } /// flowgraph.c remove_redundant_jumps @@ -5757,7 +5753,7 @@ fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResul loop { // Convergence is guaranteed because the number of redundant jumps and // nops only decreases. - let removed_nops = remove_redundant_nops(blocks)?; + let removed_nops = remove_redundant_nops(blocks); let removed_jumps = remove_redundant_jumps(blocks)?; if removed_nops + removed_jumps == 0 { break; @@ -5892,19 +5888,20 @@ fn cfg_builder_addop(g: &mut CfgBuilder, info: InstructionInfo) -> crate::Intern /// flowgraph.c cfg_builder_check fn cfg_builder_check(g: &CfgBuilder) -> bool { debug_assert!(g.entry != BlockIdx::NULL); - debug_assert!(!g.blocks[g.entry.idx()].instructions.is_empty()); + debug_assert!(g.blocks[g.entry.idx()].instruction_used != 0); let mut block = g.block_list; while block != BlockIdx::NULL { debug_assert!(block.idx() < g.blocks.len()); let block_ref = &g.blocks[block.idx()]; - let has_instr_array = - !block_ref.instructions.is_empty() || !block_ref.cpython_spare_instr_slots.is_empty(); + let has_instr_array = block_ref.instruction_allocation > 0; if has_instr_array { - debug_assert!(block_ref.instruction_allocation > 0); - debug_assert!(block_ref.instruction_allocation >= block_ref.instructions.len()); + debug_assert_eq!( + block_ref.instructions.len(), + block_ref.instruction_allocation + ); + debug_assert!(block_ref.instruction_allocation >= block_ref.instruction_used); } else { - debug_assert!(block_ref.instructions.is_empty()); - debug_assert_eq!(block_ref.instruction_allocation, 0); + debug_assert_eq!(block_ref.instruction_used, 0); } block = block_ref.allocation_next; } @@ -5958,7 +5955,7 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResu block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; - for i in 0..blocks[block_idx.idx()].instructions.len() { + for i in 0..blocks[block_idx.idx()].instruction_used { let info = &mut blocks[block_idx.idx()].instructions[i]; debug_assert_eq!(info.target, BlockIdx::NULL); if info.instr.has_target() { @@ -6108,7 +6105,7 @@ fn maybe_push( fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &mut Vec) { let idx = block_idx.idx(); let mut unsafe_mask = blocks[idx].unsafe_locals_mask; - let instr_count = blocks[idx].instructions.len(); + let instr_count = blocks[idx].instruction_used; for i in 0..instr_count { let (instr, arg, except_handler) = { @@ -6182,7 +6179,7 @@ fn fast_scan_many_locals(blocks: &mut [Block], nlocals: usize) -> crate::Interna let mut current = BlockIdx(0); while current != BlockIdx::NULL { blocknum += 1; - for i in 0..blocks[current.idx()].instructions.len() { + for i in 0..blocks[current.idx()].instruction_used { let info = &mut blocks[current.idx()].instructions[i]; debug_assert!(!matches!(info.instr.real(), Some(Instruction::ExtendedArg))); let arg = u32::from(info.arg) as usize; @@ -6253,7 +6250,7 @@ fn add_checks_for_loads_of_uninitialized_variables( /// Follow chain of empty blocks to find first non-empty block. fn next_nonempty_block(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { - while idx != BlockIdx::NULL && blocks[idx.idx()].instructions.is_empty() { + while idx != BlockIdx::NULL && blocks[idx.idx()].instruction_used == 0 { idx = blocks[idx.idx()].next; } idx @@ -6332,7 +6329,7 @@ fn is_exit_or_eval_check_without_lineno(block: &Block) -> bool { /// flowgraph.c basicblock_has_eval_break fn basicblock_has_eval_break(block: &Block) -> bool { let mut i = 0; - while i < block.instructions.len() { + while i < block.instruction_used { if block.instructions[i].instr.has_eval_break() { return true; } @@ -6344,7 +6341,7 @@ fn basicblock_has_eval_break(block: &Block) -> bool { /// flowgraph.c basicblock_has_no_lineno fn basicblock_has_no_lineno(block: &Block) -> bool { let mut i = 0; - while i < block.instructions.len() { + while i < block.instruction_used { if instruction_lineno(&block.instructions[i]) >= 0 { return false; } @@ -6378,7 +6375,7 @@ fn compute_reachable_predecessors(blocks: &mut [Block]) -> crate::InternalResult blocks[next.idx()].predecessors += 1; } - let instr_count = blocks[idx].instructions.len(); + let instr_count = blocks[idx].instruction_used; for i in 0..instr_count { let instr = blocks[idx].instructions[i]; if is_jump(&instr) || is_block_push(&instr) { @@ -6460,7 +6457,7 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec) -> crate::InternalRes let next = blocks[b.idx()].next; if bb_has_fallthrough(&blocks[b.idx()]) && next != BlockIdx::NULL - && !blocks[b.idx()].instructions.is_empty() + && blocks[b.idx()].instruction_used != 0 { if is_exit_or_eval_check_without_lineno(&blocks[next.idx()]) { let last = @@ -6487,7 +6484,7 @@ fn propagate_line_numbers(blocks: &mut [Block]) { }; let mut prev_location = no_instruction_location(); - for i in 0..blocks[idx].instructions.len() { + for i in 0..blocks[idx].instruction_used { if instruction_is_no_location(&blocks[idx].instructions[i]) { instr_set_location(&mut blocks[idx].instructions[i], prev_location); } else { @@ -6499,7 +6496,7 @@ fn propagate_line_numbers(blocks: &mut [Block]) { if bb_has_fallthrough(&blocks[idx]) { debug_assert!(next != BlockIdx::NULL); if next != BlockIdx::NULL && blocks[next.idx()].predecessors == 1 { - if !blocks[next.idx()].instructions.is_empty() { + if blocks[next.idx()].instruction_used != 0 { if instruction_is_no_location(&blocks[next.idx()].instructions[0]) { instr_set_location(&mut blocks[next.idx()].instructions[0], prev_location); } @@ -6602,7 +6599,7 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe let mut handler = except_stack_top(&stack, blocks); let mut last_yield_except_depth: i32 = -1; - let instr_count = blocks[bi].instructions.len(); + let instr_count = blocks[bi].instruction_used; for i in 0..instr_count { let info = blocks[bi].instructions[i]; let instr = info.instr; @@ -6685,7 +6682,7 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block]) -> crate::InternalResult< while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; - for i in 0..block.instructions.len() { + for i in 0..block.instruction_used { let info = &mut block.instructions[i]; if is_block_push(info) { set_to_nop(info); @@ -6779,7 +6776,7 @@ pub(crate) fn fix_cell_offsets( while block_idx != BlockIdx::NULL { let next = blocks[block_idx.idx()].next; let block = &mut blocks[block_idx.idx()]; - for i in 0..block.instructions.len() { + for i in 0..block.instruction_used { let inst = &mut block.instructions[i]; debug_assert!( !matches!(inst.instr.real(), Some(Instruction::ExtendedArg)), @@ -6854,6 +6851,11 @@ mod tests { instr } + fn test_block_push(block: &mut Block, info: InstructionInfo) { + let off = basicblock_next_instr(block).expect("test block instruction slot"); + block.instructions[off] = info; + } + fn test_code_info(block: Block) -> CodeInfo { CodeInfo { flags: CodeFlags::empty(), @@ -6996,15 +6998,15 @@ mod tests { let mut block = Block::default(); let mut stale = test_instr(Instruction::Nop, 11); stale.except_handler = Some(handler); - block.cpython_spare_instr_slots.push(stale); + test_block_push(&mut block, stale); + basicblock_clear(&mut block); basicblock_addop(&mut block, test_instr(Instruction::PopTop, 12)) .expect("basicblock_addop succeeds"); // CPython `basicblock_addop()` writes opcode/oparg/target/location into // the reused `b_instr[b_iused]` slot, but does not clear `i_except`. - assert_eq!(block.cpython_spare_instr_slots.len(), 0); - assert_eq!(block.instructions.len(), 1); + assert_eq!(block.instruction_used, 1); assert_eq!(block.instructions[0].except_handler, Some(handler)); assert_eq!(block.instructions[0].target, BlockIdx::NULL); } @@ -7032,10 +7034,11 @@ mod tests { preserve_lasti: false, }; let mut block = Block::default(); - block.instructions.push(test_instr(Instruction::Nop, 21)); + test_block_push(&mut block, test_instr(Instruction::Nop, 21)); let mut stale = test_instr(Instruction::Nop, 22); stale.except_handler = Some(handler); - block.cpython_spare_instr_slots.push(stale); + test_block_push(&mut block, stale); + block.instruction_used = 1; basicblock_insert_instruction(&mut block, 0, test_instr(Instruction::PopTop, 23)) .expect("basicblock_insert_instruction succeeds"); @@ -7043,8 +7046,7 @@ mod tests { // CPython `basicblock_insert_instruction()` also obtains a slot with // `basicblock_next_instr()`, then overwrites the inserted position with // the provided instruction copy, including its `i_except` value. - assert_eq!(block.cpython_spare_instr_slots.len(), 0); - assert_eq!(block.instructions.len(), 2); + assert_eq!(block.instruction_used, 2); assert_eq!(block.instructions[0].except_handler, None); } @@ -7057,7 +7059,7 @@ mod tests { let mut block = Block::default(); let mut stale = test_instr(Instruction::PopTop, 31); stale.except_handler = Some(handler); - block.instructions.push(stale); + test_block_push(&mut block, stale); basicblock_clear(&mut block); basicblock_addop(&mut block, test_instr(Instruction::Nop, 32)) @@ -7066,8 +7068,7 @@ mod tests { // CPython `remove_unreachable()` sets `b_iused = 0` without clearing the // backing `b_instr` slot. A later `basicblock_addop()` reuses that slot // and does not overwrite `i_except`. - assert_eq!(block.cpython_spare_instr_slots.len(), 0); - assert_eq!(block.instructions.len(), 1); + assert_eq!(block.instruction_used, 1); assert_eq!(block.instructions[0].except_handler, Some(handler)); } @@ -7080,7 +7081,7 @@ mod tests { handler_block: BlockIdx::new(i as u32 + 1), preserve_lasti: false, }); - block.instructions.push(stale); + test_block_push(&mut block, stale); } basicblock_clear(&mut block); @@ -7090,7 +7091,7 @@ mod tests { } let handlers = block - .instructions + .used_instructions() .iter() .map(|instr| { instr @@ -7103,7 +7104,6 @@ mod tests { handlers, [BlockIdx::new(1), BlockIdx::new(2), BlockIdx::new(3)] ); - assert_eq!(block.cpython_spare_instr_slots.len(), 0); } #[test] @@ -7115,19 +7115,17 @@ mod tests { let mut blocks = vec![Block::default(), Block::default()]; let mut stale = test_instr(Instruction::Nop, 41); stale.except_handler = Some(handler); - blocks[0].cpython_spare_instr_slots.push(stale); + test_block_push(&mut blocks[0], stale); + basicblock_clear(&mut blocks[0]); - blocks[1] - .instructions - .push(test_instr(Instruction::PopTop, 42)); + test_block_push(&mut blocks[1], test_instr(Instruction::PopTop, 42)); basicblock_append_block_instructions(&mut blocks, BlockIdx::new(0), BlockIdx::new(1)) .expect("basicblock_append_block_instructions succeeds"); // CPython `basicblock_append_instructions()` obtains a slot with // `basicblock_next_instr()`, then overwrites it with the copied // instruction, including `i_except`. - assert_eq!(blocks[0].cpython_spare_instr_slots.len(), 0); - assert_eq!(blocks[0].instructions.len(), 1); + assert_eq!(blocks[0].instruction_used, 1); assert_eq!(blocks[0].instructions[0].except_handler, None); } @@ -7139,7 +7137,7 @@ mod tests { assert_eq!(info.target, BlockIdx::new(1)); let mut blocks = vec![Block::default(), Block::default()]; - blocks[0].instructions.push(info); + test_block_push(&mut blocks[0], info); blocks[0].next = BlockIdx::new(1); let mut instr_sequence = instruction_sequence_new(); @@ -7152,7 +7150,7 @@ mod tests { #[should_panic(expected = "target_block != BlockIdx::NULL")] fn cfg_to_instruction_sequence_requires_target_for_target_opcodes() { let mut block = Block::default(); - block.instructions.push(test_jump(BlockIdx::NULL, 51)); + test_block_push(&mut block, test_jump(BlockIdx::NULL, 51)); let mut blocks = vec![block]; let mut instr_sequence = instruction_sequence_new(); @@ -7173,7 +7171,9 @@ mod tests { store.arg = OpArg::new(0); let mut pop = test_instr(Instruction::PopTop, 60); pop.lineno_override = Some(NO_LOCATION_OVERRIDE); - block.instructions.extend([swap, store, pop]); + for info in [swap, store, pop] { + test_block_push(&mut block, info); + } apply_static_swaps_block(&mut block).expect("apply_static_swaps_block succeeds"); @@ -7205,7 +7205,9 @@ mod tests { store.arg = OpArg::new(0); store.lineno_override = Some(NO_LOCATION_OVERRIDE); let pop = test_instr(Instruction::PopTop, 71); - block.instructions.extend([swap, store, pop]); + for info in [swap, store, pop] { + test_block_push(&mut block, info); + } apply_static_swaps_block(&mut block).expect("apply_static_swaps_block succeeds"); @@ -7228,16 +7230,19 @@ mod tests { #[test] fn optimize_load_const_tracks_cpython_copy_of_load_const() { let mut block = Block::default(); - block.instructions.push(test_instr( - Instruction::LoadConst { - consti: Arg::marker(), - }, - 80, - )); + test_block_push( + &mut block, + test_instr( + Instruction::LoadConst { + consti: Arg::marker(), + }, + 80, + ), + ); let mut copy = test_instr(Instruction::Copy { i: Arg::marker() }, 80); copy.arg = OpArg::new(1); - block.instructions.push(copy); - block.instructions.push(test_instr(Instruction::ToBool, 80)); + test_block_push(&mut block, copy); + test_block_push(&mut block, test_instr(Instruction::ToBool, 80)); let mut code = test_code_info(block); let (const_idx, _) = code.metadata.consts.insert_full(ConstantData::Tuple { @@ -7275,17 +7280,20 @@ mod tests { #[test] fn optimize_load_fast_records_no_input_opcode_ref_at_cpython_produced_index() { let mut block = Block::default(); - block.instructions.push(test_instr( - Instruction::LoadFast { - var_num: Arg::marker(), - }, - 10, - )); - block.instructions.push(test_instr(Instruction::GetLen, 10)); + test_block_push( + &mut block, + test_instr( + Instruction::LoadFast { + var_num: Arg::marker(), + }, + 10, + ), + ); + test_block_push(&mut block, test_instr(Instruction::GetLen, 10)); let mut swap = test_instr(Instruction::Swap { i: Arg::marker() }, 10); swap.arg = OpArg::new(2); - block.instructions.push(swap); - block.instructions.push(test_instr(Instruction::PopTop, 10)); + test_block_push(&mut block, swap); + test_block_push(&mut block, test_instr(Instruction::PopTop, 10)); let mut code = test_code_info(block); optimize_load_fast(&mut code.blocks).expect("optimize_load_fast succeeds"); @@ -7338,7 +7346,9 @@ mod tests { ); build.arg = OpArg::new(2); let mut block = Block::default(); - block.instructions.extend([immortal, mortal, build]); + for info in [immortal, mortal, build] { + test_block_push(&mut block, info); + } assert!( fold_tuple_of_constants(&mut metadata, &mut block, 2) @@ -7375,12 +7385,10 @@ mod tests { blocks[1].cpython_label = InstructionSequenceLabel::from_index(1); blocks[2].cpython_label = InstructionSequenceLabel::from_index(2); blocks[0].next = BlockIdx::new(1); - blocks[0].instructions.push(test_cond_jump(exit, 10)); + test_block_push(&mut blocks[0], test_cond_jump(exit, 10)); blocks[1].next = exit; - blocks[1].instructions.push(test_jump(exit, 20)); - blocks[2] - .instructions - .push(test_instr(Instruction::ReturnValue, 30)); + test_block_push(&mut blocks[1], test_jump(exit, 20)); + test_block_push(&mut blocks[2], test_instr(Instruction::ReturnValue, 30)); blocks[2].instructions[0].lineno_override = Some(NO_LOCATION_OVERRIDE); compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); @@ -7405,12 +7413,10 @@ mod tests { #[test] fn propagate_line_numbers_treats_next_location_like_cpython() { let mut block = Block::default(); - block.instructions.push(test_instr(Instruction::Nop, 10)); - block.instructions.push(test_instr(Instruction::PopTop, 20)); + test_block_push(&mut block, test_instr(Instruction::Nop, 10)); + test_block_push(&mut block, test_instr(Instruction::PopTop, 20)); block.instructions[1].lineno_override = Some(NEXT_LOCATION_OVERRIDE); - block - .instructions - .push(test_instr(Instruction::ReturnValue, 30)); + test_block_push(&mut block, test_instr(Instruction::ReturnValue, 30)); block.instructions[2].lineno_override = Some(NO_LOCATION_OVERRIDE); let mut blocks = vec![block]; @@ -7435,16 +7441,11 @@ mod tests { fn propagate_line_numbers_updates_empty_jump_target_raw_slot_like_cpython() { let mut blocks = vec![Block::default(), Block::default(), Block::default()]; blocks[0].next = BlockIdx::new(2); - blocks[0] - .instructions - .push(test_cond_jump(BlockIdx::new(1), 10)); - blocks[1] - .cpython_spare_instr_slots - .push(test_instr(Instruction::Nop, 20)); - blocks[1].cpython_spare_instr_slots[0].lineno_override = Some(NO_LOCATION_OVERRIDE); - blocks[2] - .instructions - .push(test_instr(Instruction::ReturnValue, 30)); + test_block_push(&mut blocks[0], test_cond_jump(BlockIdx::new(1), 10)); + test_block_push(&mut blocks[1], test_instr(Instruction::Nop, 20)); + blocks[1].instructions[0].lineno_override = Some(NO_LOCATION_OVERRIDE); + basicblock_clear(&mut blocks[1]); + test_block_push(&mut blocks[2], test_instr(Instruction::ReturnValue, 30)); compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); propagate_line_numbers(&mut blocks); @@ -7453,23 +7454,20 @@ mod tests { // for jump targets without checking `b_iused`. If // `remove_redundant_nops()` emptied the target, that writes the stale // backing slot rather than an active instruction. - assert_eq!( - instruction_lineno(&blocks[1].cpython_spare_instr_slots[0]), - 10 - ); + assert_eq!(instruction_lineno(&blocks[1].instructions[0]), 10); } #[test] fn basicblock_has_no_lineno_treats_next_location_like_cpython() { let mut block = Block::default(); - block.instructions.push(test_instr(Instruction::Nop, 10)); + test_block_push(&mut block, test_instr(Instruction::Nop, 10)); block.instructions[0].lineno_override = Some(NEXT_LOCATION_OVERRIDE); // CPython `basicblock_has_no_lineno()` treats every negative lineno as // no line number, including `NEXT_LOCATION` (`lineno == -2`). assert!(basicblock_has_no_lineno(&block)); - block.instructions.push(test_instr(Instruction::PopTop, 11)); + test_block_push(&mut block, test_instr(Instruction::PopTop, 11)); assert!(!basicblock_has_no_lineno(&block)); } @@ -7487,12 +7485,10 @@ mod tests { blocks[0].next = BlockIdx::new(1); blocks[1].next = BlockIdx::new(2); blocks[2].next = BlockIdx::new(3); - blocks[0].instructions.push(test_jump(BlockIdx::new(1), 10)); - blocks[1].instructions.push(test_jump(BlockIdx::new(2), 20)); - blocks[2].instructions.push(test_jump(BlockIdx::new(3), 30)); - blocks[3] - .instructions - .push(test_instr(Instruction::ReturnValue, 40)); + test_block_push(&mut blocks[0], test_jump(BlockIdx::new(1), 10)); + test_block_push(&mut blocks[1], test_jump(BlockIdx::new(2), 20)); + test_block_push(&mut blocks[2], test_jump(BlockIdx::new(3), 30)); + test_block_push(&mut blocks[3], test_instr(Instruction::ReturnValue, 40)); let mut metadata = test_code_info(Block::default()).metadata; optimize_basic_block(&mut blocks, &mut metadata, BlockIdx::new(0)) @@ -7500,7 +7496,7 @@ mod tests { // CPython `optimize_basic_block()` continues after `jump_thread()`, so // the appended jump is immediately checked against the next jump target. - let threaded = blocks[0].instructions.last().expect("threaded jump"); + let threaded = basicblock_last_instr(&blocks[0]).expect("threaded jump"); assert!(matches!( threaded.instr.pseudo(), Some(PseudoInstruction::Jump { .. }) From 4ee67baf0f14466addff457ea183b3d3d8f9621d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:06:58 +0900 Subject: [PATCH 049/131] Drop Rust-only recorded CFG precheck --- crates/codegen/src/ir.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 1dd05ec61e7..22aa9148dd0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1690,29 +1690,9 @@ impl CodeInfo { Ok(instr_sequence) } - /// flowgraph.c cfg_builder_check - fn debug_check_recorded_cfg_builder(&self) { - debug_assert!(!self.blocks.is_empty()); - debug_assert!(self.blocks[0].instruction_used != 0); - debug_assert!(!self.instr_sequence.instrs.is_empty()); - debug_assert!(self.current_block.idx() < self.blocks.len()); - for block in &self.blocks { - if block.next != BlockIdx::NULL { - debug_assert!(block.next.idx() < self.blocks.len()); - } - for instr in &block.instructions[..block.instruction_used] { - debug_assert!(!instr.instr.is_assembler()); - if instr.target != BlockIdx::NULL { - debug_assert!(instr.target.idx() < self.blocks.len()); - } - } - } - } - fn prepare_cfg_from_codegen(&mut self) -> crate::InternalResult { // CPython compile.c optimize_and_assemble_code_unit passes // u_instr_sequence directly into flowgraph.c _PyCfg_FromInstructionSequence(). - self.debug_check_recorded_cfg_builder(); self.take_recorded_instr_sequence() } } From 43aaa35236c960e3bb865efed83c3ca275744815 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:10:46 +0900 Subject: [PATCH 050/131] Model CPython instruction sequence storage --- crates/codegen/src/ir.rs | 87 ++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 34 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 22aa9148dd0..f26761958a3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -511,8 +511,12 @@ const INSTRUCTION_SEQUENCE_UNSET_LABEL: i32 = -111; #[derive(Clone)] pub(crate) struct InstructionSequence { + /// CPython `instr_sequence.s_instrs`, including allocated slots beyond `s_used`. instrs: Vec, + /// CPython `instr_sequence.s_allocated`, the allocated size of `s_instrs`. instr_allocation: usize, + /// CPython `instr_sequence.s_used`, the number of used entries in `s_instrs`. + instr_used: usize, next_free_label: usize, label_map: Option>, label_map_allocation: usize, @@ -530,6 +534,7 @@ fn instruction_sequence_new() -> InstructionSequence { InstructionSequence { instrs: Vec::new(), instr_allocation: 0, + instr_used: 0, next_free_label: 0, label_map: None, label_map_allocation: 0, @@ -539,7 +544,7 @@ fn instruction_sequence_new() -> InstructionSequence { /// instruction_sequence.c instr_sequence_next_inst fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::InternalResult { - let idx = seq.instrs.len(); + let idx = seq.instr_used; let new_allocation = c_array_ensure_capacity(seq.instr_allocation, idx + 1, INITIAL_INSTR_SEQUENCE_SIZE)?; if new_allocation > seq.instr_allocation { @@ -548,20 +553,26 @@ fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::Inter .try_reserve_exact(new_allocation - seq.instrs.capacity()) .map_err(|_| InternalError::MalformedControlFlowGraph)?; } + if new_allocation > seq.instrs.len() { + seq.instrs.resize( + new_allocation, + InstructionSequenceEntry::new( + InstructionInfo { + instr: Instruction::Cache.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: None, + }, + ZERO_EXCEPTION_HANDLER_INFO, + ), + ); + } seq.instr_allocation = new_allocation; } - seq.instrs.push(InstructionSequenceEntry::new( - InstructionInfo { - instr: Instruction::Cache.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - lineno_override: None, - }, - ZERO_EXCEPTION_HANDLER_INFO, - )); + seq.instr_used += 1; Ok(idx) } @@ -631,8 +642,7 @@ fn instruction_sequence_use_label( for i in old_len..label_map.len() { label_map[i] = INSTRUCTION_SEQUENCE_UNSET_LABEL; } - label_map[label.idx()] = - i32::try_from(seq.instrs.len()).expect("instruction offset fits in int"); + label_map[label.idx()] = i32::try_from(seq.instr_used).expect("instruction offset fits in int"); Ok(()) } @@ -651,7 +661,11 @@ fn instruction_sequence_addop( fn instruction_sequence_last_info_mut( seq: &mut InstructionSequence, ) -> Option<&mut InstructionInfo> { - seq.instrs.last_mut().map(|entry| &mut entry.info) + if seq.instr_used == 0 { + None + } else { + Some(&mut seq.instrs[seq.instr_used - 1].info) + } } /// instruction_sequence.c _PyInstructionSequence_InsertInstruction @@ -661,7 +675,7 @@ fn instruction_sequence_insert_instruction( pos: usize, info: InstructionInfo, ) -> crate::InternalResult<()> { - debug_assert!(pos <= seq.instrs.len()); + debug_assert!(pos <= seq.instr_used); instruction_sequence_debug_check_addop(&info); let last_idx = instruction_sequence_next_inst(seq)?; for i in (pos..last_idx).rev() { @@ -688,7 +702,7 @@ fn instruction_sequence_apply_label_map( let Some(label_map) = instrs.label_map.as_ref() else { return Ok(()); }; - for i in 0..instrs.instrs.len() { + for i in 0..instrs.instr_used { let entry = &mut instrs.instrs[i]; if entry.info.instr.has_target() { let label = usize::try_from(u32::from(entry.info.arg)) @@ -813,7 +827,7 @@ fn is_pseudo_target(pseudo: PseudoOpcode, target: Opcode) -> bool { fn resolve_unconditional_jumps( instr_sequence: &mut InstructionSequence, ) -> crate::InternalResult<()> { - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { let instr = &mut instr_sequence.instrs[i].info; let is_forward = i32::try_from(u32::from(instr.arg)) .map_err(|_| InternalError::MalformedControlFlowGraph)? @@ -872,7 +886,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let mut end_offset; // The offset (in code units) of END_SEND from SEND in the yield-from sequence. const END_SEND_OFFSET: i32 = 5; - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { let instr = &mut instr_sequence.instrs[i]; let opcode = instr.info.instr.expect_real(); if opcode.has_jump() { @@ -883,7 +897,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let mut extended_arg_recompile; loop { let mut totsize = 0i32; - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { let instr = &mut instr_sequence.instrs[i]; instr.i_offset = totsize; let isize = instr_size(&instr.info); @@ -897,7 +911,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte extended_arg_recompile = false; let mut offset = 0i32; - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { let isize = instr_size(&instr_sequence.instrs[i].info); // Jump offsets are computed relative to the instruction pointer // after fetching the jump instruction. @@ -1059,7 +1073,7 @@ fn assemble_location_info( first_line: i32, debug_ranges: bool, ) -> crate::InternalResult> { - for i in (0..instr_sequence.instrs.len()).rev() { + for i in (0..instr_sequence.instr_used).rev() { let loc = instruction_linetable_location(&instr_sequence.instrs[i].info); if same_location(loc, next_linetable_location()) { if instr_sequence.instrs[i] @@ -1070,7 +1084,7 @@ fn assemble_location_info( { instr_sequence.instrs[i].info.lineno_override = Some(NO_LOCATION_OVERRIDE); } else { - debug_assert!(i < instr_sequence.instrs.len() - 1); + debug_assert!(i < instr_sequence.instr_used - 1); let next = instr_sequence.instrs[i + 1].info; instr_set_loc( &mut instr_sequence.instrs[i].info, @@ -1086,7 +1100,7 @@ fn assemble_location_info( let mut prev_line = first_line; let mut loc = no_linetable_location(); let mut size = 0; - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { let entry = &instr_sequence.instrs[i]; let instr_loc = instruction_linetable_location(&entry.info); if !same_location(loc, instr_loc) { @@ -1112,14 +1126,15 @@ fn assemble_emit( let mut instructions = Vec::new(); vec_try_reserve_exact(&mut instructions, num_instructions)?; - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { let instr = &mut instr_sequence.instrs[i].info; assemble_emit_instr(&mut instructions, instr); } let linetable = assemble_location_info(instr_sequence, first_line, debug_ranges)?; - let exceptiontable = assemble_exception_table(&instr_sequence.instrs)?; + let exceptiontable = + assemble_exception_table(&instr_sequence.instrs[..instr_sequence.instr_used])?; Ok(AssembledCode { instructions, @@ -5974,19 +5989,20 @@ fn cfg_from_instruction_sequence( instruction_sequence_apply_label_map(&mut instr_sequence)?; let mut builder = cfg_builder_new()?; - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { instr_sequence.instrs[i].i_target = 0; } - for i in 0..instr_sequence.instrs.len() { + for i in 0..instr_sequence.instr_used { if instr_sequence.instrs[i].info.instr.has_target() { let target_offset = usize::try_from(u32::from(instr_sequence.instrs[i].info.arg)) .expect("instruction-sequence target index fits in usize"); - debug_assert!(target_offset < instr_sequence.instrs.len()); + debug_assert!(target_offset < instr_sequence.instr_used); instr_sequence.instrs[target_offset].i_target = 1; } } let InstructionSequence { instrs, + instr_used, label_map, annotations_code, .. @@ -5996,7 +6012,7 @@ fn cfg_from_instruction_sequence( let mut offset = 0i32; let mut i = 0; - while i < instrs.len() { + while i < instr_used { let mut entry = instrs[i]; if matches!( entry.info.instr.pseudo(), @@ -6005,7 +6021,7 @@ fn cfg_from_instruction_sequence( if let Some(annotations_code) = &annotations_code { debug_assert!(annotations_code.label_map.is_none()); debug_assert!(annotations_code.annotations_code.is_none()); - for j in 0..annotations_code.instrs.len() { + for j in 0..annotations_code.instr_used { let ann_entry = annotations_code.instrs[j]; debug_assert!(!ann_entry.info.instr.has_target()); let mut info = ann_entry.info; @@ -6013,8 +6029,7 @@ fn cfg_from_instruction_sequence( cfg_builder_addop(&mut builder, info)?; } offset += annotations_code - .instrs - .len() + .instr_used .to_i32() .ok_or(InternalError::MalformedControlFlowGraph)? - 1; @@ -6940,11 +6955,15 @@ mod tests { instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 10 + i)).unwrap(); } assert_eq!(seq.instr_allocation, INITIAL_INSTR_SEQUENCE_SIZE); + assert_eq!(seq.instrs.len(), seq.instr_allocation); + assert_eq!(seq.instr_used, 99); // CPython calls `_Py_CArray_EnsureCapacity(s_used + 1)`, so the 100th // instruction expands a 100-slot array to 200 before returning offset 99. instruction_sequence_addop(&mut seq, test_instr(Instruction::Nop, 109)).unwrap(); assert_eq!(seq.instr_allocation, INITIAL_INSTR_SEQUENCE_SIZE * 2); + assert_eq!(seq.instrs.len(), seq.instr_allocation); + assert_eq!(seq.instr_used, 100); } #[test] From 6e046228da95e6bf99eddef2aa7b2177b58d1f6d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:11:23 +0900 Subject: [PATCH 051/131] Propagate instruction sequence offset overflow --- crates/codegen/src/ir.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f26761958a3..5ff49b0e1aa 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -642,7 +642,10 @@ fn instruction_sequence_use_label( for i in old_len..label_map.len() { label_map[i] = INSTRUCTION_SEQUENCE_UNSET_LABEL; } - label_map[label.idx()] = i32::try_from(seq.instr_used).expect("instruction offset fits in int"); + label_map[label.idx()] = seq + .instr_used + .to_i32() + .ok_or(InternalError::MalformedControlFlowGraph)?; Ok(()) } @@ -683,7 +686,9 @@ fn instruction_sequence_insert_instruction( } seq.instrs[pos].info = info; if let Some(label_map) = &mut seq.label_map { - let pos = i32::try_from(pos).expect("instruction offset fits in int"); + let pos = pos + .to_i32() + .ok_or(InternalError::MalformedControlFlowGraph)?; for lbl in 0..label_map.len() { if label_map[lbl] >= pos { label_map[lbl] += 1; From a962997f2ead3103dd6730a55516906aa39ffe08 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:13:12 +0900 Subject: [PATCH 052/131] Model CPython instruction sequence labels --- crates/codegen/src/ir.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5ff49b0e1aa..b2e9a083559 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -460,8 +460,8 @@ fn is_label(label: InstructionSequenceLabel) -> bool { impl InstructionSequenceLabel { pub(crate) const NO_LABEL: Self = Self(-1); - pub(crate) fn from_index(index: usize) -> Self { - Self(i32::try_from(index).expect("instruction-sequence label id fits in int")) + pub(crate) fn from_index(index: i32) -> Self { + Self(index) } pub(crate) fn is_jump_target_label(self) -> bool { @@ -469,7 +469,8 @@ impl InstructionSequenceLabel { } pub(crate) fn idx(self) -> usize { - usize::try_from(self.0).expect("instruction-sequence label id is non-negative") + debug_assert!(self.0 >= 0); + self.0 as usize } } @@ -517,7 +518,8 @@ pub(crate) struct InstructionSequence { instr_allocation: usize, /// CPython `instr_sequence.s_used`, the number of used entries in `s_instrs`. instr_used: usize, - next_free_label: usize, + /// CPython `instr_sequence.s_next_free_label`. + next_free_label: i32, label_map: Option>, label_map_allocation: usize, annotations_code: Option>, @@ -579,7 +581,7 @@ fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::Inter /// instruction_sequence.c _PyInstructionSequence_NewLabel fn instruction_sequence_new_label(seq: &mut InstructionSequence) -> InstructionSequenceLabel { seq.next_free_label += 1; - InstructionSequenceLabel::from_index(seq.next_free_label) + InstructionSequenceLabel(seq.next_free_label) } /// instruction_sequence.c _PyInstructionSequence_Addop asserts. @@ -5195,7 +5197,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> } mark_cold(blocks)?; - let mut next_label = (get_max_label(blocks) + 1) as usize; + let mut next_label = get_max_label(blocks) + 1; // If a cold block falls through to a warm block, add an explicit jump let mut block_idx = BlockIdx(0); @@ -7484,7 +7486,7 @@ mod tests { Block::default(), ]; for (i, block) in blocks.iter_mut().enumerate() { - block.cpython_label = InstructionSequenceLabel::from_index(i); + block.cpython_label = InstructionSequenceLabel::from_index(i as i32); } blocks[0].next = BlockIdx::new(1); blocks[1].next = BlockIdx::new(2); From 41e0e42798d92411bd1d33301b3eb8a1611fc1d5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:15:34 +0900 Subject: [PATCH 053/131] Match CPython jump offset arithmetic --- crates/codegen/src/ir.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index b2e9a083559..2093c916686 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -944,25 +944,19 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte .ok_or(InternalError::MalformedControlFlowGraph)?, ); if matches!(op, Instruction::EndAsyncFor) { - oparg = offset - .checked_sub(oparg + END_SEND_OFFSET) - .expect("END_ASYNC_FOR target must be before instruction") + oparg = offset - oparg - END_SEND_OFFSET; } else if oparg < offset { debug_assert!(matches!( op.into(), Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt )); - oparg = offset - .checked_sub(oparg) - .expect("backward jump target must be before instruction") + oparg = offset - oparg; } else { debug_assert!(!matches!( op.into(), Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt )); - oparg = oparg - .checked_sub(offset) - .expect("forward jump target must be after instruction") + oparg -= offset; } info.arg = OpArg::new( oparg From e66c76c24c03d3c895d4dcffe6226b33d7b480bb Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:18:45 +0900 Subject: [PATCH 054/131] Match CPython exception table arithmetic --- crates/codegen/src/ir.rs | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 2093c916686..ef191741ba7 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4955,7 +4955,7 @@ fn next_linetable_location() -> LineTableLocation { fn assemble_emit_exception_table_item(table: &mut Vec, value: i32, mut msb: u8) { debug_assert!((msb | 128) == 128); debug_assert!((0..(1 << 30)).contains(&value)); - let value = u32::try_from(value).expect("exception table item is non-negative"); + let value = value as u32; const CONTINUATION_BIT: u8 = 64; if value >= 1 << 24 { table.push(((value >> 24) as u8) | CONTINUATION_BIT | msb); @@ -4989,14 +4989,9 @@ fn assemble_emit_exception_table_entry( let size = end - start; debug_assert!(end > start); let target = handler_offset; - let mut depth = handler - .start_depth - .checked_sub(1) - .expect("exception handler start depth includes exception item"); + let mut depth = handler.start_depth - 1; if handler.preserve_lasti > 0 { - depth = depth - .checked_sub(1) - .expect("preserve_lasti handler start depth includes lasti"); + depth -= 1; } debug_assert!(depth >= 0); let depth_lasti = (depth << 1) | handler.preserve_lasti; @@ -5024,9 +5019,7 @@ fn assemble_exception_table( let instr = &instrs[i]; if instr.except_handler.h_label != handler.h_label { if handler.h_label >= 0 { - let handler_label = - usize::try_from(handler.h_label).expect("handler label is non-negative"); - let handler_offset = instrs[handler_label].i_offset; + let handler_offset = instrs[handler.h_label as usize].i_offset; assemble_emit_exception_table_entry( &mut table, start, @@ -5038,15 +5031,11 @@ fn assemble_exception_table( start = ioffset; handler = instr.except_handler; } - ioffset += instr_size(&instr.info) - .to_i32() - .expect("assembled instruction offset fits in i32"); + ioffset += instr_size(&instr.info) as i32; } if handler.h_label >= 0 { - let handler_label = - usize::try_from(handler.h_label).expect("handler label is non-negative"); - let handler_offset = instrs[handler_label].i_offset; + let handler_offset = instrs[handler.h_label as usize].i_offset; assemble_emit_exception_table_entry(&mut table, start, ioffset, handler_offset, handler)?; } From d1bf4dfb177792b5710d0b4dd380615a6708da40 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:20:53 +0900 Subject: [PATCH 055/131] Match CPython label index arithmetic --- crates/codegen/src/ir.rs | 64 +++++++++++----------------------------- 1 file changed, 17 insertions(+), 47 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index ef191741ba7..03afa1ba027 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -701,7 +701,7 @@ fn instruction_sequence_insert_instruction( } /// instruction_sequence.c _PyInstructionSequence_ApplyLabelMap -#[allow(clippy::needless_range_loop)] +#[allow(clippy::needless_range_loop, clippy::unnecessary_wraps)] fn instruction_sequence_apply_label_map( instrs: &mut InstructionSequence, ) -> crate::InternalResult<()> { @@ -712,27 +712,17 @@ fn instruction_sequence_apply_label_map( for i in 0..instrs.instr_used { let entry = &mut instrs.instrs[i]; if entry.info.instr.has_target() { - let label = usize::try_from(u32::from(entry.info.arg)) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - let target = *label_map - .get(label) - .ok_or(InternalError::MalformedControlFlowGraph)?; - if target < 0 { - return Err(InternalError::MalformedControlFlowGraph); - } - entry.info.arg = OpArg::new( - target - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + let label = u32::from(entry.info.arg) as usize; + debug_assert!(label < label_map.len()); + let target = label_map[label]; + debug_assert!(target >= 0); + entry.info.arg = OpArg::new(target as u32); } let handler = &mut entry.except_handler; if handler.h_label >= 0 { - let label = usize::try_from(handler.h_label) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - handler.h_label = *label_map - .get(label) - .ok_or(InternalError::MalformedControlFlowGraph)?; + let label = handler.h_label as usize; + debug_assert!(label < label_map.len()); + handler.h_label = label_map[label]; } } } @@ -5944,13 +5934,9 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResu let info = &mut blocks[block_idx.idx()].instructions[i]; debug_assert_eq!(info.target, BlockIdx::NULL); if info.instr.has_target() { - let lbl = i32::try_from(u32::from(info.arg)) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - if lbl < 0 || lbl > max_label { - return Err(InternalError::MalformedControlFlowGraph); - } - let target = label_to_block - [usize::try_from(lbl).map_err(|_| InternalError::MalformedControlFlowGraph)?]; + let lbl = u32::from(info.arg) as i32; + debug_assert!(lbl >= 0 && lbl <= max_label); + let target = label_to_block[lbl as usize]; debug_assert!(target != BlockIdx::NULL); debug_assert_eq!( blocks[target.idx()].cpython_label, @@ -5984,8 +5970,7 @@ fn cfg_from_instruction_sequence( } for i in 0..instr_sequence.instr_used { if instr_sequence.instrs[i].info.instr.has_target() { - let target_offset = usize::try_from(u32::from(instr_sequence.instrs[i].info.arg)) - .expect("instruction-sequence target index fits in usize"); + let target_offset = u32::from(instr_sequence.instrs[i].info.arg) as usize; debug_assert!(target_offset < instr_sequence.instr_used); instr_sequence.instrs[target_offset].i_target = 1; } @@ -6018,11 +6003,7 @@ fn cfg_from_instruction_sequence( info.target = BlockIdx::NULL; cfg_builder_addop(&mut builder, info)?; } - offset += annotations_code - .instr_used - .to_i32() - .ok_or(InternalError::MalformedControlFlowGraph)? - - 1; + offset += annotations_code.instr_used as i32 - 1; } else { offset -= 1; } @@ -6031,11 +6012,7 @@ fn cfg_from_instruction_sequence( } if entry.i_target != 0 { - let label_id = i - .to_i32() - .ok_or(InternalError::MalformedControlFlowGraph)? - .checked_add(offset) - .ok_or(InternalError::MalformedControlFlowGraph)?; + let label_id = i as i32 + offset; let label = InstructionSequenceLabel(label_id); cfg_builder_use_label(&mut builder, label)?; } @@ -6043,15 +6020,8 @@ fn cfg_from_instruction_sequence( let opcode = entry.info.instr; let mut oparg = entry.info.arg; if opcode.has_target() { - let target_offset = i32::try_from(u32::from(oparg)) - .map_err(|_| InternalError::MalformedControlFlowGraph)? - .checked_add(offset) - .ok_or(InternalError::MalformedControlFlowGraph)?; - oparg = OpArg::new( - target_offset - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + let target_offset = u32::from(oparg) as i32 + offset; + oparg = OpArg::new(target_offset as u32); } entry.info.instr = opcode; entry.info.arg = oparg; From 1599e471dfcf27f0692d593796cac4ff579e055e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:22:02 +0900 Subject: [PATCH 056/131] Match CPython instruction offset casts --- crates/codegen/src/ir.rs | 37 ++++++++++--------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 03afa1ba027..6f819ae9100 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -644,10 +644,7 @@ fn instruction_sequence_use_label( for i in old_len..label_map.len() { label_map[i] = INSTRUCTION_SEQUENCE_UNSET_LABEL; } - label_map[label.idx()] = seq - .instr_used - .to_i32() - .ok_or(InternalError::MalformedControlFlowGraph)?; + label_map[label.idx()] = seq.instr_used as i32; Ok(()) } @@ -688,9 +685,7 @@ fn instruction_sequence_insert_instruction( } seq.instrs[pos].info = info; if let Some(label_map) = &mut seq.label_map { - let pos = pos - .to_i32() - .ok_or(InternalError::MalformedControlFlowGraph)?; + let pos = pos as i32; for lbl in 0..label_map.len() { if label_map[lbl] >= pos { label_map[lbl] += 1; @@ -757,11 +752,7 @@ fn cfg_to_instruction_sequence( debug_assert!(target_block != BlockIdx::NULL); let lbl = blocks[target_block.idx()].cpython_label; debug_assert!(is_label(lbl)); - blocks[block_idx.idx()].instructions[i].arg = OpArg::new( - lbl.idx() - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + blocks[block_idx.idx()].instructions[i].arg = OpArg::new(lbl.0 as u32); } let mut info = blocks[block_idx.idx()].instructions[i]; @@ -775,8 +766,7 @@ fn cfg_to_instruction_sequence( debug_assert!(is_label(lbl)); let start_depth = blocks[handler.handler_block.idx()].start_depth; debug_assert!(start_depth >= 0); - hi.h_label = i32::try_from(lbl.idx()) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; + hi.h_label = lbl.0; hi.start_depth = start_depth; hi.preserve_lasti = i32::from(handler.preserve_lasti); } else { @@ -793,7 +783,7 @@ fn cfg_to_instruction_sequence( /// assemble.c instr_size fn instr_size(instr: &InstructionInfo) -> usize { let opcode = instr.instr.expect_real(); - let oparg = i32::try_from(u32::from(instr.arg)).expect("oparg fits in int"); + let oparg = u32::from(instr.arg) as i32; debug_assert!( instr.instr.has_arg() || oparg == 0, "CPython assemble.c instr_size requires OPCODE_HAS_ARG or oparg == 0" @@ -826,9 +816,7 @@ fn resolve_unconditional_jumps( ) -> crate::InternalResult<()> { for i in 0..instr_sequence.instr_used { let instr = &mut instr_sequence.instrs[i].info; - let is_forward = i32::try_from(u32::from(instr.arg)) - .map_err(|_| InternalError::MalformedControlFlowGraph)? - > i.to_i32().ok_or(InternalError::MalformedControlFlowGraph)?; + let is_forward = (u32::from(instr.arg) as i32) > i as i32; match instr.instr { AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { debug_assert!(is_pseudo_target(PseudoOpcode::Jump, Opcode::JumpForward)); @@ -887,8 +875,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let instr = &mut instr_sequence.instrs[i]; let opcode = instr.info.instr.expect_real(); if opcode.has_jump() { - instr.i_target = i32::try_from(u32::from(instr.info.arg)) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; + instr.i_target = u32::from(instr.info.arg) as i32; } } let mut extended_arg_recompile; @@ -898,9 +885,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let instr = &mut instr_sequence.instrs[i]; instr.i_offset = totsize; let isize = instr_size(&instr.info); - totsize += isize - .to_i32() - .ok_or(InternalError::MalformedControlFlowGraph)?; + totsize += isize as i32; } end_offset = totsize .to_u32() @@ -912,9 +897,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let isize = instr_size(&instr_sequence.instrs[i].info); // Jump offsets are computed relative to the instruction pointer // after fetching the jump instruction. - offset += isize - .to_i32() - .ok_or(InternalError::MalformedControlFlowGraph)?; + offset += isize as i32; let opcode = instr_sequence.instrs[i].info.instr.expect_real(); if opcode.has_jump() { @@ -1016,7 +999,7 @@ fn instruction_linetable_location(info: &InstructionInfo) -> LineTableLocation { /// assemble.c write_instr fn write_instr(instructions: &mut Vec, info: &InstructionInfo, ilen: usize) { let opcode = info.instr.expect_real(); - let oparg = i32::try_from(u32::from(info.arg)).expect("oparg fits in int"); + let oparg = u32::from(info.arg) as i32; debug_assert!( info.instr.has_arg() || oparg == 0, "CPython assemble.c write_instr requires OPCODE_HAS_ARG or oparg == 0" From 77add0dce506bac5cbf2503c1257a5f5b7215f53 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:22:52 +0900 Subject: [PATCH 057/131] Match CPython jump offset indexing --- crates/codegen/src/ir.rs | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 6f819ae9100..55f5020e7d0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -866,7 +866,7 @@ fn resolve_unconditional_jumps( } /// assemble.c resolve_jump_offsets -#[allow(clippy::needless_range_loop)] +#[allow(clippy::needless_range_loop, clippy::unnecessary_wraps)] fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::InternalResult { let mut end_offset; // The offset (in code units) of END_SEND from SEND in the yield-from sequence. @@ -887,9 +887,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let isize = instr_size(&instr.info); totsize += isize as i32; } - end_offset = totsize - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?; + end_offset = totsize as u32; extended_arg_recompile = false; let mut offset = 0i32; @@ -901,21 +899,12 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let opcode = instr_sequence.instrs[i].info.instr.expect_real(); if opcode.has_jump() { - let target_index = usize::try_from(instr_sequence.instrs[i].i_target) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - let target_offset = instr_sequence - .instrs - .get(target_index) - .ok_or(InternalError::MalformedControlFlowGraph)? - .i_offset; + let target = instr_sequence.instrs[i].i_target; + let target_offset = instr_sequence.instrs[target as usize].i_offset; let info = &mut instr_sequence.instrs[i].info; let op = opcode; let mut oparg = target_offset; - info.arg = OpArg::new( - oparg - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + info.arg = OpArg::new(oparg as u32); if matches!(op, Instruction::EndAsyncFor) { oparg = offset - oparg - END_SEND_OFFSET; } else if oparg < offset { @@ -931,11 +920,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte )); oparg -= offset; } - info.arg = OpArg::new( - oparg - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + info.arg = OpArg::new(oparg as u32); if instr_size(info) != isize { extended_arg_recompile = true; } From 3efaa6f60d77108aa1ac0e1f4829c34c182ebfff Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:23:44 +0900 Subject: [PATCH 058/131] Match CPython oparg locals casts --- crates/codegen/src/ir.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 55f5020e7d0..34447597b33 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3184,9 +3184,7 @@ fn is_swappable(instr: &AnyInstruction) -> bool { fn stores_to(info: &InstructionInfo) -> i32 { match info.instr.into() { AnyOpcode::Real(Opcode::StoreFast) - | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) => { - i32::try_from(u32::from(info.arg)).expect("STORE_FAST oparg fits in int") - } + | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) => u32::from(info.arg) as i32, _ => -1, } } @@ -6705,7 +6703,7 @@ pub(crate) fn fix_cell_offsets( !matches!(inst.instr.real(), Some(Instruction::ExtendedArg)), "fix_cell_offsets is called before extended args are generated" ); - let oldoffset = i32::try_from(u32::from(inst.arg)).expect("oparg fits in int"); + let oldoffset = u32::from(inst.arg) as i32; match inst.instr { AnyInstruction::Real( Instruction::MakeCell { .. } @@ -6716,9 +6714,7 @@ pub(crate) fn fix_cell_offsets( ) | AnyInstruction::Pseudo(PseudoInstruction::LoadClosure { .. }) => { debug_assert!(oldoffset >= 0); - debug_assert!( - oldoffset < noffsets.to_i32().expect("localsplus offset fits in int") - ); + debug_assert!(oldoffset < noffsets as i32); let oldoffset = usize::try_from(oldoffset).expect("localsplus offset is non-negative"); let fixed_offset = cellfixedoffsets[oldoffset]; From 1b252730d96e6a2833a7b28aa982127f03ff1df0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:25:29 +0900 Subject: [PATCH 059/131] Match CPython localsplus offset arithmetic --- crates/codegen/src/ir.rs | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 34447597b33..62379458ad8 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1137,8 +1137,7 @@ fn compute_localsplus_info( max = if count == usize::MAX { usize::MAX } else { - max.checked_add(count) - .expect("localsplus argument count fits in usize") + max + count }; while pos < max && pos < nlocals { let name = umd @@ -1177,15 +1176,13 @@ fn compute_localsplus_info( debug_assert!(offset < nlocalsplus); cellvars.push(name.to_owned()); localspluskinds[offset] = CO_FAST_CELL; - cellvar_offset = offset.to_i32().expect("localsplus cell offset fits in int"); + cellvar_offset = offset as i32; } for i in 0..nfrees { let offset = ncells + i + nlocals - numdropped; debug_assert!(offset < nlocalsplus); - debug_assert!( - offset.to_i32().expect("localsplus free offset fits in int") > cellvar_offset - ); + debug_assert!((offset as i32) > cellvar_offset); localspluskinds[offset] = CO_FAST_FREE; } @@ -6650,9 +6647,7 @@ pub(crate) fn build_cellfixedoffsets( vec_try_reserve_exact(&mut fixed, noffsets)?; fixed.resize(noffsets, 0); for i in 0..noffsets { - fixed[i] = (nlocals + i) - .to_i32() - .expect("localsplus offset fits in int"); + fixed[i] = (nlocals + i) as i32; } for oldindex in 0..ncellvars { let varname = metadata @@ -6660,7 +6655,7 @@ pub(crate) fn build_cellfixedoffsets( .get_index(oldindex) .expect("cellvar index is in range"); if let Some(varindex) = metadata.varnames.get_index_of(varname) { - let argoffset = varindex.to_i32().expect("localsplus offset fits in int"); + let argoffset = varindex as i32; fixed[oldindex] = argoffset; } } @@ -6682,12 +6677,8 @@ pub(crate) fn fix_cell_offsets( let mut numdropped = 0usize; for i in 0..noffsets { - if cellfixedoffsets[i] - == (i + nlocals) - .to_i32() - .expect("localsplus index fits in int") - { - cellfixedoffsets[i] -= numdropped.to_i32().expect("too many dropped cell vars"); + if cellfixedoffsets[i] == (i + nlocals) as i32 { + cellfixedoffsets[i] -= numdropped as i32; } else { numdropped += 1; } @@ -6715,15 +6706,9 @@ pub(crate) fn fix_cell_offsets( | AnyInstruction::Pseudo(PseudoInstruction::LoadClosure { .. }) => { debug_assert!(oldoffset >= 0); debug_assert!(oldoffset < noffsets as i32); - let oldoffset = - usize::try_from(oldoffset).expect("localsplus offset is non-negative"); - let fixed_offset = cellfixedoffsets[oldoffset]; + let fixed_offset = cellfixedoffsets[oldoffset as usize]; debug_assert!(fixed_offset >= 0); - inst.arg = OpArg::new( - fixed_offset - .to_u32() - .expect("localsplus offset is non-negative"), - ); + inst.arg = OpArg::new(fixed_offset as u32); } _ => {} } From c074ef4eae85d206367126b4ba73ffe82440647b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:26:12 +0900 Subject: [PATCH 060/131] Match CPython cell prefix indexing --- crates/codegen/src/ir.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 62379458ad8..b44b4699d0d 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1907,8 +1907,7 @@ fn insert_prefix_instructions( vec_try_reserve_exact(&mut sorted, nvars)?; sorted.resize(nvars, 0i32); for i in 0..ncellvars { - sorted[usize::try_from(cellfixedoffsets[i]) - .expect("localsplus offset is non-negative")] = i as i32 + 1; + sorted[cellfixedoffsets[i] as usize] = i as i32 + 1; } let mut ncellsused = 0; let mut i = 0; @@ -1923,7 +1922,7 @@ fn insert_prefix_instructions( ncellsused, InstructionInfo { instr: Instruction::MakeCell { i: Arg::marker() }.into(), - arg: OpArg::new(oldindex.to_u32().expect("cell index is non-negative")), + arg: OpArg::new(oldindex as u32), target: BlockIdx::NULL, location: SourceLocation::default(), end_location: SourceLocation::default(), From 4274e1bdd8ef5398f1c8d6799e1b10a335d6fd73 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:27:31 +0900 Subject: [PATCH 061/131] Match CPython C array growth arithmetic --- crates/codegen/src/ir.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index b44b4699d0d..76dcc2a3322 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -312,6 +312,7 @@ fn empty_instruction_info() -> InstructionInfo { } /// codegen.c _Py_CArray_EnsureCapacity +#[allow(clippy::unnecessary_wraps)] fn c_array_ensure_capacity( allocated_entries: usize, idx: usize, @@ -319,19 +320,15 @@ fn c_array_ensure_capacity( ) -> crate::InternalResult { if allocated_entries == 0 { let new_alloc = if idx >= initial_num_entries { - idx.checked_add(initial_num_entries) - .ok_or(InternalError::MalformedControlFlowGraph)? + idx + initial_num_entries } else { initial_num_entries }; Ok(new_alloc) } else if idx >= allocated_entries { - let doubled = allocated_entries - .checked_mul(2) - .ok_or(InternalError::MalformedControlFlowGraph)?; + let doubled = allocated_entries << 1; let new_alloc = if idx >= doubled { - idx.checked_add(initial_num_entries) - .ok_or(InternalError::MalformedControlFlowGraph)? + idx + initial_num_entries } else { doubled }; From a8be3a076e84f9add0dbb96d33789535a008f4e3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:29:08 +0900 Subject: [PATCH 062/131] Match CPython label map allocation arithmetic --- crates/codegen/src/ir.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 76dcc2a3322..5f79512d22c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5864,13 +5864,8 @@ fn cfg_builder_check_size(g: &CfgBuilder) -> crate::InternalResult<()> { /// flowgraph.c translate_jump_labels_to_targets fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResult<()> { let max_label = get_max_label(blocks); - let label_count = max_label - .checked_add(1) - .and_then(|count| count.to_usize()) - .ok_or(InternalError::MalformedControlFlowGraph)?; - label_count - .checked_mul(core::mem::size_of::()) - .ok_or(InternalError::MalformedControlFlowGraph)?; + let label_count = (max_label + 1) as usize; + let _mapsize = core::mem::size_of::() * label_count; let mut label_to_block = Vec::new(); vec_try_reserve_exact(&mut label_to_block, label_count)?; label_to_block.resize(label_count, BlockIdx::NULL); From a301c3aa73be7f06a89f133244f7e6ab252fdd03 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:30:19 +0900 Subject: [PATCH 063/131] Match CPython label map size tracking --- crates/codegen/src/ir.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5f79512d22c..da351c3cce2 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -609,7 +609,7 @@ fn instruction_sequence_use_label( seq: &mut InstructionSequence, label: InstructionSequenceLabel, ) -> crate::InternalResult<()> { - let old_len = seq.label_map.as_ref().map_or(0, Vec::len); + let old_size = seq.label_map_allocation; let new_allocation = c_array_ensure_capacity( seq.label_map_allocation, label.idx(), @@ -638,7 +638,7 @@ fn instruction_sequence_use_label( if label_map.len() < seq.label_map_allocation { label_map.resize(seq.label_map_allocation, INSTRUCTION_SEQUENCE_UNSET_LABEL); } - for i in old_len..label_map.len() { + for i in old_size..seq.label_map_allocation { label_map[i] = INSTRUCTION_SEQUENCE_UNSET_LABEL; } label_map[label.idx()] = seq.instr_used as i32; From a3a8a9f6178ba1e8d280c49b6fd2e1cb0638b36e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:31:08 +0900 Subject: [PATCH 064/131] Use CPython label map size in sequence passes --- crates/codegen/src/ir.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index da351c3cce2..6e7264c0d6e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -683,7 +683,7 @@ fn instruction_sequence_insert_instruction( seq.instrs[pos].info = info; if let Some(label_map) = &mut seq.label_map { let pos = pos as i32; - for lbl in 0..label_map.len() { + for lbl in 0..seq.label_map_allocation { if label_map[lbl] >= pos { label_map[lbl] += 1; } @@ -705,7 +705,7 @@ fn instruction_sequence_apply_label_map( let entry = &mut instrs.instrs[i]; if entry.info.instr.has_target() { let label = u32::from(entry.info.arg) as usize; - debug_assert!(label < label_map.len()); + debug_assert!(label < instrs.label_map_allocation); let target = label_map[label]; debug_assert!(target >= 0); entry.info.arg = OpArg::new(target as u32); @@ -713,7 +713,7 @@ fn instruction_sequence_apply_label_map( let handler = &mut entry.except_handler; if handler.h_label >= 0 { let label = handler.h_label as usize; - debug_assert!(label < label_map.len()); + debug_assert!(label < instrs.label_map_allocation); handler.h_label = label_map[label]; } } From 2f976437c677021287f8a12e31eef0106e852e2d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:32:33 +0900 Subject: [PATCH 065/131] Assert CPython label map clearing invariants --- crates/codegen/src/ir.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 6e7264c0d6e..18f3959f625 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5933,10 +5933,12 @@ fn cfg_from_instruction_sequence( instrs, instr_used, label_map, + label_map_allocation, annotations_code, .. } = instr_sequence; debug_assert!(label_map.is_none()); + debug_assert_eq!(label_map_allocation, 0); let mut offset = 0i32; @@ -5949,6 +5951,7 @@ fn cfg_from_instruction_sequence( ) { if let Some(annotations_code) = &annotations_code { debug_assert!(annotations_code.label_map.is_none()); + debug_assert_eq!(annotations_code.label_map_allocation, 0); debug_assert!(annotations_code.annotations_code.is_none()); for j in 0..annotations_code.instr_used { let ann_entry = annotations_code.instrs[j]; From 518e447288b0809b39fc2fb8ef8ff065076a9e44 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:35:20 +0900 Subject: [PATCH 066/131] Match CPython label oparg assignment --- crates/codegen/src/ir.rs | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 18f3959f625..e2d68ce40eb 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1366,16 +1366,9 @@ fn instruction_sequence_label_map_resolve_label_to_block( }) } -fn instruction_sequence_label_oparg( - label: InstructionSequenceLabel, -) -> crate::InternalResult { +fn instruction_sequence_label_oparg(label: InstructionSequenceLabel) -> OpArg { debug_assert!(is_label(label)); - Ok(OpArg::new( - label - .idx() - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - )) + OpArg::new(label.idx() as u32) } fn instruction_sequence_label_map_use_label_at_block( @@ -1487,12 +1480,7 @@ impl CodeInfo { &mut self.instr_sequence, info.target, )?; - info.arg = OpArg::new( - label - .idx() - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + info.arg = instruction_sequence_label_oparg(label); info.target = BlockIdx::NULL; } instruction_sequence_addop(&mut self.instr_sequence, info)?; @@ -1507,12 +1495,7 @@ impl CodeInfo { if !info.instr.has_target() { return Err(InternalError::MalformedControlFlowGraph); } - info.arg = OpArg::new( - target_label - .idx() - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); + info.arg = instruction_sequence_label_oparg(target_label); info.target = BlockIdx::NULL; instruction_sequence_addop(&mut self.instr_sequence, info)?; Ok(()) @@ -1624,7 +1607,7 @@ impl CodeInfo { delta: Arg::marker(), } .into(), - arg: instruction_sequence_label_oparg(handler_label)?, + arg: instruction_sequence_label_oparg(handler_label), target: BlockIdx::NULL, location: SourceLocation::default(), end_location: SourceLocation::default(), @@ -5151,7 +5134,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> &mut blocks[explicit_jump.idx()], InstructionInfo { instr: PseudoOpcode::JumpNoInterrupt.into(), - arg: instruction_sequence_label_oparg(jump_label)?, + arg: instruction_sequence_label_oparg(jump_label), target: BlockIdx::NULL, location: SourceLocation::default(), end_location: SourceLocation::default(), @@ -5279,7 +5262,7 @@ fn basicblock_add_jump( debug_assert!(target != BlockIdx::NULL); let label = blocks[target.idx()].cpython_label; debug_assert!(is_label(label)); - let arg = instruction_sequence_label_oparg(label)?; + let arg = instruction_sequence_label_oparg(label); let block = &mut blocks[bi]; basicblock_addop( block, From a63f61fd6ec5d7dfa1f1e7fdaf02c10b09ed5860 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:37:15 +0900 Subject: [PATCH 067/131] Match CPython compiler direct arithmetic --- crates/codegen/src/ir.rs | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e2d68ce40eb..361e4cd1b3e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1077,8 +1077,7 @@ fn assemble_emit( first_line: i32, debug_ranges: bool, ) -> crate::InternalResult { - let num_instructions = - usize::try_from(end_offset).map_err(|_| InternalError::MalformedControlFlowGraph)?; + let num_instructions = end_offset as usize; let mut instructions = Vec::new(); vec_try_reserve_exact(&mut instructions, num_instructions)?; @@ -1945,17 +1944,8 @@ fn prepare_localsplus( debug_assert!(nlocals < int_max); debug_assert!(ncellvars < int_max); debug_assert!(nfreevars < int_max); - debug_assert!( - nlocals - .checked_add(ncellvars) - .is_some_and(|sum| sum < int_max) - ); - debug_assert!( - nlocals - .checked_add(ncellvars) - .and_then(|sum| sum.checked_add(nfreevars)) - .is_some_and(|sum| sum < int_max) - ); + debug_assert!(int_max - nlocals - ncellvars > 0); + debug_assert!(int_max - nlocals - ncellvars - nfreevars > 0); let mut nlocalsplus = nlocals + ncellvars + nfreevars; let mut cellfixedoffsets = build_cellfixedoffsets(metadata)?; @@ -3973,8 +3963,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let instr_count = blocks[block_i].instruction_used; instr_flags[..instr_count].fill(0); debug_assert!(blocks[block_i].start_depth >= 0); - let start_depth = usize::try_from(blocks[block_i].start_depth) - .expect("visited block has non-negative start depth"); + let start_depth = blocks[block_i].start_depth as usize; ref_stack_clear(&mut refs); for _ in 0..start_depth { push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL)?; @@ -4553,10 +4542,7 @@ type RefStack = Vec; /// flowgraph.c ref_stack_push fn ref_stack_push(stack: &mut RefStack, r: Ref) -> crate::InternalResult<()> { if stack.len() == stack.capacity() { - let doubled = stack - .capacity() - .checked_mul(2) - .ok_or(InternalError::MalformedControlFlowGraph)?; + let doubled = stack.capacity() * 2; let new_cap = 32.max(doubled); stack .try_reserve_exact(new_cap - stack.capacity()) From a8d4d15c504c5d88830505b9c09f4339c11eb70e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:38:24 +0900 Subject: [PATCH 068/131] Match CPython load fast local casts --- crates/codegen/src/ir.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 361e4cd1b3e..cbd820d5bd0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4602,14 +4602,11 @@ fn store_local(instr_flags: &mut [u8], refs: &RefStack, local: isize, r: Ref) { fn decode_packed_fast_locals(arg: OpArg) -> (isize, isize) { let packed = u32::from(arg); - ( - isize::try_from((packed >> 4) & 0xF).expect("local index fits in isize"), - isize::try_from(packed & 0xF).expect("local index fits in isize"), - ) + (((packed >> 4) & 0xF) as isize, (packed & 0xF) as isize) } fn local_as_ref_local(local: usize) -> isize { - isize::try_from(local).expect("local index fits in isize") + local as isize } /// flowgraph.c load_fast_push_block From c38475aa5a1b506f3759ab32dbd7c77df8032f16 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:39:12 +0900 Subject: [PATCH 069/131] Match CPython load fast depth assert --- crates/codegen/src/ir.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index cbd820d5bd0..e1f224cce44 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4617,10 +4617,8 @@ fn load_fast_push_block( start_depth: usize, ) { debug_assert!(target != BlockIdx::NULL); - debug_assert_eq!( - usize::try_from(blocks[target.idx()].start_depth).ok(), - Some(start_depth), - ); + debug_assert!(blocks[target.idx()].start_depth >= 0); + debug_assert_eq!(blocks[target.idx()].start_depth as usize, start_depth,); if !blocks[target.idx()].visited { blocks[target.idx()].visited = true; worklist.push(target); From 8b3f726af424355cc96859960806a15f5c1cc214 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:42:26 +0900 Subject: [PATCH 070/131] Match CPython resume depth flagging --- crates/codegen/src/ir.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e1f224cce44..102947cda81 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6513,14 +6513,14 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe } else if matches!(instr.real(), Some(Instruction::YieldValue { .. })) { blocks[bi].instructions[i].except_handler = handler; last_yield_except_depth = stack.len() as i32; - } else if let Some(Instruction::Resume { context }) = instr.real() { + } else if let Some(Instruction::Resume { context: _ }) = instr.real() { blocks[bi].instructions[i].except_handler = handler; - let location = context.get(arg).location(); - if !matches!(location, oparg::ResumeLocation::AtFuncStart) { + let resume_arg = u32::from(arg); + if resume_arg != u32::from(oparg::ResumeLocation::AtFuncStart) { debug_assert!(last_yield_except_depth >= 0); if last_yield_except_depth == 1 { blocks[bi].instructions[i].arg = - OpArg::new(oparg::ResumeContext::new(location, true).as_u32()); + OpArg::new(resume_arg | oparg::ResumeContext::DEPTH1_MASK); } last_yield_except_depth = -1; } From f021dfc9416facad0458689fc9fe943f13cc28a6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:43:34 +0900 Subject: [PATCH 071/131] Match CPython stack depth arithmetic --- crates/codegen/src/ir.rs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 102947cda81..47f88b17562 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4212,13 +4212,7 @@ fn calculate_stackdepth(blocks: &mut [Block]) -> crate::InternalResult { let ins = blocks[idx].instructions[i]; let instr = &ins.instr; let effects = get_stack_effects(*instr, ins.arg, 0)?; - let new_depth = depth.checked_add(effects.net).ok_or({ - if effects.net < 0 { - InternalError::StackUnderflow - } else { - InternalError::StackOverflow - } - })?; + let new_depth = depth + effects.net; if new_depth < 0 { return Err(InternalError::StackUnderflow); } @@ -4226,13 +4220,7 @@ fn calculate_stackdepth(blocks: &mut [Block]) -> crate::InternalResult { if instr.has_target() && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) { debug_assert!(ins.target != BlockIdx::NULL); let effects = get_stack_effects(*instr, ins.arg, 1)?; - let target_depth = depth.checked_add(effects.net).ok_or({ - if effects.net < 0 { - InternalError::StackUnderflow - } else { - InternalError::StackOverflow - } - })?; + let target_depth = depth + effects.net; debug_assert!(target_depth >= 0); maxdepth = maxdepth.max(depth); stackdepth_push(&mut stack, blocks, ins.target, target_depth)?; From 6ab160eab044cdc79082b441cd86b47febc36715 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:46:03 +0900 Subject: [PATCH 072/131] Drop Rust-only stack overflow error --- crates/codegen/src/error.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/codegen/src/error.rs b/crates/codegen/src/error.rs index 5e66d6cf1b8..744f1d1dce1 100644 --- a/crates/codegen/src/error.rs +++ b/crates/codegen/src/error.rs @@ -38,7 +38,6 @@ impl fmt::Display for CodegenError { #[derive(Debug)] #[non_exhaustive] pub enum InternalError { - StackOverflow, StackUnderflow, InconsistentStackDepth, MalformedControlFlowGraph, @@ -48,7 +47,6 @@ pub enum InternalError { impl Display for InternalError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::StackOverflow => write!(f, "stack overflow"), Self::StackUnderflow => write!(f, "stack underflow"), Self::InconsistentStackDepth => write!(f, "inconsistent stack depth"), Self::MalformedControlFlowGraph => write!(f, "malformed control flow graph."), From 72ee5d8d032f82fb2100a619b705c00dca63b1cd Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:46:39 +0900 Subject: [PATCH 073/131] Return CPython stack depth directly --- crates/codegen/src/ir.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 47f88b17562..66e333d630f 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4239,7 +4239,7 @@ fn calculate_stackdepth(blocks: &mut [Block]) -> crate::InternalResult { } let stackdepth = maxdepth; - u32::try_from(stackdepth).map_err(|_| InternalError::StackUnderflow) + Ok(stackdepth as u32) } #[cfg(test)] From 0afe20211d1cdd77135b8c87e1757ffae78de2c3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:51:26 +0900 Subject: [PATCH 074/131] Match CPython C array growth errors --- crates/codegen/src/ir.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 66e333d630f..741ca2f371c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -312,7 +312,6 @@ fn empty_instruction_info() -> InstructionInfo { } /// codegen.c _Py_CArray_EnsureCapacity -#[allow(clippy::unnecessary_wraps)] fn c_array_ensure_capacity( allocated_entries: usize, idx: usize, @@ -320,15 +319,19 @@ fn c_array_ensure_capacity( ) -> crate::InternalResult { if allocated_entries == 0 { let new_alloc = if idx >= initial_num_entries { - idx + initial_num_entries + idx.checked_add(initial_num_entries) + .ok_or(InternalError::MalformedControlFlowGraph)? } else { initial_num_entries }; Ok(new_alloc) } else if idx >= allocated_entries { - let doubled = allocated_entries << 1; + let doubled = allocated_entries + .checked_mul(2) + .ok_or(InternalError::MalformedControlFlowGraph)?; let new_alloc = if idx >= doubled { - idx + initial_num_entries + idx.checked_add(initial_num_entries) + .ok_or(InternalError::MalformedControlFlowGraph)? } else { doubled }; @@ -5625,7 +5628,8 @@ fn no_redundant_jumps(blocks: &[Block]) -> bool { if instruction_lineno(last) == instruction_lineno(&blocks[next.idx()].instructions[0]) { - panic!("redundant jump has same line as fallthrough target"); + debug_assert!(false, "redundant jump has same line as fallthrough target"); + return false; } } } From 52e06c8221e78307d1c84ce737fbc58a9e14545b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:53:56 +0900 Subject: [PATCH 075/131] Match CPython instruction insert asserts --- crates/codegen/src/ir.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 741ca2f371c..a947802a5e2 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -678,7 +678,6 @@ fn instruction_sequence_insert_instruction( info: InstructionInfo, ) -> crate::InternalResult<()> { debug_assert!(pos <= seq.instr_used); - instruction_sequence_debug_check_addop(&info); let last_idx = instruction_sequence_next_inst(seq)?; for i in (pos..last_idx).rev() { seq.instrs[i + 1] = seq.instrs[i]; From 02d5c057853986bac94de7072d3cc3f21b305286 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 13:57:11 +0900 Subject: [PATCH 076/131] Match CPython unreachable pseudo jump --- crates/codegen/src/ir.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a947802a5e2..4f1ec617069 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -854,10 +854,9 @@ fn resolve_unconditional_jumps( } } _ => { - debug_assert!( - !(instr.instr.has_jump() && matches!(instr.instr, AnyInstruction::Pseudo(_))), - "CPython resolve_unconditional_jumps expects no remaining pseudo jump" - ); + if instr.instr.has_jump() && matches!(instr.instr, AnyInstruction::Pseudo(_)) { + unreachable!("remaining pseudo jump in resolve_unconditional_jumps"); + } } } } From b2a58670df053bc9b8925e7e1c1a3a24c391c064 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:00:22 +0900 Subject: [PATCH 077/131] Match CPython CFG size guard --- crates/codegen/src/ir.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4f1ec617069..c95d2f87929 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5809,9 +5809,9 @@ fn cfg_builder_check_size(g: &CfgBuilder) -> crate::InternalResult<()> { block = g.blocks[block.idx()].allocation_next; } debug_assert_eq!(nblocks, g.blocks.len()); - nblocks - .checked_mul(core::mem::size_of::()) - .ok_or(InternalError::MalformedControlFlowGraph)?; + if nblocks > usize::MAX / core::mem::size_of::() { + return Err(InternalError::MalformedControlFlowGraph); + } Ok(()) } From f86c378da3d093000fc51a96d9f0116cd2383e1b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:02:38 +0900 Subject: [PATCH 078/131] Match CPython superinstruction assert --- crates/codegen/src/ir.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index c95d2f87929..17f8e953c65 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4503,10 +4503,7 @@ fn insert_superinstructions(blocks: &mut [Block]) -> usize { } let res = remove_redundant_nops(blocks); #[cfg(debug_assertions)] - { - let no_redundant = no_redundant_nops(blocks); - debug_assert!(no_redundant); - } + assert!(no_redundant_nops(blocks)); res } From aa2be0311bc0c0a5c0200f3cb995f3d7646edf7c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:04:01 +0900 Subject: [PATCH 079/131] Match CPython redundant jump assert --- crates/codegen/src/ir.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 17f8e953c65..70490625cc9 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5623,7 +5623,11 @@ fn no_redundant_jumps(blocks: &[Block]) -> bool { if instruction_lineno(last) == instruction_lineno(&blocks[next.idx()].instructions[0]) { - debug_assert!(false, "redundant jump has same line as fallthrough target"); + assert_ne!( + instruction_lineno(last), + instruction_lineno(&blocks[next.idx()].instructions[0]), + "redundant jump has same line as fallthrough target" + ); return false; } } From 6cea997a3f6f8692878efccd5a21b080f7494869 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:05:02 +0900 Subject: [PATCH 080/131] Match CPython stackdepth errors --- crates/codegen/src/error.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/error.rs b/crates/codegen/src/error.rs index 744f1d1dce1..4d21bb001e0 100644 --- a/crates/codegen/src/error.rs +++ b/crates/codegen/src/error.rs @@ -47,8 +47,8 @@ pub enum InternalError { impl Display for InternalError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::StackUnderflow => write!(f, "stack underflow"), - Self::InconsistentStackDepth => write!(f, "inconsistent stack depth"), + Self::StackUnderflow => write!(f, "Invalid CFG, stack underflow"), + Self::InconsistentStackDepth => write!(f, "Invalid CFG, inconsistent stackdepth"), Self::MalformedControlFlowGraph => write!(f, "malformed control flow graph."), Self::MissingSymbol(s) => write!( f, From 527481e1e4d663000b87268b72637882bcade00d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:08:42 +0900 Subject: [PATCH 081/131] Match CPython jump offset flow --- crates/codegen/src/ir.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 70490625cc9..6bf05e2f2d5 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -865,8 +865,7 @@ fn resolve_unconditional_jumps( /// assemble.c resolve_jump_offsets #[allow(clippy::needless_range_loop, clippy::unnecessary_wraps)] -fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::InternalResult { - let mut end_offset; +fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::InternalResult<()> { // The offset (in code units) of END_SEND from SEND in the yield-from sequence. const END_SEND_OFFSET: i32 = 5; for i in 0..instr_sequence.instr_used { @@ -885,8 +884,6 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte let isize = instr_size(&instr.info); totsize += isize as i32; } - end_offset = totsize as u32; - extended_arg_recompile = false; let mut offset = 0i32; for i in 0..instr_sequence.instr_used { @@ -930,7 +927,7 @@ fn resolve_jump_offsets(instr_sequence: &mut InstructionSequence) -> crate::Inte } } - Ok(end_offset) + Ok(()) } struct AssembledCode { @@ -1074,13 +1071,15 @@ fn assemble_location_info( /// assemble.c assemble_emit fn assemble_emit( instr_sequence: &mut InstructionSequence, - end_offset: u32, first_line: i32, debug_ranges: bool, ) -> crate::InternalResult { - let num_instructions = end_offset as usize; + const DEFAULT_CODE_SIZE: usize = 128; let mut instructions = Vec::new(); - vec_try_reserve_exact(&mut instructions, num_instructions)?; + vec_try_reserve_exact( + &mut instructions, + DEFAULT_CODE_SIZE / core::mem::size_of::(), + )?; for i in 0..instr_sequence.instr_used { let instr = &mut instr_sequence.instrs[i].info; @@ -1791,10 +1790,9 @@ impl CodeInfo { } = metadata; resolve_unconditional_jumps(&mut instr_sequence)?; - let end_offset = resolve_jump_offsets(&mut instr_sequence)?; + resolve_jump_offsets(&mut instr_sequence)?; let assembled = assemble_emit( &mut instr_sequence, - end_offset, first_line_number.get() as i32, opts.debug_ranges, )?; From 616ae7d9829c31f3b8c5e1b9797c93fe5e48a396 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:10:50 +0900 Subject: [PATCH 082/131] Match CPython assembler buffer defaults --- crates/codegen/src/ir.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 6bf05e2f2d5..80c6bb59139 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -39,6 +39,9 @@ pub(crate) const NO_LOCATION_OVERRIDE: i32 = -1; const MAX_INT_SIZE: u64 = 128; const MAX_COLLECTION_SIZE: usize = 256; +const DEFAULT_CODE_SIZE: usize = 128; +const DEFAULT_LNOTAB_SIZE: usize = 16; +const DEFAULT_CNOTAB_SIZE: usize = 32; const DEFAULT_BLOCK_SIZE: usize = 16; const INITIAL_INSTR_SEQUENCE_SIZE: usize = 100; const INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE: usize = 10; @@ -1051,6 +1054,7 @@ fn assemble_location_info( } let mut linetable = Vec::new(); + vec_try_reserve_exact(&mut linetable, DEFAULT_CNOTAB_SIZE)?; let mut prev_line = first_line; let mut loc = no_linetable_location(); let mut size = 0; @@ -1074,7 +1078,6 @@ fn assemble_emit( first_line: i32, debug_ranges: bool, ) -> crate::InternalResult { - const DEFAULT_CODE_SIZE: usize = 128; let mut instructions = Vec::new(); vec_try_reserve_exact( &mut instructions, @@ -4904,6 +4907,7 @@ fn assemble_exception_table( instrs: &[InstructionSequenceEntry], ) -> crate::InternalResult> { let mut table = Vec::new(); + vec_try_reserve_exact(&mut table, DEFAULT_LNOTAB_SIZE)?; let mut handler = InstructionSequenceExceptHandlerInfo { h_label: NO_EXCEPTION_HANDLER_LABEL, start_depth: -1, From 06ce5fed6124ec67a7b97d5575fda4e676101697 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:14:01 +0900 Subject: [PATCH 083/131] Match CPython bytecode emit growth --- crates/codegen/src/ir.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 80c6bb59139..09104a02829 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1018,9 +1018,31 @@ fn write_instr(instructions: &mut Vec, info: &InstructionInfo, ilen: u } /// assemble.c assemble_emit_instr -fn assemble_emit_instr(instructions: &mut Vec, info: &mut InstructionInfo) { +fn assemble_emit_instr( + instructions: &mut Vec, + info: &mut InstructionInfo, +) -> crate::InternalResult<()> { let size = instr_size(info); + let required = instructions + .len() + .checked_add(size) + .ok_or(InternalError::MalformedControlFlowGraph)?; + if required >= instructions.capacity() { + let additional = instructions + .capacity() + .checked_mul(core::mem::size_of::()) + .and_then(|len| { + if len > usize::MAX / 2 { + None + } else { + Some(instructions.capacity()) + } + }) + .ok_or(InternalError::MalformedControlFlowGraph)?; + vec_try_reserve_exact(instructions, additional)?; + } write_instr(instructions, info, size); + Ok(()) } /// assemble.c assemble_location_info @@ -1086,7 +1108,7 @@ fn assemble_emit( for i in 0..instr_sequence.instr_used { let instr = &mut instr_sequence.instrs[i].info; - assemble_emit_instr(&mut instructions, instr); + assemble_emit_instr(&mut instructions, instr)?; } let linetable = assemble_location_info(instr_sequence, first_line, debug_ranges)?; From ed3c754621182884df5f9deb141a4e781970af88 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:17:30 +0900 Subject: [PATCH 084/131] Match CPython assembler entry growth --- crates/codegen/src/ir.rs | 81 ++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 09104a02829..5e4a7649719 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1028,18 +1028,7 @@ fn assemble_emit_instr( .checked_add(size) .ok_or(InternalError::MalformedControlFlowGraph)?; if required >= instructions.capacity() { - let additional = instructions - .capacity() - .checked_mul(core::mem::size_of::()) - .and_then(|len| { - if len > usize::MAX / 2 { - None - } else { - Some(instructions.capacity()) - } - }) - .ok_or(InternalError::MalformedControlFlowGraph)?; - vec_try_reserve_exact(instructions, additional)?; + vec_try_resize_to_double_capacity(instructions)?; } write_instr(instructions, info, size); Ok(()) @@ -4681,6 +4670,19 @@ fn vec_try_reserve_exact(vec: &mut Vec, additional: usize) -> crate::Inter .map_err(|_| InternalError::MalformedControlFlowGraph) } +fn vec_try_resize_to_double_capacity(vec: &mut Vec) -> crate::InternalResult<()> { + let capacity = vec.capacity(); + debug_assert!(capacity > 0); + if capacity == 0 || capacity > usize::MAX / 2 { + return Err(InternalError::MalformedControlFlowGraph); + } + let new_capacity = capacity * 2; + let additional = new_capacity + .checked_sub(vec.len()) + .ok_or(InternalError::MalformedControlFlowGraph)?; + vec_try_reserve_exact(vec, additional) +} + /// assemble.c write_location_first_byte fn write_location_first_byte(linetable: &mut Vec, code: u8, length: usize) { linetable.extend(write_location_entry_start(code, length)); @@ -4714,12 +4716,11 @@ fn write_location_info_short_form( length: usize, column: i32, end_column: i32, -) -> crate::InternalResult<()> { +) { debug_assert!(length > 0 && length <= 8); debug_assert!(column < 80); debug_assert!(end_column >= column); debug_assert!(end_column - column < 16); - vec_try_reserve_exact(linetable, 2)?; let column_low_bits = column & 7; let column_group = column >> 3; let code = PyCodeLocationInfoKind::Short0 as u8 + column_group as u8; @@ -4728,7 +4729,6 @@ fn write_location_info_short_form( linetable, ((column_low_bits as u8) << 4) | ((end_column - column) as u8), ); - Ok(()) } /// assemble.c write_location_info_oneline_form @@ -4738,17 +4738,15 @@ fn write_location_info_oneline_form( line_delta: i32, column: i32, end_column: i32, -) -> crate::InternalResult<()> { +) { debug_assert!(length > 0 && length <= 8); debug_assert!((0..3).contains(&line_delta)); debug_assert!(column < 128); debug_assert!(end_column < 128); - vec_try_reserve_exact(linetable, 3)?; let code = PyCodeLocationInfoKind::OneLine0 as u8 + line_delta as u8; write_location_first_byte(linetable, code, length); write_location_byte(linetable, column as u8); write_location_byte(linetable, end_column as u8); - Ok(()) } /// assemble.c write_location_info_long_form @@ -4757,9 +4755,8 @@ fn write_location_info_long_form( loc: LineTableLocation, length: usize, line_delta: i32, -) -> crate::InternalResult<()> { +) { debug_assert!(length > 0 && length <= 8); - vec_try_reserve_exact(linetable, 25)?; write_location_first_byte(linetable, PyCodeLocationInfoKind::Long as u8, length); write_location_signed_varint(linetable, line_delta); debug_assert!(loc.end_line >= loc.line); @@ -4776,26 +4773,17 @@ fn write_location_info_long_form( (loc.end_col as u32) + 1 }, ); - Ok(()) } /// assemble.c write_location_info_none -fn write_location_info_none(linetable: &mut Vec, length: usize) -> crate::InternalResult<()> { - vec_try_reserve_exact(linetable, 1)?; +fn write_location_info_none(linetable: &mut Vec, length: usize) { write_location_first_byte(linetable, PyCodeLocationInfoKind::None as u8, length); - Ok(()) } /// assemble.c write_location_info_no_column -fn write_location_info_no_column( - linetable: &mut Vec, - length: usize, - line_delta: i32, -) -> crate::InternalResult<()> { - vec_try_reserve_exact(linetable, 7)?; +fn write_location_info_no_column(linetable: &mut Vec, length: usize, line_delta: i32) { write_location_first_byte(linetable, PyCodeLocationInfoKind::NoColumns as u8, length); write_location_signed_varint(linetable, line_delta); - Ok(()) } /// assemble.c write_location_info_entry @@ -4806,8 +4794,19 @@ fn write_location_info_entry( prev_line: &mut i32, debug_ranges: bool, ) -> crate::InternalResult<()> { + const THEORETICAL_MAX_ENTRY_SIZE: usize = 25; + if linetable + .len() + .checked_add(THEORETICAL_MAX_ENTRY_SIZE) + .ok_or(InternalError::MalformedControlFlowGraph)? + >= linetable.capacity() + { + debug_assert!(linetable.capacity() > THEORETICAL_MAX_ENTRY_SIZE); + vec_try_resize_to_double_capacity(linetable)?; + } if loc.line == NO_LOCATION_OVERRIDE { - return write_location_info_none(linetable, length); + write_location_info_none(linetable, length); + return Ok(()); } let line_delta = loc.line - *prev_line; @@ -4816,23 +4815,24 @@ fn write_location_info_entry( if !debug_ranges || ((column < 0 || end_column < 0) && (loc.end_line == loc.line || loc.end_line < 0)) { - write_location_info_no_column(linetable, length, line_delta)?; + write_location_info_no_column(linetable, length, line_delta); *prev_line = loc.line; return Ok(()); } if loc.end_line == loc.line { if line_delta == 0 && column < 80 && end_column - column < 16 && end_column >= column { - return write_location_info_short_form(linetable, length, column, end_column); + write_location_info_short_form(linetable, length, column, end_column); + return Ok(()); } if (0..3).contains(&line_delta) && column < 128 && end_column < 128 { - write_location_info_oneline_form(linetable, length, line_delta, column, end_column)?; + write_location_info_oneline_form(linetable, length, line_delta, column, end_column); *prev_line = loc.line; return Ok(()); } } - write_location_info_long_form(linetable, loc, length, line_delta)?; + write_location_info_long_form(linetable, loc, length, line_delta); *prev_line = loc.line; Ok(()) } @@ -4907,7 +4907,14 @@ fn assemble_emit_exception_table_entry( handler: InstructionSequenceExceptHandlerInfo, ) -> crate::InternalResult<()> { const MAX_SIZE_OF_ENTRY: usize = 20; - vec_try_reserve_exact(table, MAX_SIZE_OF_ENTRY)?; + if table + .len() + .checked_add(MAX_SIZE_OF_ENTRY) + .ok_or(InternalError::MalformedControlFlowGraph)? + >= table.capacity() + { + vec_try_resize_to_double_capacity(table)?; + } let size = end - start; debug_assert!(end > start); let target = handler_offset; From af7fa6494a2102700b49663812eac1859970efd0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:19:25 +0900 Subject: [PATCH 085/131] Match CPython assembler growth overflow check --- crates/codegen/src/ir.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5e4a7649719..75cfd6c95bd 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4673,7 +4673,10 @@ fn vec_try_reserve_exact(vec: &mut Vec, additional: usize) -> crate::Inter fn vec_try_resize_to_double_capacity(vec: &mut Vec) -> crate::InternalResult<()> { let capacity = vec.capacity(); debug_assert!(capacity > 0); - if capacity == 0 || capacity > usize::MAX / 2 { + let len = capacity + .checked_mul(core::mem::size_of::()) + .ok_or(InternalError::MalformedControlFlowGraph)?; + if capacity == 0 || len > usize::MAX / 2 { return Err(InternalError::MalformedControlFlowGraph); } let new_capacity = capacity * 2; From 4b8cfda7ad814258e192a7c30c2e86fcc8dba96d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:24:11 +0900 Subject: [PATCH 086/131] Match CPython remove_unreachable structure --- crates/codegen/src/ir.rs | 89 ++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 48 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 75cfd6c95bd..d6edac11800 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1972,9 +1972,45 @@ fn prepare_localsplus( /// flowgraph.c remove_unreachable fn remove_unreachable(blocks: &mut [Block]) -> crate::InternalResult<()> { - compute_reachable_predecessors(blocks)?; - let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + blocks[block_idx.idx()].predecessors = 0; + block_idx = blocks[block_idx.idx()].next; + } + + let mut stack = make_cfg_traversal_stack(blocks)?; + blocks[0].predecessors = 1; + stack.push(BlockIdx(0)); + blocks[0].visited = true; + while let Some(current) = stack.pop() { + let idx = current.idx(); + let next = blocks[idx].next; + if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { + if !blocks[next.idx()].visited { + debug_assert_eq!(blocks[next.idx()].predecessors, 0); + stack.push(next); + blocks[next.idx()].visited = true; + } + blocks[next.idx()].predecessors += 1; + } + + let instr_count = blocks[idx].instruction_used; + for i in 0..instr_count { + let instr = blocks[idx].instructions[i]; + if is_jump(&instr) || is_block_push(&instr) { + let target = instr.target; + debug_assert!(target != BlockIdx::NULL); + let target_idx = target.idx(); + if !blocks[target_idx].visited { + stack.push(target); + blocks[target_idx].visited = true; + } + blocks[target_idx].predecessors += 1; + } + } + } + + block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let i = block_idx.idx(); let next = blocks[i].next; @@ -6251,49 +6287,6 @@ fn basicblock_has_no_lineno(block: &Block) -> bool { true } -/// flowgraph.c remove_unreachable computes `b_predecessors` by traversing only -/// blocks reachable from entry. -fn compute_reachable_predecessors(blocks: &mut [Block]) -> crate::InternalResult<()> { - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - blocks[block_idx.idx()].predecessors = 0; - block_idx = blocks[block_idx.idx()].next; - } - - let mut stack = make_cfg_traversal_stack(blocks)?; - blocks[0].predecessors = 1; - stack.push(BlockIdx(0)); - blocks[0].visited = true; - while let Some(current) = stack.pop() { - let idx = current.idx(); - let next = blocks[idx].next; - if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { - if !blocks[next.idx()].visited { - debug_assert_eq!(blocks[next.idx()].predecessors, 0); - stack.push(next); - blocks[next.idx()].visited = true; - } - blocks[next.idx()].predecessors += 1; - } - - let instr_count = blocks[idx].instruction_used; - for i in 0..instr_count { - let instr = blocks[idx].instructions[i]; - if is_jump(&instr) || is_block_push(&instr) { - let target = instr.target; - debug_assert!(target != BlockIdx::NULL); - let target_idx = target.idx(); - if !blocks[target_idx].visited { - stack.push(target); - blocks[target_idx].visited = true; - } - blocks[target_idx].predecessors += 1; - } - } - } - Ok(()) -} - /// flowgraph.c copy_basicblock fn copy_basicblock( blocks: &mut Vec, @@ -7282,7 +7275,7 @@ mod tests { test_block_push(&mut blocks[2], test_instr(Instruction::ReturnValue, 30)); blocks[2].instructions[0].lineno_override = Some(NO_LOCATION_OVERRIDE); - compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); + remove_unreachable(&mut blocks).expect("remove_unreachable succeeds"); resolve_line_numbers(&mut blocks, OneIndexed::MIN).expect("resolve_line_numbers succeeds"); // CPython `duplicate_exits_without_lineno()` copies a shared exit block @@ -7311,7 +7304,7 @@ mod tests { block.instructions[2].lineno_override = Some(NO_LOCATION_OVERRIDE); let mut blocks = vec![block]; - compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); + remove_unreachable(&mut blocks).expect("remove_unreachable succeeds"); propagate_line_numbers(&mut blocks); // CPython `propagate_line_numbers()` only copies over NO_LOCATION @@ -7338,7 +7331,7 @@ mod tests { basicblock_clear(&mut blocks[1]); test_block_push(&mut blocks[2], test_instr(Instruction::ReturnValue, 30)); - compute_reachable_predecessors(&mut blocks).expect("compute predecessors succeeds"); + remove_unreachable(&mut blocks).expect("remove_unreachable succeeds"); propagate_line_numbers(&mut blocks); // CPython `propagate_line_numbers()` directly reads `target->b_instr[0]` From 9ec26384188e98a692ae760e451c5fc561abfd9c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:27:37 +0900 Subject: [PATCH 087/131] Match CPython static swap flow --- crates/codegen/src/ir.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d6edac11800..335be077204 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3331,10 +3331,6 @@ fn apply_static_swaps(instructions: &mut [InstructionInfo], mut i: isize) { _ => return, }; - if swap_arg < 2 { - return; - } - let Some(j) = next_swappable_instruction(instructions, idx, -1) else { return; }; From 32c82c70ce3cea53ad91e440e7cdfd0c7f5aec89 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:28:31 +0900 Subject: [PATCH 088/131] Inline CPython code unit preprocessing --- crates/codegen/src/ir.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 335be077204..85aa4bd514e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1670,7 +1670,9 @@ fn optimize_code_unit( ) -> crate::InternalResult<()> { // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) *blocks = cfg_from_instruction_sequence(instr_sequence)?; - optimize_code_unit_preprocess(blocks)?; + translate_jump_labels_to_targets(blocks)?; + mark_except_handlers(blocks)?; + label_exception_targets(blocks)?; optimize_cfg(metadata, blocks, metadata.firstlineno)?; remove_unused_consts(blocks, &mut metadata.consts)?; add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams)?; @@ -4355,7 +4357,9 @@ impl CodeInfo { "after_cfg_from_instruction_sequence".to_owned(), self.debug_block_dump(), )); - optimize_code_unit_preprocess(&mut self.blocks)?; + translate_jump_labels_to_targets(&mut self.blocks)?; + mark_except_handlers(&mut self.blocks)?; + label_exception_targets(&mut self.blocks)?; check_cfg(&self.blocks)?; inline_small_or_no_lineno_blocks(&mut self.blocks)?; trace.push(( @@ -5925,14 +5929,6 @@ fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResu Ok(()) } -/// flowgraph.c _PyCfg_OptimizeCodeUnit preprocessing -fn optimize_code_unit_preprocess(blocks: &mut [Block]) -> crate::InternalResult<()> { - translate_jump_labels_to_targets(blocks)?; - mark_except_handlers(blocks)?; - label_exception_targets(blocks)?; - Ok(()) -} - /// flowgraph.c _PyCfg_FromInstructionSequence fn cfg_from_instruction_sequence( mut instr_sequence: InstructionSequence, From 3f55473c66100d003141a499302af998237e0d95 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:31:39 +0900 Subject: [PATCH 089/131] Match CPython C array growth checks --- crates/codegen/src/ir.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 85aa4bd514e..109e1d84af0 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -315,7 +315,7 @@ fn empty_instruction_info() -> InstructionInfo { } /// codegen.c _Py_CArray_EnsureCapacity -fn c_array_ensure_capacity( +fn c_array_ensure_capacity( allocated_entries: usize, idx: usize, initial_num_entries: usize, @@ -329,6 +329,9 @@ fn c_array_ensure_capacity( }; Ok(new_alloc) } else if idx >= allocated_entries { + let oldsize = allocated_entries + .checked_mul(core::mem::size_of::()) + .ok_or(InternalError::MalformedControlFlowGraph)?; let doubled = allocated_entries .checked_mul(2) .ok_or(InternalError::MalformedControlFlowGraph)?; @@ -338,6 +341,12 @@ fn c_array_ensure_capacity( } else { doubled }; + let newsize = new_alloc + .checked_mul(core::mem::size_of::()) + .ok_or(InternalError::MalformedControlFlowGraph)?; + if oldsize > usize::MAX >> 1 || newsize == 0 { + return Err(InternalError::MalformedControlFlowGraph); + } Ok(new_alloc) } else { Ok(allocated_entries) @@ -347,8 +356,11 @@ fn c_array_ensure_capacity( /// flowgraph.c basicblock_next_instr fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { let off = block.instruction_used; - let new_allocation = - c_array_ensure_capacity(block.instruction_allocation, off + 1, DEFAULT_BLOCK_SIZE)?; + let new_allocation = c_array_ensure_capacity::( + block.instruction_allocation, + off + 1, + DEFAULT_BLOCK_SIZE, + )?; if new_allocation > block.instruction_allocation { if new_allocation > block.instructions.len() { block @@ -550,8 +562,11 @@ fn instruction_sequence_new() -> InstructionSequence { /// instruction_sequence.c instr_sequence_next_inst fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::InternalResult { let idx = seq.instr_used; - let new_allocation = - c_array_ensure_capacity(seq.instr_allocation, idx + 1, INITIAL_INSTR_SEQUENCE_SIZE)?; + let new_allocation = c_array_ensure_capacity::( + seq.instr_allocation, + idx + 1, + INITIAL_INSTR_SEQUENCE_SIZE, + )?; if new_allocation > seq.instr_allocation { if new_allocation > seq.instrs.capacity() { seq.instrs @@ -616,7 +631,7 @@ fn instruction_sequence_use_label( label: InstructionSequenceLabel, ) -> crate::InternalResult<()> { let old_size = seq.label_map_allocation; - let new_allocation = c_array_ensure_capacity( + let new_allocation = c_array_ensure_capacity::( seq.label_map_allocation, label.idx(), INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, From ff0e10632423880fd18173718a43886d50bdf62c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:32:49 +0900 Subject: [PATCH 090/131] Match CPython label map size guard --- crates/codegen/src/ir.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 109e1d84af0..5bed4eb61b5 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5905,7 +5905,9 @@ fn cfg_builder_check_size(g: &CfgBuilder) -> crate::InternalResult<()> { fn translate_jump_labels_to_targets(blocks: &mut [Block]) -> crate::InternalResult<()> { let max_label = get_max_label(blocks); let label_count = (max_label + 1) as usize; - let _mapsize = core::mem::size_of::() * label_count; + if label_count > usize::MAX / core::mem::size_of::() { + return Err(InternalError::MalformedControlFlowGraph); + } let mut label_to_block = Vec::new(); vec_try_reserve_exact(&mut label_to_block, label_count)?; label_to_block.resize(label_count, BlockIdx::NULL); From b158977976701217926fe3e483d9c4655cafea6d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:35:43 +0900 Subject: [PATCH 091/131] Match CPython load-fast flow --- crates/codegen/src/ir.rs | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5bed4eb61b5..17b611e9e7c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4057,7 +4057,8 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { push_ref(&mut refs, i as isize, local)?; } AnyInstruction::Real(Instruction::LoadFastLoadFast { .. }) => { - let (local1, local2) = decode_packed_fast_locals(info.arg); + let local1 = (arg_u32 >> 4) as isize; + let local2 = (arg_u32 & 15) as isize; push_ref(&mut refs, i as isize, local1)?; push_ref(&mut refs, i as isize, local2)?; } @@ -4071,17 +4072,15 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { ); } AnyInstruction::Real(Instruction::StoreFastLoadFast { .. }) => { - let (store_local_idx, load_local_idx) = decode_packed_fast_locals(info.arg); let r = ref_stack_pop(&mut refs); - store_local(&mut instr_flags, &refs, store_local_idx, r); - push_ref(&mut refs, i as isize, load_local_idx)?; + store_local(&mut instr_flags, &refs, (arg_u32 >> 4) as isize, r); + push_ref(&mut refs, i as isize, (arg_u32 & 15) as isize)?; } AnyInstruction::Real(Instruction::StoreFastStoreFast { .. }) => { - let (local1, local2) = decode_packed_fast_locals(info.arg); let r1 = ref_stack_pop(&mut refs); - store_local(&mut instr_flags, &refs, local1, r1); + store_local(&mut instr_flags, &refs, (arg_u32 >> 4) as isize, r1); let r2 = ref_stack_pop(&mut refs); - store_local(&mut instr_flags, &refs, local2, r2); + store_local(&mut instr_flags, &refs, (arg_u32 & 15) as isize, r2); } AnyInstruction::Real(Instruction::Copy { i: _ }) => { let depth = arg_u32 as usize; @@ -4153,19 +4152,16 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { load_fast_push_block(&mut worklist, blocks, target, refs.len() + 1); push_ref(&mut refs, i as isize, NOT_LOCAL)?; } - AnyInstruction::Real(Instruction::LoadAttr { namei }) => { + AnyInstruction::Real( + Instruction::LoadAttr { .. } | Instruction::LoadSuperAttr { .. }, + ) => { let self_ref = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL)?; - if namei.get(info.arg).is_method() { - push_ref(&mut refs, self_ref.instr, self_ref.local)?; + if matches!(instr.real(), Some(Instruction::LoadSuperAttr { .. })) { + let _ = ref_stack_pop(&mut refs); + let _ = ref_stack_pop(&mut refs); } - } - AnyInstruction::Real(Instruction::LoadSuperAttr { namei }) => { - let self_ref = ref_stack_pop(&mut refs); - let _ = ref_stack_pop(&mut refs); - let _ = ref_stack_pop(&mut refs); push_ref(&mut refs, i as isize, NOT_LOCAL)?; - if namei.get(info.arg).is_load_method() { + if arg_u32 & 1 != 0 { push_ref(&mut refs, self_ref.instr, self_ref.local)?; } } @@ -4649,11 +4645,6 @@ fn store_local(instr_flags: &mut [u8], refs: &RefStack, local: isize, r: Ref) { } } -fn decode_packed_fast_locals(arg: OpArg) -> (isize, isize) { - let packed = u32::from(arg); - (((packed >> 4) & 0xF) as isize, (packed & 0xF) as isize) -} - fn local_as_ref_local(local: usize) -> isize { local as isize } From 01ad7f581585f2ea5a56e117590873bafb139b70 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:38:17 +0900 Subject: [PATCH 092/131] Simplify CPython CFG condition flow --- crates/codegen/src/ir.rs | 85 +++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 17b611e9e7c..829adfdac0a 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5249,7 +5249,6 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) -> crate::InternalResult<()> } /// flowgraph.c check_cfg -#[allow(clippy::collapsible_if)] fn check_cfg(blocks: &[Block]) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { @@ -5257,10 +5256,8 @@ fn check_cfg(blocks: &[Block]) -> crate::InternalResult<()> { for i in 0..block.instruction_used { let opcode = block.instructions[i].instr; debug_assert!(!opcode.is_assembler()); - if opcode.is_terminator() { - if i != block.instruction_used - 1 { - return Err(InternalError::MalformedControlFlowGraph); - } + if opcode.is_terminator() && i != block.instruction_used - 1 { + return Err(InternalError::MalformedControlFlowGraph); } } block_idx = block.next; @@ -5685,27 +5682,26 @@ fn remove_redundant_jumps(blocks: &mut [Block]) -> crate::InternalResult /// flowgraph.c no_redundant_jumps #[cfg(debug_assertions)] -#[allow(clippy::collapsible_if)] fn no_redundant_jumps(blocks: &[Block]) -> bool { let mut current = BlockIdx(0); while current != BlockIdx::NULL { let block = &blocks[current.idx()]; - if let Some(last) = basicblock_last_instr(block) { - if last.instr.is_unconditional_jump() { - let next = next_nonempty_block(blocks, block.next); - let jump_target = next_nonempty_block(blocks, last.target); - if jump_target == next { - assert!(next != BlockIdx::NULL); - if instruction_lineno(last) - == instruction_lineno(&blocks[next.idx()].instructions[0]) - { - assert_ne!( - instruction_lineno(last), - instruction_lineno(&blocks[next.idx()].instructions[0]), - "redundant jump has same line as fallthrough target" - ); - return false; - } + if let Some(last) = basicblock_last_instr(block) + && last.instr.is_unconditional_jump() + { + let next = next_nonempty_block(blocks, block.next); + let jump_target = next_nonempty_block(blocks, last.target); + if jump_target == next { + assert!(next != BlockIdx::NULL); + if instruction_lineno(last) + == instruction_lineno(&blocks[next.idx()].instructions[0]) + { + assert_ne!( + instruction_lineno(last), + instruction_lineno(&blocks[next.idx()].instructions[0]), + "redundant jump has same line as fallthrough target" + ); + return false; } } } @@ -6038,7 +6034,6 @@ fn maybe_push( } /// flowgraph.c scan_block_for_locals -#[allow(clippy::collapsible_if)] fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &mut Vec) { let idx = block_idx.idx(); let mut unsafe_mask = blocks[idx].unsafe_locals_mask; @@ -6095,12 +6090,12 @@ fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &m } let last = basicblock_last_instr(&blocks[idx]).copied(); - if let Some(last) = last { - if is_jump(&last) { - let target = last.target; - debug_assert!(target != BlockIdx::NULL); - maybe_push(blocks, worklist, target, unsafe_mask); - } + if let Some(last) = last + && is_jump(&last) + { + let target = last.target; + debug_assert!(target != BlockIdx::NULL); + maybe_push(blocks, worklist, target, unsafe_mask); } } @@ -6310,7 +6305,6 @@ fn get_max_label(blocks: &[Block]) -> i32 { lbl } -#[allow(clippy::collapsible_if)] fn duplicate_exits_without_lineno(blocks: &mut Vec) -> crate::InternalResult<()> { let mut next_lbl = get_max_label(blocks) + 1; @@ -6352,22 +6346,19 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec) -> crate::InternalRes if bb_has_fallthrough(&blocks[b.idx()]) && next != BlockIdx::NULL && blocks[b.idx()].instruction_used != 0 + && is_exit_or_eval_check_without_lineno(&blocks[next.idx()]) { - if is_exit_or_eval_check_without_lineno(&blocks[next.idx()]) { - let last = - *basicblock_last_instr(&blocks[b.idx()]).expect("block has instructions"); - instr_set_location( - &mut blocks[next.idx()].instructions[0], - instr_location(&last), - ); - } + let last = *basicblock_last_instr(&blocks[b.idx()]).expect("block has instructions"); + instr_set_location( + &mut blocks[next.idx()].instructions[0], + instr_location(&last), + ); } b = blocks[b.idx()].next; } Ok(()) } -#[allow(clippy::collapsible_if)] fn propagate_line_numbers(blocks: &mut [Block]) { let mut current = BlockIdx(0); while current != BlockIdx::NULL { @@ -6387,15 +6378,13 @@ fn propagate_line_numbers(blocks: &mut [Block]) { } let next = blocks[idx].next; - if bb_has_fallthrough(&blocks[idx]) { - debug_assert!(next != BlockIdx::NULL); - if next != BlockIdx::NULL && blocks[next.idx()].predecessors == 1 { - if blocks[next.idx()].instruction_used != 0 { - if instruction_is_no_location(&blocks[next.idx()].instructions[0]) { - instr_set_location(&mut blocks[next.idx()].instructions[0], prev_location); - } - } - } + if bb_has_fallthrough(&blocks[idx]) + && next != BlockIdx::NULL + && blocks[next.idx()].predecessors == 1 + && blocks[next.idx()].instruction_used != 0 + && instruction_is_no_location(&blocks[next.idx()].instructions[0]) + { + instr_set_location(&mut blocks[next.idx()].instructions[0], prev_location); } if is_jump(&last) { From 63171441483fb196538ea5f774ccff1decc4f8b4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:41:22 +0900 Subject: [PATCH 093/131] Align exception fallthrough propagation --- crates/codegen/src/ir.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 829adfdac0a..7bca6ba0af9 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6537,13 +6537,11 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe } let next = blocks[bi].next; - if bb_has_fallthrough(&blocks[bi]) { - debug_assert!(next != BlockIdx::NULL); - if !blocks[next.idx()].visited { - blocks[next.idx()].except_stack = Some(stack); - todo.push(next); - blocks[next.idx()].visited = true; - } + if bb_has_fallthrough(&blocks[bi]) && next != BlockIdx::NULL && !blocks[next.idx()].visited + { + blocks[next.idx()].except_stack = Some(stack); + todo.push(next); + blocks[next.idx()].visited = true; } } #[cfg(debug_assertions)] From 821149a1dcf001571464ca259dece9eb50c5fc0c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:46:12 +0900 Subject: [PATCH 094/131] Match CPython pseudo target table --- crates/codegen/src/ir.rs | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7bca6ba0af9..7758ec12f43 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -813,19 +813,35 @@ fn instr_size(instr: &InstructionInfo) -> usize { /// pycore_opcode_metadata.h is_pseudo_target fn is_pseudo_target(pseudo: PseudoOpcode, target: Opcode) -> bool { - matches!( - (pseudo, target), - ( - PseudoOpcode::Jump, - Opcode::JumpForward | Opcode::JumpBackward - ) | ( - PseudoOpcode::JumpNoInterrupt, - Opcode::JumpForward | Opcode::JumpBackwardNoInterrupt - ) | (PseudoOpcode::LoadClosure, Opcode::LoadFast) - | (PseudoOpcode::StoreFastMaybeNull, Opcode::StoreFast) - ) + match pseudo { + PseudoOpcode::LoadClosure => matches!(target, Opcode::LoadFast), + PseudoOpcode::StoreFastMaybeNull => matches!(target, Opcode::StoreFast), + PseudoOpcode::AnnotationsPlaceholder + | PseudoOpcode::SetupFinally + | PseudoOpcode::SetupCleanup + | PseudoOpcode::SetupWith + | PseudoOpcode::PopBlock => matches!(target, Opcode::Nop), + PseudoOpcode::Jump => matches!(target, Opcode::JumpForward | Opcode::JumpBackward), + PseudoOpcode::JumpNoInterrupt => { + matches!( + target, + Opcode::JumpForward | Opcode::JumpBackwardNoInterrupt + ) + } + PseudoOpcode::JumpIfFalse => { + matches!( + target, + Opcode::Copy | Opcode::ToBool | Opcode::PopJumpIfFalse + ) + } + PseudoOpcode::JumpIfTrue => { + matches!( + target, + Opcode::Copy | Opcode::ToBool | Opcode::PopJumpIfTrue + ) + } + } } - /// assemble.c resolve_unconditional_jumps #[allow(clippy::unnecessary_wraps)] fn resolve_unconditional_jumps( From d9ab6b70f89e2b2dc763558becc31d99b5852026 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:51:02 +0900 Subject: [PATCH 095/131] Match CPython annotations CFG assert --- crates/codegen/src/ir.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 7758ec12f43..d6c648d4b84 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5989,7 +5989,6 @@ fn cfg_from_instruction_sequence( if let Some(annotations_code) = &annotations_code { debug_assert!(annotations_code.label_map.is_none()); debug_assert_eq!(annotations_code.label_map_allocation, 0); - debug_assert!(annotations_code.annotations_code.is_none()); for j in 0..annotations_code.instr_used { let ann_entry = annotations_code.instrs[j]; debug_assert!(!ann_entry.info.instr.has_target()); From 5afb44f4dca2fa9bf99d0c623561dd8a092be3c0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:56:41 +0900 Subject: [PATCH 096/131] Match CPython inverted op assert --- crates/codegen/src/ir.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d6c648d4b84..c2cbb151c3a 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3789,10 +3789,12 @@ fn optimize_basic_block( if matches!(nextop, Some(Instruction::UnaryNot)) => { set_to_nop(&mut blocks[bi].instructions[i]); + let inverted = u32::from(inst.arg) ^ 1; + debug_assert!(inverted == 0 || inverted == 1); instr_set_op1( &mut blocks[bi].instructions[i + 1], inst.instr, - OpArg::new(u32::from(inst.arg) ^ 1), + OpArg::new(inverted), ); i += 1; continue; From d65f063b16aed3b6a87dce0cd285fdedbd32b637 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 14:58:32 +0900 Subject: [PATCH 097/131] Match CPython many-locals guard --- crates/codegen/src/ir.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index c2cbb151c3a..3cffed9bc07 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6135,6 +6135,7 @@ fn fast_scan_many_locals(blocks: &mut [Block], nlocals: usize) -> crate::Interna if arg < LOCAL_UNSAFE_MASK_BITS { continue; } + debug_assert!(arg >= LOCAL_UNSAFE_MASK_BITS); match info.instr { AnyInstruction::Real( Instruction::DeleteFast { .. } | Instruction::LoadFastAndClear { .. }, From 9bb6e42e64ddf9973169f4f41d098575baed2d97 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:13:08 +0900 Subject: [PATCH 098/131] Reject deopt opcodes in CFG stack effects --- crates/codegen/src/ir.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 3cffed9bc07..b1d53b57131 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4714,6 +4714,9 @@ fn get_stack_effects( oparg: OpArg, jump: i32, ) -> crate::InternalResult { + if instr.real().is_some_and(|op| op.deopt().is_some()) { + return Err(InternalError::MalformedControlFlowGraph); + } let oparg = u32::from(oparg); let net = if instr.is_block_push() && jump == 0 { 0 From bdfe5233ec7dffb8ff3ad0b366b03ff415508571 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:18:28 +0900 Subject: [PATCH 099/131] Match CPython invalid stack effect error --- crates/codegen/src/error.rs | 2 ++ crates/codegen/src/ir.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/error.rs b/crates/codegen/src/error.rs index 4d21bb001e0..fb848354e86 100644 --- a/crates/codegen/src/error.rs +++ b/crates/codegen/src/error.rs @@ -40,6 +40,7 @@ impl fmt::Display for CodegenError { pub enum InternalError { StackUnderflow, InconsistentStackDepth, + InvalidStackEffect, MalformedControlFlowGraph, MissingSymbol(String), } @@ -49,6 +50,7 @@ impl Display for InternalError { match self { Self::StackUnderflow => write!(f, "Invalid CFG, stack underflow"), Self::InconsistentStackDepth => write!(f, "Invalid CFG, inconsistent stackdepth"), + Self::InvalidStackEffect => write!(f, "Invalid stack effect"), Self::MalformedControlFlowGraph => write!(f, "malformed control flow graph."), Self::MissingSymbol(s) => write!( f, diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index b1d53b57131..5a8e9dfaac3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4715,7 +4715,7 @@ fn get_stack_effects( jump: i32, ) -> crate::InternalResult { if instr.real().is_some_and(|op| op.deopt().is_some()) { - return Err(InternalError::MalformedControlFlowGraph); + return Err(InternalError::InvalidStackEffect); } let oparg = u32::from(oparg); let net = if instr.is_block_push() && jump == 0 { From cdcfac2269816d2bc08b1da1acc6f8793933ad90 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:20:54 +0900 Subject: [PATCH 100/131] Test CPython deopt stack effect guard --- crates/codegen/src/ir.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5a8e9dfaac3..938e2775f3f 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6779,6 +6779,15 @@ mod tests { } } + #[test] + fn get_stack_effects_rejects_cpython_deopt_opcodes() { + match get_stack_effects(Instruction::BinaryOpAddInt.into(), OpArg::new(0), 0) { + Err(InternalError::InvalidStackEffect) => {} + Err(err) => panic!("unexpected stack-effect error: {err}"), + Ok(_) => panic!("CPython get_stack_effects rejects specialized deopt opcodes"), + } + } + #[test] fn instruction_sequence_label_shadow_preserves_cpython_offset_aliases() { let mut seq = instruction_sequence_new(); From 77d971aef22d5b9b5832a0d5f018113cdf7ff292 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:47:50 +0900 Subject: [PATCH 101/131] Match CPython load-fast extended-arg assert --- crates/codegen/src/ir.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 938e2775f3f..acd91cfc0ca 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4053,6 +4053,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let info = blocks[block_i].instructions[i]; let instr = info.instr; let arg_u32 = u32::from(info.arg); + debug_assert!(!matches!(instr.real(), Some(Instruction::ExtendedArg))); match instr { AnyInstruction::Real(Instruction::DeleteFast { var_num }) => { From d342eadbe4e4f0d6584eb66a84687b113baba6d5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:51:04 +0900 Subject: [PATCH 102/131] Match CPython instruction allocation asserts --- crates/codegen/src/ir.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index acd91cfc0ca..6f5df1247bf 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -373,6 +373,7 @@ fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { } block.instruction_allocation = new_allocation; } + debug_assert!(block.instruction_allocation > off); block.instruction_used += 1; Ok(off) } @@ -561,6 +562,7 @@ fn instruction_sequence_new() -> InstructionSequence { /// instruction_sequence.c instr_sequence_next_inst fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::InternalResult { + debug_assert!(!seq.instrs.is_empty() || seq.instr_used == 0); let idx = seq.instr_used; let new_allocation = c_array_ensure_capacity::( seq.instr_allocation, @@ -592,6 +594,7 @@ fn instruction_sequence_next_inst(seq: &mut InstructionSequence) -> crate::Inter } seq.instr_allocation = new_allocation; } + debug_assert!(seq.instr_allocation > idx); seq.instr_used += 1; Ok(idx) } From bc92972d55bea5dd24eaa65be98cc727e66238de Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:53:50 +0900 Subject: [PATCH 103/131] Match CPython basicblock last-instr asserts --- crates/codegen/src/ir.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 6f5df1247bf..a7db9552c7d 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -380,7 +380,9 @@ fn basicblock_next_instr(block: &mut Block) -> crate::InternalResult { /// flowgraph.c basicblock_last_instr fn basicblock_last_instr(block: &Block) -> Option<&InstructionInfo> { + debug_assert!(block.instruction_allocation >= block.instruction_used); if block.instruction_used > 0 { + debug_assert!(!block.instructions.is_empty()); Some(&block.instructions[block.instruction_used - 1]) } else { None @@ -389,7 +391,9 @@ fn basicblock_last_instr(block: &Block) -> Option<&InstructionInfo> { /// flowgraph.c basicblock_last_instr fn basicblock_last_instr_mut(block: &mut Block) -> Option<&mut InstructionInfo> { + debug_assert!(block.instruction_allocation >= block.instruction_used); if block.instruction_used > 0 { + debug_assert!(!block.instructions.is_empty()); Some(&mut block.instructions[block.instruction_used - 1]) } else { None From 32f01b78a209515b47d642974388e8ab2823739f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:56:57 +0900 Subject: [PATCH 104/131] Match CPython opcode range asserts --- crates/codegen/src/ir.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a7db9552c7d..f2c236c067c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -45,11 +45,21 @@ const DEFAULT_CNOTAB_SIZE: usize = 32; const DEFAULT_BLOCK_SIZE: usize = 16; const INITIAL_INSTR_SEQUENCE_SIZE: usize = 100; const INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE: usize = 10; +const MAX_REAL_OPCODE: u16 = 254; +const MAX_OPCODE: u16 = 511; const MAX_TOTAL_ITEMS: isize = 1024; const MAX_STR_SIZE: usize = 4096; const MIN_CONST_SEQUENCE_SIZE: usize = 3; const STACK_USE_GUIDELINE: usize = 30; +/// pycore_opcode_utils.h IS_WITHIN_OPCODE_RANGE +fn is_within_opcode_range(opcode: AnyOpcode) -> bool { + match opcode { + AnyOpcode::Real(opcode) => u16::from(opcode.as_u8()) <= MAX_REAL_OPCODE, + AnyOpcode::Pseudo(opcode) => opcode.as_u16() <= MAX_OPCODE, + } +} + #[derive(Clone, Debug, Default)] pub struct ConstantPool { constants: Vec, @@ -402,6 +412,8 @@ fn basicblock_last_instr_mut(block: &mut Block) -> Option<&mut InstructionInfo> /// flowgraph.c basicblock_addop fn basicblock_addop(block: &mut Block, mut info: InstructionInfo) -> crate::InternalResult<()> { + let opcode = AnyOpcode::from(info.instr); + debug_assert!(is_within_opcode_range(opcode)); debug_assert!(!info.instr.is_assembler()); debug_assert!( info.instr.has_arg() || info.instr.has_target() || u32::from(info.arg) == 0, @@ -612,6 +624,7 @@ fn instruction_sequence_new_label(seq: &mut InstructionSequence) -> InstructionS /// instruction_sequence.c _PyInstructionSequence_Addop asserts. fn instruction_sequence_debug_check_addop(info: &InstructionInfo) { let opcode = AnyOpcode::from(info.instr); + debug_assert!(is_within_opcode_range(opcode)); debug_assert!( opcode.has_arg() || info.instr.has_target() || u32::from(info.arg) == 0, "CPython _PyInstructionSequence_Addop requires either OPCODE_HAS_ARG, HAS_TARGET, or oparg == 0" From 1a8c7fa6c32955902d37820bf901d3fb002546e3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 15:58:32 +0900 Subject: [PATCH 105/131] Assert CPython fallthrough line propagation invariant --- crates/codegen/src/ir.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f2c236c067c..a1ae94080ee 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6420,13 +6420,15 @@ fn propagate_line_numbers(blocks: &mut [Block]) { } let next = blocks[idx].next; - if bb_has_fallthrough(&blocks[idx]) - && next != BlockIdx::NULL - && blocks[next.idx()].predecessors == 1 - && blocks[next.idx()].instruction_used != 0 - && instruction_is_no_location(&blocks[next.idx()].instructions[0]) - { - instr_set_location(&mut blocks[next.idx()].instructions[0], prev_location); + if bb_has_fallthrough(&blocks[idx]) { + debug_assert!(next != BlockIdx::NULL); + if next != BlockIdx::NULL + && blocks[next.idx()].predecessors == 1 + && blocks[next.idx()].instruction_used != 0 + && instruction_is_no_location(&blocks[next.idx()].instructions[0]) + { + instr_set_location(&mut blocks[next.idx()].instructions[0], prev_location); + } } if is_jump(&last) { From 41c972a00abb8dff975c1f8394ef0b851b665594 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:00:17 +0900 Subject: [PATCH 106/131] Assert CPython CFG target offset sign --- crates/codegen/src/ir.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a1ae94080ee..9e998e860ef 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6040,6 +6040,7 @@ fn cfg_from_instruction_sequence( let mut oparg = entry.info.arg; if opcode.has_target() { let target_offset = u32::from(oparg) as i32 + offset; + debug_assert!(target_offset >= 0); oparg = OpArg::new(target_offset as u32); } entry.info.instr = opcode; From 2f8e23fd3f41d56f70afafc4043f65cd73abdf5f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:01:22 +0900 Subject: [PATCH 107/131] Assert CPython exception fallthrough invariant --- crates/codegen/src/ir.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 9e998e860ef..4e46afbe649 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6582,11 +6582,13 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe } let next = blocks[bi].next; - if bb_has_fallthrough(&blocks[bi]) && next != BlockIdx::NULL && !blocks[next.idx()].visited - { - blocks[next.idx()].except_stack = Some(stack); - todo.push(next); - blocks[next.idx()].visited = true; + if bb_has_fallthrough(&blocks[bi]) { + debug_assert!(next != BlockIdx::NULL); + if next != BlockIdx::NULL && !blocks[next.idx()].visited { + blocks[next.idx()].except_stack = Some(stack); + todo.push(next); + blocks[next.idx()].visited = true; + } } } #[cfg(debug_assertions)] From eca1b1ada5521b3e15dc4799f0d3cabb479eaf83 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:03:19 +0900 Subject: [PATCH 108/131] Assert CPython exception stack bounds --- crates/codegen/src/ir.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4e46afbe649..cb8c5883fac 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6461,11 +6461,13 @@ fn make_except_stack() -> crate::InternalResult> { stack .try_reserve_exact(CO_MAXBLOCKS + 2) .map_err(|_| InternalError::MalformedControlFlowGraph)?; + debug_assert!(stack.is_empty()); Ok(stack) } /// flowgraph.c copy_except_stack fn copy_except_stack(stack: &[BlockIdx]) -> crate::InternalResult> { + debug_assert!(stack.len() <= CO_MAXBLOCKS + 1); let mut copy = Vec::new(); copy.try_reserve_exact(CO_MAXBLOCKS + 2) .map_err(|_| InternalError::MalformedControlFlowGraph)?; @@ -6475,6 +6477,7 @@ fn copy_except_stack(stack: &[BlockIdx]) -> crate::InternalResult> /// flowgraph.c except_stack_top fn except_stack_top(stack: &[BlockIdx], blocks: &[Block]) -> Option { + debug_assert!(stack.len() <= CO_MAXBLOCKS + 1); let handler_block = stack.last().copied()?; Some(ExceptHandlerInfo { handler_block, @@ -6500,6 +6503,7 @@ fn push_except_block( } debug_assert!(stack.len() <= CO_MAXBLOCKS); stack.push(target); + debug_assert!(stack.len() <= CO_MAXBLOCKS + 1); except_stack_top(stack, blocks) } @@ -6507,6 +6511,7 @@ fn push_except_block( fn pop_except_block(stack: &mut Vec, blocks: &[Block]) -> Option { debug_assert!(!stack.is_empty()); stack.pop().expect("non-empty exception stack"); + debug_assert!(stack.len() <= CO_MAXBLOCKS); except_stack_top(stack, blocks) } From 265d5265bb29ff4a65e8256952ec5a5ae1827959 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:04:19 +0900 Subject: [PATCH 109/131] Assert CPython traversal stack allocation --- crates/codegen/src/ir.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index cb8c5883fac..f6adf0ffef3 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5775,10 +5775,12 @@ fn make_cfg_traversal_stack(blocks: &mut [Block]) -> crate::InternalResult 0); let mut stack = Vec::new(); stack .try_reserve_exact(nblocks) .map_err(|_| InternalError::MalformedControlFlowGraph)?; + debug_assert!(stack.capacity() >= nblocks); Ok(stack) } From 9526995396e1fc7415e19343a27b9073caabcf34 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:08:29 +0900 Subject: [PATCH 110/131] Match CPython label-map allocation in shadow --- crates/codegen/src/ir.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index f6adf0ffef3..3968bf95978 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1343,13 +1343,23 @@ fn instruction_sequence_label_map_register_label( map: &mut InstructionSequenceLabelMap, label: InstructionSequenceLabel, ) -> crate::InternalResult<()> { - if map.cpython_block_by_label.len() <= label.idx() { - map.cpython_block_by_label - .try_reserve(label.idx() + 1 - map.cpython_block_by_label.len()) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; + debug_assert!(is_label(label)); + let old_size = map.cpython_block_by_label.len(); + let new_allocation = c_array_ensure_capacity::( + old_size, + label.idx(), + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + )?; + if new_allocation > old_size { + if new_allocation > map.cpython_block_by_label.capacity() { + map.cpython_block_by_label + .try_reserve_exact(new_allocation - map.cpython_block_by_label.capacity()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + } map.cpython_block_by_label - .resize(label.idx() + 1, BlockIdx::NULL); + .resize(new_allocation, BlockIdx::NULL); } + debug_assert!(map.cpython_block_by_label.len() > label.idx()); Ok(()) } @@ -1364,6 +1374,7 @@ fn instruction_sequence_label_map_ensure_label_for_block( return Ok(block_label); } let label = instruction_sequence_new_label(seq); + debug_assert_eq!(label.0, seq.next_free_label); instruction_sequence_label_map_register_label(map, label)?; map.cpython_block_by_label[label.idx()] = block; map.block_labels[block.idx()] = label; @@ -1445,6 +1456,7 @@ fn instruction_sequence_label_map_use_label_at_block( return Ok(()); } let from_label = instruction_sequence_label_map_ensure_label_for_block(map, seq, from)?; + debug_assert!(map.cpython_block_by_label.len() > from_label.idx()); let to_block = instruction_sequence_label_map_resolve_label(map, to); if to_block == BlockIdx::NULL { debug_assert!( @@ -1472,6 +1484,7 @@ fn instruction_sequence_label_map_push_unmapped_label( seq: &mut InstructionSequence, ) -> crate::InternalResult<()> { let label = instruction_sequence_new_label(seq); + debug_assert_eq!(label.0, seq.next_free_label); instruction_sequence_label_map_register_label(map, label)?; let block = BlockIdx( map.block_labels @@ -6827,6 +6840,10 @@ mod tests { let mut labels = InstructionSequenceLabelMap::new(); instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq).unwrap(); instruction_sequence_label_map_push_unmapped_label(&mut labels, &mut seq).unwrap(); + assert_eq!( + labels.cpython_block_by_label.len(), + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE + ); let first = BlockIdx::new(1); let second = BlockIdx::new(2); From 72837ed2e6961033923b8b0b9ea2ad78c315acd8 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:11:30 +0900 Subject: [PATCH 111/131] Mirror CPython label-map sentinel fill --- crates/codegen/src/ir.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 3968bf95978..c4e7c59ed9b 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1358,6 +1358,9 @@ fn instruction_sequence_label_map_register_label( } map.cpython_block_by_label .resize(new_allocation, BlockIdx::NULL); + for i in old_size..map.cpython_block_by_label.len() { + map.cpython_block_by_label[i] = BlockIdx::NULL; + } } debug_assert!(map.cpython_block_by_label.len() > label.idx()); Ok(()) From a7aa5d8432c6614150a94d6fc6ede11f6491ee80 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:13:48 +0900 Subject: [PATCH 112/131] Match CPython CFG builder allocation asserts --- crates/codegen/src/ir.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index c4e7c59ed9b..ab0c4c50fd1 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5916,6 +5916,7 @@ fn cfg_builder_check(g: &CfgBuilder) -> bool { let block_ref = &g.blocks[block.idx()]; let has_instr_array = block_ref.instruction_allocation > 0; if has_instr_array { + debug_assert!(block_ref.instruction_allocation > 0); debug_assert_eq!( block_ref.instructions.len(), block_ref.instruction_allocation @@ -5923,6 +5924,7 @@ fn cfg_builder_check(g: &CfgBuilder) -> bool { debug_assert!(block_ref.instruction_allocation >= block_ref.instruction_used); } else { debug_assert_eq!(block_ref.instruction_used, 0); + debug_assert_eq!(block_ref.instruction_allocation, 0); } block = block_ref.allocation_next; } From 2012265be8d41621d9bc683c5a0d38dce1c5063c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:21:39 +0900 Subject: [PATCH 113/131] Match CPython exception stack structure --- crates/codegen/src/ir.rs | 141 +++++++++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 36 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index ab0c4c50fd1..3032bea1a4f 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1268,7 +1268,7 @@ pub struct Block { /// CPython `basicblock.b_ialloc`, the allocated size of `b_instr`. instruction_allocation: usize, /// Exception stack at start of block, used by label_exception_targets (b_exceptstack) - except_stack: Option>, + except_stack: Option, /// CPython `basicblock.b_instr`, including allocated slots beyond `b_iused`. pub instructions: Vec, pub next: BlockIdx, @@ -1327,6 +1327,13 @@ impl Block { pub(crate) const START_DEPTH_UNSET: i32 = i32::MIN; const CO_MAXBLOCKS: usize = 20; +/// flowgraph.c struct _PyCfgExceptStack +#[derive(Clone, Debug)] +struct CfgExceptStack { + handlers: Vec, + depth: usize, +} + #[derive(Clone, Debug)] pub(crate) struct InstructionSequenceLabelMap { block_labels: Vec, @@ -6476,29 +6483,37 @@ fn resolve_line_numbers( } /// flowgraph.c make_except_stack -fn make_except_stack() -> crate::InternalResult> { - let mut stack = Vec::new(); - stack +fn make_except_stack() -> crate::InternalResult { + let mut handlers = Vec::new(); + handlers .try_reserve_exact(CO_MAXBLOCKS + 2) .map_err(|_| InternalError::MalformedControlFlowGraph)?; - debug_assert!(stack.is_empty()); - Ok(stack) + handlers.resize(CO_MAXBLOCKS + 2, BlockIdx::NULL); + debug_assert_eq!(handlers[0], BlockIdx::NULL); + Ok(CfgExceptStack { handlers, depth: 0 }) } /// flowgraph.c copy_except_stack -fn copy_except_stack(stack: &[BlockIdx]) -> crate::InternalResult> { - debug_assert!(stack.len() <= CO_MAXBLOCKS + 1); - let mut copy = Vec::new(); - copy.try_reserve_exact(CO_MAXBLOCKS + 2) +fn copy_except_stack(stack: &CfgExceptStack) -> crate::InternalResult { + debug_assert!(stack.depth <= CO_MAXBLOCKS + 1); + let mut handlers = Vec::new(); + handlers + .try_reserve_exact(CO_MAXBLOCKS + 2) .map_err(|_| InternalError::MalformedControlFlowGraph)?; - copy.extend_from_slice(stack); - Ok(copy) + handlers.extend_from_slice(&stack.handlers); + Ok(CfgExceptStack { + handlers, + depth: stack.depth, + }) } /// flowgraph.c except_stack_top -fn except_stack_top(stack: &[BlockIdx], blocks: &[Block]) -> Option { - debug_assert!(stack.len() <= CO_MAXBLOCKS + 1); - let handler_block = stack.last().copied()?; +fn except_stack_top(stack: &CfgExceptStack, blocks: &[Block]) -> Option { + debug_assert!(stack.depth <= CO_MAXBLOCKS + 1); + let handler_block = stack.handlers[stack.depth]; + if handler_block == BlockIdx::NULL { + return None; + } Some(ExceptHandlerInfo { handler_block, preserve_lasti: blocks[handler_block.idx()].preserve_lasti, @@ -6507,7 +6522,7 @@ fn except_stack_top(stack: &[BlockIdx], blocks: &[Block]) -> Option, + stack: &mut CfgExceptStack, setup: InstructionInfo, blocks: &mut [Block], ) -> Option { @@ -6521,17 +6536,18 @@ fn push_except_block( ) { blocks[target.idx()].preserve_lasti = true; } - debug_assert!(stack.len() <= CO_MAXBLOCKS); - stack.push(target); - debug_assert!(stack.len() <= CO_MAXBLOCKS + 1); + debug_assert!(stack.depth <= CO_MAXBLOCKS); + stack.depth += 1; + stack.handlers[stack.depth] = target; + debug_assert!(stack.depth <= CO_MAXBLOCKS + 1); except_stack_top(stack, blocks) } /// flowgraph.c pop_except_block -fn pop_except_block(stack: &mut Vec, blocks: &[Block]) -> Option { - debug_assert!(!stack.is_empty()); - stack.pop().expect("non-empty exception stack"); - debug_assert!(stack.len() <= CO_MAXBLOCKS); +fn pop_except_block(stack: &mut CfgExceptStack, blocks: &[Block]) -> Option { + debug_assert!(stack.depth > 0); + stack.depth -= 1; + debug_assert!(stack.depth <= CO_MAXBLOCKS); except_stack_top(stack, blocks) } @@ -6545,12 +6561,15 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe while let Some(block_idx) = todo.pop() { let bi = block_idx.idx(); debug_assert!(blocks[bi].visited); - let mut stack = blocks[bi] - .except_stack - .take() - .expect("visited exception block has an except stack"); - let mut handler = except_stack_top(&stack, blocks); + let mut stack = Some( + blocks[bi] + .except_stack + .take() + .expect("visited exception block has an except stack"), + ); + let mut handler = except_stack_top(stack.as_ref().expect("active exception stack"), blocks); let mut last_yield_except_depth: i32 = -1; + let mut stack_transferred = false; let instr_count = blocks[bi].instruction_used; for i in 0..instr_count { @@ -6562,13 +6581,19 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe if is_block_push(&info) { debug_assert!(target != BlockIdx::NULL); if !blocks[target.idx()].visited { - blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)?); + blocks[target.idx()].except_stack = Some(copy_except_stack( + stack.as_ref().expect("active exception stack"), + )?); todo.push(target); blocks[target.idx()].visited = true; } - handler = push_except_block(&mut stack, info, blocks); + handler = push_except_block( + stack.as_mut().expect("active exception stack"), + info, + blocks, + ); } else if instr.is_pop_block() { - handler = pop_except_block(&mut stack, blocks); + handler = pop_except_block(stack.as_mut().expect("active exception stack"), blocks); set_to_nop(&mut blocks[bi].instructions[i]); } else if is_jump(&blocks[bi].instructions[i]) { blocks[bi].instructions[i].except_handler = handler; @@ -6580,16 +6605,23 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe debug_assert!(target != BlockIdx::NULL); if !blocks[target.idx()].visited { if bb_has_fallthrough(&blocks[bi]) { - blocks[target.idx()].except_stack = Some(copy_except_stack(&stack)?); + blocks[target.idx()].except_stack = Some(copy_except_stack( + stack.as_ref().expect("active exception stack"), + )?); } else { - blocks[target.idx()].except_stack = Some(core::mem::take(&mut stack)); + blocks[target.idx()].except_stack = stack.take(); + stack_transferred = true; + todo.push(target); + blocks[target.idx()].visited = true; + break; } todo.push(target); blocks[target.idx()].visited = true; } } else if matches!(instr.real(), Some(Instruction::YieldValue { .. })) { blocks[bi].instructions[i].except_handler = handler; - last_yield_except_depth = stack.len() as i32; + last_yield_except_depth = + stack.as_ref().expect("active exception stack").depth as i32; } else if let Some(Instruction::Resume { context: _ }) = instr.real() { blocks[bi].instructions[i].except_handler = handler; let resume_arg = u32::from(arg); @@ -6607,10 +6639,10 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) -> crate::InternalRe } let next = blocks[bi].next; - if bb_has_fallthrough(&blocks[bi]) { + if !stack_transferred && bb_has_fallthrough(&blocks[bi]) { debug_assert!(next != BlockIdx::NULL); if next != BlockIdx::NULL && !blocks[next.idx()].visited { - blocks[next.idx()].except_stack = Some(stack); + blocks[next.idx()].except_stack = stack.take(); todo.push(next); blocks[next.idx()].visited = true; } @@ -6872,6 +6904,43 @@ mod tests { ); } + #[test] + fn except_stack_tracks_cpython_depth_and_handler_slots() { + let mut stack = make_except_stack().unwrap(); + assert_eq!(stack.depth, 0); + assert_eq!(stack.handlers.len(), CO_MAXBLOCKS + 2); + assert_eq!(stack.handlers[0], BlockIdx::NULL); + + let mut blocks = vec![Block::default(), Block::default()]; + assert!(except_stack_top(&stack, &blocks).is_none()); + + let setup = InstructionInfo { + instr: PseudoInstruction::SetupWith { + delta: Arg::marker(), + } + .into(), + arg: OpArg::new(0), + target: BlockIdx::new(1), + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: None, + }; + let handler = push_except_block(&mut stack, setup, &mut blocks).unwrap(); + assert_eq!(stack.depth, 1); + assert_eq!(stack.handlers[1], BlockIdx::new(1)); + assert_eq!(handler.handler_block, BlockIdx::new(1)); + assert!(handler.preserve_lasti); + assert!(blocks[1].preserve_lasti); + + let copy = copy_except_stack(&stack).unwrap(); + assert_eq!(copy.depth, stack.depth); + assert_eq!(copy.handlers, stack.handlers); + + assert!(pop_except_block(&mut stack, &blocks).is_none()); + assert_eq!(stack.depth, 0); + } + #[test] fn instruction_sequence_insert_preserves_cpython_slot_metadata() { let handler = InstructionSequenceExceptHandlerInfo { From 069d44c6e3ac16f0126717843058f2601cfb22f9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:25:21 +0900 Subject: [PATCH 114/131] Match CPython ref stack structure --- crates/codegen/src/ir.rs | 87 +++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 23 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 3032bea1a4f..da6792d417c 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4075,7 +4075,10 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { .try_reserve_exact(max_instrs) .map_err(|_| InternalError::MalformedControlFlowGraph)?; instr_flags.resize(max_instrs, 0u8); - let mut refs = Vec::new(); + let mut refs = RefStack { + refs: Vec::new(), + size: 0, + }; let mut worklist = make_cfg_traversal_stack(blocks)?; worklist.push(BlockIdx(0)); blocks[0].start_depth = 0; @@ -4147,14 +4150,14 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { AnyInstruction::Real(Instruction::Copy { i: _ }) => { let depth = arg_u32 as usize; assert!(depth > 0); - assert!(refs.len() >= depth); - let r = ref_stack_at(&refs, refs.len() - depth); + assert!(refs.size >= depth); + let r = ref_stack_at(&refs, refs.size - depth); push_ref(&mut refs, r.instr, r.local)?; } AnyInstruction::Real(Instruction::Swap { i: _ }) => { let depth = arg_u32 as usize; assert!(depth >= 2); - assert!(refs.len() >= depth); + assert!(refs.size >= depth); ref_stack_swap_top(&mut refs, depth); } AnyInstruction::Real( @@ -4211,7 +4214,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { AnyInstruction::Real(Instruction::ForIter { .. }) => { let target = info.target; debug_assert!(target != BlockIdx::NULL); - load_fast_push_block(&mut worklist, blocks, target, refs.len() + 1); + load_fast_push_block(&mut worklist, blocks, target, refs.size + 1); push_ref(&mut refs, i as isize, NOT_LOCAL)?; } AnyInstruction::Real( @@ -4237,7 +4240,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { AnyInstruction::Real(Instruction::Send { .. }) => { let target = info.target; debug_assert!(target != BlockIdx::NULL); - load_fast_push_block(&mut worklist, blocks, target, refs.len()); + load_fast_push_block(&mut worklist, blocks, target, refs.size); let _ = ref_stack_pop(&mut refs); push_ref(&mut refs, i as isize, NOT_LOCAL)?; } @@ -4248,8 +4251,8 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let target = info.target; if instr.has_target() { debug_assert!(target != BlockIdx::NULL); - debug_assert!(refs.len() >= num_popped); - let target_depth = refs.len() - num_popped + num_pushed; + debug_assert!(refs.size >= num_popped); + let target_depth = refs.size - num_popped + num_pushed; load_fast_push_block(&mut worklist, blocks, target, target_depth); } if !is_block_push(&info) { @@ -4272,10 +4275,10 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { && !term.instr.is_scope_exit() { debug_assert!(bb_has_fallthrough(&blocks[block_i])); - load_fast_push_block(&mut worklist, blocks, fallthrough, refs.len()); + load_fast_push_block(&mut worklist, blocks, fallthrough, refs.size); } - for i in 0..refs.len() { + for i in 0..refs.size { let r = ref_stack_at(&refs, i); if r.instr != DUMMY_INSTR { instr_flags[r.instr as usize] |= LoadFastInstrFlag::RefUnconsumed as u8; @@ -4644,42 +4647,51 @@ struct Ref { } /// flowgraph.c ref_stack -type RefStack = Vec; +struct RefStack { + refs: Vec, + size: usize, +} /// flowgraph.c ref_stack_push fn ref_stack_push(stack: &mut RefStack, r: Ref) -> crate::InternalResult<()> { - if stack.len() == stack.capacity() { - let doubled = stack.capacity() * 2; + if stack.size == stack.refs.len() { + let doubled = stack.refs.len() * 2; let new_cap = 32.max(doubled); stack - .try_reserve_exact(new_cap - stack.capacity()) + .refs + .try_reserve_exact(new_cap - stack.refs.len()) .map_err(|_| InternalError::MalformedControlFlowGraph)?; + stack.refs.resize(new_cap, Ref { instr: 0, local: 0 }); } - stack.push(r); + stack.refs[stack.size] = r; + stack.size += 1; Ok(()) } /// flowgraph.c ref_stack_pop fn ref_stack_pop(stack: &mut RefStack) -> Ref { - stack.pop().expect("ref stack underflow") + assert!(stack.size > 0); + stack.size -= 1; + stack.refs[stack.size] } /// flowgraph.c ref_stack_swap_top fn ref_stack_swap_top(stack: &mut RefStack, off: usize) { - assert!(off >= 2 && stack.len() >= off); - let top = stack.len() - 1; - let other = stack.len() - off; - stack.swap(top, other); + assert!(off >= 2 && stack.size >= off); + let top = stack.size - 1; + let other = stack.size - off; + stack.refs.swap(top, other); } /// flowgraph.c ref_stack_at fn ref_stack_at(stack: &RefStack, idx: usize) -> Ref { - stack.get(idx).copied().expect("ref stack index in bounds") + assert!(idx < stack.size); + stack.refs[idx] } /// flowgraph.c ref_stack_clear fn ref_stack_clear(stack: &mut RefStack) { - stack.clear(); + stack.size = 0; } /// flowgraph.c optimize_load_fast PUSH_REF @@ -4689,7 +4701,7 @@ fn push_ref(stack: &mut RefStack, instr: isize, local: isize) -> crate::Internal /// flowgraph.c kill_local fn kill_local(instr_flags: &mut [u8], refs: &RefStack, local: isize) { - for i in 0..refs.len() { + for i in 0..refs.size { let r = ref_stack_at(refs, i); if r.local != local { continue; @@ -6941,6 +6953,35 @@ mod tests { assert_eq!(stack.depth, 0); } + #[test] + fn ref_stack_tracks_cpython_size_and_allocated_refs() { + let mut stack = RefStack { + refs: Vec::new(), + size: 0, + }; + ref_stack_push(&mut stack, Ref { instr: 7, local: 3 }).unwrap(); + assert_eq!(stack.size, 1); + assert_eq!(stack.refs.len(), 32); + assert_eq!(ref_stack_at(&stack, 0).instr, 7); + assert_eq!(ref_stack_at(&stack, 0).local, 3); + + ref_stack_clear(&mut stack); + assert_eq!(stack.size, 0); + assert_eq!(stack.refs.len(), 32); + + ref_stack_push( + &mut stack, + Ref { + instr: DUMMY_INSTR, + local: NOT_LOCAL, + }, + ) + .unwrap(); + assert_eq!(stack.size, 1); + assert_eq!(ref_stack_pop(&mut stack).instr, DUMMY_INSTR); + assert_eq!(stack.size, 0); + } + #[test] fn instruction_sequence_insert_preserves_cpython_slot_metadata() { let handler = InstructionSequenceExceptHandlerInfo { From 45ef619695569cccade874c83c920bce6405f1c2 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:30:48 +0900 Subject: [PATCH 115/131] Match CPython CFG traversal stack structure --- crates/codegen/src/ir.rs | 55 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index da6792d417c..5203594456e 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1334,6 +1334,26 @@ struct CfgExceptStack { depth: usize, } +/// flowgraph.c `basicblock **stack` +#[derive(Clone, Debug)] +struct CfgTraversalStack { + stack: Vec, +} + +impl CfgTraversalStack { + fn push(&mut self, block: BlockIdx) { + self.stack.push(block); + } + + fn pop(&mut self) -> Option { + self.stack.pop() + } + + fn capacity(&self) -> usize { + self.stack.capacity() + } +} + #[derive(Clone, Debug)] pub(crate) struct InstructionSequenceLabelMap { block_labels: Vec, @@ -4725,7 +4745,7 @@ fn local_as_ref_local(local: usize) -> isize { /// flowgraph.c load_fast_push_block fn load_fast_push_block( - worklist: &mut Vec, + worklist: &mut CfgTraversalStack, blocks: &mut [Block], target: BlockIdx, start_depth: usize, @@ -4740,7 +4760,7 @@ fn load_fast_push_block( } fn stackdepth_push( - stack: &mut Vec, + stack: &mut CfgTraversalStack, blocks: &mut [Block], target: BlockIdx, depth: i32, @@ -5801,7 +5821,7 @@ fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResul } /// flowgraph.c make_cfg_traversal_stack -fn make_cfg_traversal_stack(blocks: &mut [Block]) -> crate::InternalResult> { +fn make_cfg_traversal_stack(blocks: &mut [Block]) -> crate::InternalResult { debug_assert!(!blocks.is_empty()); let mut nblocks = 0; let mut current = BlockIdx(0); @@ -5815,6 +5835,7 @@ fn make_cfg_traversal_stack(blocks: &mut [Block]) -> crate::InternalResult= nblocks); Ok(stack) } @@ -6097,7 +6118,7 @@ fn cfg_from_instruction_sequence( /// flowgraph.c maybe_push fn maybe_push( blocks: &mut [Block], - worklist: &mut Vec, + worklist: &mut CfgTraversalStack, block: BlockIdx, unsafe_mask: u64, ) { @@ -6115,7 +6136,11 @@ fn maybe_push( } /// flowgraph.c scan_block_for_locals -fn scan_block_for_locals(blocks: &mut [Block], block_idx: BlockIdx, worklist: &mut Vec) { +fn scan_block_for_locals( + blocks: &mut [Block], + block_idx: BlockIdx, + worklist: &mut CfgTraversalStack, +) { let idx = block_idx.idx(); let mut unsafe_mask = blocks[idx].unsafe_locals_mask; let instr_count = blocks[idx].instruction_used; @@ -6982,6 +7007,26 @@ mod tests { assert_eq!(stack.size, 0); } + #[test] + fn cfg_traversal_stack_resets_visited_and_allocates_for_blocks() { + let mut blocks = vec![Block::default(), Block::default()]; + blocks[0].next = BlockIdx::new(1); + blocks[0].visited = true; + blocks[1].visited = true; + + let mut stack = make_cfg_traversal_stack(&mut blocks).unwrap(); + assert!(!blocks[0].visited); + assert!(!blocks[1].visited); + assert!(stack.capacity() >= 2); + assert_eq!(stack.pop(), None); + + stack.push(BlockIdx::new(1)); + stack.push(BlockIdx::new(0)); + assert_eq!(stack.pop(), Some(BlockIdx::new(0))); + assert_eq!(stack.pop(), Some(BlockIdx::new(1))); + assert_eq!(stack.pop(), None); + } + #[test] fn instruction_sequence_insert_preserves_cpython_slot_metadata() { let handler = InstructionSequenceExceptHandlerInfo { From c947c0506f526dcec09d22821a6fb992ba8bf085 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:32:20 +0900 Subject: [PATCH 116/131] Mirror CPython CFG traversal stack pointer --- crates/codegen/src/ir.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 5203594456e..b8e3f630454 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1338,19 +1338,26 @@ struct CfgExceptStack { #[derive(Clone, Debug)] struct CfgTraversalStack { stack: Vec, + sp: usize, } impl CfgTraversalStack { fn push(&mut self, block: BlockIdx) { - self.stack.push(block); + debug_assert!(self.sp < self.stack.len()); + self.stack[self.sp] = block; + self.sp += 1; } fn pop(&mut self) -> Option { - self.stack.pop() + if self.sp == 0 { + return None; + } + self.sp -= 1; + Some(self.stack[self.sp]) } fn capacity(&self) -> usize { - self.stack.capacity() + self.stack.len() } } @@ -5835,8 +5842,9 @@ fn make_cfg_traversal_stack(blocks: &mut [Block]) -> crate::InternalResult= nblocks); + stack.resize(nblocks, BlockIdx::NULL); + let stack = CfgTraversalStack { stack, sp: 0 }; + debug_assert_eq!(stack.capacity(), nblocks); Ok(stack) } From 82d90b44957adb7f749d79fc72d436bb69ee55f6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:35:27 +0900 Subject: [PATCH 117/131] Use CPython fixed exception handler stack --- crates/codegen/src/ir.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index b8e3f630454..22c5e89a007 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1330,7 +1330,7 @@ const CO_MAXBLOCKS: usize = 20; /// flowgraph.c struct _PyCfgExceptStack #[derive(Clone, Debug)] struct CfgExceptStack { - handlers: Vec, + handlers: [BlockIdx; CO_MAXBLOCKS + 2], depth: usize, } @@ -6528,26 +6528,19 @@ fn resolve_line_numbers( } /// flowgraph.c make_except_stack +#[allow(clippy::unnecessary_wraps)] fn make_except_stack() -> crate::InternalResult { - let mut handlers = Vec::new(); - handlers - .try_reserve_exact(CO_MAXBLOCKS + 2) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - handlers.resize(CO_MAXBLOCKS + 2, BlockIdx::NULL); + let handlers = [BlockIdx::NULL; CO_MAXBLOCKS + 2]; debug_assert_eq!(handlers[0], BlockIdx::NULL); Ok(CfgExceptStack { handlers, depth: 0 }) } /// flowgraph.c copy_except_stack +#[allow(clippy::unnecessary_wraps)] fn copy_except_stack(stack: &CfgExceptStack) -> crate::InternalResult { debug_assert!(stack.depth <= CO_MAXBLOCKS + 1); - let mut handlers = Vec::new(); - handlers - .try_reserve_exact(CO_MAXBLOCKS + 2) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - handlers.extend_from_slice(&stack.handlers); Ok(CfgExceptStack { - handlers, + handlers: stack.handlers, depth: stack.depth, }) } From d4130eb40d7ba39221e7cfcc4b1b3a84ac8b2639 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:36:32 +0900 Subject: [PATCH 118/131] Mirror CPython ref stack capacity field --- crates/codegen/src/ir.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 22c5e89a007..278c14c2eeb 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4105,6 +4105,7 @@ fn optimize_load_fast(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut refs = RefStack { refs: Vec::new(), size: 0, + capacity: 0, }; let mut worklist = make_cfg_traversal_stack(blocks)?; worklist.push(BlockIdx(0)); @@ -4677,18 +4678,21 @@ struct Ref { struct RefStack { refs: Vec, size: usize, + capacity: usize, } /// flowgraph.c ref_stack_push fn ref_stack_push(stack: &mut RefStack, r: Ref) -> crate::InternalResult<()> { - if stack.size == stack.refs.len() { - let doubled = stack.refs.len() * 2; + debug_assert_eq!(stack.refs.len(), stack.capacity); + if stack.size == stack.capacity { + let doubled = stack.capacity * 2; let new_cap = 32.max(doubled); stack .refs - .try_reserve_exact(new_cap - stack.refs.len()) + .try_reserve_exact(new_cap - stack.capacity) .map_err(|_| InternalError::MalformedControlFlowGraph)?; stack.refs.resize(new_cap, Ref { instr: 0, local: 0 }); + stack.capacity = new_cap; } stack.refs[stack.size] = r; stack.size += 1; @@ -6984,15 +6988,18 @@ mod tests { let mut stack = RefStack { refs: Vec::new(), size: 0, + capacity: 0, }; ref_stack_push(&mut stack, Ref { instr: 7, local: 3 }).unwrap(); assert_eq!(stack.size, 1); + assert_eq!(stack.capacity, 32); assert_eq!(stack.refs.len(), 32); assert_eq!(ref_stack_at(&stack, 0).instr, 7); assert_eq!(ref_stack_at(&stack, 0).local, 3); ref_stack_clear(&mut stack); assert_eq!(stack.size, 0); + assert_eq!(stack.capacity, 32); assert_eq!(stack.refs.len(), 32); ref_stack_push( From c3d660cba89b9a397774e6a314ed7bd9912a5ab0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:38:25 +0900 Subject: [PATCH 119/131] Match CPython swap optimizer scratch stack --- crates/codegen/src/ir.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 278c14c2eeb..a937c2cd736 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3287,7 +3287,8 @@ fn optimize_lists_and_sets( Ok(true) } -const SWAP_OPTIMIZE_VISITED: i32 = -1; +/// flowgraph.c VISITED +const VISITED: i32 = -1; /// flowgraph.c SWAPPABLE fn is_swappable(instr: &AnyInstruction) -> bool { @@ -3365,9 +3366,10 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::In stack .try_reserve_exact(depth) .map_err(|_| InternalError::MalformedControlFlowGraph)?; + stack.resize(depth, 0); let mut i = 0; while i < depth { - stack.push(i as i32); + stack[i] = i as i32; i += 1; } @@ -3383,7 +3385,7 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::In let mut current = len as isize - 1; for i in 0..depth { - if stack[i] == SWAP_OPTIMIZE_VISITED || stack[i] == i as i32 { + if stack[i] == VISITED || stack[i] == i as i32 { continue; } let mut j = i; @@ -3395,12 +3397,12 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::In out.arg = OpArg::new((j + 1) as u32); current -= 1; } - if stack[j] == SWAP_OPTIMIZE_VISITED { + if stack[j] == VISITED { debug_assert_eq!(j, i); break; } let next_j = stack[j] as usize; - stack[j] = SWAP_OPTIMIZE_VISITED; + stack[j] = VISITED; j = next_j; } } From d88e575e8b2181016c5606217c2bfd2cb30b616a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:39:54 +0900 Subject: [PATCH 120/131] Align static swap helpers with CPython blocks --- crates/codegen/src/ir.rs | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index a937c2cd736..2f8a9836a79 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3309,17 +3309,13 @@ fn stores_to(info: &InstructionInfo) -> i32 { } /// flowgraph.c next_swappable_instruction -fn next_swappable_instruction( - instructions: &[InstructionInfo], - mut i: usize, - lineno: i32, -) -> Option { +fn next_swappable_instruction(block: &Block, mut i: usize, lineno: i32) -> Option { loop { i += 1; - if i >= instructions.len() { + if i >= block.instruction_used { return None; } - let info = &instructions[i]; + let info = &block.instructions[i]; let info_lineno = instruction_lineno(info); if lineno >= 0 && info_lineno != lineno { return None; @@ -3416,17 +3412,18 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::In } /// flowgraph.c apply_static_swaps -fn apply_static_swaps(instructions: &mut [InstructionInfo], mut i: isize) { +fn apply_static_swaps(block: &mut Block, mut i: isize) { while i >= 0 { let idx = i as usize; - let swap_arg = match instructions[idx].instr.real() { - Some(Instruction::Swap { .. }) => u32::from(instructions[idx].arg), + debug_assert!(idx < block.instruction_used); + let swap_arg = match block.instructions[idx].instr.real() { + Some(Instruction::Swap { .. }) => u32::from(block.instructions[idx].arg), Some(Instruction::Nop | Instruction::PopTop | Instruction::StoreFast { .. }) => { i -= 1; continue; } _ if matches!( - instructions[idx].instr.pseudo(), + block.instructions[idx].instr.pseudo(), Some(PseudoInstruction::StoreFastMaybeNull { .. }) ) => { @@ -3436,27 +3433,27 @@ fn apply_static_swaps(instructions: &mut [InstructionInfo], mut i: isize) { _ => return, }; - let Some(j) = next_swappable_instruction(instructions, idx, -1) else { + let Some(j) = next_swappable_instruction(block, idx, -1) else { return; }; - let lineno = instruction_lineno(&instructions[j]); + let lineno = instruction_lineno(&block.instructions[j]); let mut k = j; for _ in 1..swap_arg { - let Some(next) = next_swappable_instruction(instructions, k, lineno) else { + let Some(next) = next_swappable_instruction(block, k, lineno) else { return; }; k = next; } - let store_j = stores_to(&instructions[j]); - let store_k = stores_to(&instructions[k]); + let store_j = stores_to(&block.instructions[j]); + let store_k = stores_to(&block.instructions[k]); if store_j >= 0 || store_k >= 0 { if store_j == store_k { return; } let mut idx = j + 1; while idx < k { - let store_idx = stores_to(&instructions[idx]); + let store_idx = stores_to(&block.instructions[idx]); if store_idx >= 0 && (store_idx == store_j || store_idx == store_k) { return; } @@ -3464,8 +3461,8 @@ fn apply_static_swaps(instructions: &mut [InstructionInfo], mut i: isize) { } } - set_to_nop(&mut instructions[idx]); - instructions.swap(j, k); + set_to_nop(&mut block.instructions[idx]); + block.instructions.swap(j, k); i -= 1; } } @@ -3478,9 +3475,8 @@ fn apply_static_swaps_block(block: &mut Block) -> crate::InternalResult<()> { block.instructions[i].instr.real(), Some(Instruction::Swap { .. }) ) { - let instructions = &mut block.instructions[..block.instruction_used]; - swaptimize(instructions, &mut i)?; - apply_static_swaps(instructions, i as isize); + swaptimize(&mut block.instructions[..block.instruction_used], &mut i)?; + apply_static_swaps(block, i as isize); } i += 1; } From 9abe87b1895f52faaa369f529852e740947665c2 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:41:34 +0900 Subject: [PATCH 121/131] Align swaptimize signature with CPython --- crates/codegen/src/ir.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 2f8a9836a79..9a015161510 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -3331,19 +3331,19 @@ fn next_swappable_instruction(block: &Block, mut i: usize, lineno: i32) -> Optio } /// flowgraph.c swaptimize -fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::InternalResult<()> { +fn swaptimize(block: &mut Block, ix: &mut usize) -> crate::InternalResult<()> { debug_assert!(matches!( - instructions[*ix].instr.real(), + block.instructions[*ix].instr.real(), Some(Instruction::Swap { .. }) )); - let mut depth = u32::from(instructions[*ix].arg) as usize; + let mut depth = u32::from(block.instructions[*ix].arg) as usize; let mut len = 1usize; let mut more = false; - let limit = instructions.len() - *ix; + let limit = block.instruction_used - *ix; while len < limit { - match instructions[*ix + len].instr.real() { + match block.instructions[*ix + len].instr.real() { Some(Instruction::Swap { .. }) => { - depth = depth.max(u32::from(instructions[*ix + len].arg) as usize); + depth = depth.max(u32::from(block.instructions[*ix + len].arg) as usize); more = true; len += 1; } @@ -3371,7 +3371,7 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::In i = 0; while i < len { - let info = &instructions[*ix + i]; + let info = &block.instructions[*ix + i]; if matches!(info.instr.real(), Some(Instruction::Swap { .. })) { let oparg = u32::from(info.arg) as usize; stack.swap(0, oparg - 1); @@ -3388,7 +3388,7 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::In loop { if j != 0 { debug_assert!(current >= 0); - let out = &mut instructions[*ix + current as usize]; + let out = &mut block.instructions[*ix + current as usize]; out.instr = Opcode::Swap.into(); out.arg = OpArg::new((j + 1) as u32); current -= 1; @@ -3404,7 +3404,7 @@ fn swaptimize(instructions: &mut [InstructionInfo], ix: &mut usize) -> crate::In } while current >= 0 { - set_to_nop(&mut instructions[*ix + current as usize]); + set_to_nop(&mut block.instructions[*ix + current as usize]); current -= 1; } *ix += len - 1; @@ -3475,7 +3475,7 @@ fn apply_static_swaps_block(block: &mut Block) -> crate::InternalResult<()> { block.instructions[i].instr.real(), Some(Instruction::Swap { .. }) ) { - swaptimize(&mut block.instructions[..block.instruction_used], &mut i)?; + swaptimize(block, &mut i)?; apply_static_swaps(block, i as isize); } i += 1; From 5673b2069fdcac2e03f16caaf6c88764bd1e5c4c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:44:25 +0900 Subject: [PATCH 122/131] Match CPython redundant pair pass result --- crates/codegen/src/ir.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 9a015161510..56d2c386e95 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1815,7 +1815,7 @@ fn optimize_cfg( optimize_basic_block(blocks, metadata, block_idx)?; block_idx = next_block; } - remove_redundant_nops_and_pairs(blocks); + remove_redundant_nops_and_pairs(blocks)?; // CPython optimize_cfg() removes newly-unreachable blocks and // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes // unused constants. @@ -3921,7 +3921,8 @@ fn optimize_basic_block( /// flowgraph.c remove_redundant_nops_and_pairs #[allow(clippy::if_same_then_else, clippy::useless_let_if_seq)] -fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) { +#[allow(clippy::unnecessary_wraps)] +fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) -> crate::InternalResult<()> { let mut done = false; while !done { @@ -3985,6 +3986,7 @@ fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) { block_idx = block.next; } } + Ok(()) } /// flowgraph.c remove_unused_consts @@ -4485,7 +4487,7 @@ impl CodeInfo { "after_optimize_basic_block".to_owned(), self.debug_block_dump(), )); - remove_redundant_nops_and_pairs(&mut self.blocks); + remove_redundant_nops_and_pairs(&mut self.blocks)?; remove_unreachable(&mut self.blocks)?; remove_redundant_nops_and_jumps(&mut self.blocks)?; #[cfg(debug_assertions)] From 4cf0668e3e160cf44f41a1a40d37f0c6b4923fd0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:45:24 +0900 Subject: [PATCH 123/131] Match CPython inline pass result --- crates/codegen/src/ir.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 56d2c386e95..c7bbeb65ebe 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -5654,7 +5654,7 @@ fn basicblock_inline_small_or_no_lineno_blocks( } /// flowgraph.c inline_small_or_no_lineno_blocks -fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) -> crate::InternalResult<()> { +fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) -> crate::InternalResult { loop { let mut changes = false; let mut current = BlockIdx(0); @@ -5668,10 +5668,9 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) -> crate::InternalResu current = next; } if !changes { - break; + return Ok(changes); } } - Ok(()) } /// flowgraph.c basicblock_remove_redundant_nops From 84c5bd67e8a3697acced7aa803a2b2182eb89d1d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:50:03 +0900 Subject: [PATCH 124/131] Match CPython redundant NOP pass results --- crates/codegen/src/ir.rs | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index c7bbeb65ebe..1c98a9ec019 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1780,7 +1780,7 @@ fn optimize_code_unit( // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before // later jump normalization / block reordering can create adjacencies // that never exist at this stage in flowgraph.c. - insert_superinstructions(blocks); + insert_superinstructions(blocks)?; push_cold_blocks_to_end(blocks)?; // CPython resolves line numbers again after cold-block extraction. resolve_line_numbers(blocks, metadata.firstlineno)?; @@ -3931,7 +3931,7 @@ fn remove_redundant_nops_and_pairs(blocks: &mut [Block]) -> crate::InternalResul let mut block_idx = BlockIdx::new(0); while block_idx != BlockIdx::NULL { - basicblock_remove_redundant_nops(blocks, block_idx); + basicblock_remove_redundant_nops(blocks, block_idx)?; if is_label(blocks[block_idx.idx()].cpython_label) { instr = None; } @@ -4500,7 +4500,7 @@ impl CodeInfo { let nlocals = self.metadata.varnames.len(); let nparams = self.nparams; add_checks_for_loads_of_uninitialized_variables(&mut self.blocks, nlocals, nparams)?; - insert_superinstructions(&mut self.blocks); + insert_superinstructions(&mut self.blocks)?; push_cold_blocks_to_end(&mut self.blocks)?; trace.push(( "after_push_cold_before_chain_reorder".to_owned(), @@ -4600,7 +4600,7 @@ fn make_super_instruction( } /// flowgraph.c insert_superinstructions -fn insert_superinstructions(blocks: &mut [Block]) -> usize { +fn insert_superinstructions(blocks: &mut [Block]) -> crate::InternalResult { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next_block = blocks[block_idx.idx()].next; @@ -4653,10 +4653,10 @@ fn insert_superinstructions(blocks: &mut [Block]) -> usize { } block_idx = next_block; } - let res = remove_redundant_nops(blocks); + let res = remove_redundant_nops(blocks)?; #[cfg(debug_assertions)] assert!(no_redundant_nops(blocks)); - res + Ok(res) } /// flowgraph.c LoadFastInstrFlag @@ -5674,7 +5674,11 @@ fn inline_small_or_no_lineno_blocks(blocks: &mut [Block]) -> crate::InternalResu } /// flowgraph.c basicblock_remove_redundant_nops -fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) -> usize { +#[allow(clippy::unnecessary_wraps)] +fn basicblock_remove_redundant_nops( + blocks: &mut [Block], + block_idx: BlockIdx, +) -> crate::InternalResult { let bi = block_idx.idx(); let mut dest = 0; let mut prev_lineno = -1i32; @@ -5738,25 +5742,30 @@ fn basicblock_remove_redundant_nops(blocks: &mut [Block], block_idx: BlockIdx) - debug_assert!(dest <= instr_count); let num_removed = instr_count - dest; blocks[bi].instruction_used = dest; - num_removed + Ok(num_removed) } /// flowgraph.c remove_redundant_nops -fn remove_redundant_nops(blocks: &mut [Block]) -> usize { +#[allow(clippy::unnecessary_wraps)] +fn remove_redundant_nops(blocks: &mut [Block]) -> crate::InternalResult { let mut changes = 0; let mut current = BlockIdx(0); while current != BlockIdx::NULL { let next = blocks[current.idx()].next; - changes += basicblock_remove_redundant_nops(blocks, current); + let change = basicblock_remove_redundant_nops(blocks, current)?; + changes += change; current = next; } - changes + Ok(changes) } /// flowgraph.c no_redundant_nops #[cfg(debug_assertions)] fn no_redundant_nops(blocks: &mut [Block]) -> bool { - remove_redundant_nops(blocks) == 0 + match remove_redundant_nops(blocks) { + Ok(0) => true, + Ok(_) | Err(_) => false, + } } /// flowgraph.c remove_redundant_jumps @@ -5821,7 +5830,7 @@ fn remove_redundant_nops_and_jumps(blocks: &mut [Block]) -> crate::InternalResul loop { // Convergence is guaranteed because the number of redundant jumps and // nops only decreases. - let removed_nops = remove_redundant_nops(blocks); + let removed_nops = remove_redundant_nops(blocks)?; let removed_jumps = remove_redundant_jumps(blocks)?; if removed_nops + removed_jumps == 0 { break; From d27e3ae6d1ed171cbc6d02dca291d8b67a3f136a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 16:59:48 +0900 Subject: [PATCH 125/131] Fix bytecode metadata after upstream rebase --- crates/codegen/src/ir.rs | 5 ++++- crates/compiler-core/src/bytecode/opcode_metadata.rs | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 1c98a9ec019..aa10c18f307 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -4801,7 +4801,10 @@ fn get_stack_effects( oparg: OpArg, jump: i32, ) -> crate::InternalResult { - if instr.real().is_some_and(|op| op.deopt().is_some()) { + if instr + .real() + .is_some_and(|op| op.as_opcode().deopt().is_some()) + { return Err(InternalError::InvalidStackEffect); } let oparg = u32::from(oparg); diff --git a/crates/compiler-core/src/bytecode/opcode_metadata.rs b/crates/compiler-core/src/bytecode/opcode_metadata.rs index 2b48500c251..ccd49917f87 100644 --- a/crates/compiler-core/src/bytecode/opcode_metadata.rs +++ b/crates/compiler-core/src/bytecode/opcode_metadata.rs @@ -720,6 +720,9 @@ impl super::PseudoOpcode { | Self::JumpIfTrue | Self::JumpNoInterrupt | Self::LoadClosure + | Self::SetupCleanup + | Self::SetupFinally + | Self::SetupWith | Self::StoreFastMaybeNull ) } From 4d11097a28ad81b7c86c969e22fe12bd72a956f4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 17:03:59 +0900 Subject: [PATCH 126/131] Match CPython opcode stack metadata --- .../compiler-core/src/bytecode/instruction.rs | 21 ++++++++++++++----- .../src/bytecode/opcode_metadata.rs | 20 ++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 802dd1ece2b..8e539efb7a2 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -63,11 +63,6 @@ macro_rules! define_opcodes { $(Self::$op_name => $op_id,)* } } - /// Stack effect of [`Self::stack_effect_info`]. - #[must_use] - $opcode_vis fn stack_effect(&self, oparg: u32) -> i32 { - self.stack_effect_info(oparg).effect() - } } impl From<$opcode_name> for $instr_name { @@ -770,6 +765,12 @@ impl Opcode { false } + /// Stack effect of [`Self::stack_effect_info`]. + #[must_use] + pub fn stack_effect(&self, oparg: u32) -> i32 { + self.stack_effect_info(oparg).effect() + } + /// Stack effect when the instruction takes its branch (jump=true). /// /// CPython equivalent: `stack_effect(opcode, oparg, jump=True)`. @@ -830,6 +831,16 @@ impl PseudoOpcode { self.has_jump() || self.is_block_push() } + /// flowgraph.c get_stack_effects block-push non-jump case. + #[must_use] + pub fn stack_effect(&self, oparg: u32) -> i32 { + if self.is_block_push() { + 0 + } else { + self.stack_effect_info(oparg).effect() + } + } + /// Handler entry effect for SETUP_* pseudo ops. /// /// Fallthrough effect is 0 (NOPs), but when the branch is taken the diff --git a/crates/compiler-core/src/bytecode/opcode_metadata.rs b/crates/compiler-core/src/bytecode/opcode_metadata.rs index ccd49917f87..1cf8126fa6c 100644 --- a/crates/compiler-core/src/bytecode/opcode_metadata.rs +++ b/crates/compiler-core/src/bytecode/opcode_metadata.rs @@ -441,10 +441,7 @@ impl super::Opcode { Self::UnaryInvert => (1, 1), Self::UnaryNegative => (1, 1), Self::UnaryNot => (1, 1), - Self::WithExceptStart => ( - 7, // TODO: Differs from CPython `6` - 6, // TODO: Differs from CPython `5` - ), + Self::WithExceptStart => (6, 5), Self::BinaryOp => (1, 2), Self::BuildInterpolation => (1, 2 + (oparg & 1)), Self::BuildList => (1, oparg), @@ -775,18 +772,9 @@ impl super::PseudoOpcode { Self::JumpNoInterrupt => (0, 0), Self::LoadClosure => (1, 0), Self::PopBlock => (0, 0), - Self::SetupCleanup => ( - 0, // TODO: Differs from CPython `2` - 0, - ), - Self::SetupFinally => ( - 0, // TODO: Differs from CPython `1` - 0, - ), - Self::SetupWith => ( - 0, // TODO: Differs from CPython `1` - 0, - ), + Self::SetupCleanup => (2, 0), + Self::SetupFinally => (1, 0), + Self::SetupWith => (1, 0), Self::StoreFastMaybeNull => (0, 1), }; From 17eb608cf4a6fd2b3607edfef9786b60c6ebe2ab Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 19:47:38 +0900 Subject: [PATCH 127/131] Add CPython identifiers to cspell dictionary Add CNOTAB, LNOTAB, ialloc, ioffset, iused, nblocks, ncellsused, ncellvars, nextop, noffsets, nvars, swaptimize, untargeted to .cspell.dict/cpython.txt for the new CFG/assembler code in crates/codegen/src/ir.rs. --- .cspell.dict/cpython.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index ffbed52121b..84265bda609 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -37,6 +37,7 @@ CFWS CLASSDEREF classdict cmpop +CNOTAB codedepth CODEUNIT CONIN @@ -98,12 +99,14 @@ HASUNION heaptype hexdigit HIGHRES +ialloc IFUNC IMMUTABLETYPE INCREF inlinedepth inplace inpos +ioffset isbytecode ishidden ismine @@ -111,6 +114,7 @@ ISPOINTER isoctal iteminfo Itertool +iused keeped kwnames kwonlyarg @@ -122,6 +126,7 @@ linearise lineful lineiterator linetable +LNOTAB loadfast localsplus localspluskinds @@ -137,17 +142,22 @@ multibytecodec nameobj nameop nargsf +nblocks ncells +ncellsused +ncellvars nconsts newargs newfree NEWLOCALS newsemlockobject +nextop nfrees nkwargs nkwelts nlocalsplus nointerrupt +noffsets Nondescriptor noninteger nops @@ -155,6 +165,7 @@ noraise nseen NSIGNALS numer +nvars opname opnames orelse @@ -220,6 +231,7 @@ subparams subscr sval swappedbytes +swaptimize sysdict tbstderr templatelib @@ -240,6 +252,7 @@ uncollectable Unhandle unparse unparser +untargeted untracking VARKEYWORDS varkwarg From 4390e6b9cecb350cc5812ec45f188ce55ee27f0f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 20:30:43 +0900 Subject: [PATCH 128/131] Fix CI failures from bytecode-parity work - clippy: drop redundant `test_` prefix on three test functions and remove an unnecessary `u32` cast in basicblock_clear_reuses_cpython_spare_slots_in_offset_order - insta: regenerate nested_double_async_with snapshot to match the new CFG output that drops unreferenced labels after the redundant-NOP pass - regrtest: drop `@expectedFailure` markers from test_func_args, test_meth_args (test_compile), test_disassemble_with, test_disassemble_try_finally (test_dis), and test_except_star (test_monitoring) which now pass --- Lib/test/test_compile.py | 2 -- Lib/test/test_dis.py | 2 -- Lib/test/test_monitoring.py | 1 - crates/codegen/src/compile.rs | 2 +- crates/codegen/src/ir.rs | 2 +- ...code__tests__nested_double_async_with.snap | 26 +++++++++---------- crates/vm/src/vm/python_run.rs | 4 +-- 7 files changed, 17 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 052d2bfc041..4d117be1b88 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2580,7 +2580,6 @@ def test_set(self): def test_dict(self): self.check_stack_size("{" + "x:x, " * self.N + "x:x}") - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_func_args(self): self.check_stack_size("f(" + "x, " * self.N + ")") @@ -2588,7 +2587,6 @@ def test_func_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("f(" + ", ".join(kwargs) + ")") - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_meth_args(self): self.check_stack_size("o.m(" + "x, " * self.N + ")") diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index cedad5a0fba..7360ead1144 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1211,14 +1211,12 @@ def test_disassemble_coroutine(self): def test_disassemble_fstring(self): self.do_disassembly_test(_fstring, dis_fstring) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_disassemble_with(self): self.do_disassembly_test(_with, dis_with) def test_disassemble_asyncwith(self): self.do_disassembly_test(_asyncwith, dis_asyncwith) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_disassemble_try_finally(self): self.do_disassembly_test(_tryfinally, dis_tryfinally) self.do_disassembly_test(_tryfinallyconst, dis_tryfinallyconst) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 5125701202b..1f72b552c6c 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1624,7 +1624,6 @@ def whilefunc(n=0): ('branch left', 'func', 44, 50), ('branch right', 'func', 28, 70)]) - @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython def test_except_star(self): class Foo: diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 5de5084beab..c3b86ce46bc 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -33823,7 +33823,7 @@ async def f(self, asyncio, sys, task, timeout_handle, sleep): } #[test] - fn test_try_else_attribute_probe_end_allows_following_loads_borrow() { + fn try_else_attribute_probe_end_allows_following_loads_borrow() { let code = compile_exec( "\ def f(self): diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index aa10c18f307..e7087a211e4 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -7208,7 +7208,7 @@ mod tests { for i in 0..3 { let mut stale = test_instr(Instruction::Nop, 35 + i); stale.except_handler = Some(ExceptHandlerInfo { - handler_block: BlockIdx::new(i as u32 + 1), + handler_block: BlockIdx::new(i + 1), preserve_lasti: false, }); test_block_push(&mut block, stale); diff --git a/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__nested_double_async_with.snap b/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__nested_double_async_with.snap index 34b5ce7e5c9..1b0ca25c15d 100644 --- a/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__nested_double_async_with.snap +++ b/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__nested_double_async_with.snap @@ -1,5 +1,6 @@ --- source: crates/stdlib/src/_opcode.rs +assertion_line: 300 expression: "dis(r#\"\nasync def test():\n for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\"#)" --- 0 RESUME 0 @@ -146,29 +147,29 @@ Disassembly of ", line 1>: L34: PUSH_EXC_INFO WITH_EXCEPT_START TO_BOOL - POP_JUMP_IF_TRUE 2 (to L37) - L35: NOT_TAKEN - L36: RERAISE 2 - L37: POP_TOP - L38: POP_EXCEPT + POP_JUMP_IF_TRUE 2 (to L35) + NOT_TAKEN + RERAISE 2 + L35: POP_TOP + L36: POP_EXCEPT POP_TOP POP_TOP POP_TOP JUMP_BACKWARD 205 (to L2) - -- L39: COPY 3 + -- L37: COPY 3 POP_EXCEPT RERAISE 1 - L40: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + L38: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) RERAISE 1 ExceptionTable: - L1 to L3 -> L40 [0] lasti + L1 to L3 -> L38 [0] lasti L3 to L4 -> L34 [3] lasti L5 to L7 -> L27 [3] L7 to L8 -> L12 [7] L8 to L10 -> L27 [3] L10 to L11 -> L14 [5] lasti - L11 to L12 -> L40 [0] lasti + L11 to L12 -> L38 [0] lasti L12 to L13 -> L27 [3] L14 to L16 -> L24 [7] lasti L16 to L17 -> L18 [10] @@ -181,7 +182,6 @@ ExceptionTable: L29 to L30 -> L34 [3] lasti L30 to L32 -> L32 [4] lasti L32 to L33 -> L34 [3] lasti - L33 to L34 -> L40 [0] lasti - L34 to L35 -> L39 [5] lasti - L36 to L38 -> L39 [5] lasti - L38 to L40 -> L40 [0] lasti + L33 to L34 -> L38 [0] lasti + L34 to L36 -> L37 [5] lasti + L36 to L38 -> L38 [0] lasti diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index a4142deb371..ccb295465ec 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -208,7 +208,7 @@ mod tests { } #[test] - fn test_block_expr_return_function_def() { + fn block_expr_return_function_def() { interpreter().enter(|vm| { let scope = vm.new_scope_with_builtins(); let value = @@ -220,7 +220,7 @@ mod tests { } #[test] - fn test_block_expr_return_class_def() { + fn block_expr_return_class_def() { interpreter().enter(|vm| { let scope = vm.new_scope_with_builtins(); let value = From b05b634fb838bfe316a84edc0fc8fe08b5d6bb13 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 21:29:51 +0900 Subject: [PATCH 129/131] Resync generated opcode metadata Empty conf.toml since WithExceptStart and Setup{Cleanup,Finally,With} stack effects already match CPython, so the TODO override entries are stale and only cause CI hook diffs. Regenerate opcode_metadata.rs and drop the matching SetupCleanup/ SetupFinally/SetupWith assertions on PseudoOpcode::has_arg(); their `HAS_ARG` flag comes from pseudo definitions in bytecodes.c that the upstream analyzer does not propagate through PseudoInstruction.properties, so the generated has_arg() excludes them. has_target() still covers these block-push pseudos via is_block_push(). --- crates/compiler-core/src/bytecode/instruction.rs | 5 ----- crates/compiler-core/src/bytecode/opcode_metadata.rs | 3 --- tools/opcode_metadata/conf.toml | 11 ----------- 3 files changed, 19 deletions(-) diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 8e539efb7a2..eaf055f9950 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -1430,14 +1430,9 @@ mod tests { assert!(PseudoOpcode::JumpIfTrue.has_arg()); assert!(PseudoOpcode::JumpNoInterrupt.has_arg()); assert!(PseudoOpcode::LoadClosure.has_arg()); - assert!(PseudoOpcode::SetupCleanup.has_arg()); - assert!(PseudoOpcode::SetupFinally.has_arg()); - assert!(PseudoOpcode::SetupWith.has_arg()); assert!(PseudoOpcode::StoreFastMaybeNull.has_arg()); assert!(!PseudoOpcode::AnnotationsPlaceholder.has_arg()); assert!(!PseudoOpcode::PopBlock.has_arg()); - - assert!(AnyInstruction::from(PseudoOpcode::SetupFinally).has_arg()); } #[test] diff --git a/crates/compiler-core/src/bytecode/opcode_metadata.rs b/crates/compiler-core/src/bytecode/opcode_metadata.rs index 1cf8126fa6c..2076b11661f 100644 --- a/crates/compiler-core/src/bytecode/opcode_metadata.rs +++ b/crates/compiler-core/src/bytecode/opcode_metadata.rs @@ -717,9 +717,6 @@ impl super::PseudoOpcode { | Self::JumpIfTrue | Self::JumpNoInterrupt | Self::LoadClosure - | Self::SetupCleanup - | Self::SetupFinally - | Self::SetupWith | Self::StoreFastMaybeNull ) } diff --git a/tools/opcode_metadata/conf.toml b/tools/opcode_metadata/conf.toml index 6b2f0afe195..e69de29bb2d 100644 --- a/tools/opcode_metadata/conf.toml +++ b/tools/opcode_metadata/conf.toml @@ -1,11 +0,0 @@ -[WithExceptStart] -stack_effect = { pushed = "7", popped = "6" } - -[SetupCleanup] -stack_effect = { pushed = "0" } - -[SetupFinally] -stack_effect = { pushed = "0" } - -[SetupWith] -stack_effect = { pushed = "0" } From bd9c09c33460ee65900f05e6dc33cf05d4108c66 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 22:23:26 +0900 Subject: [PATCH 130/131] Drop is_block_push has_arg invariant The CPython invariant `assert(OPCODE_HAS_ARG(op) || !IS_BLOCK_PUSH(op))` relies on SETUP_{FINALLY,CLEANUP,WITH} carrying `HAS_ARG_FLAG` in CPython's metadata. The autogen tool reads pseudo-opcode properties from target instructions and does not propagate the pseudo's own HAS_ARG flag, so PseudoOpcode::has_arg() omits these three opcodes. Drop the debug_assert that fired inside py_freeze proc-macro expansion. --- crates/codegen/src/ir.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e7087a211e4..f22efe8e52d 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -6358,7 +6358,6 @@ fn is_jump(instr: &InstructionInfo) -> bool { /// flowgraph.c is_block_push fn is_block_push(instr: &InstructionInfo) -> bool { - debug_assert!(instr.instr.has_arg() || !instr.instr.is_block_push()); instr.instr.is_block_push() } From 6ea698512d25693b20148f7b0128092ffe42a5da Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 27 May 2026 23:38:27 +0900 Subject: [PATCH 131/131] Auto-generate has_eval_break and route AnyInstruction has_arg/has_const via macro Add fn_has_eval_break to generate_rs_opcode_metadata.py using CPython's Properties.eval_breaker, removing the hand-written matches! body for Opcode::has_eval_break and PseudoOpcode::has_eval_break. Forward has_arg/has_const from Instruction and PseudoInstruction to their opcode, so AnyInstruction can use either_real_pseudo! like the other has_* accessors instead of an open-coded match. --- .../compiler-core/src/bytecode/instruction.rs | 68 +++++-------------- .../src/bytecode/opcode_metadata.rs | 36 ++++++++++ .../generate_rs_opcode_metadata.py | 6 ++ 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index eaf055f9950..69714a0fe66 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -171,6 +171,16 @@ macro_rules! define_opcodes { self.as_opcode().has_jump() } + #[must_use] + $instr_vis const fn has_arg(&self) -> bool { + self.as_opcode().has_arg() + } + + #[must_use] + $instr_vis const fn has_const(&self) -> bool { + self.as_opcode().has_const() + } + #[must_use] $instr_vis const fn has_eval_break(&self) -> bool { self.as_opcode().has_eval_break() @@ -730,36 +740,6 @@ impl Opcode { self.has_jump() || self.is_block_push() } - /// Does this opcode have CPython's `HAS_EVAL_BREAK_FLAG` set. - #[must_use] - pub const fn has_eval_break(&self) -> bool { - matches!( - self, - Self::Call - | Self::CallBuiltinClass - | Self::CallBuiltinFast - | Self::CallBuiltinFastWithKeywords - | Self::CallBuiltinO - | Self::CallFunctionEx - | Self::CallKwNonPy - | Self::CallMethodDescriptorFast - | Self::CallMethodDescriptorFastWithKeywords - | Self::CallMethodDescriptorNoargs - | Self::CallMethodDescriptorO - | Self::CallNonPyGeneral - | Self::CallStr1 - | Self::CallTuple1 - | Self::InstrumentedCall - | Self::InstrumentedCallFunctionEx - | Self::InstrumentedJumpBackward - | Self::InstrumentedResume - | Self::JumpBackward - | Self::JumpBackwardJit - | Self::JumpBackwardNoJit - | Self::Resume - ) - } - #[must_use] pub const fn is_block_push(&self) -> bool { false @@ -802,12 +782,6 @@ impl PseudoOpcode { matches!(self, Self::Jump | Self::JumpNoInterrupt) } - /// Does this pseudo opcode have CPython's `HAS_EVAL_BREAK_FLAG` set. - #[must_use] - pub const fn has_eval_break(&self) -> bool { - matches!(self, Self::Jump) - } - #[must_use] pub const fn is_assembler(&self) -> bool { false @@ -925,21 +899,15 @@ impl AnyInstruction { pub const fn has_jump(&self) -> bool ); - #[must_use] - pub const fn has_arg(&self) -> bool { - match self { - Self::Real(instr) => instr.as_opcode().has_arg(), - Self::Pseudo(instr) => instr.as_opcode().has_arg(), - } - } + either_real_pseudo!( + #[must_use] + pub const fn has_arg(&self) -> bool + ); - #[must_use] - pub const fn has_const(&self) -> bool { - match self { - Self::Real(instr) => instr.as_opcode().has_const(), - Self::Pseudo(instr) => instr.as_opcode().has_const(), - } - } + either_real_pseudo!( + #[must_use] + pub const fn has_const(&self) -> bool + ); either_real_pseudo!( #[must_use] diff --git a/crates/compiler-core/src/bytecode/opcode_metadata.rs b/crates/compiler-core/src/bytecode/opcode_metadata.rs index 2076b11661f..64c8d3c5330 100644 --- a/crates/compiler-core/src/bytecode/opcode_metadata.rs +++ b/crates/compiler-core/src/bytecode/opcode_metadata.rs @@ -282,6 +282,36 @@ impl super::Opcode { ) } + /// Does this opcode have 'HAS_EVAL_BREAK_FLAG' set. + #[must_use] + pub const fn has_eval_break(self) -> bool { + matches!( + self, + Self::CallFunctionEx + | Self::Call + | Self::JumpBackward + | Self::Resume + | Self::CallBuiltinClass + | Self::CallBuiltinFast + | Self::CallBuiltinFastWithKeywords + | Self::CallBuiltinO + | Self::CallKwNonPy + | Self::CallMethodDescriptorFast + | Self::CallMethodDescriptorFastWithKeywords + | Self::CallMethodDescriptorNoargs + | Self::CallMethodDescriptorO + | Self::CallNonPyGeneral + | Self::CallStr1 + | Self::CallTuple1 + | Self::JumpBackwardJit + | Self::JumpBackwardNoJit + | Self::InstrumentedResume + | Self::InstrumentedCall + | Self::InstrumentedCallFunctionEx + | Self::InstrumentedJumpBackward + ) + } + /// Does this opcode have 'HAS_FREE_FLAG' set. #[must_use] pub const fn has_free(self) -> bool { @@ -727,6 +757,12 @@ impl super::PseudoOpcode { false } + /// Does this opcode have 'HAS_EVAL_BREAK_FLAG' set. + #[must_use] + pub const fn has_eval_break(self) -> bool { + matches!(self, Self::Jump) + } + /// Does this opcode have 'HAS_FREE_FLAG' set. #[must_use] pub const fn has_free(self) -> bool { diff --git a/tools/opcode_metadata/generate_rs_opcode_metadata.py b/tools/opcode_metadata/generate_rs_opcode_metadata.py index 1d6ce2c0e97..df2476c5e08 100644 --- a/tools/opcode_metadata/generate_rs_opcode_metadata.py +++ b/tools/opcode_metadata/generate_rs_opcode_metadata.py @@ -66,6 +66,12 @@ def fn_has_free(self) -> str: def fn_has_local(self) -> str: return self.gen_fn_has_attr("has_local", "uses_locals", "HAS_LOCAL_FLAG") + @property + def fn_has_eval_break(self) -> str: + return self.gen_fn_has_attr( + "has_eval_break", "eval_breaker", "HAS_EVAL_BREAK_FLAG" + ) + @property def fn_is_instrumented(self) -> str: arms = "|".join(