diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index e7b50659e8e..5728ddfbefe 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -437,22 +437,6 @@ fn basicblock_insert_instruction( Ok(()) } -/// flowgraph.c basicblock_append_instructions -fn basicblock_append_block_instructions( - blocks: &mut Blocks, - to: BlockIdx, - from: BlockIdx, -) -> crate::InternalResult<()> { - debug_assert_ne!(to, from); - let from_len = blocks[from].instruction_used; - for i in 0..from_len { - let info = blocks[from].instructions[i]; - let off = basicblock_next_instr(&mut blocks[to])?; - blocks[to].instructions[off] = info; - } - Ok(()) -} - /// flowgraph.c direct `b_iused = 0` fn basicblock_clear(block: &mut Block) { block.instruction_used = 0; @@ -753,60 +737,6 @@ fn instruction_sequence_apply_label_map( Ok(()) } -/// flowgraph.c _PyCfg_ToInstructionSequence -fn cfg_to_instruction_sequence( - blocks: &mut Blocks, - instr_sequence: &mut InstructionSequence, -) -> crate::InternalResult<()> { - let mut label_id = 0; - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - blocks[block_idx.idx()].cpython_label = InstructionSequenceLabel::from_index(label_id); - label_id += 1; - block_idx = blocks[block_idx.idx()].next; - } - - block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let block_label = blocks[block_idx.idx()].cpython_label; - debug_assert!(is_label(block_label)); - instruction_sequence_use_label(instr_sequence, block_label)?; - - let instr_count = blocks[block_idx.idx()].instruction_used; - for i in 0..instr_count { - if blocks[block_idx.idx()].instructions[i].instr.has_target() { - let target_block = blocks[block_idx.idx()].instructions[i].target; - debug_assert!(target_block != BlockIdx::NULL); - let lbl = blocks[target_block.idx()].cpython_label; - debug_assert!(is_label(lbl)); - blocks[block_idx.idx()].instructions[i].arg = OpArg::new(lbl.0 as u32); - } - - let mut info = blocks[block_idx.idx()].instructions[i]; - info.target = BlockIdx::NULL; - let except_handler = info.except_handler.take(); - let entry = instruction_sequence_addop(instr_sequence, info)?; - let hi = &mut entry.except_handler; - if let Some(handler) = except_handler { - debug_assert!(handler.handler_block != BlockIdx::NULL); - let lbl = blocks[handler.handler_block.idx()].cpython_label; - debug_assert!(is_label(lbl)); - let start_depth = blocks[handler.handler_block.idx()].start_depth; - debug_assert!(start_depth >= 0); - hi.h_label = lbl.0; - hi.start_depth = start_depth; - hi.preserve_lasti = i32::from(handler.preserve_lasti); - } else { - hi.h_label = NO_EXCEPTION_HANDLER_LABEL; - } - } - block_idx = blocks[block_idx.idx()].next; - } - - instruction_sequence_apply_label_map(instr_sequence)?; - Ok(()) -} - /// assemble.c instr_size fn instr_size(instr: &InstructionInfo) -> usize { let opcode = instr.instr.expect_real(); @@ -1307,6 +1237,7 @@ impl Block { #[derive(Clone, Debug, Default)] pub struct Blocks(Vec); +// Vec like methods impl Blocks { pub fn try_reserve( &mut self, @@ -1320,3097 +1251,3389 @@ impl Blocks { } } -impl From> for Blocks { - fn from(value: Vec) -> Self { - Self(value) - } -} +// CPython functions -impl From> for Blocks { - fn from(value: Box<[Block]>) -> Self { - Self(value.into()) - } -} +impl Blocks { + /// # See also + /// [CPython's remove_unreachable](https://github.com/python/cpython/blob/v3.14.6/Python/flowgraph.c#L995-L1041) + pub fn remove_unreachable(&mut self) -> crate::InternalResult<()> { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + self[block_idx].predecessors = 0; + block_idx = self[block_idx].next; + } + + let mut stack = self.make_cfg_traversal_stack()?; + self[0].predecessors = 1; + stack.push(BlockIdx(0)); + self[0].visited = true; + while let Some(current) = stack.pop() { + let idx = current.idx(); + let next = self[idx].next; + if next != BlockIdx::NULL && bb_has_fallthrough(&self[idx]) { + if !self[next].visited { + debug_assert_eq!(self[next].predecessors, 0); + stack.push(next); + self[next].visited = true; + } + self[next].predecessors += 1; + } -impl From<&[Block]> for Blocks { - fn from(value: &[Block]) -> Self { - Self(value.to_vec()) - } -} + let instr_count = self[idx].instruction_used; + for i in 0..instr_count { + let instr = self[idx].instructions[i]; + if is_jump(&instr) || is_block_push(&instr) { + let target = instr.target; + debug_assert!(target != BlockIdx::NULL); + let target_idx = target.idx(); + if !self[target_idx].visited { + stack.push(target); + self[target_idx].visited = true; + } + self[target_idx].predecessors += 1; + } + } + } -impl From<&mut [Block]> for Blocks { - fn from(value: &mut [Block]) -> Self { - Self(value.to_vec()) + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next = self[block_idx].next; + if self[block_idx].predecessors == 0 { + let block = &mut self[block_idx]; + basicblock_clear(block); + block.except_handler = false; + } + block_idx = next; + } + Ok(()) } -} -impl From<[Block; N]> for Blocks { - fn from(value: [Block; N]) -> Self { - Self(value.into()) - } -} + /// flowgraph.c basicblock_append_instructions + fn basicblock_append_block_instructions( + &mut self, + to: BlockIdx, + from: BlockIdx, + ) -> crate::InternalResult<()> { + debug_assert_ne!(to, from); -impl From<&[Block; N]> for Blocks { - fn from(value: &[Block; N]) -> Self { - Self(value.to_vec()) + let from_len = self[from].instruction_used; + for i in 0..from_len { + let info = self[from].instructions[i]; + let off = basicblock_next_instr(&mut self[to])?; + self[to].instructions[off] = info; + } + + Ok(()) } -} -impl Deref for Blocks { - type Target = [Block]; + /// flowgraph.c copy_basicblock + fn copy_basicblock(&mut self, block_idx: BlockIdx) -> crate::InternalResult { + debug_assert!(bb_no_fallthrough(&self[block_idx])); - fn deref(&self) -> &Self::Target { - &self.0 + let result = blocks_new_block(self)?; + self.basicblock_append_block_instructions(result, block_idx)?; + Ok(result) } -} -impl DerefMut for Blocks { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} + fn duplicate_exits_without_lineno(&mut self) -> crate::InternalResult<()> { + let mut next_lbl = get_max_label(self) + 1; -impl Index for Blocks { - type Output = Block; + let entryblock = BlockIdx(0); + let mut b = entryblock; + while b != BlockIdx::NULL { + let Some(last) = basicblock_last_instr(&self[b]).copied() else { + b = self[b].next; + continue; + }; - fn index(&self, idx: usize) -> &Self::Output { - &self.0[idx] - } -} + if is_jump(&last) { + debug_assert!(last.target != BlockIdx::NULL); -impl IndexMut for Blocks { - fn index_mut(&mut self, idx: usize) -> &mut Self::Output { - &mut self.0[idx] - } -} + let target = next_nonempty_block(self, last.target); -impl Index for Blocks { - type Output = Block; + debug_assert!(target != BlockIdx::NULL); - fn index(&self, block_idx: BlockIdx) -> &Self::Output { - &self.0[block_idx.as_usize()] + if is_exit_or_eval_check_without_lineno(&self[target]) + && self[target].predecessors > 1 + { + let new_target = self.copy_basicblock(target)?; + instr_set_location( + &mut self[new_target].instructions[0], + instr_location(&last), + ); + let last_mut = basicblock_last_instr_mut(&mut self[b]).unwrap(); + last_mut.target = new_target; + self[target].predecessors -= 1; + self[new_target].predecessors = 1; + self[new_target].next = self[target].next; + self[new_target].cpython_label = InstructionSequenceLabel(next_lbl); + next_lbl += 1; + self[target].next = new_target; + } + } + b = self[b].next; + } + + b = entryblock; + while b != BlockIdx::NULL { + let next = self[b].next; + if bb_has_fallthrough(&self[b]) + && next != BlockIdx::NULL + && self[b].instruction_used != 0 + && is_exit_or_eval_check_without_lineno(&self[next]) + { + let last = *basicblock_last_instr(&self[b]).expect("block has instructions"); + instr_set_location(&mut self[next].instructions[0], instr_location(&last)); + } + b = self[b].next; + } + + Ok(()) } -} -impl IndexMut for Blocks { - fn index_mut(&mut self, block_idx: BlockIdx) -> &mut Self::Output { - &mut self.0[block_idx.as_usize()] + fn resolve_line_numbers(&mut self, _firstlineno: OneIndexed) -> crate::InternalResult<()> { + self.duplicate_exits_without_lineno()?; + self.propagate_line_numbers(); + Ok(()) } -} -pub(crate) const START_DEPTH_UNSET: i32 = i32::MIN; -const CO_MAXBLOCKS: usize = 20; + /// flowgraph.c optimize_basic_block + fn optimize_basic_block( + &mut self, + metadata: &mut CodeUnitMetadata, + block_idx: BlockIdx, + ) -> crate::InternalResult<()> { + let mut nop = InstructionInfo { + instr: Instruction::Nop.into(), + arg: OpArg::NULL, + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: None, + }; + instr_set_op0(&mut nop, Instruction::Nop.into()); + let mut i = 0; + while i < self[block_idx].instruction_used { + let inst = self[block_idx].instructions[i]; + debug_assert!(!inst.instr.is_assembler()); + let target = if inst.instr.has_target() { + let target = inst.target; + debug_assert!(target != BlockIdx::NULL); + debug_assert!(self[target.idx()].instruction_used != 0); + debug_assert!(!self[target.idx()].instructions[0].instr.is_assembler()); + self[target.idx()].instructions[0] + } else { + nop + }; -/// flowgraph.c struct _PyCfgExceptStack -#[derive(Clone, Debug)] -struct CfgExceptStack { - handlers: [BlockIdx; CO_MAXBLOCKS + 2], - depth: usize, -} + let nextop = self[block_idx] + .instructions + .get(i + 1) + .and_then(|next| next.instr.real()); -/// flowgraph.c `basicblock **stack` -#[derive(Clone, Debug)] -struct CfgTraversalStack { - stack: Vec, - sp: usize, -} + match inst.instr { + AnyInstruction::Real(Instruction::BuildTuple { .. }) => { + let oparg = u32::from(inst.arg); + if matches!(nextop, Some(Instruction::UnpackSequence { .. })) + && u32::from(self[block_idx].instructions[i + 1].arg) == oparg + { + match oparg { + 1 => { + set_to_nop(&mut self[block_idx].instructions[i]); + set_to_nop(&mut self[block_idx].instructions[i + 1]); + i += 1; + continue; + } + 2 | 3 => { + set_to_nop(&mut self[block_idx].instructions[i]); + self[block_idx].instructions[i + 1].instr = Opcode::Swap.into(); + i += 1; + continue; + } + _ => {} + } + } + fold_tuple_of_constants(metadata, &mut self[block_idx], i)?; + } + AnyInstruction::Real( + Instruction::BuildList { .. } | Instruction::BuildSet { .. }, + ) => { + optimize_lists_and_sets(metadata, &mut self[block_idx], i, nextop)?; + } + AnyInstruction::Real( + Instruction::PopJumpIfNotNone { .. } | Instruction::PopJumpIfNone { .. }, + ) if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) + && jump_thread(self, block_idx, i, &target, inst.instr)? => + { + continue; + } + AnyInstruction::Real(Instruction::PopJumpIfFalse { .. }) + if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) + && jump_thread(self, block_idx, i, &target, inst.instr)? => + { + continue; + } + AnyInstruction::Real(Instruction::PopJumpIfTrue { .. }) + if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) + && jump_thread(self, block_idx, i, &target, inst.instr)? => + { + continue; + } + AnyInstruction::Pseudo( + pseudo @ (PseudoInstruction::JumpIfFalse { .. } + | PseudoInstruction::JumpIfTrue { .. }), + ) => { + let opcode = pseudo.into(); + match target.instr.pseudo().map(Into::into) { + Some(PseudoOpcode::Jump) + if jump_thread(self, block_idx, i, &target, opcode)? => + { + continue; + } + Some(PseudoOpcode::JumpIfFalse) + if matches!( + opcode, + AnyInstruction::Pseudo(PseudoInstruction::JumpIfFalse { .. }) + ) && jump_thread(self, block_idx, i, &target, opcode)? => + { + continue; + } + Some(PseudoOpcode::JumpIfTrue) + if matches!( + opcode, + AnyInstruction::Pseudo(PseudoInstruction::JumpIfTrue { .. }) + ) && jump_thread(self, block_idx, i, &target, opcode)? => + { + continue; + } + Some(PseudoOpcode::JumpIfFalse | PseudoOpcode::JumpIfTrue) => { + let next = self[inst.target.idx()].next; + debug_assert!(next != BlockIdx::NULL); + debug_assert!(next != inst.target); + self[block_idx].instructions[i].target = next; + continue; + } + _ => {} + } + } + AnyInstruction::Pseudo( + PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. }, + ) => match target.instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::Jump) + if jump_thread(self, block_idx, i, &target, PseudoOpcode::Jump.into())? => + { + continue; + } + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) + if jump_thread(self, block_idx, i, &target, inst.instr)? => + { + continue; + } + _ => {} + }, + // CPython leaves FOR_ITER jump threading disabled. + AnyInstruction::Real(Instruction::ForIter { .. }) => {} + AnyInstruction::Real(Instruction::StoreFast { .. }) + if matches!(nextop, Some(Instruction::StoreFast { .. })) + && u32::from(inst.arg) + == u32::from(self[block_idx].instructions[i + 1].arg) + && instruction_lineno(&self[block_idx].instructions[i]) + == instruction_lineno(&self[block_idx].instructions[i + 1]) => + { + self[block_idx].instructions[i].instr = Instruction::PopTop.into(); + self[block_idx].instructions[i].arg = OpArg::NULL; + } + AnyInstruction::Real(Instruction::Swap { .. }) if u32::from(inst.arg) == 1 => { + set_to_nop(&mut self[block_idx].instructions[i]); + } + AnyInstruction::Real(Instruction::LoadGlobal { .. }) + if matches!(nextop, Some(Instruction::PushNull)) + && (u32::from(inst.arg) & 1) == 0 => + { + instr_set_op1( + &mut self[block_idx].instructions[i], + inst.instr, + OpArg::new(u32::from(inst.arg) | 1), + ); + set_to_nop(&mut self[block_idx].instructions[i + 1]); + } + AnyInstruction::Real(Instruction::CompareOp { .. }) + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut self[block_idx].instructions[i]); + instr_set_op1( + &mut self[block_idx].instructions[i + 1], + inst.instr, + OpArg::new(u32::from(inst.arg) | oparg::COMPARE_OP_BOOL_MASK), + ); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::ContainsOp { .. } | Instruction::IsOp { .. }) + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut self[block_idx].instructions[i]); + instr_set_op1( + &mut self[block_idx].instructions[i + 1], + inst.instr, + inst.arg, + ); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::ContainsOp { .. } | Instruction::IsOp { .. }) + if matches!(nextop, Some(Instruction::UnaryNot)) => + { + set_to_nop(&mut self[block_idx].instructions[i]); + let inverted = u32::from(inst.arg) ^ 1; + debug_assert!(inverted == 0 || inverted == 1); + instr_set_op1( + &mut self[block_idx].instructions[i + 1], + inst.instr, + OpArg::new(inverted), + ); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::ToBool) + if matches!(nextop, Some(Instruction::ToBool)) => + { + set_to_nop(&mut self[block_idx].instructions[i]); + i += 1; + continue; + } + AnyInstruction::Real(Instruction::UnaryNot) => { + if matches!(nextop, Some(Instruction::ToBool)) { + set_to_nop(&mut self[block_idx].instructions[i]); + instr_set_op0(&mut self[block_idx].instructions[i + 1], inst.instr); + i += 1; + continue; + } + if matches!(nextop, Some(Instruction::UnaryNot)) { + set_to_nop(&mut self[block_idx].instructions[i]); + set_to_nop(&mut self[block_idx].instructions[i + 1]); + i += 1; + continue; + } + fold_const_unaryop(metadata, &mut self[block_idx], i)?; + } + AnyInstruction::Real(Instruction::UnaryInvert | Instruction::UnaryNegative) => { + fold_const_unaryop(metadata, &mut self[block_idx], i)?; + } + AnyInstruction::Real(Instruction::CallIntrinsic1 { func }) => { + match func.get(inst.arg) { + IntrinsicFunction1::ListToTuple => { + if matches!(nextop, Some(Instruction::GetIter)) { + set_to_nop(&mut self[block_idx].instructions[i]); + } else { + fold_constant_intrinsic_list_to_tuple( + metadata, + &mut self[block_idx], + i, + )?; + } + } + IntrinsicFunction1::UnaryPositive => { + fold_const_unaryop(metadata, &mut self[block_idx], i)?; + } + _ => {} + } + } + AnyInstruction::Real(Instruction::BinaryOp { .. }) => { + fold_const_binop(metadata, &mut self[block_idx], i)?; + } + _ => {} + } -impl CfgTraversalStack { - fn push(&mut self, block: BlockIdx) { - debug_assert!(self.sp < self.stack.len()); - self.stack[self.sp] = block; - self.sp += 1; + i += 1; + } + apply_static_swaps_block(&mut self[block_idx])?; + Ok(()) } - fn pop(&mut self) -> Option { - if self.sp == 0 { - return None; + /// flowgraph.c _PyCfg_ToInstructionSequence + fn cfg_to_instruction_sequence( + &mut self, + instr_sequence: &mut InstructionSequence, + ) -> crate::InternalResult<()> { + let mut label_id = 0; + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + self[block_idx].cpython_label = InstructionSequenceLabel::from_index(label_id); + label_id += 1; + block_idx = self[block_idx].next; } - self.sp -= 1; - Some(self.stack[self.sp]) - } - fn capacity(&self) -> usize { - self.stack.len() - } -} - -#[derive(Clone, Debug)] -pub(crate) struct InstructionSequenceLabelMap { - block_labels: Vec, - /// Codegen-side shadow of CPython's instruction-sequence label map. - /// - /// `_PyInstructionSequence_UseLabel()` can map multiple labels to the same - /// instruction offset before `_PyCfg_FromInstructionSequence()` materializes - /// CFG blocks. The codegen CFG path keeps the same aliasing by resolving - /// those labels to the block that owns the shared offset. - cpython_block_by_label: Vec, -} + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let block_label = self[block_idx].cpython_label; + debug_assert!(is_label(block_label)); + instruction_sequence_use_label(instr_sequence, block_label)?; + + let instr_count = self[block_idx].instruction_used; + for i in 0..instr_count { + if self[block_idx].instructions[i].instr.has_target() { + let target_block = self[block_idx].instructions[i].target; + debug_assert!(target_block != BlockIdx::NULL); + let lbl = self[target_block].cpython_label; + debug_assert!(is_label(lbl)); + self[block_idx].instructions[i].arg = OpArg::new(lbl.0 as u32); + } -fn instruction_sequence_label_map_register_label( - map: &mut InstructionSequenceLabelMap, - label: InstructionSequenceLabel, -) -> crate::InternalResult<()> { - debug_assert!(is_label(label)); - let old_size = map.cpython_block_by_label.len(); - let new_allocation = c_array_ensure_capacity::( - old_size, - label.idx(), - INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, - )?; - if new_allocation > old_size { - if new_allocation > map.cpython_block_by_label.capacity() { - map.cpython_block_by_label - .try_reserve_exact(new_allocation - map.cpython_block_by_label.capacity()) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - } - map.cpython_block_by_label - .resize(new_allocation, BlockIdx::NULL); - for i in old_size..map.cpython_block_by_label.len() { - map.cpython_block_by_label[i] = BlockIdx::NULL; + let mut info = self[block_idx].instructions[i]; + info.target = BlockIdx::NULL; + let except_handler = info.except_handler.take(); + let entry = instruction_sequence_addop(instr_sequence, info)?; + let hi = &mut entry.except_handler; + if let Some(handler) = except_handler { + debug_assert!(handler.handler_block != BlockIdx::NULL); + let lbl = self[handler.handler_block].cpython_label; + debug_assert!(is_label(lbl)); + let start_depth = self[handler.handler_block].start_depth; + debug_assert!(start_depth >= 0); + hi.h_label = lbl.0; + hi.start_depth = start_depth; + hi.preserve_lasti = i32::from(handler.preserve_lasti); + } else { + hi.h_label = NO_EXCEPTION_HANDLER_LABEL; + } + } + block_idx = self[block_idx].next; } - } - debug_assert!(map.cpython_block_by_label.len() > label.idx()); - Ok(()) -} -fn instruction_sequence_label_map_ensure_label_for_block( - map: &mut InstructionSequenceLabelMap, - seq: &mut InstructionSequence, - block: BlockIdx, -) -> crate::InternalResult { - debug_assert_ne!(block, BlockIdx::NULL); - let block_label = map.block_labels[block.idx()]; - if is_label(block_label) { - return Ok(block_label); + instruction_sequence_apply_label_map(instr_sequence)?; + Ok(()) } - let label = instruction_sequence_new_label(seq); - debug_assert_eq!(label.0, seq.next_free_label); - instruction_sequence_label_map_register_label(map, label)?; - map.cpython_block_by_label[label.idx()] = block; - map.block_labels[block.idx()] = label; - Ok(label) -} -fn instruction_sequence_label_map_label_for_block( - map: &InstructionSequenceLabelMap, - block: BlockIdx, -) -> InstructionSequenceLabel { - debug_assert_ne!(block, BlockIdx::NULL); - map.block_labels - .get(block.idx()) - .copied() - .unwrap_or(InstructionSequenceLabel::NO_LABEL) -} + fn optimize_load_fast(&mut self) -> crate::InternalResult<()> { + let mut max_instrs = 0; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + max_instrs = max_instrs.max(self[current].instruction_used); + current = self[current].next; + } -fn instruction_sequence_label_map_block_for_label( - map: &InstructionSequenceLabelMap, - label: InstructionSequenceLabel, -) -> Option { - if !is_label(label) { - return None; - } - map.cpython_block_by_label - .get(label.idx()) - .copied() - .filter(|&block| block != BlockIdx::NULL) -} + let mut instr_flags = Vec::new(); + instr_flags + .try_reserve_exact(max_instrs) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + instr_flags.resize(max_instrs, 0u8); + let mut refs = RefStack { + refs: Vec::new(), + size: 0, + capacity: 0, + }; + let mut worklist = self.make_cfg_traversal_stack()?; + worklist.push(BlockIdx(0)); + self[0].start_depth = 0; + self[0].visited = true; + while let Some(block_idx) = worklist.pop() { + let instr_count = self[block_idx].instruction_used; + instr_flags[..instr_count].fill(0); + debug_assert!(self[block_idx].start_depth >= 0); + let start_depth = self[block_idx].start_depth as usize; + ref_stack_clear(&mut refs); + for _ in 0..start_depth { + push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL)?; + } -fn instruction_sequence_label_map_resolve_label( - map: &InstructionSequenceLabelMap, - block: BlockIdx, -) -> BlockIdx { - if block == BlockIdx::NULL { - return BlockIdx::NULL; - } - let label = instruction_sequence_label_map_label_for_block(map, block); - if !is_label(label) { - return block; - } - instruction_sequence_label_map_block_for_label(map, label).unwrap_or_else(|| { - debug_assert!( - false, - "CPython instruction-sequence label must map to a codegen CFG block" - ); - BlockIdx::NULL - }) -} + for i in 0..instr_count { + let info = self[block_idx].instructions[i]; + let instr = info.instr; + let arg_u32 = u32::from(info.arg); + debug_assert!(!matches!(instr.real(), Some(Instruction::ExtendedArg))); + + match instr { + AnyInstruction::Real(Instruction::DeleteFast { var_num }) => { + kill_local( + &mut instr_flags, + &refs, + local_as_ref_local(usize::from(var_num.get(info.arg))), + ); + } + AnyInstruction::Real(Instruction::LoadFast { var_num }) => { + push_ref( + &mut refs, + i as isize, + local_as_ref_local(usize::from(var_num.get(info.arg))), + )?; + } + AnyInstruction::Real(Instruction::LoadFastAndClear { var_num }) => { + let local = local_as_ref_local(usize::from(var_num.get(info.arg))); + kill_local(&mut instr_flags, &refs, local); + push_ref(&mut refs, i as isize, local)?; + } + AnyInstruction::Real(Instruction::LoadFastLoadFast { .. }) => { + let local1 = (arg_u32 >> 4) as isize; + let local2 = (arg_u32 & 15) as isize; + push_ref(&mut refs, i as isize, local1)?; + push_ref(&mut refs, i as isize, local2)?; + } + AnyInstruction::Real(Instruction::StoreFast { var_num }) => { + let r = ref_stack_pop(&mut refs); + store_local( + &mut instr_flags, + &refs, + local_as_ref_local(usize::from(var_num.get(info.arg))), + r, + ); + } + AnyInstruction::Real(Instruction::StoreFastLoadFast { .. }) => { + let r = ref_stack_pop(&mut refs); + store_local(&mut instr_flags, &refs, (arg_u32 >> 4) as isize, r); + push_ref(&mut refs, i as isize, (arg_u32 & 15) as isize)?; + } + AnyInstruction::Real(Instruction::StoreFastStoreFast { .. }) => { + let r1 = ref_stack_pop(&mut refs); + store_local(&mut instr_flags, &refs, (arg_u32 >> 4) as isize, r1); + let r2 = ref_stack_pop(&mut refs); + store_local(&mut instr_flags, &refs, (arg_u32 & 15) as isize, r2); + } + AnyInstruction::Real(Instruction::Copy { i: _ }) => { + let depth = arg_u32 as usize; + assert!(depth > 0); + assert!(refs.size >= depth); + let r = ref_stack_at(&refs, refs.size - depth); + push_ref(&mut refs, r.instr, r.local)?; + } + AnyInstruction::Real(Instruction::Swap { i: _ }) => { + let depth = arg_u32 as usize; + assert!(depth >= 2); + assert!(refs.size >= depth); + ref_stack_swap_top(&mut refs, depth); + } + AnyInstruction::Real( + Instruction::FormatSimple + | Instruction::GetAnext + | Instruction::GetLen + | Instruction::GetYieldFromIter + | Instruction::ImportFrom { .. } + | Instruction::MatchKeys + | Instruction::MatchMapping + | Instruction::MatchSequence + | Instruction::WithExceptStart, + ) => { + let effect = instr.stack_effect_info(arg_u32); + let net_pushed = effect.pushed() as isize - effect.popped() as isize; + debug_assert!(net_pushed >= 0); + // CPython optimize_load_fast() shadows the outer + // instruction index in this produced-value loop. + for produced in 0..net_pushed { + push_ref(&mut refs, produced, NOT_LOCAL)?; + } + } + AnyInstruction::Real( + Instruction::DictMerge { .. } + | Instruction::DictUpdate { .. } + | Instruction::ListAppend { .. } + | Instruction::ListExtend { .. } + | Instruction::MapAdd { .. } + | Instruction::Reraise { .. } + | Instruction::SetAdd { .. } + | Instruction::SetUpdate { .. }, + ) => { + let effect = instr.stack_effect_info(arg_u32); + let net_popped = effect.popped() as isize - effect.pushed() as isize; + debug_assert!(net_popped > 0); + for _ in 0..net_popped { + let _ = ref_stack_pop(&mut refs); + } + } + AnyInstruction::Real( + Instruction::EndSend | Instruction::SetFunctionAttribute { .. }, + ) => { + let effect = instr.stack_effect_info(arg_u32); + debug_assert_eq!(effect.popped(), 2); + debug_assert_eq!(effect.pushed(), 1); + let tos = ref_stack_pop(&mut refs); + let _ = ref_stack_pop(&mut refs); + push_ref(&mut refs, tos.instr, tos.local)?; + } + AnyInstruction::Real(Instruction::CheckExcMatch) => { + let _ = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; + } + AnyInstruction::Real(Instruction::ForIter { .. }) => { + let target = info.target; + debug_assert!(target != BlockIdx::NULL); + load_fast_push_block(&mut worklist, self, target, refs.size + 1); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; + } + AnyInstruction::Real( + Instruction::LoadAttr { .. } | Instruction::LoadSuperAttr { .. }, + ) => { + let self_ref = ref_stack_pop(&mut refs); + if matches!(instr.real(), Some(Instruction::LoadSuperAttr { .. })) { + let _ = ref_stack_pop(&mut refs); + let _ = ref_stack_pop(&mut refs); + } + push_ref(&mut refs, i as isize, NOT_LOCAL)?; + if arg_u32 & 1 != 0 { + push_ref(&mut refs, self_ref.instr, self_ref.local)?; + } + } + AnyInstruction::Real( + Instruction::LoadSpecial { .. } | Instruction::PushExcInfo, + ) => { + let tos = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; + push_ref(&mut refs, tos.instr, tos.local)?; + } + AnyInstruction::Real(Instruction::Send { .. }) => { + let target = info.target; + debug_assert!(target != BlockIdx::NULL); + load_fast_push_block(&mut worklist, self, target, refs.size); + let _ = ref_stack_pop(&mut refs); + push_ref(&mut refs, i as isize, NOT_LOCAL)?; + } + _ => { + let effect = instr.stack_effect_info(arg_u32); + let num_popped = effect.popped() as usize; + let num_pushed = effect.pushed() as usize; + let target = info.target; + if instr.has_target() { + debug_assert!(target != BlockIdx::NULL); + debug_assert!(refs.size >= num_popped); + let target_depth = refs.size - num_popped + num_pushed; + load_fast_push_block(&mut worklist, self, target, target_depth); + } + if !is_block_push(&info) { + for _ in 0..num_popped { + let _ = ref_stack_pop(&mut refs); + } + for _ in 0..num_pushed { + push_ref(&mut refs, i as isize, NOT_LOCAL)?; + } + } + } + } + } -fn instruction_sequence_label_map_resolve_label_to_block( - map: &InstructionSequenceLabelMap, - label: InstructionSequenceLabel, -) -> BlockIdx { - if !is_label(label) { - return BlockIdx::NULL; - } - instruction_sequence_label_map_block_for_label(map, label).unwrap_or_else(|| { - debug_assert!( - false, - "CPython instruction-sequence label must map to a codegen CFG block" - ); - BlockIdx::NULL - }) -} + let fallthrough = self[block_idx].next; + let term = basicblock_last_instr(&self[block_idx]).copied(); + if let Some(term) = term + && fallthrough != BlockIdx::NULL + && !term.instr.is_unconditional_jump() + && !term.instr.is_scope_exit() + { + debug_assert!(bb_has_fallthrough(&self[block_idx])); + load_fast_push_block(&mut worklist, self, fallthrough, refs.size); + } -fn instruction_sequence_label_oparg(label: InstructionSequenceLabel) -> OpArg { - debug_assert!(is_label(label)); - OpArg::new(label.idx() as u32) -} + for i in 0..refs.size { + let r = ref_stack_at(&refs, i); + if r.instr != DUMMY_INSTR { + instr_flags[r.instr as usize] |= LoadFastInstrFlag::RefUnconsumed as u8; + } + } -fn instruction_sequence_label_map_use_label_at_block( - map: &mut InstructionSequenceLabelMap, - seq: &mut InstructionSequence, - from: BlockIdx, - to: BlockIdx, -) -> crate::InternalResult<()> { - if from == BlockIdx::NULL || from == to { - return Ok(()); - } - let from_label = instruction_sequence_label_map_ensure_label_for_block(map, seq, from)?; - debug_assert!(map.cpython_block_by_label.len() > from_label.idx()); - let to_block = instruction_sequence_label_map_resolve_label(map, to); - if to_block == BlockIdx::NULL { - debug_assert!( - false, - "CPython label target must map to a codegen CFG block" - ); - return Ok(()); + let block = &mut self[block_idx]; + let iused = block.instruction_used; + let mut i = 0; + while i < iused { + let info = &mut block.instructions[i]; + if instr_flags[i] != 0 { + i += 1; + continue; + } + + match info.instr.real_opcode() { + Some(Opcode::LoadFast) => { + info.instr = Opcode::LoadFastBorrow.into(); + } + Some(Opcode::LoadFastLoadFast) => { + info.instr = Opcode::LoadFastBorrowLoadFastBorrow.into(); + } + _ => {} + } + i += 1; + } + } + + Ok(()) } - map.cpython_block_by_label[from_label.idx()] = to_block; - Ok(()) -} -fn instruction_sequence_label_map_push_unlabeled_block( - map: &mut InstructionSequenceLabelMap, -) -> crate::InternalResult<()> { - map.block_labels - .try_reserve(1) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - map.block_labels.push(InstructionSequenceLabel::NO_LABEL); - Ok(()) -} + fn propagate_line_numbers(&mut self) { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + let Some(last) = basicblock_last_instr(&self[current]).copied() else { + current = self[current].next; + continue; + }; -fn instruction_sequence_label_map_push_unmapped_label( - map: &mut InstructionSequenceLabelMap, - seq: &mut InstructionSequence, -) -> crate::InternalResult<()> { - let label = instruction_sequence_new_label(seq); - debug_assert_eq!(label.0, seq.next_free_label); - instruction_sequence_label_map_register_label(map, label)?; - let block = BlockIdx( - map.block_labels - .len() - .to_u32() - .ok_or(InternalError::MalformedControlFlowGraph)?, - ); - map.cpython_block_by_label[label.idx()] = block; - map.block_labels - .try_reserve(1) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - map.block_labels.push(label); - Ok(()) -} + let mut prev_location = no_instruction_location(); + for i in 0..self[current].instruction_used { + if instruction_is_no_location(&self[current].instructions[i]) { + instr_set_location(&mut self[current].instructions[i], prev_location); + } else { + prev_location = instr_location(&self[current].instructions[i]); + } + } -impl InstructionSequenceLabelMap { - pub(crate) fn new() -> Self { - Self { - block_labels: vec![InstructionSequenceLabel::NO_LABEL], - cpython_block_by_label: Vec::new(), + let next = self[current].next; + if bb_has_fallthrough(&self[current]) { + debug_assert!(next != BlockIdx::NULL); + if next != BlockIdx::NULL + && self[next].predecessors == 1 + && self[next].instruction_used != 0 + && instruction_is_no_location(&self[next].instructions[0]) + { + instr_set_location(&mut self[next].instructions[0], prev_location); + } + } + + if is_jump(&last) { + let target = last.target; + debug_assert!(target != BlockIdx::NULL); + if self[target].predecessors == 1 { + let instr = basicblock_raw_first_instr_mut(&mut self[target]); + if instruction_is_no_location(instr) { + instr_set_location(instr, prev_location); + } + } + } + current = self[current].next; } } -} -pub struct CodeInfo { - pub flags: CodeFlags, - pub source_path: String, - pub private: Option, // For private name mangling, mostly for class + /// flowgraph.c remove_redundant_nops_and_pairs + fn remove_redundant_nops_and_pairs(&mut self) -> crate::InternalResult<()> { + let mut done = false; - pub blocks: Blocks, - pub current_block: BlockIdx, - pub(crate) instr_sequence: InstructionSequence, - pub(crate) instr_sequence_label_map: InstructionSequenceLabelMap, - pub(crate) annotations_instr_sequence: Option, + while !done { + done = true; + let mut instr: Option<(BlockIdx, usize)> = None; + let mut block_idx = BlockIdx::new(0); - pub metadata: CodeUnitMetadata, + while block_idx != BlockIdx::NULL { + basicblock_remove_redundant_nops(self, block_idx)?; + if is_label(self[block_idx].cpython_label) { + instr = None; + } - // For class scopes: attributes accessed via self.X - pub static_attributes: Option>, + let len = self[block_idx].instruction_used; + for instr_idx in 0..len { + let prev_instr = instr; + instr = Some((block_idx, instr_idx)); + let instr_info = self[block_idx].instructions[instr_idx]; + let mut prev_opcode = None; + let prev_oparg = if let Some((prev_block, prev_instr_idx)) = prev_instr { + let prev_info = self[prev_block].instructions[prev_instr_idx]; + prev_opcode = prev_info.instr.real_opcode(); + match prev_info.instr.real() { + Some(Instruction::Copy { i }) => i.get(prev_info.arg), + _ => u32::from(prev_info.arg), + } + } else { + 0 + }; - // True if compiling an inlined comprehension - pub in_inlined_comp: bool, + let opcode = instr_info.instr.real_opcode(); + let is_redundant_pair = matches!(opcode, Some(Opcode::PopTop)) + && (matches!(prev_opcode, Some(Opcode::LoadConst | Opcode::LoadSmallInt)) + || (prev_oparg == 1 && matches!(prev_opcode, Some(Opcode::Copy)))); + + if is_redundant_pair { + let (prev_block, prev_instr_idx) = + prev_instr.expect("redundant pair has previous"); + set_to_nop(&mut self[prev_block].instructions[prev_instr_idx]); + set_to_nop(&mut self[block_idx].instructions[instr_idx]); + done = false; + } + } - // Block stack for tracking nested control structures - pub fblock: Vec, + let instr_is_jump = instr.is_some_and(|(instr_block, instr_idx)| { + is_jump(&self[instr_block].instructions[instr_idx]) + }); - // Reference to the symbol table for this scope - pub symbol_table_index: usize, - // CPython compile.c uses PyList_GET_SIZE(u->u_ste->ste_varnames) - // when calling flowgraph.c _PyCfg_OptimizeCodeUnit(). - pub nparams: usize, + let block = &self[block_idx]; + if instr_is_jump || !bb_has_fallthrough(block) { + instr = None; + } + block_idx = block.next; + } + } + Ok(()) + } - // PEP 649: Track nesting depth inside conditional blocks (if/for/while/etc.) - // u_in_conditional_block - pub in_conditional_block: u32, + /// flowgraph.c calculate_stackdepth + fn calculate_stackdepth(&mut self) -> crate::InternalResult { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + self[current.idx()].start_depth = START_DEPTH_UNSET; + current = self[current.idx()].next; + } + let mut stack = self.make_cfg_traversal_stack()?; + let mut maxdepth = 0i32; + stackdepth_push(&mut stack, self, BlockIdx(0), 0)?; + while let Some(block_idx) = stack.pop() { + let mut depth = self[block_idx].start_depth; + debug_assert!(depth >= 0); + let mut next = self[block_idx].next; + let instr_count = self[block_idx].instruction_used; + for i in 0..instr_count { + let ins = self[block_idx].instructions[i]; + let instr = &ins.instr; + let effects = get_stack_effects(*instr, ins.arg, 0)?; + let new_depth = depth + effects.net; + if new_depth < 0 { + return Err(InternalError::StackUnderflow); + } + maxdepth = maxdepth.max(depth); + if instr.has_target() && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) { + debug_assert!(ins.target != BlockIdx::NULL); + let effects = get_stack_effects(*instr, ins.arg, 1)?; + let target_depth = depth + effects.net; + debug_assert!(target_depth >= 0); + maxdepth = maxdepth.max(depth); + stackdepth_push(&mut stack, self, ins.target, target_depth)?; + } + depth = new_depth; + debug_assert!(!instr.is_assembler()); + if instr.is_unconditional_jump() || instr.is_scope_exit() { + next = BlockIdx::NULL; + break; + } + } - // PEP 649: Next index for conditional annotation tracking - // u_next_conditional_annotation_index - pub next_conditional_annotation_index: u32, -} + if next != BlockIdx::NULL { + debug_assert!(bb_has_fallthrough(&self[block_idx])); + stackdepth_push(&mut stack, self, next, depth)?; + } + } -impl CodeInfo { - pub(crate) fn addop_to_instr_sequence( - &mut self, - mut info: InstructionInfo, - ) -> crate::InternalResult<()> { - if info.instr.has_target() && info.target != BlockIdx::NULL { - let label = instruction_sequence_label_map_ensure_label_for_block( - &mut self.instr_sequence_label_map, - &mut self.instr_sequence, - info.target, - )?; - info.arg = instruction_sequence_label_oparg(label); - info.target = BlockIdx::NULL; + let stackdepth = maxdepth; + Ok(stackdepth as u32) + } + + /// flowgraph.c make_cfg_traversal_stack + fn make_cfg_traversal_stack(&mut self) -> crate::InternalResult { + debug_assert!(!self.is_empty()); + + let mut nblocks = 0; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + self[current].visited = false; + nblocks += 1; + current = self[current].next; } - instruction_sequence_addop(&mut self.instr_sequence, info)?; - Ok(()) + debug_assert!(nblocks > 0); + let mut stack = Vec::new(); + stack + .try_reserve_exact(nblocks) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + stack.resize(nblocks, BlockIdx::NULL); + let stack = CfgTraversalStack { stack, sp: 0 }; + debug_assert_eq!(stack.capacity(), nblocks); + Ok(stack) } - pub(crate) fn addop_to_instr_sequence_with_target_label( - &mut self, - mut info: InstructionInfo, - target_label: InstructionSequenceLabel, - ) -> crate::InternalResult<()> { - if !info.instr.has_target() { - return Err(InternalError::MalformedControlFlowGraph); + /// flowgraph.c normalize_jumps + fn normalize_jumps(&mut self) -> crate::InternalResult<()> { + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + self[current].visited = false; + current = self[current].next; } - info.arg = instruction_sequence_label_oparg(target_label); - info.target = BlockIdx::NULL; - instruction_sequence_addop(&mut self.instr_sequence, info)?; + + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + self[current].visited = true; + normalize_jumps_in_block(self, current)?; + current = self[current].next; + } + Ok(()) } - pub(crate) fn addop_to_current_block( - &mut self, - info: InstructionInfo, - ) -> crate::InternalResult<()> { - basicblock_addop(&mut self.blocks[self.current_block.idx()], info) - } + /// flowgraph.c remove_unused_consts + #[allow(clippy::needless_range_loop)] + fn remove_unused_consts(&mut self, consts: &mut ConstantPool) -> crate::InternalResult<()> { + let nconsts = consts.len(); + if nconsts == 0 { + return Ok(()); + } - pub(crate) fn last_current_block_instr_mut(&mut self) -> Option<&mut InstructionInfo> { - basicblock_last_instr_mut(&mut self.blocks[self.current_block.idx()]) - } + let mut index_map = Vec::new(); + index_map + .try_reserve_exact(nconsts) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + index_map.resize(nconsts, 0isize); + for i in 1..nconsts { + index_map[i] = -1; + } + // The first constant may be docstring; keep it always. + index_map[0] = 0; - pub(crate) fn set_last_instr_sequence_lineno_override(&mut self, lineno_override: i32) { - if let Some(last) = instruction_sequence_last_info_mut(&mut self.instr_sequence) { - last.lineno_override = Some(lineno_override); + // Mark used consts. + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let block = &self[block_idx]; + for i in 0..block.instruction_used { + let instr = &block.instructions[i]; + if instr.instr.has_const() { + let index = u32::from(instr.arg) as usize; + debug_assert!(index < nconsts); + index_map[index] = index as isize; + } + } + block_idx = block.next; } - } - pub(crate) fn use_instr_sequence_label( - &mut self, - block: BlockIdx, - ) -> crate::InternalResult<()> { - let label = instruction_sequence_label_map_ensure_label_for_block( - &mut self.instr_sequence_label_map, - &mut self.instr_sequence, - block, - )?; - instruction_sequence_use_label(&mut self.instr_sequence, label) - } + // Now index_map[i] == i if consts[i] is used, -1 otherwise. + // Condense consts. + let mut n_used_consts = 0; + for i in 0..nconsts { + if index_map[i] != -1 { + debug_assert_eq!(index_map[i], i as isize); + index_map[n_used_consts] = index_map[i]; + n_used_consts += 1; + } + } - pub(crate) fn new_instr_sequence_label(&mut self) -> InstructionSequenceLabel { - instruction_sequence_new_label(&mut self.instr_sequence) - } + if n_used_consts == nconsts { + return Ok(()); + } - pub(crate) fn use_raw_instr_sequence_label( - &mut self, - label: InstructionSequenceLabel, - ) -> crate::InternalResult<()> { - instruction_sequence_use_label(&mut self.instr_sequence, label) - } + // Move all used consts to the beginning of the consts list. + debug_assert!(n_used_consts < nconsts); + for i in 0..n_used_consts { + let old_index = index_map[i] as usize; + debug_assert!(i <= old_index && old_index < nconsts); + if i != old_index { + let value = consts.constants[old_index].clone(); + consts.constants[i] = value; + } + } - pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) -> crate::InternalResult<()> { - let label = instruction_sequence_label_map_ensure_label_for_block( - &mut self.instr_sequence_label_map, - &mut self.instr_sequence, - block, - )?; - self.blocks[block.idx()].cpython_label = label; + // Truncate the consts list at its new size. + consts.constants.truncate(n_used_consts); + + // Adjust const indices in the bytecode. + let mut reverse_index_map = Vec::new(); + reverse_index_map + .try_reserve_exact(nconsts) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + reverse_index_map.resize(nconsts, 0isize); + for i in 0..nconsts { + reverse_index_map[i] = -1; + } + for i in 0..n_used_consts { + let old_index = index_map[i]; + debug_assert!(old_index != -1); + let old_index = old_index as usize; + debug_assert_eq!(reverse_index_map[old_index], -1); + reverse_index_map[old_index] = i as isize; + } + + block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = self[block_idx.idx()].next; + let block = &mut self[block_idx]; + for i in 0..block.instruction_used { + let instr = &mut block.instructions[i]; + if instr.instr.has_const() { + let index = u32::from(instr.arg) as usize; + debug_assert!(reverse_index_map[index] >= 0); + debug_assert!(reverse_index_map[index] < n_used_consts as isize); + instr.arg = OpArg::new(reverse_index_map[index] as u32); + } + } + block_idx = next_block; + } Ok(()) } - pub(crate) fn resolve_instr_sequence_label(&self, block: BlockIdx) -> BlockIdx { - instruction_sequence_label_map_resolve_label(&self.instr_sequence_label_map, block) - } + /// flowgraph.c insert_superinstructions + fn insert_superinstructions(&mut self) -> crate::InternalResult { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = self[block_idx].next; + let block = &mut self[block_idx]; + for i in 0..block.instruction_used { + let nextop = (i + 1 < block.instruction_used) + .then(|| block.instructions[i + 1].instr.real_opcode()) + .flatten(); + + match (block.instructions[i].instr.real_opcode(), nextop) { + (Some(Opcode::LoadFast), _) => { + if matches!(nextop, Some(Opcode::LoadFast)) { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + make_super_instruction( + &mut inst1[0], + &mut rest[0], + Opcode::LoadFastLoadFast.into(), + ); + } + } - pub(crate) fn block_for_instr_sequence_label( - &self, - label: InstructionSequenceLabel, - ) -> BlockIdx { - instruction_sequence_label_map_resolve_label_to_block(&self.instr_sequence_label_map, label) - } + (Some(Opcode::StoreFast), Some(Opcode::LoadFast)) => { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + make_super_instruction( + &mut inst1[0], + &mut rest[0], + Opcode::StoreFastLoadFast.into(), + ); + } - pub(crate) fn use_instr_sequence_label_at_block( - &mut self, - from: BlockIdx, - to: BlockIdx, - ) -> crate::InternalResult<()> { - instruction_sequence_label_map_use_label_at_block( - &mut self.instr_sequence_label_map, - &mut self.instr_sequence, - from, - to, - ) - } + (Some(Opcode::StoreFast), Some(Opcode::StoreFast)) => { + let (inst1, rest) = block.instructions[i..].split_at_mut(1); + make_super_instruction( + &mut inst1[0], + &mut rest[0], + Opcode::StoreFastStoreFast.into(), + ); + } - pub(crate) fn instr_sequence_label_for_block( - &mut self, - block: BlockIdx, - ) -> crate::InternalResult { - if block == BlockIdx::NULL { - Ok(InstructionSequenceLabel::NO_LABEL) - } else { - instruction_sequence_label_map_ensure_label_for_block( - &mut self.instr_sequence_label_map, - &mut self.instr_sequence, - block, - ) + (_, _) => {} + } + } + + block_idx = next_block; } + + let res = remove_redundant_nops(self)?; + + #[cfg(debug_assertions)] + assert!(no_redundant_nops(self)); + + Ok(res) } +} - pub(crate) fn insert_start_setup_cleanup( - &mut self, - handler_block: BlockIdx, - ) -> crate::InternalResult<()> { - let handler_label = instruction_sequence_label_map_ensure_label_for_block( - &mut self.instr_sequence_label_map, - &mut self.instr_sequence, - handler_block, - )?; - instruction_sequence_insert_instruction( - &mut self.instr_sequence, - 0, - InstructionInfo { - instr: PseudoOpcode::SetupCleanup.into(), - arg: instruction_sequence_label_oparg(handler_label), - target: BlockIdx::NULL, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - lineno_override: Some(NO_LOCATION_OVERRIDE), - }, - ) +impl From> for Blocks { + fn from(value: Vec) -> Self { + Self(value) } +} - pub(crate) fn push_unmapped_instr_sequence_label(&mut self) -> crate::InternalResult<()> { - instruction_sequence_label_map_push_unmapped_label( - &mut self.instr_sequence_label_map, - &mut self.instr_sequence, - ) +impl From> for Blocks { + fn from(value: Box<[Block]>) -> Self { + Self(value.into()) } +} - pub(crate) fn push_unlabeled_instr_sequence_block(&mut self) -> crate::InternalResult<()> { - instruction_sequence_label_map_push_unlabeled_block(&mut self.instr_sequence_label_map) +impl From<&[Block]> for Blocks { + fn from(value: &[Block]) -> Self { + Self(value.to_vec()) } +} - fn take_recorded_instr_sequence(&mut self) -> crate::InternalResult { - let mut instr_sequence = - core::mem::replace(&mut self.instr_sequence, instruction_sequence_new()); - if let Some(mut annotations_instr_sequence) = self.annotations_instr_sequence.take() { - instruction_sequence_apply_label_map(&mut annotations_instr_sequence)?; - instruction_sequence_set_annotations_code( - &mut instr_sequence, - Some(Box::new(annotations_instr_sequence)), - ); - } - Ok(instr_sequence) +impl From<&mut [Block]> for Blocks { + fn from(value: &mut [Block]) -> Self { + Self(value.to_vec()) } +} - fn prepare_cfg_from_codegen(&mut self) -> crate::InternalResult { - // CPython compile.c optimize_and_assemble_code_unit passes - // u_instr_sequence directly into flowgraph.c _PyCfg_FromInstructionSequence(). - self.take_recorded_instr_sequence() +impl From<[Block; N]> for Blocks { + fn from(value: [Block; N]) -> Self { + Self(value.into()) } } -fn optimize_code_unit( - metadata: &mut CodeUnitMetadata, - blocks: &mut Blocks, - instr_sequence: InstructionSequence, - nlocals: usize, - nparams: usize, -) -> crate::InternalResult<()> { - // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) - *blocks = cfg_from_instruction_sequence(instr_sequence)?; - translate_jump_labels_to_targets(blocks)?; - mark_except_handlers(blocks)?; - label_exception_targets(blocks)?; - optimize_cfg(metadata, blocks, metadata.firstlineno)?; - remove_unused_consts(blocks, &mut metadata.consts)?; - add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams)?; - // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before - // later jump normalization / block reordering can create adjacencies - // that never exist at this stage in flowgraph.c. - insert_superinstructions(blocks)?; - push_cold_blocks_to_end(blocks)?; - // CPython resolves line numbers again after cold-block extraction. - resolve_line_numbers(blocks, metadata.firstlineno)?; - Ok(()) +impl From<&[Block; N]> for Blocks { + fn from(value: &[Block; N]) -> Self { + Self(value.to_vec()) + } } -fn optimize_cfg( - metadata: &mut CodeUnitMetadata, - blocks: &mut Blocks, - firstlineno: OneIndexed, -) -> crate::InternalResult<()> { - // flowgraph.c optimize_cfg - // CPython optimize_cfg() starts with check_cfg() and raises - // SystemError if a jump or scope exit is not the last instruction in - // its block. - check_cfg(blocks)?; - inline_small_or_no_lineno_blocks(blocks)?; - // CPython does not re-run instruction-sequence label-map/CFG conversion - // after this point. Unreferenced label blocks left by jump inlining - // remain block boundaries and can preserve line-marker NOPs. - remove_unreachable(blocks)?; - // CPython optimize_cfg resolves line numbers before local checks and - // superinstruction insertion, so fusion decisions see propagated - // source locations. - resolve_line_numbers(blocks, firstlineno)?; - // CPython optimize_cfg() runs optimize_load_const() and then - // optimize_basic_block() after line numbers are resolved. - optimize_load_const(metadata, blocks)?; - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = blocks[block_idx].next; - optimize_basic_block(blocks, metadata, block_idx)?; - block_idx = next_block; +impl Deref for Blocks { + type Target = [Block]; + + fn deref(&self) -> &Self::Target { + &self.0 } - remove_redundant_nops_and_pairs(blocks)?; - // CPython optimize_cfg() removes newly-unreachable blocks and - // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes - // unused constants. - remove_unreachable(blocks)?; - remove_redundant_nops_and_jumps(blocks)?; - #[cfg(debug_assertions)] - assert!(no_redundant_jumps(blocks)); - Ok(()) } -fn optimized_cfg_to_instruction_sequence( - metadata: &CodeUnitMetadata, - flags: CodeFlags, - blocks: &mut Blocks, -) -> crate::InternalResult<(u32, usize, InstructionSequence)> { - // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) - convert_pseudo_conditional_jumps(blocks)?; - let max_stackdepth = calculate_stackdepth(blocks)?; - debug_assert!(!is_generator(flags) || max_stackdepth != 0); - let nlocalsplus = prepare_localsplus(metadata, blocks, flags)?; - // Match CPython order: pseudo ops are lowered after stackdepth and - // localsplus preparation, before normalize_jumps. - convert_pseudo_ops(blocks)?; - normalize_jumps(blocks)?; - #[cfg(debug_assertions)] - assert!(no_redundant_jumps(blocks)); - // optimize_load_fast: after normalize_jumps - optimize_load_fast(blocks)?; +impl DerefMut for Blocks { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} - let mut instr_sequence = instruction_sequence_new(); - cfg_to_instruction_sequence(blocks, &mut instr_sequence)?; - Ok((max_stackdepth, nlocalsplus, instr_sequence)) +impl Index for Blocks { + type Output = Block; + + fn index(&self, idx: usize) -> &Self::Output { + &self.0[idx] + } } -impl CodeInfo { - pub fn finalize_code( - mut self, - opts: &crate::compile::CompileOpts, - ) -> crate::InternalResult { - let instr_sequence = self.prepare_cfg_from_codegen()?; - let nlocals = self.metadata.varnames.len(); - let nparams = self.nparams; - optimize_code_unit( - &mut self.metadata, - &mut self.blocks, - instr_sequence, - nlocals, - nparams, - )?; - let (max_stackdepth, nlocalsplus, mut instr_sequence) = - optimized_cfg_to_instruction_sequence(&self.metadata, self.flags, &mut self.blocks)?; - let localsplusinfo = compute_localsplus_info(&self.metadata, nlocalsplus, self.flags)?; +impl IndexMut for Blocks { + fn index_mut(&mut self, idx: usize) -> &mut Self::Output { + &mut self.0[idx] + } +} - let Self { - flags, - source_path, - private: _, // private is only used during compilation +impl Index for Blocks { + type Output = Block; - blocks: _, - current_block: _, - instr_sequence: _, - instr_sequence_label_map: _, - annotations_instr_sequence: _, - metadata, - static_attributes: _, - in_inlined_comp: _, - fblock: _, - symbol_table_index: _, - nparams: _, - in_conditional_block: _, - next_conditional_annotation_index: _, - } = self; + fn index(&self, block_idx: BlockIdx) -> &Self::Output { + &self.0[block_idx.as_usize()] + } +} - let CodeUnitMetadata { - name: obj_name, - qualname, - consts: constants, - names: name_cache, - varnames: varname_cache, - cellvars: _, - freevars: freevar_cache, - fast_hidden: _, - fast_hidden_final: _, - argcount: arg_count, - posonlyargcount: posonlyarg_count, - kwonlyargcount: kwonlyarg_count, - firstlineno: first_line_number, - } = metadata; +impl IndexMut for Blocks { + fn index_mut(&mut self, block_idx: BlockIdx) -> &mut Self::Output { + &mut self.0[block_idx.as_usize()] + } +} - resolve_unconditional_jumps(&mut instr_sequence)?; - resolve_jump_offsets(&mut instr_sequence)?; - let assembled = assemble_emit( - &mut instr_sequence, - first_line_number.get() as i32, - opts.debug_ranges, - )?; - let locations = rustpython_compiler_core::marshal::linetable_to_locations( - &assembled.linetable, - first_line_number.get() as i32, - assembled.instructions.len(), - ); +pub(crate) const START_DEPTH_UNSET: i32 = i32::MIN; +const CO_MAXBLOCKS: usize = 20; - Ok(CodeObject { - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number: Some(first_line_number), - obj_name: obj_name.clone(), - qualname: qualname.unwrap_or(obj_name), +/// flowgraph.c struct _PyCfgExceptStack +#[derive(Clone, Debug)] +struct CfgExceptStack { + handlers: [BlockIdx; CO_MAXBLOCKS + 2], + depth: usize, +} - max_stackdepth, - instructions: CodeUnits::from(assembled.instructions), - locations, - constants: constants.into_iter().collect(), - names: name_cache.into_iter().collect(), - varnames: varname_cache.into_iter().collect(), - cellvars: localsplusinfo.cellvars, - freevars: freevar_cache.into_iter().collect(), - localspluskinds: localsplusinfo.kinds, - linetable: assembled.linetable, - exceptiontable: assembled.exceptiontable, - }) +/// flowgraph.c `basicblock **stack` +#[derive(Clone, Debug)] +struct CfgTraversalStack { + stack: Vec, + sp: usize, +} + +impl CfgTraversalStack { + fn push(&mut self, block: BlockIdx) { + debug_assert!(self.sp < self.stack.len()); + self.stack[self.sp] = block; + self.sp += 1; + } + + fn pop(&mut self) -> Option { + if self.sp == 0 { + return None; + } + self.sp -= 1; + Some(self.stack[self.sp]) + } + + fn capacity(&self) -> usize { + self.stack.len() } } -/// flowgraph.c IS_GENERATOR -fn is_generator(flags: CodeFlags) -> bool { - flags.intersects(CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR) +#[derive(Clone, Debug)] +pub(crate) struct InstructionSequenceLabelMap { + block_labels: Vec, + /// Codegen-side shadow of CPython's instruction-sequence label map. + /// + /// `_PyInstructionSequence_UseLabel()` can map multiple labels to the same + /// instruction offset before `_PyCfg_FromInstructionSequence()` materializes + /// CFG blocks. The codegen CFG path keeps the same aliasing by resolving + /// those labels to the block that owns the shared offset. + cpython_block_by_label: Vec, } -/// flowgraph.c insert_prefix_instructions -fn insert_prefix_instructions( - metadata: &CodeUnitMetadata, - blocks: &mut Blocks, - cellfixedoffsets: &[i32], - nfreevars: usize, - flags: CodeFlags, +fn instruction_sequence_label_map_register_label( + map: &mut InstructionSequenceLabelMap, + label: InstructionSequenceLabel, ) -> crate::InternalResult<()> { - debug_assert!(!blocks.is_empty()); - let entry = &mut blocks[0]; - let ncellvars = metadata.cellvars.len(); - let firstlineno = metadata.firstlineno; - debug_assert!(firstlineno.get() > 0); - - if is_generator(flags) { - let location = SourceLocation { - line: firstlineno, - character_offset: OneIndexed::MIN, - }; - basicblock_insert_instruction( - entry, - 0, - InstructionInfo { - instr: Instruction::ReturnGenerator.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location, - end_location: location, - except_handler: None, - lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), - }, - )?; - basicblock_insert_instruction( - entry, - 1, - InstructionInfo { - instr: Instruction::PopTop.into(), - arg: OpArg::new(0), - target: BlockIdx::NULL, - location, - end_location: location, - except_handler: None, - lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), - }, - )?; - } - - if ncellvars > 0 { - let nvars = metadata.varnames.len() + ncellvars; - let mut sorted = Vec::new(); - vec_try_reserve_exact(&mut sorted, nvars)?; - sorted.resize(nvars, 0i32); - for i in 0..ncellvars { - sorted[cellfixedoffsets[i] as usize] = i as i32 + 1; + debug_assert!(is_label(label)); + let old_size = map.cpython_block_by_label.len(); + let new_allocation = c_array_ensure_capacity::( + old_size, + label.idx(), + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + )?; + if new_allocation > old_size { + if new_allocation > map.cpython_block_by_label.capacity() { + map.cpython_block_by_label + .try_reserve_exact(new_allocation - map.cpython_block_by_label.capacity()) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; } - let mut ncellsused = 0; - let mut i = 0; - while ncellsused < ncellvars { - let oldindex = sorted[i] - 1; - i += 1; - if oldindex == -1 { - continue; - } - basicblock_insert_instruction( - entry, - ncellsused, - InstructionInfo { - instr: Opcode::MakeCell.into(), - arg: OpArg::new(oldindex as u32), - target: BlockIdx::NULL, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - lineno_override: Some(NO_LOCATION_OVERRIDE), - }, - )?; - ncellsused += 1; + map.cpython_block_by_label + .resize(new_allocation, BlockIdx::NULL); + for i in old_size..map.cpython_block_by_label.len() { + map.cpython_block_by_label[i] = BlockIdx::NULL; } } + debug_assert!(map.cpython_block_by_label.len() > label.idx()); + Ok(()) +} - if nfreevars > 0 { - basicblock_insert_instruction( - entry, - 0, - InstructionInfo { - instr: Opcode::CopyFreeVars.into(), - arg: OpArg::new(nfreevars as u32), - target: BlockIdx::NULL, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - lineno_override: Some(NO_LOCATION_OVERRIDE), - }, - )?; +fn instruction_sequence_label_map_ensure_label_for_block( + map: &mut InstructionSequenceLabelMap, + seq: &mut InstructionSequence, + block: BlockIdx, +) -> crate::InternalResult { + debug_assert_ne!(block, BlockIdx::NULL); + let block_label = map.block_labels[block.idx()]; + if is_label(block_label) { + return Ok(block_label); } - Ok(()) + let label = instruction_sequence_new_label(seq); + debug_assert_eq!(label.0, seq.next_free_label); + instruction_sequence_label_map_register_label(map, label)?; + map.cpython_block_by_label[label.idx()] = block; + map.block_labels[block.idx()] = label; + Ok(label) } -/// flowgraph.c prepare_localsplus -fn prepare_localsplus( - metadata: &CodeUnitMetadata, - blocks: &mut Blocks, - flags: CodeFlags, -) -> crate::InternalResult { - let nlocals = metadata.varnames.len(); - let ncellvars = metadata.cellvars.len(); - let nfreevars = metadata.freevars.len(); - let int_max = i32::MAX as usize; - debug_assert!(nlocals < int_max); - debug_assert!(ncellvars < int_max); - debug_assert!(nfreevars < int_max); - debug_assert!(int_max - nlocals - ncellvars > 0); - debug_assert!(int_max - nlocals - ncellvars - nfreevars > 0); - let mut nlocalsplus = nlocals + ncellvars + nfreevars; - let mut cellfixedoffsets = build_cellfixedoffsets(metadata)?; +fn instruction_sequence_label_map_label_for_block( + map: &InstructionSequenceLabelMap, + block: BlockIdx, +) -> InstructionSequenceLabel { + debug_assert_ne!(block, BlockIdx::NULL); + map.block_labels + .get(block.idx()) + .copied() + .unwrap_or(InstructionSequenceLabel::NO_LABEL) +} - // This must be called before fix_cell_offsets(). - insert_prefix_instructions(metadata, blocks, &cellfixedoffsets, nfreevars, flags)?; +fn instruction_sequence_label_map_block_for_label( + map: &InstructionSequenceLabelMap, + label: InstructionSequenceLabel, +) -> Option { + if !is_label(label) { + return None; + } + map.cpython_block_by_label + .get(label.idx()) + .copied() + .filter(|&block| block != BlockIdx::NULL) +} - let numdropped = fix_cell_offsets(metadata, blocks, &mut cellfixedoffsets); - nlocalsplus -= numdropped; - Ok(nlocalsplus) +fn instruction_sequence_label_map_resolve_label( + map: &InstructionSequenceLabelMap, + block: BlockIdx, +) -> BlockIdx { + if block == BlockIdx::NULL { + return BlockIdx::NULL; + } + let label = instruction_sequence_label_map_label_for_block(map, block); + if !is_label(label) { + return block; + } + instruction_sequence_label_map_block_for_label(map, label).unwrap_or_else(|| { + debug_assert!( + false, + "CPython instruction-sequence label must map to a codegen CFG block" + ); + BlockIdx::NULL + }) } -/// flowgraph.c remove_unreachable -fn remove_unreachable(blocks: &mut Blocks) -> crate::InternalResult<()> { - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - blocks[block_idx].predecessors = 0; - block_idx = blocks[block_idx].next; +fn instruction_sequence_label_map_resolve_label_to_block( + map: &InstructionSequenceLabelMap, + label: InstructionSequenceLabel, +) -> BlockIdx { + if !is_label(label) { + return BlockIdx::NULL; } + instruction_sequence_label_map_block_for_label(map, label).unwrap_or_else(|| { + debug_assert!( + false, + "CPython instruction-sequence label must map to a codegen CFG block" + ); + BlockIdx::NULL + }) +} - let mut stack = make_cfg_traversal_stack(blocks)?; - blocks[0].predecessors = 1; - stack.push(BlockIdx(0)); - blocks[0].visited = true; - while let Some(current) = stack.pop() { - let idx = current.idx(); - let next = blocks[idx].next; - if next != BlockIdx::NULL && bb_has_fallthrough(&blocks[idx]) { - if !blocks[next].visited { - debug_assert_eq!(blocks[next].predecessors, 0); - stack.push(next); - blocks[next].visited = true; - } - blocks[next].predecessors += 1; - } +fn instruction_sequence_label_oparg(label: InstructionSequenceLabel) -> OpArg { + debug_assert!(is_label(label)); + OpArg::new(label.idx() as u32) +} - let instr_count = blocks[idx].instruction_used; - for i in 0..instr_count { - let instr = blocks[idx].instructions[i]; - if is_jump(&instr) || is_block_push(&instr) { - let target = instr.target; - debug_assert!(target != BlockIdx::NULL); - let target_idx = target.idx(); - if !blocks[target_idx].visited { - stack.push(target); - blocks[target_idx].visited = true; - } - blocks[target_idx].predecessors += 1; - } - } +fn instruction_sequence_label_map_use_label_at_block( + map: &mut InstructionSequenceLabelMap, + seq: &mut InstructionSequence, + from: BlockIdx, + to: BlockIdx, +) -> crate::InternalResult<()> { + if from == BlockIdx::NULL || from == to { + return Ok(()); } - - block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let i = block_idx.idx(); - let next = blocks[i].next; - if blocks[i].predecessors == 0 { - let block = &mut blocks[i]; - basicblock_clear(block); - block.except_handler = false; - } - block_idx = next; + let from_label = instruction_sequence_label_map_ensure_label_for_block(map, seq, from)?; + debug_assert!(map.cpython_block_by_label.len() > from_label.idx()); + let to_block = instruction_sequence_label_map_resolve_label(map, to); + if to_block == BlockIdx::NULL { + debug_assert!( + false, + "CPython label target must map to a codegen CFG block" + ); + return Ok(()); } + map.cpython_block_by_label[from_label.idx()] = to_block; Ok(()) } -/// flowgraph.c eval_const_unaryop -fn eval_const_unaryop( - operand: &ConstantData, - op: Instruction, - intrinsic: Option, -) -> Option { - match (operand, op, intrinsic) { - (ConstantData::Integer { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Integer { value: -value }) - } - (ConstantData::Float { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Float { value: -value }) - } - (ConstantData::Complex { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Complex { value: -value }) - } - (ConstantData::Boolean { value }, Instruction::UnaryNegative, None) => { - Some(ConstantData::Integer { - value: BigInt::from(-i32::from(*value)), - }) - } - (ConstantData::Integer { value }, Instruction::UnaryInvert, None) => { - Some(ConstantData::Integer { value: !value }) - } - (ConstantData::Boolean { .. }, Instruction::UnaryInvert, None) => None, - (_, Instruction::UnaryNot, None) => Some(ConstantData::Boolean { - value: !operand.truthiness(), - }), - ( - ConstantData::Integer { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Integer { - value: value.clone(), - }), - ( - ConstantData::Float { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Float { value: *value }), - ( - ConstantData::Boolean { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Integer { - value: BigInt::from(i32::from(*value)), - }), - ( - ConstantData::Complex { value }, - Instruction::CallIntrinsic1 { .. }, - Some(oparg::IntrinsicFunction1::UnaryPositive), - ) => Some(ConstantData::Complex { value: *value }), - _ => None, - } -} - -fn load_const_truthiness( - instr: Instruction, - arg: OpArg, - metadata: &CodeUnitMetadata, -) -> Option { - match instr { - Instruction::LoadConst { consti } => { - let constant = &metadata.consts[consti.get(arg).as_usize()]; - Some(constant.truthiness()) - } - Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), - _ => None, - } -} - -/// flowgraph.c add_const -fn add_const( - metadata: &mut CodeUnitMetadata, - constant: ConstantData, -) -> crate::InternalResult { - Ok(metadata.consts.try_insert_full(constant)?.0) +fn instruction_sequence_label_map_push_unlabeled_block( + map: &mut InstructionSequenceLabelMap, +) -> crate::InternalResult<()> { + map.block_labels + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + map.block_labels.push(InstructionSequenceLabel::NO_LABEL); + Ok(()) } -fn instr_make_load_const( - metadata: &mut CodeUnitMetadata, - instr: &mut InstructionInfo, - constant: ConstantData, +fn instruction_sequence_label_map_push_unmapped_label( + map: &mut InstructionSequenceLabelMap, + seq: &mut InstructionSequence, ) -> crate::InternalResult<()> { - if maybe_instr_make_load_smallint(instr, &constant) { - return Ok(()); - } - - let const_idx = add_const(metadata, constant)?; - instr_set_op1( - instr, - Opcode::LoadConst.into(), - OpArg::new(const_idx as u32), + let label = instruction_sequence_new_label(seq); + debug_assert_eq!(label.0, seq.next_free_label); + instruction_sequence_label_map_register_label(map, label)?; + let block = BlockIdx( + map.block_labels + .len() + .to_u32() + .ok_or(InternalError::MalformedControlFlowGraph)?, ); + map.cpython_block_by_label[label.idx()] = block; + map.block_labels + .try_reserve(1) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + map.block_labels.push(label); Ok(()) } -/// flowgraph.c fold_const_unaryop -fn fold_const_unaryop( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, -) -> crate::InternalResult { - let instr = &block.instructions[i]; - let (op, intrinsic) = match instr.instr.real() { - Some(Instruction::UnaryNegative) => (Instruction::UnaryNegative, None), - Some(Instruction::UnaryInvert) => (Instruction::UnaryInvert, None), - Some(Instruction::UnaryNot) => (Instruction::UnaryNot, None), - Some(Instruction::CallIntrinsic1 { func }) - if matches!( - func.get(instr.arg), - oparg::IntrinsicFunction1::UnaryPositive - ) => - { - (Opcode::CallIntrinsic1.into(), Some(func.get(instr.arg))) - } - _ => return Ok(false), - }; - let Some(operand_index) = (if let Some(start) = i.checked_sub(1) { - get_const_loading_instrs(block, start, 1)? - } else { - None - }) - .and_then(|indices| indices.into_iter().next()) else { - return Ok(false); - }; - let operand = get_const_value(metadata, &block.instructions[operand_index]); - let Some(operand) = operand else { - return Ok(false); - }; - let Some(folded_const) = eval_const_unaryop(&operand, op, intrinsic) else { - return Ok(false); - }; - nop_out(block, &[operand_index]); - instr_make_load_const(metadata, &mut block.instructions[i], folded_const)?; - Ok(true) -} - -/// flowgraph.c get_const_loading_instrs -fn get_const_loading_instrs( - block: &Block, - mut start: usize, - size: usize, -) -> crate::InternalResult>> { - let mut indices = Vec::new(); - indices - .try_reserve_exact(size) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - loop { - if start >= block.instruction_used { - return Ok(None); - } - let instr = &block.instructions[start]; - if !matches!(instr.instr.real(), Some(Instruction::Nop)) { - if !loads_const(instr) { - return Ok(None); - } - indices.push(start); - if indices.len() == size { - break; - } +impl InstructionSequenceLabelMap { + pub(crate) fn new() -> Self { + Self { + block_labels: vec![InstructionSequenceLabel::NO_LABEL], + cpython_block_by_label: Vec::new(), } - let Some(prev) = start.checked_sub(1) else { - return Ok(None); - }; - start = prev; } - indices.reverse(); - Ok(Some(indices)) } -/// flowgraph.c nop_out -fn nop_out(block: &mut Block, instrs: &[usize]) { - for &i in instrs { - nop_out_no_location(&mut block.instructions[i]); - } -} +pub struct CodeInfo { + pub flags: CodeFlags, + pub source_path: String, + pub private: Option, // For private name mangling, mostly for class -/// flowgraph.c fold_const_binop -fn fold_const_binop( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, -) -> crate::InternalResult { - use oparg::BinaryOperator as BinOp; + pub blocks: Blocks, + pub current_block: BlockIdx, + pub(crate) instr_sequence: InstructionSequence, + pub(crate) instr_sequence_label_map: InstructionSequenceLabelMap, + pub(crate) annotations_instr_sequence: Option, - let Some(Opcode::BinaryOp) = block.instructions[i].instr.real_opcode() else { - return Ok(false); - }; + pub metadata: CodeUnitMetadata, - let Some(operand_indices) = (if let Some(start) = i.checked_sub(1) { - get_const_loading_instrs(block, start, 2)? - } else { - None - }) else { - return Ok(false); - }; + // For class scopes: attributes accessed via self.X + pub static_attributes: Option>, - let op_raw = u32::from(block.instructions[i].arg); - let Ok(op) = BinOp::try_from(op_raw) else { - return Ok(false); - }; + // True if compiling an inlined comprehension + pub in_inlined_comp: bool, - let left = get_const_value(metadata, &block.instructions[operand_indices[0]]); - let right = get_const_value(metadata, &block.instructions[operand_indices[1]]); - let (Some(left_val), Some(right_val)) = (left, right) else { - return Ok(false); - }; + // Block stack for tracking nested control structures + pub fblock: Vec, - let Some(result_const) = eval_const_binop(&left_val, &right_val, op) else { - return Ok(false); - }; + // Reference to the symbol table for this scope + pub symbol_table_index: usize, + // CPython compile.c uses PyList_GET_SIZE(u->u_ste->ste_varnames) + // when calling flowgraph.c _PyCfg_OptimizeCodeUnit(). + pub nparams: usize, - nop_out(block, &operand_indices); - instr_make_load_const(metadata, &mut block.instructions[i], result_const)?; - Ok(true) -} + // PEP 649: Track nesting depth inside conditional blocks (if/for/while/etc.) + // u_in_conditional_block + pub in_conditional_block: u32, -/// flowgraph.c loads_const -fn loads_const(info: &InstructionInfo) -> bool { - info.instr.has_const() || matches!(info.instr.real_opcode(), Some(Opcode::LoadSmallInt)) + // PEP 649: Next index for conditional annotation tracking + // u_next_conditional_annotation_index + pub next_conditional_annotation_index: u32, } -/// flowgraph.c get_const_value -fn get_const_value(metadata: &CodeUnitMetadata, info: &InstructionInfo) -> Option { - match info.instr.real_opcode() { - Some(Opcode::LoadSmallInt) => { - let v = u32::from(info.arg) as i32; - Some(ConstantData::Integer { - value: BigInt::from(v), - }) - } - _ if info.instr.has_const() => { - let idx = u32::from(info.arg) as usize; - metadata.consts.get_index(idx).cloned() +impl CodeInfo { + pub(crate) fn addop_to_instr_sequence( + &mut self, + mut info: InstructionInfo, + ) -> crate::InternalResult<()> { + if info.instr.has_target() && info.target != BlockIdx::NULL { + let label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + info.target, + )?; + info.arg = instruction_sequence_label_oparg(label); + info.target = BlockIdx::NULL; } - _ => None, + instruction_sequence_addop(&mut self.instr_sequence, info)?; + Ok(()) } -} -/// flowgraph.c const_folding_check_complexity -fn const_folding_check_complexity(obj: &ConstantData, mut limit: isize) -> Option { - if let ConstantData::Tuple { elements } = obj { - limit -= isize::try_from(elements.len()).ok()?; - if limit < 0 { - return None; - } - for element in elements { - limit = const_folding_check_complexity(element, limit)?; + pub(crate) fn addop_to_instr_sequence_with_target_label( + &mut self, + mut info: InstructionInfo, + target_label: InstructionSequenceLabel, + ) -> crate::InternalResult<()> { + if !info.instr.has_target() { + return Err(InternalError::MalformedControlFlowGraph); } + info.arg = instruction_sequence_label_oparg(target_label); + info.target = BlockIdx::NULL; + instruction_sequence_addop(&mut self.instr_sequence, info)?; + Ok(()) } - Some(limit) -} -fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Option { - let mut result = Wtf8Buf::new(); - result.try_reserve_exact(value.len().checked_mul(n)?).ok()?; - for _ in 0..n { - result.push_wtf8(value); + pub(crate) fn addop_to_current_block( + &mut self, + info: InstructionInfo, + ) -> crate::InternalResult<()> { + basicblock_addop(&mut self.blocks[self.current_block.idx()], info) } - Some(result) -} -fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { - let n = n.to_isize()?; - if item_size != 0 && (n < 0 || n as usize > MAX_STR_SIZE / item_size) { - return None; + pub(crate) fn last_current_block_instr_mut(&mut self) -> Option<&mut InstructionInfo> { + basicblock_last_instr_mut(&mut self.blocks[self.current_block.idx()]) } - Some(n.max(0) as usize) -} -/// flowgraph.c const_folding_safe_multiply -fn const_folding_safe_multiply(left: &ConstantData, right: &ConstantData) -> Option { - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - if !l.is_zero() && !r.is_zero() && l.bits() + r.bits() > MAX_INT_SIZE { - return None; - } - Some(ConstantData::Integer { value: l * r }) - } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - Some(ConstantData::Float { value: l * r }) - } - (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) => { - let n = checked_repeat_count(n, s.code_points().count())?; - Some(ConstantData::Str { - value: repeat_wtf8(s, n)?, - }) - } - (ConstantData::Integer { .. }, ConstantData::Str { .. }) => { - const_folding_safe_multiply(right, left) - } - (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) => { - let n = checked_repeat_count(n, b.len())?; - let mut value = Vec::new(); - value.try_reserve_exact(b.len().checked_mul(n)?).ok()?; - for _ in 0..n { - value.extend_from_slice(b); - } - Some(ConstantData::Bytes { value }) - } - (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) => { - const_folding_safe_multiply(right, left) - } - (ConstantData::Tuple { elements }, ConstantData::Integer { value: n }) => { - let n = n.to_usize()?; - if n != 0 && !elements.is_empty() { - if n > MAX_COLLECTION_SIZE / elements.len() { - return None; - } - const_folding_check_complexity( - &ConstantData::Tuple { - elements: elements.clone(), - }, - MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, - )?; - } - let mut result = Vec::new(); - result - .try_reserve_exact(elements.len().checked_mul(n)?) - .ok()?; - for _ in 0..n { - result.extend(elements.iter().cloned()); - } - Some(ConstantData::Tuple { elements: result }) - } - (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) => { - const_folding_safe_multiply(right, left) + pub(crate) fn set_last_instr_sequence_lineno_override(&mut self, lineno_override: i32) { + if let Some(last) = instruction_sequence_last_info_mut(&mut self.instr_sequence) { + last.lineno_override = Some(lineno_override); } - _ => None, } -} -/// flowgraph.c const_folding_safe_power -fn const_folding_safe_power(left: &ConstantData, right: &ConstantData) -> Option { - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - if r < &BigInt::from(0) { - if l.is_zero() { - return None; - } - let base = l.to_f64()?; - if !base.is_finite() { - return None; - } - let result = if let Some(exp) = r.to_i32() { - base.powi(exp) - } else { - base.powf(r.to_f64()?) - }; - if !result.is_finite() { - return None; - } - return Some(ConstantData::Float { value: result }); - } - let exp: u64 = r.try_into().ok()?; - let exp_usize = usize::try_from(exp).ok()?; - if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE / exp { - return None; - } - Some(ConstantData::Integer { - value: num_traits::pow::pow(l.clone(), exp_usize), - }) - } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - let result = l.powf(*r); - result - .is_finite() - .then_some(ConstantData::Float { value: result }) - } - _ => None, + pub(crate) fn use_instr_sequence_label( + &mut self, + block: BlockIdx, + ) -> crate::InternalResult<()> { + let label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + block, + )?; + instruction_sequence_use_label(&mut self.instr_sequence, label) } -} -/// flowgraph.c const_folding_safe_lshift -fn const_folding_safe_lshift(left: &ConstantData, right: &ConstantData) -> Option { - let (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) = (left, right) - else { - return None; - }; - let shift: u64 = r.try_into().ok()?; - let shift_usize = usize::try_from(shift).ok()?; - if shift > MAX_INT_SIZE || (!l.is_zero() && l.bits() > MAX_INT_SIZE - shift) { - return None; + pub(crate) fn new_instr_sequence_label(&mut self) -> InstructionSequenceLabel { + instruction_sequence_new_label(&mut self.instr_sequence) } - Some(ConstantData::Integer { - value: l << shift_usize, - }) -} -/// flowgraph.c const_folding_safe_mod -fn const_folding_safe_mod(left: &ConstantData, right: &ConstantData) -> Option { - if matches!(left, ConstantData::Str { .. } | ConstantData::Bytes { .. }) { - return None; + pub(crate) fn use_raw_instr_sequence_label( + &mut self, + label: InstructionSequenceLabel, + ) -> crate::InternalResult<()> { + instruction_sequence_use_label(&mut self.instr_sequence, label) } - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - if r.is_zero() { - return None; - } - let rem = l.clone() % r.clone(); - let value = if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { - rem + r - } else { - rem - }; - Some(ConstantData::Integer { value }) - } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - let (_, modulo) = float_div_mod(*l, *r)?; - Some(ConstantData::Float { value: modulo }) - } - _ => None, + pub(crate) fn mark_cpython_cfg_label(&mut self, block: BlockIdx) -> crate::InternalResult<()> { + let label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + block, + )?; + self.blocks[block.idx()].cpython_label = label; + Ok(()) } -} -fn float_div_mod(left: f64, right: f64) -> Option<(f64, f64)> { - if right == 0.0 { - return None; + pub(crate) fn resolve_instr_sequence_label(&self, block: BlockIdx) -> BlockIdx { + instruction_sequence_label_map_resolve_label(&self.instr_sequence_label_map, block) } - let mut modulo = left % right; - let div = (left - modulo) / right; - let floordiv = if modulo != 0.0 { - let div = if (right < 0.0) != (modulo < 0.0) { - modulo += right; - div - 1.0 - } else { - div - }; - let mut floordiv = div.floor(); - if div - floordiv > 0.5 { - floordiv += 1.0; - } - floordiv - } else { - modulo = 0.0f64.copysign(right); - 0.0f64.copysign(left / right) - }; + pub(crate) fn block_for_instr_sequence_label( + &self, + label: InstructionSequenceLabel, + ) -> BlockIdx { + instruction_sequence_label_map_resolve_label_to_block(&self.instr_sequence_label_map, label) + } - Some((floordiv, modulo)) -} - -/// flowgraph.c eval_const_binop complex result construction -fn eval_const_complex_const(value: Complex) -> Option { - (value.re.is_finite() && value.im.is_finite()).then_some(ConstantData::Complex { value }) -} - -/// flowgraph.c eval_const_binop complex operations -fn eval_const_complex_binop( - left: Complex, - right: Complex, - op: oparg::BinaryOperator, -) -> Option { - use oparg::BinaryOperator as BinOp; - - let value = match op { - BinOp::Add => left + right, - BinOp::Subtract => { - let re = left.re - right.re; - // Preserve CPython's signed-zero behavior for real-zero - // minus zero-complex expressions such as `0 - 0j`. - let im = if left.re == 0.0 - && left.im == 0.0 - && right.re == 0.0 - && right.im == 0.0 - && !right.im.is_sign_negative() - { - -0.0 - } else { - left.im - right.im - }; - Complex::new(re, im) - } - BinOp::Multiply => left * right, - BinOp::TrueDivide => { - if right == Complex::new(0.0, 0.0) { - return None; - } - left / right - } - BinOp::Power => { - if left == Complex::new(0.0, 0.0) { - if right.im != 0.0 || right.re < 0.0 { - return None; - } - - return eval_const_complex_const(if right.re == 0.0 { - Complex::new(1.0, 0.0) - } else { - Complex::new(0.0, 0.0) - }); - } + pub(crate) fn use_instr_sequence_label_at_block( + &mut self, + from: BlockIdx, + to: BlockIdx, + ) -> crate::InternalResult<()> { + instruction_sequence_label_map_use_label_at_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + from, + to, + ) + } - if right.im == 0.0 - && right.re.fract() == 0.0 - && right.re >= f64::from(i32::MIN) - && right.re <= f64::from(i32::MAX) - { - left.powi(right.re as i32) - } else { - left.powc(right) - } + pub(crate) fn instr_sequence_label_for_block( + &mut self, + block: BlockIdx, + ) -> crate::InternalResult { + if block == BlockIdx::NULL { + Ok(InstructionSequenceLabel::NO_LABEL) + } else { + instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + block, + ) } - _ => return None, - }; - eval_const_complex_const(value) -} + } -/// flowgraph.c eval_const_binop subscript index conversion -fn constant_as_index(value: &ConstantData) -> Option { - match value { - ConstantData::Integer { value } => value.to_i64().or_else(|| { - if value < &BigInt::from(0) { - Some(i64::MIN) - } else { - Some(i64::MAX) - } - }), - ConstantData::Boolean { value } => Some(i64::from(*value)), - _ => None, + pub(crate) fn insert_start_setup_cleanup( + &mut self, + handler_block: BlockIdx, + ) -> crate::InternalResult<()> { + let handler_label = instruction_sequence_label_map_ensure_label_for_block( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + handler_block, + )?; + instruction_sequence_insert_instruction( + &mut self.instr_sequence, + 0, + InstructionInfo { + instr: PseudoOpcode::SetupCleanup.into(), + arg: instruction_sequence_label_oparg(handler_label), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(NO_LOCATION_OVERRIDE), + }, + ) } -} -/// flowgraph.c eval_const_binop subscript slice bound conversion -fn slice_bound(value: &ConstantData) -> Option> { - match value { - ConstantData::None => Some(None), - _ => constant_as_index(value).map(Some), + pub(crate) fn push_unmapped_instr_sequence_label(&mut self) -> crate::InternalResult<()> { + instruction_sequence_label_map_push_unmapped_label( + &mut self.instr_sequence_label_map, + &mut self.instr_sequence, + ) } -} -/// flowgraph.c eval_const_binop subscript slice index adjustment -fn adjusted_slice_indices(len: usize, slice: &[ConstantData; 3]) -> Option> { - let len = i64::try_from(len).ok()?; - let start = slice_bound(&slice[0])?; - let stop = slice_bound(&slice[1])?; - let step = slice_bound(&slice[2])?.unwrap_or(1); - if step == 0 || step == i64::MIN { - return None; + pub(crate) fn push_unlabeled_instr_sequence_block(&mut self) -> crate::InternalResult<()> { + instruction_sequence_label_map_push_unlabeled_block(&mut self.instr_sequence_label_map) } - let step_is_negative = step < 0; - let lower = if step_is_negative { -1 } else { 0 }; - let upper = if step_is_negative { len - 1 } else { len }; - let adjust = |value: Option, default: i64| { - let mut value = value.unwrap_or(default); - if value < 0 { - value = value.saturating_add(len); - if value < 0 { - value = lower; - } - } else if value >= len { - value = upper; + fn take_recorded_instr_sequence(&mut self) -> crate::InternalResult { + let mut instr_sequence = + core::mem::replace(&mut self.instr_sequence, instruction_sequence_new()); + if let Some(mut annotations_instr_sequence) = self.annotations_instr_sequence.take() { + instruction_sequence_apply_label_map(&mut annotations_instr_sequence)?; + instruction_sequence_set_annotations_code( + &mut instr_sequence, + Some(Box::new(annotations_instr_sequence)), + ); } - value - }; - let start = adjust(start, if step_is_negative { upper } else { lower }); - let stop = adjust(stop, if step_is_negative { lower } else { upper }); + Ok(instr_sequence) + } - let mut index = i128::from(start); - let stop = i128::from(stop); - let step = i128::from(step); - let slice_len = if step > 0 { - if index < stop { - usize::try_from((stop - index - 1) / step + 1).ok()? - } else { - 0 - } - } else if index > stop { - usize::try_from((index - stop - 1) / -step + 1).ok()? - } else { - 0 - }; - let mut indices = Vec::new(); - indices.try_reserve_exact(slice_len).ok()?; - if step > 0 { - while index < stop { - indices.push(usize::try_from(index).ok()?); - index += step; - } - } else { - while index > stop { - indices.push(usize::try_from(index).ok()?); - index += step; - } + fn prepare_cfg_from_codegen(&mut self) -> crate::InternalResult { + // CPython compile.c optimize_and_assemble_code_unit passes + // u_instr_sequence directly into flowgraph.c _PyCfg_FromInstructionSequence(). + self.take_recorded_instr_sequence() } - Some(indices) } -/// flowgraph.c eval_const_binop subscript index adjustment -fn adjusted_const_index(len: usize, index: &ConstantData) -> Option { - let len = i64::try_from(len).ok()?; - let index = constant_as_index(index)?; - let index = if index < 0 { - index.saturating_add(len) - } else { - index - }; - if index < 0 || index >= len { - return None; - } - usize::try_from(index).ok() +fn optimize_code_unit( + metadata: &mut CodeUnitMetadata, + blocks: &mut Blocks, + instr_sequence: InstructionSequence, + nlocals: usize, + nparams: usize, +) -> crate::InternalResult<()> { + // Phase 1: _PyCfg_OptimizeCodeUnit (flowgraph.c) + *blocks = cfg_from_instruction_sequence(instr_sequence)?; + translate_jump_labels_to_targets(blocks)?; + mark_except_handlers(blocks)?; + label_exception_targets(blocks)?; + optimize_cfg(metadata, blocks, metadata.firstlineno)?; + blocks.remove_unused_consts(&mut metadata.consts)?; + add_checks_for_loads_of_uninitialized_variables(blocks, nlocals, nparams)?; + // CPython inserts superinstructions in _PyCfg_OptimizeCodeUnit, before + // later jump normalization / block reordering can create adjacencies + // that never exist at this stage in flowgraph.c. + blocks.insert_superinstructions()?; + push_cold_blocks_to_end(blocks)?; + // CPython resolves line numbers again after cold-block extraction. + blocks.resolve_line_numbers(metadata.firstlineno)?; + Ok(()) } -/// flowgraph.c eval_const_binop NB_SUBSCR -fn eval_const_subscript(container: &ConstantData, index: &ConstantData) -> Option { - match (container, index) { - ( - ConstantData::Str { value }, - ConstantData::Integer { .. } | ConstantData::Boolean { .. }, - ) => { - let string = value.to_string(); - if string.contains(char::REPLACEMENT_CHARACTER) { - return None; - } - let mut chars = Vec::new(); - chars.try_reserve_exact(string.chars().count()).ok()?; - chars.extend(string.chars()); - let index = adjusted_const_index(chars.len(), index)?; - Some(ConstantData::Str { - value: chars[index].to_string().into(), - }) - } - (ConstantData::Str { value }, ConstantData::Slice { elements }) => { - let string = value.to_string(); - if string.contains(char::REPLACEMENT_CHARACTER) { - return None; - } - let mut chars = Vec::new(); - chars.try_reserve_exact(string.chars().count()).ok()?; - chars.extend(string.chars()); - let indices = adjusted_slice_indices(chars.len(), elements)?; - let capacity = indices.iter().try_fold(0usize, |capacity, &index| { - capacity.checked_add(chars[index].len_utf8()) - })?; - let mut result = String::new(); - result.try_reserve_exact(capacity).ok()?; - for index in indices { - result.push(chars[index]); +fn optimize_cfg( + metadata: &mut CodeUnitMetadata, + blocks: &mut Blocks, + firstlineno: OneIndexed, +) -> crate::InternalResult<()> { + // flowgraph.c optimize_cfg + // CPython optimize_cfg() starts with check_cfg() and raises + // SystemError if a jump or scope exit is not the last instruction in + // its block. + check_cfg(blocks)?; + inline_small_or_no_lineno_blocks(blocks)?; + // CPython does not re-run instruction-sequence label-map/CFG conversion + // after this point. Unreferenced label blocks left by jump inlining + // remain block boundaries and can preserve line-marker NOPs. + blocks.remove_unreachable()?; + // CPython optimize_cfg resolves line numbers before local checks and + // superinstruction insertion, so fusion decisions see propagated + // source locations. + blocks.resolve_line_numbers(firstlineno)?; + // CPython optimize_cfg() runs optimize_load_const() and then + // optimize_basic_block() after line numbers are resolved. + optimize_load_const(metadata, blocks)?; + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = blocks[block_idx].next; + blocks.optimize_basic_block(metadata, block_idx)?; + block_idx = next_block; + } + blocks.remove_redundant_nops_and_pairs()?; + // CPython optimize_cfg() removes newly-unreachable blocks and + // redundant NOP/jump chains before _PyCfg_OptimizeCodeUnit() prunes + // unused constants. + blocks.remove_unreachable()?; + remove_redundant_nops_and_jumps(blocks)?; + #[cfg(debug_assertions)] + assert!(no_redundant_jumps(blocks)); + Ok(()) +} + +fn optimized_cfg_to_instruction_sequence( + metadata: &CodeUnitMetadata, + flags: CodeFlags, + blocks: &mut Blocks, +) -> crate::InternalResult<(u32, usize, InstructionSequence)> { + // Phase 2: _PyCfg_OptimizedCfgToInstructionSequence (flowgraph.c) + convert_pseudo_conditional_jumps(blocks)?; + let max_stackdepth = blocks.calculate_stackdepth()?; + debug_assert!(!is_generator(flags) || max_stackdepth != 0); + let nlocalsplus = prepare_localsplus(metadata, blocks, flags)?; + // Match CPython order: pseudo ops are lowered after stackdepth and + // localsplus preparation, before normalize_jumps. + convert_pseudo_ops(blocks)?; + blocks.normalize_jumps()?; + #[cfg(debug_assertions)] + assert!(no_redundant_jumps(blocks)); + // optimize_load_fast: after normalize_jumps + blocks.optimize_load_fast()?; + + let mut instr_sequence = instruction_sequence_new(); + blocks.cfg_to_instruction_sequence(&mut instr_sequence)?; + Ok((max_stackdepth, nlocalsplus, instr_sequence)) +} + +impl CodeInfo { + pub fn finalize_code( + mut self, + opts: &crate::compile::CompileOpts, + ) -> crate::InternalResult { + let instr_sequence = self.prepare_cfg_from_codegen()?; + let nlocals = self.metadata.varnames.len(); + let nparams = self.nparams; + optimize_code_unit( + &mut self.metadata, + &mut self.blocks, + instr_sequence, + nlocals, + nparams, + )?; + let (max_stackdepth, nlocalsplus, mut instr_sequence) = + optimized_cfg_to_instruction_sequence(&self.metadata, self.flags, &mut self.blocks)?; + let localsplusinfo = compute_localsplus_info(&self.metadata, nlocalsplus, self.flags)?; + + let Self { + flags, + source_path, + private: _, // private is only used during compilation + + blocks: _, + current_block: _, + instr_sequence: _, + instr_sequence_label_map: _, + annotations_instr_sequence: _, + metadata, + static_attributes: _, + in_inlined_comp: _, + fblock: _, + symbol_table_index: _, + nparams: _, + in_conditional_block: _, + next_conditional_annotation_index: _, + } = self; + + let CodeUnitMetadata { + name: obj_name, + qualname, + consts: constants, + names: name_cache, + varnames: varname_cache, + cellvars: _, + freevars: freevar_cache, + fast_hidden: _, + fast_hidden_final: _, + argcount: arg_count, + posonlyargcount: posonlyarg_count, + kwonlyargcount: kwonlyarg_count, + firstlineno: first_line_number, + } = metadata; + + resolve_unconditional_jumps(&mut instr_sequence)?; + resolve_jump_offsets(&mut instr_sequence)?; + let assembled = assemble_emit( + &mut instr_sequence, + first_line_number.get() as i32, + opts.debug_ranges, + )?; + let locations = rustpython_compiler_core::marshal::linetable_to_locations( + &assembled.linetable, + first_line_number.get() as i32, + assembled.instructions.len(), + ); + + Ok(CodeObject { + flags, + posonlyarg_count, + arg_count, + kwonlyarg_count, + source_path, + first_line_number: Some(first_line_number), + obj_name: obj_name.clone(), + qualname: qualname.unwrap_or(obj_name), + + max_stackdepth, + instructions: CodeUnits::from(assembled.instructions), + locations, + constants: constants.into_iter().collect(), + names: name_cache.into_iter().collect(), + varnames: varname_cache.into_iter().collect(), + cellvars: localsplusinfo.cellvars, + freevars: freevar_cache.into_iter().collect(), + localspluskinds: localsplusinfo.kinds, + linetable: assembled.linetable, + exceptiontable: assembled.exceptiontable, + }) + } +} + +/// flowgraph.c IS_GENERATOR +fn is_generator(flags: CodeFlags) -> bool { + flags.intersects(CodeFlags::GENERATOR | CodeFlags::COROUTINE | CodeFlags::ASYNC_GENERATOR) +} + +/// flowgraph.c insert_prefix_instructions +fn insert_prefix_instructions( + metadata: &CodeUnitMetadata, + blocks: &mut Blocks, + cellfixedoffsets: &[i32], + nfreevars: usize, + flags: CodeFlags, +) -> crate::InternalResult<()> { + debug_assert!(!blocks.is_empty()); + let entry = &mut blocks[0]; + let ncellvars = metadata.cellvars.len(); + let firstlineno = metadata.firstlineno; + debug_assert!(firstlineno.get() > 0); + + if is_generator(flags) { + let location = SourceLocation { + line: firstlineno, + character_offset: OneIndexed::MIN, + }; + basicblock_insert_instruction( + entry, + 0, + InstructionInfo { + instr: Instruction::ReturnGenerator.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location, + end_location: location, + except_handler: None, + lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), + }, + )?; + basicblock_insert_instruction( + entry, + 1, + InstructionInfo { + instr: Instruction::PopTop.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location, + end_location: location, + except_handler: None, + lineno_override: Some(LINE_ONLY_LOCATION_OVERRIDE), + }, + )?; + } + + if ncellvars > 0 { + let nvars = metadata.varnames.len() + ncellvars; + let mut sorted = Vec::new(); + vec_try_reserve_exact(&mut sorted, nvars)?; + sorted.resize(nvars, 0i32); + for i in 0..ncellvars { + sorted[cellfixedoffsets[i] as usize] = i as i32 + 1; + } + let mut ncellsused = 0; + let mut i = 0; + while ncellsused < ncellvars { + let oldindex = sorted[i] - 1; + i += 1; + if oldindex == -1 { + continue; } - Some(ConstantData::Str { - value: result.into(), - }) + basicblock_insert_instruction( + entry, + ncellsused, + InstructionInfo { + instr: Opcode::MakeCell.into(), + arg: OpArg::new(oldindex as u32), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(NO_LOCATION_OVERRIDE), + }, + )?; + ncellsused += 1; } - ( - ConstantData::Bytes { value }, - ConstantData::Integer { .. } | ConstantData::Boolean { .. }, - ) => { - let index = adjusted_const_index(value.len(), index)?; + } + + if nfreevars > 0 { + basicblock_insert_instruction( + entry, + 0, + InstructionInfo { + instr: Opcode::CopyFreeVars.into(), + arg: OpArg::new(nfreevars as u32), + target: BlockIdx::NULL, + location: SourceLocation::default(), + end_location: SourceLocation::default(), + except_handler: None, + lineno_override: Some(NO_LOCATION_OVERRIDE), + }, + )?; + } + Ok(()) +} + +/// flowgraph.c prepare_localsplus +fn prepare_localsplus( + metadata: &CodeUnitMetadata, + blocks: &mut Blocks, + flags: CodeFlags, +) -> crate::InternalResult { + let nlocals = metadata.varnames.len(); + let ncellvars = metadata.cellvars.len(); + let nfreevars = metadata.freevars.len(); + let int_max = i32::MAX as usize; + debug_assert!(nlocals < int_max); + debug_assert!(ncellvars < int_max); + debug_assert!(nfreevars < int_max); + debug_assert!(int_max - nlocals - ncellvars > 0); + debug_assert!(int_max - nlocals - ncellvars - nfreevars > 0); + let mut nlocalsplus = nlocals + ncellvars + nfreevars; + let mut cellfixedoffsets = build_cellfixedoffsets(metadata)?; + + // This must be called before fix_cell_offsets(). + insert_prefix_instructions(metadata, blocks, &cellfixedoffsets, nfreevars, flags)?; + + let numdropped = fix_cell_offsets(metadata, blocks, &mut cellfixedoffsets); + nlocalsplus -= numdropped; + Ok(nlocalsplus) +} + +/// flowgraph.c eval_const_unaryop +fn eval_const_unaryop( + operand: &ConstantData, + op: Instruction, + intrinsic: Option, +) -> Option { + match (operand, op, intrinsic) { + (ConstantData::Integer { value }, Instruction::UnaryNegative, None) => { + Some(ConstantData::Integer { value: -value }) + } + (ConstantData::Float { value }, Instruction::UnaryNegative, None) => { + Some(ConstantData::Float { value: -value }) + } + (ConstantData::Complex { value }, Instruction::UnaryNegative, None) => { + Some(ConstantData::Complex { value: -value }) + } + (ConstantData::Boolean { value }, Instruction::UnaryNegative, None) => { Some(ConstantData::Integer { - value: BigInt::from(value[index]), + value: BigInt::from(-i32::from(*value)), }) } - (ConstantData::Bytes { value }, ConstantData::Slice { elements }) => { - let indices = adjusted_slice_indices(value.len(), elements)?; - let mut result = Vec::new(); - result.try_reserve_exact(indices.len()).ok()?; - for index in indices { - result.push(value[index]); - } - Some(ConstantData::Bytes { value: result }) + (ConstantData::Integer { value }, Instruction::UnaryInvert, None) => { + Some(ConstantData::Integer { value: !value }) } + (ConstantData::Boolean { .. }, Instruction::UnaryInvert, None) => None, + (_, Instruction::UnaryNot, None) => Some(ConstantData::Boolean { + value: !operand.truthiness(), + }), ( - ConstantData::Tuple { elements }, - ConstantData::Integer { .. } | ConstantData::Boolean { .. }, - ) => { - let index = adjusted_const_index(elements.len(), index)?; - Some(elements[index].clone()) - } - (ConstantData::Tuple { elements }, ConstantData::Slice { elements: slice }) => { - let indices = adjusted_slice_indices(elements.len(), slice)?; - let mut result = Vec::new(); - result.try_reserve_exact(indices.len()).ok()?; - for index in indices { - result.push(elements[index].clone()); - } - Some(ConstantData::Tuple { elements: result }) - } + ConstantData::Integer { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Integer { + value: value.clone(), + }), + ( + ConstantData::Float { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Float { value: *value }), + ( + ConstantData::Boolean { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Integer { + value: BigInt::from(i32::from(*value)), + }), + ( + ConstantData::Complex { value }, + Instruction::CallIntrinsic1 { .. }, + Some(oparg::IntrinsicFunction1::UnaryPositive), + ) => Some(ConstantData::Complex { value: *value }), _ => None, } } -/// flowgraph.c eval_const_binop bool/int coercion -fn constant_as_int(value: &ConstantData) -> Option<(BigInt, bool)> { - match value { - ConstantData::Boolean { value } => Some((BigInt::from(u8::from(*value)), true)), - ConstantData::Integer { value } => Some((value.clone(), false)), +fn load_const_truthiness( + instr: Instruction, + arg: OpArg, + metadata: &CodeUnitMetadata, +) -> Option { + match instr { + Instruction::LoadConst { consti } => { + let constant = &metadata.consts[consti.get(arg).as_usize()]; + Some(constant.truthiness()) + } + Instruction::LoadSmallInt { i } => Some(i.get(arg) != 0), _ => None, } } -/// flowgraph.c eval_const_binop -fn eval_const_binop( - left: &ConstantData, - right: &ConstantData, - op: oparg::BinaryOperator, -) -> Option { - use oparg::BinaryOperator as BinOp; - - if matches!(op, BinOp::Subscr) { - return eval_const_subscript(left, right); - } - - if let (Some((left_int, left_is_bool)), Some((right_int, right_is_bool))) = - (constant_as_int(left), constant_as_int(right)) - && (left_is_bool || right_is_bool) - { - if left_is_bool && right_is_bool { - match op { - BinOp::And => { - return Some(ConstantData::Boolean { - value: !left_int.is_zero() & !right_int.is_zero(), - }); - } - BinOp::Or => { - return Some(ConstantData::Boolean { - value: !left_int.is_zero() | !right_int.is_zero(), - }); - } - BinOp::Xor => { - return Some(ConstantData::Boolean { - value: !left_int.is_zero() ^ !right_int.is_zero(), - }); - } - _ => {} - } - } +/// flowgraph.c add_const +fn add_const( + metadata: &mut CodeUnitMetadata, + constant: ConstantData, +) -> crate::InternalResult { + Ok(metadata.consts.try_insert_full(constant)?.0) +} - return eval_const_binop( - &ConstantData::Integer { value: left_int }, - &ConstantData::Integer { value: right_int }, - op, - ); +fn instr_make_load_const( + metadata: &mut CodeUnitMetadata, + instr: &mut InstructionInfo, + constant: ConstantData, +) -> crate::InternalResult<()> { + if maybe_instr_make_load_smallint(instr, &constant) { + return Ok(()); } - match (left, right) { - (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { - let result = match op { - BinOp::Add => l + r, - BinOp::Subtract => l - r, - BinOp::Multiply => { - return const_folding_safe_multiply(left, right); - } - BinOp::TrueDivide => { - if r.is_zero() { - return None; - } - let l_f = l.to_f64()?; - let r_f = r.to_f64()?; - let result = l_f / r_f; - if !result.is_finite() { - return None; - } - return Some(ConstantData::Float { value: result }); - } - BinOp::FloorDivide => { - if r.is_zero() { - return None; - } - // Python floor division: round towards negative infinity - let (q, rem) = (l.clone() / r.clone(), l.clone() % r.clone()); - if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { - q - 1 - } else { - q - } - } - BinOp::Remainder => return const_folding_safe_mod(left, right), - BinOp::Power => return const_folding_safe_power(left, right), - BinOp::Lshift => return const_folding_safe_lshift(left, right), - BinOp::Rshift => { - let shift: u32 = r.try_into().ok()?; - l >> (shift as usize) - } - BinOp::And => l & r, - BinOp::Or => l | r, - BinOp::Xor => l ^ r, - _ => return None, - }; - Some(ConstantData::Integer { value: result }) - } - (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { - let result = match op { - BinOp::Add => l + r, - BinOp::Subtract => l - r, - BinOp::Multiply => return const_folding_safe_multiply(left, right), - BinOp::TrueDivide => { - if *r == 0.0 { - return None; - } - l / r - } - BinOp::FloorDivide => { - let (floordiv, _) = float_div_mod(*l, *r)?; - floordiv - } - BinOp::Remainder => return const_folding_safe_mod(left, right), - BinOp::Power => return const_folding_safe_power(left, right), - _ => return None, - }; - if matches!(op, BinOp::Power) && !result.is_finite() { - return None; - } - Some(ConstantData::Float { value: result }) - } - // Int op Float or Float op Int → Float - (ConstantData::Integer { value: l }, ConstantData::Float { value: r }) => { - let l_f = l.to_f64()?; - eval_const_binop( - &ConstantData::Float { value: l_f }, - &ConstantData::Float { value: *r }, - op, - ) - } - (ConstantData::Float { value: l }, ConstantData::Integer { value: r }) => { - let r_f = r.to_f64()?; - eval_const_binop( - &ConstantData::Float { value: *l }, - &ConstantData::Float { value: r_f }, - op, - ) - } - (ConstantData::Integer { value: l }, ConstantData::Complex { value: r }) => { - eval_const_complex_binop(Complex::new(l.to_f64()?, 0.0), *r, op) - } - (ConstantData::Complex { value: l }, ConstantData::Integer { value: r }) => { - eval_const_complex_binop(*l, Complex::new(r.to_f64()?, 0.0), op) - } - (ConstantData::Float { value: l }, ConstantData::Complex { value: r }) => { - eval_const_complex_binop(Complex::new(*l, 0.0), *r, op) - } - (ConstantData::Complex { value: l }, ConstantData::Float { value: r }) => { - eval_const_complex_binop(*l, Complex::new(*r, 0.0), op) - } - (ConstantData::Complex { value: l }, ConstantData::Complex { value: r }) => { - eval_const_complex_binop(*l, *r, op) - } - // String concatenation and repetition - (ConstantData::Str { value: l }, ConstantData::Str { value: r }) - if matches!(op, BinOp::Add) => - { - let mut result = Wtf8Buf::new(); - result - .try_reserve_exact(l.len().checked_add(r.len())?) - .ok()?; - result.push_wtf8(l); - result.push_wtf8(r); - Some(ConstantData::Str { value: result }) - } - (ConstantData::Str { .. }, ConstantData::Integer { .. }) - if matches!(op, BinOp::Multiply) => - { - const_folding_safe_multiply(left, right) - } - (ConstantData::Tuple { elements: l }, ConstantData::Tuple { elements: r }) - if matches!(op, BinOp::Add) => - { - let mut result = Vec::new(); - result - .try_reserve_exact(l.len().checked_add(r.len())?) - .ok()?; - result.extend(l.iter().cloned()); - result.extend(r.iter().cloned()); - Some(ConstantData::Tuple { elements: result }) - } - (ConstantData::Tuple { .. }, ConstantData::Integer { .. }) - if matches!(op, BinOp::Multiply) => - { - const_folding_safe_multiply(left, right) - } - (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) - if matches!(op, BinOp::Multiply) => - { - const_folding_safe_multiply(left, right) - } - (ConstantData::Integer { .. }, ConstantData::Str { .. }) - if matches!(op, BinOp::Multiply) => - { - const_folding_safe_multiply(left, right) - } - (ConstantData::Bytes { value: l }, ConstantData::Bytes { value: r }) - if matches!(op, BinOp::Add) => - { - let mut result = Vec::new(); - result - .try_reserve_exact(l.len().checked_add(r.len())?) - .ok()?; - result.extend_from_slice(l); - result.extend_from_slice(r); - Some(ConstantData::Bytes { value: result }) - } - (ConstantData::Bytes { .. }, ConstantData::Integer { .. }) - if matches!(op, BinOp::Multiply) => - { - const_folding_safe_multiply(left, right) - } - (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) - if matches!(op, BinOp::Multiply) => - { - const_folding_safe_multiply(left, right) - } - _ => None, - } + let const_idx = add_const(metadata, constant)?; + instr_set_op1( + instr, + Opcode::LoadConst.into(), + OpArg::new(const_idx as u32), + ); + Ok(()) } -/// flowgraph.c fold_tuple_of_constants -fn fold_tuple_of_constants( +/// flowgraph.c fold_const_unaryop +fn fold_const_unaryop( metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize, ) -> crate::InternalResult { - let Some(Opcode::BuildTuple) = block.instructions[i].instr.real_opcode() else { - return Ok(false); + let instr = &block.instructions[i]; + let (op, intrinsic) = match instr.instr.real() { + Some(Instruction::UnaryNegative) => (Instruction::UnaryNegative, None), + Some(Instruction::UnaryInvert) => (Instruction::UnaryInvert, None), + Some(Instruction::UnaryNot) => (Instruction::UnaryNot, None), + Some(Instruction::CallIntrinsic1 { func }) + if matches!( + func.get(instr.arg), + oparg::IntrinsicFunction1::UnaryPositive + ) => + { + (Opcode::CallIntrinsic1.into(), Some(func.get(instr.arg))) + } + _ => return Ok(false), }; - - let tuple_size = u32::from(block.instructions[i].arg) as usize; - if tuple_size > STACK_USE_GUIDELINE { - return Ok(false); - } - - let Some(operand_indices) = (if tuple_size == 0 { - Some(Vec::new()) - } else if let Some(start) = i.checked_sub(1) { - get_const_loading_instrs(block, start, tuple_size)? + let Some(operand_index) = (if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, 1)? } else { None - }) else { + }) + .and_then(|indices| indices.into_iter().next()) else { + return Ok(false); + }; + let operand = get_const_value(metadata, &block.instructions[operand_index]); + let Some(operand) = operand else { + return Ok(false); + }; + let Some(folded_const) = eval_const_unaryop(&operand, op, intrinsic) else { return Ok(false); }; + nop_out(block, &[operand_index]); + instr_make_load_const(metadata, &mut block.instructions[i], folded_const)?; + Ok(true) +} - let mut elements = Vec::new(); - elements - .try_reserve_exact(tuple_size) +/// flowgraph.c get_const_loading_instrs +fn get_const_loading_instrs( + block: &Block, + mut start: usize, + size: usize, +) -> crate::InternalResult>> { + let mut indices = Vec::new(); + indices + .try_reserve_exact(size) .map_err(|_| InternalError::MalformedControlFlowGraph)?; - for &j in &operand_indices { - let Some(element) = get_const_value(metadata, &block.instructions[j]) else { - return Ok(false); + loop { + if start >= block.instruction_used { + return Ok(None); + } + let instr = &block.instructions[start]; + if !matches!(instr.instr.real(), Some(Instruction::Nop)) { + if !loads_const(instr) { + return Ok(None); + } + indices.push(start); + if indices.len() == size { + break; + } + } + let Some(prev) = start.checked_sub(1) else { + return Ok(None); }; - elements.push(element); + start = prev; } + indices.reverse(); + Ok(Some(indices)) +} - nop_out(block, &operand_indices); - instr_make_load_const( - metadata, - &mut block.instructions[i], - ConstantData::Tuple { elements }, - )?; - Ok(true) +/// flowgraph.c nop_out +fn nop_out(block: &mut Block, instrs: &[usize]) { + for &i in instrs { + nop_out_no_location(&mut block.instructions[i]); + } } -fn fold_constant_intrinsic_list_to_tuple( +/// flowgraph.c fold_const_binop +fn fold_const_binop( metadata: &mut CodeUnitMetadata, block: &mut Block, i: usize, ) -> crate::InternalResult { - let Some(Instruction::CallIntrinsic1 { func }) = block.instructions[i].instr.real() else { + use oparg::BinaryOperator as BinOp; + + let Some(Opcode::BinaryOp) = block.instructions[i].instr.real_opcode() else { return Ok(false); }; - if func.get(block.instructions[i].arg) != IntrinsicFunction1::ListToTuple { + + let Some(operand_indices) = (if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, 2)? + } else { + None + }) else { + return Ok(false); + }; + + let op_raw = u32::from(block.instructions[i].arg); + let Ok(op) = BinOp::try_from(op_raw) else { + return Ok(false); + }; + + let left = get_const_value(metadata, &block.instructions[operand_indices[0]]); + let right = get_const_value(metadata, &block.instructions[operand_indices[1]]); + let (Some(left_val), Some(right_val)) = (left, right) else { + return Ok(false); + }; + + let Some(result_const) = eval_const_binop(&left_val, &right_val, op) else { return Ok(false); + }; + + nop_out(block, &operand_indices); + instr_make_load_const(metadata, &mut block.instructions[i], result_const)?; + Ok(true) +} + +/// flowgraph.c loads_const +fn loads_const(info: &InstructionInfo) -> bool { + info.instr.has_const() || matches!(info.instr.real_opcode(), Some(Opcode::LoadSmallInt)) +} + +/// flowgraph.c get_const_value +fn get_const_value(metadata: &CodeUnitMetadata, info: &InstructionInfo) -> Option { + match info.instr.real_opcode() { + Some(Opcode::LoadSmallInt) => { + let v = u32::from(info.arg) as i32; + Some(ConstantData::Integer { + value: BigInt::from(v), + }) + } + _ if info.instr.has_const() => { + let idx = u32::from(info.arg) as usize; + metadata.consts.get_index(idx).cloned() + } + _ => None, } +} - let mut consts_found = 0usize; - let mut expect_append = true; - let mut pos = i; - while let Some(prev) = pos.checked_sub(1) { - pos = prev; - let instr = &block.instructions[pos]; - if matches!(instr.instr.real(), Some(Instruction::Nop)) { - continue; +/// flowgraph.c const_folding_check_complexity +fn const_folding_check_complexity(obj: &ConstantData, mut limit: isize) -> Option { + if let ConstantData::Tuple { elements } = obj { + limit -= isize::try_from(elements.len()).ok()?; + if limit < 0 { + return None; + } + for element in elements { + limit = const_folding_check_complexity(element, limit)?; } + } + Some(limit) +} - if matches!(instr.instr.real(), Some(Instruction::BuildList { .. })) - && u32::from(instr.arg) == 0 - { - if !expect_append { - return Ok(false); - } +fn repeat_wtf8(value: &Wtf8Buf, n: usize) -> Option { + let mut result = Wtf8Buf::new(); + result.try_reserve_exact(value.len().checked_mul(n)?).ok()?; + for _ in 0..n { + result.push_wtf8(value); + } + Some(result) +} - let mut elements = Vec::new(); - elements - .try_reserve_exact(consts_found) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - for idx in (pos..i).rev() { - if matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { - continue; - } - if loads_const(&block.instructions[idx]) { - let Some(value) = get_const_value(metadata, &block.instructions[idx]) else { - return Ok(false); - }; - elements.push(value); +fn checked_repeat_count(n: &BigInt, item_size: usize) -> Option { + let n = n.to_isize()?; + if item_size != 0 && (n < 0 || n as usize > MAX_STR_SIZE / item_size) { + return None; + } + Some(n.max(0) as usize) +} + +/// flowgraph.c const_folding_safe_multiply +fn const_folding_safe_multiply(left: &ConstantData, right: &ConstantData) -> Option { + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if !l.is_zero() && !r.is_zero() && l.bits() + r.bits() > MAX_INT_SIZE { + return None; + } + Some(ConstantData::Integer { value: l * r }) + } + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + Some(ConstantData::Float { value: l * r }) + } + (ConstantData::Str { value: s }, ConstantData::Integer { value: n }) => { + let n = checked_repeat_count(n, s.code_points().count())?; + Some(ConstantData::Str { + value: repeat_wtf8(s, n)?, + }) + } + (ConstantData::Integer { .. }, ConstantData::Str { .. }) => { + const_folding_safe_multiply(right, left) + } + (ConstantData::Bytes { value: b }, ConstantData::Integer { value: n }) => { + let n = checked_repeat_count(n, b.len())?; + let mut value = Vec::new(); + value.try_reserve_exact(b.len().checked_mul(n)?).ok()?; + for _ in 0..n { + value.extend_from_slice(b); + } + Some(ConstantData::Bytes { value }) + } + (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) => { + const_folding_safe_multiply(right, left) + } + (ConstantData::Tuple { elements }, ConstantData::Integer { value: n }) => { + let n = n.to_usize()?; + if n != 0 && !elements.is_empty() { + if n > MAX_COLLECTION_SIZE / elements.len() { + return None; } - nop_out_no_location(&mut block.instructions[idx]); + const_folding_check_complexity( + &ConstantData::Tuple { + elements: elements.clone(), + }, + MAX_TOTAL_ITEMS / isize::try_from(n).ok()?, + )?; } - debug_assert_eq!(elements.len(), consts_found); - elements.reverse(); - instr_make_load_const( - metadata, - &mut block.instructions[i], - ConstantData::Tuple { elements }, - )?; - return Ok(true); + let mut result = Vec::new(); + result + .try_reserve_exact(elements.len().checked_mul(n)?) + .ok()?; + for _ in 0..n { + result.extend(elements.iter().cloned()); + } + Some(ConstantData::Tuple { elements: result }) + } + (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) => { + const_folding_safe_multiply(right, left) } + _ => None, + } +} - if expect_append { - if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) - || u32::from(instr.arg) != 1 - { - return Ok(false); +/// flowgraph.c const_folding_safe_power +fn const_folding_safe_power(left: &ConstantData, right: &ConstantData) -> Option { + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if r < &BigInt::from(0) { + if l.is_zero() { + return None; + } + let base = l.to_f64()?; + if !base.is_finite() { + return None; + } + let result = if let Some(exp) = r.to_i32() { + base.powi(exp) + } else { + base.powf(r.to_f64()?) + }; + if !result.is_finite() { + return None; + } + return Some(ConstantData::Float { value: result }); } - } else { - if !loads_const(instr) { - return Ok(false); + let exp: u64 = r.try_into().ok()?; + let exp_usize = usize::try_from(exp).ok()?; + if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE / exp { + return None; } - consts_found += 1; + Some(ConstantData::Integer { + value: num_traits::pow::pow(l.clone(), exp_usize), + }) } - expect_append = !expect_append; + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let result = l.powf(*r); + result + .is_finite() + .then_some(ConstantData::Float { value: result }) + } + _ => None, } - - Ok(false) } -/// Port of CPython's flowgraph.c optimize_lists_and_sets(). -fn optimize_lists_and_sets( - metadata: &mut CodeUnitMetadata, - block: &mut Block, - i: usize, - nextop: Option, -) -> crate::InternalResult { - let Some(instr) = block.instructions[i].instr.real() else { - return Ok(false); +/// flowgraph.c const_folding_safe_lshift +fn const_folding_safe_lshift(left: &ConstantData, right: &ConstantData) -> Option { + let (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) = (left, right) + else { + return None; }; - let is_list = matches!(instr, Instruction::BuildList { .. }); - let is_set = matches!(instr, Instruction::BuildSet { .. }); - if !is_list && !is_set { - return Ok(false); + let shift: u64 = r.try_into().ok()?; + let shift_usize = usize::try_from(shift).ok()?; + if shift > MAX_INT_SIZE || (!l.is_zero() && l.bits() > MAX_INT_SIZE - shift) { + return None; } + Some(ConstantData::Integer { + value: l << shift_usize, + }) +} - let contains_or_iter = matches!( - nextop, - Some(Instruction::GetIter | Instruction::ContainsOp { .. }) - ); - let seq_size = u32::from(block.instructions[i].arg) as usize; - if seq_size > STACK_USE_GUIDELINE || (seq_size < MIN_CONST_SEQUENCE_SIZE && !contains_or_iter) { - return Ok(false); +/// flowgraph.c const_folding_safe_mod +fn const_folding_safe_mod(left: &ConstantData, right: &ConstantData) -> Option { + if matches!(left, ConstantData::Str { .. } | ConstantData::Bytes { .. }) { + return None; } - let Some(operand_indices) = (if seq_size == 0 { - Some(Vec::new()) - } else if let Some(start) = i.checked_sub(1) { - get_const_loading_instrs(block, start, seq_size)? - } else { - None - }) else { - if contains_or_iter && is_list { - let arg = block.instructions[i].arg; - instr_set_op1(&mut block.instructions[i], Opcode::BuildTuple.into(), arg); - return Ok(true); + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + if r.is_zero() { + return None; + } + let rem = l.clone() % r.clone(); + let value = if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { + rem + r + } else { + rem + }; + Some(ConstantData::Integer { value }) } - return Ok(false); - }; - - let mut elements = Vec::new(); - elements - .try_reserve_exact(seq_size) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - for &j in &operand_indices { - let Some(element) = get_const_value(metadata, &block.instructions[j]) else { - return Ok(false); - }; - elements.push(element); + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let (_, modulo) = float_div_mod(*l, *r)?; + Some(ConstantData::Float { value: modulo }) + } + _ => None, } +} - let const_data = if is_list { - ConstantData::Tuple { elements } - } else { - ConstantData::Frozenset { elements } - }; - let const_idx = add_const(metadata, const_data)?; - - if !contains_or_iter { - debug_assert!(i >= 2); - let folded_loc = block.instructions[i].location; - let end_loc = block.instructions[i].end_location; - - nop_out(block, &operand_indices); +fn float_div_mod(left: f64, right: f64) -> Option<(f64, f64)> { + if right == 0.0 { + return None; + } - let build_instr = if is_list { - Opcode::BuildList + let mut modulo = left % right; + let div = (left - modulo) / right; + let floordiv = if modulo != 0.0 { + let div = if (right < 0.0) != (modulo < 0.0) { + modulo += right; + div - 1.0 } else { - Opcode::BuildSet + div + }; + let mut floordiv = div.floor(); + if div - floordiv > 0.5 { + floordiv += 1.0; } - .into(); - instr_set_op1(&mut block.instructions[i - 2], build_instr, OpArg::new(0)); - block.instructions[i - 2].location = folded_loc; - block.instructions[i - 2].end_location = end_loc; - block.instructions[i - 2].lineno_override = None; + floordiv + } else { + modulo = 0.0f64.copysign(right); + 0.0f64.copysign(left / right) + }; - instr_set_op1( - &mut block.instructions[i - 1], - Opcode::LoadConst.into(), - OpArg::new(const_idx as u32), - ); + Some((floordiv, modulo)) +} - let extend_instr = if is_list { - Opcode::ListExtend - } else { - Opcode::SetUpdate - }; - instr_set_op1( - &mut block.instructions[i], - extend_instr.into(), - OpArg::new(1), - ); - return Ok(true); - } +/// flowgraph.c eval_const_binop complex result construction +fn eval_const_complex_const(value: Complex) -> Option { + (value.re.is_finite() && value.im.is_finite()).then_some(ConstantData::Complex { value }) +} - nop_out(block, &operand_indices); +/// flowgraph.c eval_const_binop complex operations +fn eval_const_complex_binop( + left: Complex, + right: Complex, + op: oparg::BinaryOperator, +) -> Option { + use oparg::BinaryOperator as BinOp; - instr_set_op1( - &mut block.instructions[i], - Opcode::LoadConst.into(), - OpArg::new(const_idx as u32), - ); - Ok(true) -} + let value = match op { + BinOp::Add => left + right, + BinOp::Subtract => { + let re = left.re - right.re; + // Preserve CPython's signed-zero behavior for real-zero + // minus zero-complex expressions such as `0 - 0j`. + let im = if left.re == 0.0 + && left.im == 0.0 + && right.re == 0.0 + && right.im == 0.0 + && !right.im.is_sign_negative() + { + -0.0 + } else { + left.im - right.im + }; + Complex::new(re, im) + } + BinOp::Multiply => left * right, + BinOp::TrueDivide => { + if right == Complex::new(0.0, 0.0) { + return None; + } + left / right + } + BinOp::Power => { + if left == Complex::new(0.0, 0.0) { + if right.im != 0.0 || right.re < 0.0 { + return None; + } -/// flowgraph.c VISITED -const VISITED: i32 = -1; + return eval_const_complex_const(if right.re == 0.0 { + Complex::new(1.0, 0.0) + } else { + Complex::new(0.0, 0.0) + }); + } -/// flowgraph.c SWAPPABLE -fn is_swappable(instr: AnyInstruction) -> bool { - matches!( - instr.into(), - AnyOpcode::Real(Opcode::StoreFast | Opcode::PopTop) - | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) - ) + if right.im == 0.0 + && right.re.fract() == 0.0 + && right.re >= f64::from(i32::MIN) + && right.re <= f64::from(i32::MAX) + { + left.powi(right.re as i32) + } else { + left.powc(right) + } + } + _ => return None, + }; + eval_const_complex_const(value) } -/// flowgraph.c STORES_TO -fn stores_to(info: &InstructionInfo) -> i32 { - match info.instr.into() { - AnyOpcode::Real(Opcode::StoreFast) - | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) => u32::from(info.arg) as i32, - _ => -1, +/// flowgraph.c eval_const_binop subscript index conversion +fn constant_as_index(value: &ConstantData) -> Option { + match value { + ConstantData::Integer { value } => value.to_i64().or_else(|| { + if value < &BigInt::from(0) { + Some(i64::MIN) + } else { + Some(i64::MAX) + } + }), + ConstantData::Boolean { value } => Some(i64::from(*value)), + _ => None, } } -/// flowgraph.c next_swappable_instruction -fn next_swappable_instruction(block: &Block, mut i: usize, lineno: i32) -> Option { - loop { - i += 1; - if i >= block.instruction_used { - return None; - } +/// flowgraph.c eval_const_binop subscript slice bound conversion +fn slice_bound(value: &ConstantData) -> Option> { + match value { + ConstantData::None => Some(None), + _ => constant_as_index(value).map(Some), + } +} - let info = &block.instructions[i]; - let info_lineno = instruction_lineno(info); +/// flowgraph.c eval_const_binop subscript slice index adjustment +fn adjusted_slice_indices(len: usize, slice: &[ConstantData; 3]) -> Option> { + let len = i64::try_from(len).ok()?; + let start = slice_bound(&slice[0])?; + let stop = slice_bound(&slice[1])?; + let step = slice_bound(&slice[2])?.unwrap_or(1); + if step == 0 || step == i64::MIN { + return None; + } - if lineno >= 0 && info_lineno != lineno { - return None; + let step_is_negative = step < 0; + let lower = if step_is_negative { -1 } else { 0 }; + let upper = if step_is_negative { len - 1 } else { len }; + let adjust = |value: Option, default: i64| { + let mut value = value.unwrap_or(default); + if value < 0 { + value = value.saturating_add(len); + if value < 0 { + value = lower; + } + } else if value >= len { + value = upper; } + value + }; + let start = adjust(start, if step_is_negative { upper } else { lower }); + let stop = adjust(stop, if step_is_negative { lower } else { upper }); - if matches!(info.instr, AnyInstruction::Real(Instruction::Nop)) { - continue; + let mut index = i128::from(start); + let stop = i128::from(stop); + let step = i128::from(step); + let slice_len = if step > 0 { + if index < stop { + usize::try_from((stop - index - 1) / step + 1).ok()? + } else { + 0 } - - if is_swappable(info.instr) { - return Some(i); + } else if index > stop { + usize::try_from((index - stop - 1) / -step + 1).ok()? + } else { + 0 + }; + let mut indices = Vec::new(); + indices.try_reserve_exact(slice_len).ok()?; + if step > 0 { + while index < stop { + indices.push(usize::try_from(index).ok()?); + index += step; + } + } else { + while index > stop { + indices.push(usize::try_from(index).ok()?); + index += step; } + } + Some(indices) +} +/// flowgraph.c eval_const_binop subscript index adjustment +fn adjusted_const_index(len: usize, index: &ConstantData) -> Option { + let len = i64::try_from(len).ok()?; + let index = constant_as_index(index)?; + let index = if index < 0 { + index.saturating_add(len) + } else { + index + }; + if index < 0 || index >= len { return None; } + usize::try_from(index).ok() } -/// flowgraph.c swaptimize -fn swaptimize(block: &mut Block, ix: &mut usize) -> crate::InternalResult<()> { - debug_assert!(matches!( - block.instructions[*ix].instr.real_opcode(), - Some(Opcode::Swap) - )); - let mut depth = u32::from(block.instructions[*ix].arg) as usize; - let mut len = 1usize; - let mut more = false; - let limit = block.instruction_used - *ix; - while len < limit { - match block.instructions[*ix + len].instr.real_opcode() { - Some(Opcode::Swap) => { - depth = depth.max(u32::from(block.instructions[*ix + len].arg) as usize); - more = true; - len += 1; - } - Some(Opcode::Nop) => { - len += 1; +/// flowgraph.c eval_const_binop NB_SUBSCR +fn eval_const_subscript(container: &ConstantData, index: &ConstantData) -> Option { + match (container, index) { + ( + ConstantData::Str { value }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let string = value.to_string(); + if string.contains(char::REPLACEMENT_CHARACTER) { + return None; } - _ => break, + let mut chars = Vec::new(); + chars.try_reserve_exact(string.chars().count()).ok()?; + chars.extend(string.chars()); + let index = adjusted_const_index(chars.len(), index)?; + Some(ConstantData::Str { + value: chars[index].to_string().into(), + }) } - } - - if !more { - return Ok(()); - } - - let mut stack = Vec::new(); - stack - .try_reserve_exact(depth) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - stack.resize(depth, 0); - let mut i = 0; - while i < depth { - stack[i] = i as i32; - i += 1; - } - - i = 0; - while i < len { - let info = &block.instructions[*ix + i]; - if matches!(info.instr.real_opcode(), Some(Opcode::Swap)) { - let oparg = u32::from(info.arg) as usize; - stack.swap(0, oparg - 1); + (ConstantData::Str { value }, ConstantData::Slice { elements }) => { + let string = value.to_string(); + if string.contains(char::REPLACEMENT_CHARACTER) { + return None; + } + let mut chars = Vec::new(); + chars.try_reserve_exact(string.chars().count()).ok()?; + chars.extend(string.chars()); + let indices = adjusted_slice_indices(chars.len(), elements)?; + let capacity = indices.iter().try_fold(0usize, |capacity, &index| { + capacity.checked_add(chars[index].len_utf8()) + })?; + let mut result = String::new(); + result.try_reserve_exact(capacity).ok()?; + for index in indices { + result.push(chars[index]); + } + Some(ConstantData::Str { + value: result.into(), + }) } - i += 1; - } - - let mut current = len as isize - 1; - for i in 0..depth { - if stack[i] == VISITED || stack[i] == i as i32 { - continue; + ( + ConstantData::Bytes { value }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let index = adjusted_const_index(value.len(), index)?; + Some(ConstantData::Integer { + value: BigInt::from(value[index]), + }) } - let mut j = i; - loop { - if j != 0 { - debug_assert!(current >= 0); - let out = &mut block.instructions[*ix + current as usize]; - out.instr = Opcode::Swap.into(); - out.arg = OpArg::new((j + 1) as u32); - current -= 1; + (ConstantData::Bytes { value }, ConstantData::Slice { elements }) => { + let indices = adjusted_slice_indices(value.len(), elements)?; + let mut result = Vec::new(); + result.try_reserve_exact(indices.len()).ok()?; + for index in indices { + result.push(value[index]); } - if stack[j] == VISITED { - debug_assert_eq!(j, i); - break; + Some(ConstantData::Bytes { value: result }) + } + ( + ConstantData::Tuple { elements }, + ConstantData::Integer { .. } | ConstantData::Boolean { .. }, + ) => { + let index = adjusted_const_index(elements.len(), index)?; + Some(elements[index].clone()) + } + (ConstantData::Tuple { elements }, ConstantData::Slice { elements: slice }) => { + let indices = adjusted_slice_indices(elements.len(), slice)?; + let mut result = Vec::new(); + result.try_reserve_exact(indices.len()).ok()?; + for index in indices { + result.push(elements[index].clone()); } - let next_j = stack[j] as usize; - stack[j] = VISITED; - j = next_j; + Some(ConstantData::Tuple { elements: result }) } + _ => None, } +} - while current >= 0 { - set_to_nop(&mut block.instructions[*ix + current as usize]); - current -= 1; +/// flowgraph.c eval_const_binop bool/int coercion +fn constant_as_int(value: &ConstantData) -> Option<(BigInt, bool)> { + match value { + ConstantData::Boolean { value } => Some((BigInt::from(u8::from(*value)), true)), + ConstantData::Integer { value } => Some((value.clone(), false)), + _ => None, } - *ix += len - 1; - Ok(()) } -/// flowgraph.c apply_static_swaps -fn apply_static_swaps(block: &mut Block, mut i: isize) { - while i >= 0 { - let idx = i as usize; - debug_assert!(idx < block.instruction_used); - let swap_arg = match block.instructions[idx].instr.real_opcode() { - Some(Opcode::Swap) => u32::from(block.instructions[idx].arg), - Some(Opcode::Nop | Opcode::PopTop | Opcode::StoreFast) => { - i -= 1; - continue; - } - _ if matches!( - block.instructions[idx].instr.pseudo_opcode(), - Some(PseudoOpcode::StoreFastMaybeNull) - ) => - { - i -= 1; - continue; +/// flowgraph.c eval_const_binop +fn eval_const_binop( + left: &ConstantData, + right: &ConstantData, + op: oparg::BinaryOperator, +) -> Option { + use oparg::BinaryOperator as BinOp; + + if matches!(op, BinOp::Subscr) { + return eval_const_subscript(left, right); + } + + if let (Some((left_int, left_is_bool)), Some((right_int, right_is_bool))) = + (constant_as_int(left), constant_as_int(right)) + && (left_is_bool || right_is_bool) + { + if left_is_bool && right_is_bool { + match op { + BinOp::And => { + return Some(ConstantData::Boolean { + value: !left_int.is_zero() & !right_int.is_zero(), + }); + } + BinOp::Or => { + return Some(ConstantData::Boolean { + value: !left_int.is_zero() | !right_int.is_zero(), + }); + } + BinOp::Xor => { + return Some(ConstantData::Boolean { + value: !left_int.is_zero() ^ !right_int.is_zero(), + }); + } + _ => {} } - _ => return, - }; + } - let Some(j) = next_swappable_instruction(block, idx, -1) else { - return; - }; - let lineno = instruction_lineno(&block.instructions[j]); - let mut k = j; - for _ in 1..swap_arg { - let Some(next) = next_swappable_instruction(block, k, lineno) else { - return; + return eval_const_binop( + &ConstantData::Integer { value: left_int }, + &ConstantData::Integer { value: right_int }, + op, + ); + } + + match (left, right) { + (ConstantData::Integer { value: l }, ConstantData::Integer { value: r }) => { + let result = match op { + BinOp::Add => l + r, + BinOp::Subtract => l - r, + BinOp::Multiply => { + return const_folding_safe_multiply(left, right); + } + BinOp::TrueDivide => { + if r.is_zero() { + return None; + } + let l_f = l.to_f64()?; + let r_f = r.to_f64()?; + let result = l_f / r_f; + if !result.is_finite() { + return None; + } + return Some(ConstantData::Float { value: result }); + } + BinOp::FloorDivide => { + if r.is_zero() { + return None; + } + // Python floor division: round towards negative infinity + let (q, rem) = (l.clone() / r.clone(), l.clone() % r.clone()); + if !rem.is_zero() && (rem < BigInt::from(0)) != (*r < BigInt::from(0)) { + q - 1 + } else { + q + } + } + BinOp::Remainder => return const_folding_safe_mod(left, right), + BinOp::Power => return const_folding_safe_power(left, right), + BinOp::Lshift => return const_folding_safe_lshift(left, right), + BinOp::Rshift => { + let shift: u32 = r.try_into().ok()?; + l >> (shift as usize) + } + BinOp::And => l & r, + BinOp::Or => l | r, + BinOp::Xor => l ^ r, + _ => return None, }; - k = next; + Some(ConstantData::Integer { value: result }) } - - let store_j = stores_to(&block.instructions[j]); - let store_k = stores_to(&block.instructions[k]); - if store_j >= 0 || store_k >= 0 { - if store_j == store_k { - return; - } - let mut idx = j + 1; - while idx < k { - let store_idx = stores_to(&block.instructions[idx]); - if store_idx >= 0 && (store_idx == store_j || store_idx == store_k) { - return; + (ConstantData::Float { value: l }, ConstantData::Float { value: r }) => { + let result = match op { + BinOp::Add => l + r, + BinOp::Subtract => l - r, + BinOp::Multiply => return const_folding_safe_multiply(left, right), + BinOp::TrueDivide => { + if *r == 0.0 { + return None; + } + l / r } - idx += 1; + BinOp::FloorDivide => { + let (floordiv, _) = float_div_mod(*l, *r)?; + floordiv + } + BinOp::Remainder => return const_folding_safe_mod(left, right), + BinOp::Power => return const_folding_safe_power(left, right), + _ => return None, + }; + if matches!(op, BinOp::Power) && !result.is_finite() { + return None; } + Some(ConstantData::Float { value: result }) + } + // Int op Float or Float op Int → Float + (ConstantData::Integer { value: l }, ConstantData::Float { value: r }) => { + let l_f = l.to_f64()?; + eval_const_binop( + &ConstantData::Float { value: l_f }, + &ConstantData::Float { value: *r }, + op, + ) + } + (ConstantData::Float { value: l }, ConstantData::Integer { value: r }) => { + let r_f = r.to_f64()?; + eval_const_binop( + &ConstantData::Float { value: *l }, + &ConstantData::Float { value: r_f }, + op, + ) + } + (ConstantData::Integer { value: l }, ConstantData::Complex { value: r }) => { + eval_const_complex_binop(Complex::new(l.to_f64()?, 0.0), *r, op) + } + (ConstantData::Complex { value: l }, ConstantData::Integer { value: r }) => { + eval_const_complex_binop(*l, Complex::new(r.to_f64()?, 0.0), op) + } + (ConstantData::Float { value: l }, ConstantData::Complex { value: r }) => { + eval_const_complex_binop(Complex::new(*l, 0.0), *r, op) + } + (ConstantData::Complex { value: l }, ConstantData::Float { value: r }) => { + eval_const_complex_binop(*l, Complex::new(*r, 0.0), op) + } + (ConstantData::Complex { value: l }, ConstantData::Complex { value: r }) => { + eval_const_complex_binop(*l, *r, op) + } + // String concatenation and repetition + (ConstantData::Str { value: l }, ConstantData::Str { value: r }) + if matches!(op, BinOp::Add) => + { + let mut result = Wtf8Buf::new(); + result + .try_reserve_exact(l.len().checked_add(r.len())?) + .ok()?; + result.push_wtf8(l); + result.push_wtf8(r); + Some(ConstantData::Str { value: result }) + } + (ConstantData::Str { .. }, ConstantData::Integer { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Tuple { elements: l }, ConstantData::Tuple { elements: r }) + if matches!(op, BinOp::Add) => + { + let mut result = Vec::new(); + result + .try_reserve_exact(l.len().checked_add(r.len())?) + .ok()?; + result.extend(l.iter().cloned()); + result.extend(r.iter().cloned()); + Some(ConstantData::Tuple { elements: result }) + } + (ConstantData::Tuple { .. }, ConstantData::Integer { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Integer { .. }, ConstantData::Tuple { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Integer { .. }, ConstantData::Str { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + (ConstantData::Bytes { value: l }, ConstantData::Bytes { value: r }) + if matches!(op, BinOp::Add) => + { + let mut result = Vec::new(); + result + .try_reserve_exact(l.len().checked_add(r.len())?) + .ok()?; + result.extend_from_slice(l); + result.extend_from_slice(r); + Some(ConstantData::Bytes { value: result }) } - - set_to_nop(&mut block.instructions[idx]); - block.instructions.swap(j, k); - i -= 1; - } -} - -/// flowgraph.c optimize_basic_block swap pass -fn apply_static_swaps_block(block: &mut Block) -> crate::InternalResult<()> { - let mut i = 0; - while i < block.instruction_used { - if matches!( - block.instructions[i].instr.real_opcode(), - Some(Opcode::Swap) - ) { - swaptimize(block, &mut i)?; - apply_static_swaps(block, i as isize); + (ConstantData::Bytes { .. }, ConstantData::Integer { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) } - i += 1; - } - Ok(()) -} - -/// flowgraph.c maybe_instr_make_load_smallint -fn maybe_instr_make_load_smallint(instr: &mut InstructionInfo, constant: &ConstantData) -> bool { - if let ConstantData::Integer { value } = constant - && let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) - { - instr_set_op1(instr, Opcode::LoadSmallInt.into(), OpArg::new(small as u32)); - return true; + (ConstantData::Integer { .. }, ConstantData::Bytes { .. }) + if matches!(op, BinOp::Multiply) => + { + const_folding_safe_multiply(left, right) + } + _ => None, } - false } -/// flowgraph.c basicblock_optimize_load_const -fn basicblock_optimize_load_const( +/// flowgraph.c fold_tuple_of_constants +fn fold_tuple_of_constants( metadata: &mut CodeUnitMetadata, block: &mut Block, -) -> crate::InternalResult<()> { - let mut i = 0; - let mut effective_opcode = None; - let mut effective_oparg = OpArg::new(0); - while i < block.instruction_used { - if matches!( - block.instructions[i].instr.real(), - Some(Instruction::LoadConst { .. }) - ) && let Some(constant) = get_const_value(metadata, &block.instructions[i]) - { - maybe_instr_make_load_smallint(&mut block.instructions[i], &constant); - } + i: usize, +) -> crate::InternalResult { + let Some(Opcode::BuildTuple) = block.instructions[i].instr.real_opcode() else { + return Ok(false); + }; - let curr = block.instructions[i]; - let curr_arg = curr.arg; + let tuple_size = u32::from(block.instructions[i].arg) as usize; + if tuple_size > STACK_USE_GUIDELINE { + return Ok(false); + } - // Only combine if the source is a real instruction. - let Some(curr_instr) = curr.instr.real() else { - i += 1; - continue; - }; + let Some(operand_indices) = (if tuple_size == 0 { + Some(Vec::new()) + } else if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, tuple_size)? + } else { + None + }) else { + return Ok(false); + }; - let is_copy_of_load_const = matches!( - (effective_opcode, curr_instr), - (Some(Instruction::LoadConst { .. }), Instruction::Copy { i }) if i.get(curr_arg) == 1 - ); - if !is_copy_of_load_const { - effective_opcode = Some(curr_instr); - effective_oparg = curr_arg; - } - let Some(const_instr) = effective_opcode else { - i += 1; - continue; + let mut elements = Vec::new(); + elements + .try_reserve_exact(tuple_size) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + for &j in &operand_indices { + let Some(element) = get_const_value(metadata, &block.instructions[j]) else { + return Ok(false); }; - let const_arg = effective_oparg; + elements.push(element); + } - if i + 1 >= block.instruction_used { - i += 1; + nop_out(block, &operand_indices); + instr_make_load_const( + metadata, + &mut block.instructions[i], + ConstantData::Tuple { elements }, + )?; + Ok(true) +} + +fn fold_constant_intrinsic_list_to_tuple( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, +) -> crate::InternalResult { + let Some(Instruction::CallIntrinsic1 { func }) = block.instructions[i].instr.real() else { + return Ok(false); + }; + if func.get(block.instructions[i].arg) != IntrinsicFunction1::ListToTuple { + return Ok(false); + } + + let mut consts_found = 0usize; + let mut expect_append = true; + let mut pos = i; + while let Some(prev) = pos.checked_sub(1) { + pos = prev; + let instr = &block.instructions[pos]; + if matches!(instr.instr.real(), Some(Instruction::Nop)) { continue; } - let next = block.instructions[i + 1]; - let next_arg = next.arg; + if matches!(instr.instr.real(), Some(Instruction::BuildList { .. })) + && u32::from(instr.arg) == 0 + { + if !expect_append { + return Ok(false); + } - if let Some(is_true) = load_const_truthiness(const_instr, const_arg, metadata) { - let const_jump = match (next.instr.real_opcode(), next.instr.pseudo_opcode()) { - (_, Some(PseudoOpcode::JumpIfTrue)) => Some((true, false)), - (_, Some(PseudoOpcode::JumpIfFalse)) => Some((false, false)), - (Some(Opcode::PopJumpIfTrue), _) => Some((true, true)), - (Some(Opcode::PopJumpIfFalse), _) => Some((false, true)), - _ => None, - }; - if let Some((jump_if_true, pops_condition)) = const_jump { - if pops_condition { - set_to_nop(&mut block.instructions[i]); + let mut elements = Vec::new(); + elements + .try_reserve_exact(consts_found) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + for idx in (pos..i).rev() { + if matches!(block.instructions[idx].instr.real(), Some(Instruction::Nop)) { + continue; } - if is_true == jump_if_true { - block.instructions[i + 1].instr = PseudoOpcode::Jump.into(); - } else { - set_to_nop(&mut block.instructions[i + 1]); + if loads_const(&block.instructions[idx]) { + let Some(value) = get_const_value(metadata, &block.instructions[idx]) else { + return Ok(false); + }; + elements.push(value); } - i += 1; - continue; + nop_out_no_location(&mut block.instructions[idx]); } + debug_assert_eq!(elements.len(), consts_found); + elements.reverse(); + instr_make_load_const( + metadata, + &mut block.instructions[i], + ConstantData::Tuple { elements }, + )?; + return Ok(true); } - // The remaining combinations require both instructions to be real. - let Some(next_instr) = next.instr.real() else { - i += 1; - continue; - }; - - if let Instruction::LoadConst { consti } = const_instr { - let constant = &metadata.consts[consti.get(const_arg).as_usize()]; - if matches!(constant, ConstantData::None) - && let Instruction::IsOp { invert } = next_instr + if expect_append { + if !matches!(instr.instr.real(), Some(Instruction::ListAppend { .. })) + || u32::from(instr.arg) != 1 { - let mut jump_idx = i + 2; - if jump_idx >= block.instruction_used { - i += 1; - continue; - } + return Ok(false); + } + } else { + if !loads_const(instr) { + return Ok(false); + } + consts_found += 1; + } + expect_append = !expect_append; + } + + Ok(false) +} + +/// Port of CPython's flowgraph.c optimize_lists_and_sets(). +fn optimize_lists_and_sets( + metadata: &mut CodeUnitMetadata, + block: &mut Block, + i: usize, + nextop: Option, +) -> crate::InternalResult { + let Some(instr) = block.instructions[i].instr.real() else { + return Ok(false); + }; + let is_list = matches!(instr, Instruction::BuildList { .. }); + let is_set = matches!(instr, Instruction::BuildSet { .. }); + if !is_list && !is_set { + return Ok(false); + } + + let contains_or_iter = matches!( + nextop, + Some(Instruction::GetIter | Instruction::ContainsOp { .. }) + ); + let seq_size = u32::from(block.instructions[i].arg) as usize; + if seq_size > STACK_USE_GUIDELINE || (seq_size < MIN_CONST_SEQUENCE_SIZE && !contains_or_iter) { + return Ok(false); + } + + let Some(operand_indices) = (if seq_size == 0 { + Some(Vec::new()) + } else if let Some(start) = i.checked_sub(1) { + get_const_loading_instrs(block, start, seq_size)? + } else { + None + }) else { + if contains_or_iter && is_list { + let arg = block.instructions[i].arg; + instr_set_op1(&mut block.instructions[i], Opcode::BuildTuple.into(), arg); + return Ok(true); + } + return Ok(false); + }; + + let mut elements = Vec::new(); + elements + .try_reserve_exact(seq_size) + .map_err(|_| InternalError::MalformedControlFlowGraph)?; + for &j in &operand_indices { + let Some(element) = get_const_value(metadata, &block.instructions[j]) else { + return Ok(false); + }; + elements.push(element); + } - if matches!( - block.instructions[jump_idx].instr.real(), - Some(Instruction::ToBool) - ) { - set_to_nop(&mut block.instructions[jump_idx]); - jump_idx += 1; - if jump_idx >= block.instruction_used { - i += 1; - continue; - } - } + let const_data = if is_list { + ConstantData::Tuple { elements } + } else { + ConstantData::Frozenset { elements } + }; + let const_idx = add_const(metadata, const_data)?; - let Some(jump_instr) = block.instructions[jump_idx].instr.real() else { - i += 1; - continue; - }; + if !contains_or_iter { + debug_assert!(i >= 2); + let folded_loc = block.instructions[i].location; + let end_loc = block.instructions[i].end_location; - let mut invert = matches!( - invert.get(next_arg), - rustpython_compiler_core::bytecode::Invert::Yes - ); - match jump_instr { - Instruction::PopJumpIfFalse { .. } => { - invert = !invert; - } - Instruction::PopJumpIfTrue { .. } => {} - _ => { - i += 1; - continue; - } - }; + nop_out(block, &operand_indices); - set_to_nop(&mut block.instructions[i]); - set_to_nop(&mut block.instructions[i + 1]); - block.instructions[jump_idx].instr = if invert { - Opcode::PopJumpIfNotNone - } else { - Opcode::PopJumpIfNone - } - .into(); - i = jump_idx; - continue; - } + let build_instr = if is_list { + Opcode::BuildList + } else { + Opcode::BuildSet } + .into(); + instr_set_op1(&mut block.instructions[i - 2], build_instr, OpArg::new(0)); + block.instructions[i - 2].location = folded_loc; + block.instructions[i - 2].end_location = end_loc; + block.instructions[i - 2].lineno_override = None; - if matches!( - const_instr, - Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } - ) && matches!(next_instr, Instruction::ToBool) - && let Some(value) = load_const_truthiness(const_instr, const_arg, metadata) - { - let const_idx = add_const(metadata, ConstantData::Boolean { value })?; - set_to_nop(&mut block.instructions[i]); - instr_set_op1( - &mut block.instructions[i + 1], - Opcode::LoadConst.into(), - OpArg::new(const_idx as u32), - ); - i += 1; - continue; - } + instr_set_op1( + &mut block.instructions[i - 1], + Opcode::LoadConst.into(), + OpArg::new(const_idx as u32), + ); - i += 1; + let extend_instr = if is_list { + Opcode::ListExtend + } else { + Opcode::SetUpdate + }; + instr_set_op1( + &mut block.instructions[i], + extend_instr.into(), + OpArg::new(1), + ); + return Ok(true); } - Ok(()) + + nop_out(block, &operand_indices); + + instr_set_op1( + &mut block.instructions[i], + Opcode::LoadConst.into(), + OpArg::new(const_idx as u32), + ); + Ok(true) } -/// flowgraph.c optimize_load_const -fn optimize_load_const( - metadata: &mut CodeUnitMetadata, - blocks: &mut Blocks, -) -> crate::InternalResult<()> { - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = blocks[block_idx.idx()].next; - let block = &mut blocks[block_idx]; - basicblock_optimize_load_const(metadata, block)?; - block_idx = next_block; +/// flowgraph.c VISITED +const VISITED: i32 = -1; + +/// flowgraph.c SWAPPABLE +fn is_swappable(instr: AnyInstruction) -> bool { + matches!( + instr.into(), + AnyOpcode::Real(Opcode::StoreFast | Opcode::PopTop) + | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) + ) +} + +/// flowgraph.c STORES_TO +fn stores_to(info: &InstructionInfo) -> i32 { + match info.instr.into() { + AnyOpcode::Real(Opcode::StoreFast) + | AnyOpcode::Pseudo(PseudoOpcode::StoreFastMaybeNull) => u32::from(info.arg) as i32, + _ => -1, } - Ok(()) } -/// flowgraph.c optimize_basic_block -fn optimize_basic_block( - blocks: &mut Blocks, - metadata: &mut CodeUnitMetadata, - block_idx: BlockIdx, -) -> crate::InternalResult<()> { - let bi = block_idx.idx(); - let mut nop = InstructionInfo { - instr: Instruction::Nop.into(), - arg: OpArg::NULL, - target: BlockIdx::NULL, - location: SourceLocation::default(), - end_location: SourceLocation::default(), - except_handler: None, - lineno_override: None, - }; - instr_set_op0(&mut nop, Instruction::Nop.into()); - let mut i = 0; - while i < blocks[bi].instruction_used { - let inst = blocks[bi].instructions[i]; - debug_assert!(!inst.instr.is_assembler()); - let target = if inst.instr.has_target() { - let target = inst.target; - debug_assert!(target != BlockIdx::NULL); - debug_assert!(blocks[target.idx()].instruction_used != 0); - debug_assert!(!blocks[target.idx()].instructions[0].instr.is_assembler()); - blocks[target.idx()].instructions[0] - } else { - nop - }; +/// flowgraph.c next_swappable_instruction +fn next_swappable_instruction(block: &Block, mut i: usize, lineno: i32) -> Option { + loop { + i += 1; + if i >= block.instruction_used { + return None; + } - let nextop = blocks[bi] - .instructions - .get(i + 1) - .and_then(|next| next.instr.real()); + let info = &block.instructions[i]; + let info_lineno = instruction_lineno(info); - match inst.instr { - AnyInstruction::Real(Instruction::BuildTuple { .. }) => { - let oparg = u32::from(inst.arg); - if matches!(nextop, Some(Instruction::UnpackSequence { .. })) - && u32::from(blocks[bi].instructions[i + 1].arg) == oparg - { - match oparg { - 1 => { - set_to_nop(&mut blocks[bi].instructions[i]); - set_to_nop(&mut blocks[bi].instructions[i + 1]); - i += 1; - continue; - } - 2 | 3 => { - set_to_nop(&mut blocks[bi].instructions[i]); - blocks[bi].instructions[i + 1].instr = Opcode::Swap.into(); - i += 1; - continue; - } - _ => {} - } - } - fold_tuple_of_constants(metadata, &mut blocks[bi], i)?; - } - AnyInstruction::Real(Instruction::BuildList { .. } | Instruction::BuildSet { .. }) => { - optimize_lists_and_sets(metadata, &mut blocks[bi], i, nextop)?; - } - AnyInstruction::Real( - Instruction::PopJumpIfNotNone { .. } | Instruction::PopJumpIfNone { .. }, - ) if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) - && jump_thread(blocks, block_idx, i, &target, inst.instr)? => - { - continue; - } - AnyInstruction::Real(Instruction::PopJumpIfFalse { .. }) - if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) - && jump_thread(blocks, block_idx, i, &target, inst.instr)? => - { - continue; - } - AnyInstruction::Real(Instruction::PopJumpIfTrue { .. }) - if matches!(target.instr.into(), AnyOpcode::Pseudo(PseudoOpcode::Jump)) - && jump_thread(blocks, block_idx, i, &target, inst.instr)? => - { - continue; - } - AnyInstruction::Pseudo( - pseudo @ (PseudoInstruction::JumpIfFalse { .. } - | PseudoInstruction::JumpIfTrue { .. }), - ) => { - let opcode = pseudo.into(); - match target.instr.pseudo().map(Into::into) { - Some(PseudoOpcode::Jump) - if jump_thread(blocks, block_idx, i, &target, opcode)? => - { - continue; - } - Some(PseudoOpcode::JumpIfFalse) - if matches!( - opcode, - AnyInstruction::Pseudo(PseudoInstruction::JumpIfFalse { .. }) - ) && jump_thread(blocks, block_idx, i, &target, opcode)? => - { - continue; - } - Some(PseudoOpcode::JumpIfTrue) - if matches!( - opcode, - AnyInstruction::Pseudo(PseudoInstruction::JumpIfTrue { .. }) - ) && jump_thread(blocks, block_idx, i, &target, opcode)? => - { - continue; - } - Some(PseudoOpcode::JumpIfFalse | PseudoOpcode::JumpIfTrue) => { - let next = blocks[inst.target.idx()].next; - debug_assert!(next != BlockIdx::NULL); - debug_assert!(next != inst.target); - blocks[bi].instructions[i].target = next; - continue; - } - _ => {} - } - } - AnyInstruction::Pseudo( - PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. }, - ) => match target.instr.into() { - AnyOpcode::Pseudo(PseudoOpcode::Jump) - if jump_thread(blocks, block_idx, i, &target, PseudoOpcode::Jump.into())? => - { - continue; - } - AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) - if jump_thread(blocks, block_idx, i, &target, inst.instr)? => - { - continue; - } - _ => {} - }, - // CPython leaves FOR_ITER jump threading disabled. - AnyInstruction::Real(Instruction::ForIter { .. }) => {} - AnyInstruction::Real(Instruction::StoreFast { .. }) - if matches!(nextop, Some(Instruction::StoreFast { .. })) - && u32::from(inst.arg) == u32::from(blocks[bi].instructions[i + 1].arg) - && instruction_lineno(&blocks[bi].instructions[i]) - == instruction_lineno(&blocks[bi].instructions[i + 1]) => - { - blocks[bi].instructions[i].instr = Instruction::PopTop.into(); - blocks[bi].instructions[i].arg = OpArg::NULL; - } - AnyInstruction::Real(Instruction::Swap { .. }) if u32::from(inst.arg) == 1 => { - set_to_nop(&mut blocks[bi].instructions[i]); - } - AnyInstruction::Real(Instruction::LoadGlobal { .. }) - if matches!(nextop, Some(Instruction::PushNull)) - && (u32::from(inst.arg) & 1) == 0 => - { - instr_set_op1( - &mut blocks[bi].instructions[i], - inst.instr, - OpArg::new(u32::from(inst.arg) | 1), - ); - set_to_nop(&mut blocks[bi].instructions[i + 1]); - } - AnyInstruction::Real(Instruction::CompareOp { .. }) - if matches!(nextop, Some(Instruction::ToBool)) => - { - set_to_nop(&mut blocks[bi].instructions[i]); - instr_set_op1( - &mut blocks[bi].instructions[i + 1], - inst.instr, - OpArg::new(u32::from(inst.arg) | oparg::COMPARE_OP_BOOL_MASK), - ); - i += 1; - continue; - } - AnyInstruction::Real(Instruction::ContainsOp { .. } | Instruction::IsOp { .. }) - if matches!(nextop, Some(Instruction::ToBool)) => - { - set_to_nop(&mut blocks[bi].instructions[i]); - instr_set_op1(&mut blocks[bi].instructions[i + 1], inst.instr, inst.arg); - i += 1; - continue; - } - AnyInstruction::Real(Instruction::ContainsOp { .. } | Instruction::IsOp { .. }) - if matches!(nextop, Some(Instruction::UnaryNot)) => - { - set_to_nop(&mut blocks[bi].instructions[i]); - let inverted = u32::from(inst.arg) ^ 1; - debug_assert!(inverted == 0 || inverted == 1); - instr_set_op1( - &mut blocks[bi].instructions[i + 1], - inst.instr, - OpArg::new(inverted), - ); - i += 1; - continue; - } - AnyInstruction::Real(Instruction::ToBool) - if matches!(nextop, Some(Instruction::ToBool)) => - { - set_to_nop(&mut blocks[bi].instructions[i]); - i += 1; - continue; - } - AnyInstruction::Real(Instruction::UnaryNot) => { - if matches!(nextop, Some(Instruction::ToBool)) { - set_to_nop(&mut blocks[bi].instructions[i]); - instr_set_op0(&mut blocks[bi].instructions[i + 1], inst.instr); - i += 1; - continue; - } - if matches!(nextop, Some(Instruction::UnaryNot)) { - set_to_nop(&mut blocks[bi].instructions[i]); - set_to_nop(&mut blocks[bi].instructions[i + 1]); - i += 1; - continue; - } - fold_const_unaryop(metadata, &mut blocks[bi], i)?; - } - AnyInstruction::Real(Instruction::UnaryInvert | Instruction::UnaryNegative) => { - fold_const_unaryop(metadata, &mut blocks[bi], i)?; - } - AnyInstruction::Real(Instruction::CallIntrinsic1 { func }) => { - match func.get(inst.arg) { - IntrinsicFunction1::ListToTuple => { - if matches!(nextop, Some(Instruction::GetIter)) { - set_to_nop(&mut blocks[bi].instructions[i]); - } else { - fold_constant_intrinsic_list_to_tuple(metadata, &mut blocks[bi], i)?; - } - } - IntrinsicFunction1::UnaryPositive => { - fold_const_unaryop(metadata, &mut blocks[bi], i)?; - } - _ => {} - } - } - AnyInstruction::Real(Instruction::BinaryOp { .. }) => { - fold_const_binop(metadata, &mut blocks[bi], i)?; - } - _ => {} + if lineno >= 0 && info_lineno != lineno { + return None; + } + + if matches!(info.instr, AnyInstruction::Real(Instruction::Nop)) { + continue; + } + + if is_swappable(info.instr) { + return Some(i); } - i += 1; + return None; } - apply_static_swaps_block(&mut blocks[block_idx])?; - Ok(()) } -/// flowgraph.c remove_redundant_nops_and_pairs -#[allow(clippy::unnecessary_wraps)] -fn remove_redundant_nops_and_pairs(blocks: &mut Blocks) -> crate::InternalResult<()> { - let mut done = false; - - while !done { - done = true; - let mut instr: Option<(BlockIdx, usize)> = None; - let mut block_idx = BlockIdx::new(0); - - while block_idx != BlockIdx::NULL { - basicblock_remove_redundant_nops(blocks, block_idx)?; - if is_label(blocks[block_idx.idx()].cpython_label) { - instr = None; - } - - let len = blocks[block_idx.idx()].instruction_used; - for instr_idx in 0..len { - let prev_instr = instr; - instr = Some((block_idx, instr_idx)); - let instr_info = blocks[block_idx.idx()].instructions[instr_idx]; - let mut prev_opcode = None; - let prev_oparg = if let Some((prev_block, prev_instr_idx)) = prev_instr { - let prev_info = blocks[prev_block.idx()].instructions[prev_instr_idx]; - prev_opcode = prev_info.instr.real_opcode(); - match prev_info.instr.real() { - Some(Instruction::Copy { i }) => i.get(prev_info.arg), - _ => u32::from(prev_info.arg), - } - } else { - 0 - }; - - let opcode = instr_info.instr.real_opcode(); - let is_redundant_pair = matches!(opcode, Some(Opcode::PopTop)) - && (matches!(prev_opcode, Some(Opcode::LoadConst | Opcode::LoadSmallInt)) - || (prev_oparg == 1 && matches!(prev_opcode, Some(Opcode::Copy)))); - - if is_redundant_pair { - let (prev_block, prev_instr_idx) = - prev_instr.expect("redundant pair has previous"); - set_to_nop(&mut blocks[prev_block].instructions[prev_instr_idx]); - set_to_nop(&mut blocks[block_idx].instructions[instr_idx]); - done = false; - } +/// flowgraph.c swaptimize +fn swaptimize(block: &mut Block, ix: &mut usize) -> crate::InternalResult<()> { + debug_assert!(matches!( + block.instructions[*ix].instr.real_opcode(), + Some(Opcode::Swap) + )); + let mut depth = u32::from(block.instructions[*ix].arg) as usize; + let mut len = 1usize; + let mut more = false; + let limit = block.instruction_used - *ix; + while len < limit { + match block.instructions[*ix + len].instr.real_opcode() { + Some(Opcode::Swap) => { + depth = depth.max(u32::from(block.instructions[*ix + len].arg) as usize); + more = true; + len += 1; } - - let instr_is_jump = instr.is_some_and(|(instr_block, instr_idx)| { - is_jump(&blocks[instr_block].instructions[instr_idx]) - }); - - let block = &blocks[block_idx]; - if instr_is_jump || !bb_has_fallthrough(block) { - instr = None; + Some(Opcode::Nop) => { + len += 1; } - block_idx = block.next; + _ => break, } } - Ok(()) -} -/// flowgraph.c remove_unused_consts -#[allow(clippy::needless_range_loop)] -fn remove_unused_consts( - blocks: &mut Blocks, - consts: &mut ConstantPool, -) -> crate::InternalResult<()> { - let nconsts = consts.len(); - if nconsts == 0 { + if !more { return Ok(()); } - let mut index_map = Vec::new(); - index_map - .try_reserve_exact(nconsts) + let mut stack = Vec::new(); + stack + .try_reserve_exact(depth) .map_err(|_| InternalError::MalformedControlFlowGraph)?; - index_map.resize(nconsts, 0isize); - for i in 1..nconsts { - index_map[i] = -1; + stack.resize(depth, 0); + let mut i = 0; + while i < depth { + stack[i] = i as i32; + i += 1; } - // The first constant may be docstring; keep it always. - index_map[0] = 0; - // Mark used consts. - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let block = &blocks[block_idx]; - for i in 0..block.instruction_used { - let instr = &block.instructions[i]; - if instr.instr.has_const() { - let index = u32::from(instr.arg) as usize; - debug_assert!(index < nconsts); - index_map[index] = index as isize; - } + i = 0; + while i < len { + let info = &block.instructions[*ix + i]; + if matches!(info.instr.real_opcode(), Some(Opcode::Swap)) { + let oparg = u32::from(info.arg) as usize; + stack.swap(0, oparg - 1); } - block_idx = block.next; + i += 1; } - // Now index_map[i] == i if consts[i] is used, -1 otherwise. - // Condense consts. - let mut n_used_consts = 0; - for i in 0..nconsts { - if index_map[i] != -1 { - debug_assert_eq!(index_map[i], i as isize); - index_map[n_used_consts] = index_map[i]; - n_used_consts += 1; + let mut current = len as isize - 1; + for i in 0..depth { + if stack[i] == VISITED || stack[i] == i as i32 { + continue; + } + let mut j = i; + loop { + if j != 0 { + debug_assert!(current >= 0); + let out = &mut block.instructions[*ix + current as usize]; + out.instr = Opcode::Swap.into(); + out.arg = OpArg::new((j + 1) as u32); + current -= 1; + } + if stack[j] == VISITED { + debug_assert_eq!(j, i); + break; + } + let next_j = stack[j] as usize; + stack[j] = VISITED; + j = next_j; } } - if n_used_consts == nconsts { - return Ok(()); + while current >= 0 { + set_to_nop(&mut block.instructions[*ix + current as usize]); + current -= 1; } + *ix += len - 1; + Ok(()) +} + +/// flowgraph.c apply_static_swaps +fn apply_static_swaps(block: &mut Block, mut i: isize) { + while i >= 0 { + let idx = i as usize; + debug_assert!(idx < block.instruction_used); + let swap_arg = match block.instructions[idx].instr.real_opcode() { + Some(Opcode::Swap) => u32::from(block.instructions[idx].arg), + Some(Opcode::Nop | Opcode::PopTop | Opcode::StoreFast) => { + i -= 1; + continue; + } + _ if matches!( + block.instructions[idx].instr.pseudo_opcode(), + Some(PseudoOpcode::StoreFastMaybeNull) + ) => + { + i -= 1; + continue; + } + _ => return, + }; - // Move all used consts to the beginning of the consts list. - debug_assert!(n_used_consts < nconsts); - for i in 0..n_used_consts { - let old_index = index_map[i] as usize; - debug_assert!(i <= old_index && old_index < nconsts); - if i != old_index { - let value = consts.constants[old_index].clone(); - consts.constants[i] = value; + let Some(j) = next_swappable_instruction(block, idx, -1) else { + return; + }; + let lineno = instruction_lineno(&block.instructions[j]); + let mut k = j; + for _ in 1..swap_arg { + let Some(next) = next_swappable_instruction(block, k, lineno) else { + return; + }; + k = next; } - } - // Truncate the consts list at its new size. - consts.constants.truncate(n_used_consts); + let store_j = stores_to(&block.instructions[j]); + let store_k = stores_to(&block.instructions[k]); + if store_j >= 0 || store_k >= 0 { + if store_j == store_k { + return; + } + let mut idx = j + 1; + while idx < k { + let store_idx = stores_to(&block.instructions[idx]); + if store_idx >= 0 && (store_idx == store_j || store_idx == store_k) { + return; + } + idx += 1; + } + } - // Adjust const indices in the bytecode. - let mut reverse_index_map = Vec::new(); - reverse_index_map - .try_reserve_exact(nconsts) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - reverse_index_map.resize(nconsts, 0isize); - for i in 0..nconsts { - reverse_index_map[i] = -1; - } - for i in 0..n_used_consts { - let old_index = index_map[i]; - debug_assert!(old_index != -1); - let old_index = old_index as usize; - debug_assert_eq!(reverse_index_map[old_index], -1); - reverse_index_map[old_index] = i as isize; + set_to_nop(&mut block.instructions[idx]); + block.instructions.swap(j, k); + i -= 1; } +} - block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = blocks[block_idx.idx()].next; - let block = &mut blocks[block_idx]; - for i in 0..block.instruction_used { - let instr = &mut block.instructions[i]; - if instr.instr.has_const() { - let index = u32::from(instr.arg) as usize; - debug_assert!(reverse_index_map[index] >= 0); - debug_assert!(reverse_index_map[index] < n_used_consts as isize); - instr.arg = OpArg::new(reverse_index_map[index] as u32); - } +/// flowgraph.c optimize_basic_block swap pass +fn apply_static_swaps_block(block: &mut Block) -> crate::InternalResult<()> { + let mut i = 0; + while i < block.instruction_used { + if matches!( + block.instructions[i].instr.real_opcode(), + Some(Opcode::Swap) + ) { + swaptimize(block, &mut i)?; + apply_static_swaps(block, i as isize); } - block_idx = next_block; + i += 1; } Ok(()) } -fn optimize_load_fast(blocks: &mut Blocks) -> crate::InternalResult<()> { - let mut max_instrs = 0; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - max_instrs = max_instrs.max(blocks[current.idx()].instruction_used); - current = blocks[current.idx()].next; +/// flowgraph.c maybe_instr_make_load_smallint +fn maybe_instr_make_load_smallint(instr: &mut InstructionInfo, constant: &ConstantData) -> bool { + if let ConstantData::Integer { value } = constant + && let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) + { + instr_set_op1(instr, Opcode::LoadSmallInt.into(), OpArg::new(small as u32)); + return true; } - let mut instr_flags = Vec::new(); - instr_flags - .try_reserve_exact(max_instrs) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - instr_flags.resize(max_instrs, 0u8); - let mut refs = RefStack { - refs: Vec::new(), - size: 0, - capacity: 0, - }; - let mut worklist = make_cfg_traversal_stack(blocks)?; - worklist.push(BlockIdx(0)); - blocks[0].start_depth = 0; - blocks[0].visited = true; - while let Some(block_idx) = worklist.pop() { - let block_i = block_idx.idx(); + false +} - let instr_count = blocks[block_i].instruction_used; - instr_flags[..instr_count].fill(0); - debug_assert!(blocks[block_i].start_depth >= 0); - let start_depth = blocks[block_i].start_depth as usize; - ref_stack_clear(&mut refs); - for _ in 0..start_depth { - push_ref(&mut refs, DUMMY_INSTR, NOT_LOCAL)?; +/// flowgraph.c basicblock_optimize_load_const +fn basicblock_optimize_load_const( + metadata: &mut CodeUnitMetadata, + block: &mut Block, +) -> crate::InternalResult<()> { + let mut i = 0; + let mut effective_opcode = None; + let mut effective_oparg = OpArg::new(0); + while i < block.instruction_used { + if matches!( + block.instructions[i].instr.real(), + Some(Instruction::LoadConst { .. }) + ) && let Some(constant) = get_const_value(metadata, &block.instructions[i]) + { + maybe_instr_make_load_smallint(&mut block.instructions[i], &constant); } - for i in 0..instr_count { - let info = blocks[block_i].instructions[i]; - let instr = info.instr; - let arg_u32 = u32::from(info.arg); - debug_assert!(!matches!(instr.real(), Some(Instruction::ExtendedArg))); - - match instr { - AnyInstruction::Real(Instruction::DeleteFast { var_num }) => { - kill_local( - &mut instr_flags, - &refs, - local_as_ref_local(usize::from(var_num.get(info.arg))), - ); - } - AnyInstruction::Real(Instruction::LoadFast { var_num }) => { - push_ref( - &mut refs, - i as isize, - local_as_ref_local(usize::from(var_num.get(info.arg))), - )?; - } - AnyInstruction::Real(Instruction::LoadFastAndClear { var_num }) => { - let local = local_as_ref_local(usize::from(var_num.get(info.arg))); - kill_local(&mut instr_flags, &refs, local); - push_ref(&mut refs, i as isize, local)?; - } - AnyInstruction::Real(Instruction::LoadFastLoadFast { .. }) => { - let local1 = (arg_u32 >> 4) as isize; - let local2 = (arg_u32 & 15) as isize; - push_ref(&mut refs, i as isize, local1)?; - push_ref(&mut refs, i as isize, local2)?; - } - AnyInstruction::Real(Instruction::StoreFast { var_num }) => { - let r = ref_stack_pop(&mut refs); - store_local( - &mut instr_flags, - &refs, - local_as_ref_local(usize::from(var_num.get(info.arg))), - r, - ); - } - AnyInstruction::Real(Instruction::StoreFastLoadFast { .. }) => { - let r = ref_stack_pop(&mut refs); - store_local(&mut instr_flags, &refs, (arg_u32 >> 4) as isize, r); - push_ref(&mut refs, i as isize, (arg_u32 & 15) as isize)?; - } - AnyInstruction::Real(Instruction::StoreFastStoreFast { .. }) => { - let r1 = ref_stack_pop(&mut refs); - store_local(&mut instr_flags, &refs, (arg_u32 >> 4) as isize, r1); - let r2 = ref_stack_pop(&mut refs); - store_local(&mut instr_flags, &refs, (arg_u32 & 15) as isize, r2); - } - AnyInstruction::Real(Instruction::Copy { i: _ }) => { - let depth = arg_u32 as usize; - assert!(depth > 0); - assert!(refs.size >= depth); - let r = ref_stack_at(&refs, refs.size - depth); - push_ref(&mut refs, r.instr, r.local)?; - } - AnyInstruction::Real(Instruction::Swap { i: _ }) => { - let depth = arg_u32 as usize; - assert!(depth >= 2); - assert!(refs.size >= depth); - ref_stack_swap_top(&mut refs, depth); - } - AnyInstruction::Real( - Instruction::FormatSimple - | Instruction::GetAnext - | Instruction::GetLen - | Instruction::GetYieldFromIter - | Instruction::ImportFrom { .. } - | Instruction::MatchKeys - | Instruction::MatchMapping - | Instruction::MatchSequence - | Instruction::WithExceptStart, - ) => { - let effect = instr.stack_effect_info(arg_u32); - let net_pushed = effect.pushed() as isize - effect.popped() as isize; - debug_assert!(net_pushed >= 0); - // CPython optimize_load_fast() shadows the outer - // instruction index in this produced-value loop. - for produced in 0..net_pushed { - push_ref(&mut refs, produced, NOT_LOCAL)?; - } - } - AnyInstruction::Real( - Instruction::DictMerge { .. } - | Instruction::DictUpdate { .. } - | Instruction::ListAppend { .. } - | Instruction::ListExtend { .. } - | Instruction::MapAdd { .. } - | Instruction::Reraise { .. } - | Instruction::SetAdd { .. } - | Instruction::SetUpdate { .. }, - ) => { - let effect = instr.stack_effect_info(arg_u32); - let net_popped = effect.popped() as isize - effect.pushed() as isize; - debug_assert!(net_popped > 0); - for _ in 0..net_popped { - let _ = ref_stack_pop(&mut refs); - } - } - AnyInstruction::Real( - Instruction::EndSend | Instruction::SetFunctionAttribute { .. }, - ) => { - let effect = instr.stack_effect_info(arg_u32); - debug_assert_eq!(effect.popped(), 2); - debug_assert_eq!(effect.pushed(), 1); - let tos = ref_stack_pop(&mut refs); - let _ = ref_stack_pop(&mut refs); - push_ref(&mut refs, tos.instr, tos.local)?; - } - AnyInstruction::Real(Instruction::CheckExcMatch) => { - let _ = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL)?; - } - AnyInstruction::Real(Instruction::ForIter { .. }) => { - let target = info.target; - debug_assert!(target != BlockIdx::NULL); - load_fast_push_block(&mut worklist, blocks, target, refs.size + 1); - push_ref(&mut refs, i as isize, NOT_LOCAL)?; - } - AnyInstruction::Real( - Instruction::LoadAttr { .. } | Instruction::LoadSuperAttr { .. }, - ) => { - let self_ref = ref_stack_pop(&mut refs); - if matches!(instr.real(), Some(Instruction::LoadSuperAttr { .. })) { - let _ = ref_stack_pop(&mut refs); - let _ = ref_stack_pop(&mut refs); - } - push_ref(&mut refs, i as isize, NOT_LOCAL)?; - if arg_u32 & 1 != 0 { - push_ref(&mut refs, self_ref.instr, self_ref.local)?; - } - } - AnyInstruction::Real( - Instruction::LoadSpecial { .. } | Instruction::PushExcInfo, - ) => { - let tos = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL)?; - push_ref(&mut refs, tos.instr, tos.local)?; - } - AnyInstruction::Real(Instruction::Send { .. }) => { - let target = info.target; - debug_assert!(target != BlockIdx::NULL); - load_fast_push_block(&mut worklist, blocks, target, refs.size); - let _ = ref_stack_pop(&mut refs); - push_ref(&mut refs, i as isize, NOT_LOCAL)?; - } - _ => { - let effect = instr.stack_effect_info(arg_u32); - let num_popped = effect.popped() as usize; - let num_pushed = effect.pushed() as usize; - let target = info.target; - if instr.has_target() { - debug_assert!(target != BlockIdx::NULL); - debug_assert!(refs.size >= num_popped); - let target_depth = refs.size - num_popped + num_pushed; - load_fast_push_block(&mut worklist, blocks, target, target_depth); - } - if !is_block_push(&info) { - for _ in 0..num_popped { - let _ = ref_stack_pop(&mut refs); - } - for _ in 0..num_pushed { - push_ref(&mut refs, i as isize, NOT_LOCAL)?; - } - } - } - } + let curr = block.instructions[i]; + let curr_arg = curr.arg; + + // Only combine if the source is a real instruction. + let Some(curr_instr) = curr.instr.real() else { + i += 1; + continue; + }; + + let is_copy_of_load_const = matches!( + (effective_opcode, curr_instr), + (Some(Instruction::LoadConst { .. }), Instruction::Copy { i }) if i.get(curr_arg) == 1 + ); + if !is_copy_of_load_const { + effective_opcode = Some(curr_instr); + effective_oparg = curr_arg; } + let Some(const_instr) = effective_opcode else { + i += 1; + continue; + }; + let const_arg = effective_oparg; - let fallthrough = blocks[block_i].next; - let term = basicblock_last_instr(&blocks[block_i]).copied(); - if let Some(term) = term - && fallthrough != BlockIdx::NULL - && !term.instr.is_unconditional_jump() - && !term.instr.is_scope_exit() - { - debug_assert!(bb_has_fallthrough(&blocks[block_i])); - load_fast_push_block(&mut worklist, blocks, fallthrough, refs.size); + if i + 1 >= block.instruction_used { + i += 1; + continue; } - for i in 0..refs.size { - let r = ref_stack_at(&refs, i); - if r.instr != DUMMY_INSTR { - instr_flags[r.instr as usize] |= LoadFastInstrFlag::RefUnconsumed as u8; - } - } + let next = block.instructions[i + 1]; + let next_arg = next.arg; - let block = &mut blocks[block_idx]; - let iused = block.instruction_used; - let mut i = 0; - while i < iused { - let info = &mut block.instructions[i]; - if instr_flags[i] != 0 { + if let Some(is_true) = load_const_truthiness(const_instr, const_arg, metadata) { + let const_jump = match (next.instr.real_opcode(), next.instr.pseudo_opcode()) { + (_, Some(PseudoOpcode::JumpIfTrue)) => Some((true, false)), + (_, Some(PseudoOpcode::JumpIfFalse)) => Some((false, false)), + (Some(Opcode::PopJumpIfTrue), _) => Some((true, true)), + (Some(Opcode::PopJumpIfFalse), _) => Some((false, true)), + _ => None, + }; + if let Some((jump_if_true, pops_condition)) = const_jump { + if pops_condition { + set_to_nop(&mut block.instructions[i]); + } + if is_true == jump_if_true { + block.instructions[i + 1].instr = PseudoOpcode::Jump.into(); + } else { + set_to_nop(&mut block.instructions[i + 1]); + } i += 1; continue; } + } + + // The remaining combinations require both instructions to be real. + let Some(next_instr) = next.instr.real() else { + i += 1; + continue; + }; + + if let Instruction::LoadConst { consti } = const_instr { + let constant = &metadata.consts[consti.get(const_arg).as_usize()]; + if matches!(constant, ConstantData::None) + && let Instruction::IsOp { invert } = next_instr + { + let mut jump_idx = i + 2; + if jump_idx >= block.instruction_used { + i += 1; + continue; + } - match info.instr.real_opcode() { - Some(Opcode::LoadFast) => { - info.instr = Opcode::LoadFastBorrow.into(); + if matches!( + block.instructions[jump_idx].instr.real(), + Some(Instruction::ToBool) + ) { + set_to_nop(&mut block.instructions[jump_idx]); + jump_idx += 1; + if jump_idx >= block.instruction_used { + i += 1; + continue; + } } - Some(Opcode::LoadFastLoadFast) => { - info.instr = Opcode::LoadFastBorrowLoadFastBorrow.into(); + + let Some(jump_instr) = block.instructions[jump_idx].instr.real() else { + i += 1; + continue; + }; + + let mut invert = matches!( + invert.get(next_arg), + rustpython_compiler_core::bytecode::Invert::Yes + ); + match jump_instr { + Instruction::PopJumpIfFalse { .. } => { + invert = !invert; + } + Instruction::PopJumpIfTrue { .. } => {} + _ => { + i += 1; + continue; + } + }; + + set_to_nop(&mut block.instructions[i]); + set_to_nop(&mut block.instructions[i + 1]); + block.instructions[jump_idx].instr = if invert { + Opcode::PopJumpIfNotNone + } else { + Opcode::PopJumpIfNone } - _ => {} + .into(); + i = jump_idx; + continue; } + } + + if matches!( + const_instr, + Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } + ) && matches!(next_instr, Instruction::ToBool) + && let Some(value) = load_const_truthiness(const_instr, const_arg, metadata) + { + let const_idx = add_const(metadata, ConstantData::Boolean { value })?; + set_to_nop(&mut block.instructions[i]); + instr_set_op1( + &mut block.instructions[i + 1], + Opcode::LoadConst.into(), + OpArg::new(const_idx as u32), + ); i += 1; + continue; } + + i += 1; } Ok(()) } -/// flowgraph.c calculate_stackdepth -fn calculate_stackdepth(blocks: &mut Blocks) -> crate::InternalResult { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - blocks[current.idx()].start_depth = START_DEPTH_UNSET; - current = blocks[current.idx()].next; - } - let mut stack = make_cfg_traversal_stack(blocks)?; - let mut maxdepth = 0i32; - stackdepth_push(&mut stack, blocks, BlockIdx(0), 0)?; - while let Some(block_idx) = stack.pop() { - let idx = block_idx.idx(); - let mut depth = blocks[idx].start_depth; - debug_assert!(depth >= 0); - let mut next = blocks[idx].next; - let instr_count = blocks[idx].instruction_used; - for i in 0..instr_count { - let ins = blocks[idx].instructions[i]; - let instr = &ins.instr; - let effects = get_stack_effects(*instr, ins.arg, 0)?; - let new_depth = depth + effects.net; - if new_depth < 0 { - return Err(InternalError::StackUnderflow); - } - maxdepth = maxdepth.max(depth); - if instr.has_target() && !matches!(instr.real(), Some(Instruction::EndAsyncFor)) { - debug_assert!(ins.target != BlockIdx::NULL); - let effects = get_stack_effects(*instr, ins.arg, 1)?; - let target_depth = depth + effects.net; - debug_assert!(target_depth >= 0); - maxdepth = maxdepth.max(depth); - stackdepth_push(&mut stack, blocks, ins.target, target_depth)?; - } - depth = new_depth; - debug_assert!(!instr.is_assembler()); - if instr.is_unconditional_jump() || instr.is_scope_exit() { - next = BlockIdx::NULL; - break; - } - } - if next != BlockIdx::NULL { - debug_assert!(bb_has_fallthrough(&blocks[idx])); - stackdepth_push(&mut stack, blocks, next, depth)?; - } +/// flowgraph.c optimize_load_const +fn optimize_load_const( + metadata: &mut CodeUnitMetadata, + blocks: &mut Blocks, +) -> crate::InternalResult<()> { + let mut block_idx = BlockIdx(0); + while block_idx != BlockIdx::NULL { + let next_block = blocks[block_idx.idx()].next; + let block = &mut blocks[block_idx]; + basicblock_optimize_load_const(metadata, block)?; + block_idx = next_block; } - - let stackdepth = maxdepth; - Ok(stackdepth as u32) + Ok(()) } #[cfg(test)] @@ -4490,8 +4713,9 @@ impl CodeInfo { "after_inline_small_or_no_lineno_blocks".to_owned(), self.debug_block_dump(), )); - remove_unreachable(&mut self.blocks)?; - resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno)?; + self.blocks.remove_unreachable()?; + self.blocks + .resolve_line_numbers(self.metadata.firstlineno)?; optimize_load_const(&mut self.metadata, &mut self.blocks)?; trace.push(( "after_optimize_load_const".to_owned(), @@ -4500,19 +4724,21 @@ impl CodeInfo { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let next_block = self.blocks[block_idx].next; - optimize_basic_block(&mut self.blocks, &mut self.metadata, block_idx)?; + self.blocks + .optimize_basic_block(&mut self.metadata, block_idx)?; block_idx = next_block; } trace.push(( "after_optimize_basic_block".to_owned(), self.debug_block_dump(), )); - remove_redundant_nops_and_pairs(&mut self.blocks)?; - remove_unreachable(&mut self.blocks)?; + self.blocks.remove_redundant_nops_and_pairs()?; + self.blocks.remove_unreachable()?; remove_redundant_nops_and_jumps(&mut self.blocks)?; #[cfg(debug_assertions)] assert!(no_redundant_jumps(&self.blocks)); - remove_unused_consts(&mut self.blocks, &mut self.metadata.consts)?; + self.blocks + .remove_unused_consts(&mut self.metadata.consts)?; trace.push(( "after_optimize_cfg_cleanup".to_owned(), self.debug_block_dump(), @@ -4520,13 +4746,14 @@ impl CodeInfo { let nlocals = self.metadata.varnames.len(); let nparams = self.nparams; add_checks_for_loads_of_uninitialized_variables(&mut self.blocks, nlocals, nparams)?; - insert_superinstructions(&mut self.blocks)?; + self.blocks.insert_superinstructions()?; push_cold_blocks_to_end(&mut self.blocks)?; trace.push(( "after_push_cold_before_chain_reorder".to_owned(), self.debug_block_dump(), )); - resolve_line_numbers(&mut self.blocks, self.metadata.firstlineno)?; + self.blocks + .resolve_line_numbers(self.metadata.firstlineno)?; trace.push(( "after_push_cold_resolve_line_numbers".to_owned(), self.debug_block_dump(), @@ -4543,7 +4770,7 @@ impl CodeInfo { self.debug_block_dump(), )); - let _max_stackdepth = calculate_stackdepth(&mut self.blocks)?; + let _max_stackdepth = self.blocks.calculate_stackdepth()?; let _nlocalsplus = prepare_localsplus(&self.metadata, &mut self.blocks, self.flags)?; convert_pseudo_ops(&mut self.blocks)?; trace.push(( @@ -4551,11 +4778,11 @@ impl CodeInfo { self.debug_block_dump(), )); - normalize_jumps(&mut self.blocks)?; + self.blocks.normalize_jumps()?; #[cfg(debug_assertions)] assert!(no_redundant_jumps(&self.blocks)); trace.push(("after_normalize_jumps".to_owned(), self.debug_block_dump())); - optimize_load_fast(&mut self.blocks)?; + self.blocks.optimize_load_fast()?; trace.push(( "after_optimize_load_fast".to_owned(), self.debug_block_dump(), @@ -4619,57 +4846,6 @@ fn make_super_instruction( set_to_nop(inst2); } -/// flowgraph.c insert_superinstructions -fn insert_superinstructions(blocks: &mut Blocks) -> crate::InternalResult { - let mut block_idx = BlockIdx(0); - while block_idx != BlockIdx::NULL { - let next_block = blocks[block_idx.idx()].next; - let block = &mut blocks[block_idx]; - for i in 0..block.instruction_used { - let nextop = (i + 1 < block.instruction_used) - .then(|| block.instructions[i + 1].instr.real()) - .flatten(); - match block.instructions[i].instr.real() { - Some(Instruction::LoadFast { .. }) => { - if matches!(nextop, Some(Instruction::LoadFast { .. })) { - let (inst1, rest) = block.instructions[i..].split_at_mut(1); - make_super_instruction( - &mut inst1[0], - &mut rest[0], - Opcode::LoadFastLoadFast.into(), - ); - } - } - Some(Instruction::StoreFast { .. }) => match nextop { - Some(Instruction::LoadFast { .. }) => { - let (inst1, rest) = block.instructions[i..].split_at_mut(1); - make_super_instruction( - &mut inst1[0], - &mut rest[0], - Opcode::StoreFastLoadFast.into(), - ); - } - Some(Instruction::StoreFast { .. }) => { - let (inst1, rest) = block.instructions[i..].split_at_mut(1); - make_super_instruction( - &mut inst1[0], - &mut rest[0], - Opcode::StoreFastStoreFast.into(), - ); - } - _ => {} - }, - _ => {} - } - } - block_idx = next_block; - } - let res = remove_redundant_nops(blocks)?; - #[cfg(debug_assertions)] - assert!(no_redundant_nops(blocks)); - Ok(res) -} - /// flowgraph.c LoadFastInstrFlag #[derive(Clone, Copy, Eq, PartialEq)] #[repr(u8)] @@ -5188,7 +5364,7 @@ pub(crate) fn mark_except_handlers(blocks: &mut Blocks) -> crate::InternalResult /// optimize_load_fast to terminate fall-through at those placeholders. /// flowgraph.c mark_warm fn mark_warm(blocks: &mut Blocks) -> crate::InternalResult<()> { - let mut stack = make_cfg_traversal_stack(blocks)?; + let mut stack = blocks.make_cfg_traversal_stack()?; stack.push(BlockIdx(0)); blocks[0].visited = true; while let Some(block_idx) = stack.pop() { @@ -5221,7 +5397,7 @@ fn mark_warm(blocks: &mut Blocks) -> crate::InternalResult<()> { fn mark_cold(blocks: &mut Blocks) -> crate::InternalResult<()> { let mut block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { - let block = &mut blocks[block_idx.idx()]; + let block = &mut blocks[block_idx]; debug_assert!(!block.cold); debug_assert!(!block.warm); block_idx = block.next; @@ -5229,7 +5405,7 @@ fn mark_cold(blocks: &mut Blocks) -> crate::InternalResult<()> { mark_warm(blocks)?; - let mut cold_stack = make_cfg_traversal_stack(blocks)?; + let mut cold_stack = blocks.make_cfg_traversal_stack()?; block_idx = BlockIdx(0); while block_idx != BlockIdx::NULL { let i = block_idx.idx(); @@ -5596,24 +5772,6 @@ fn normalize_jumps_in_block(blocks: &mut Blocks, block_idx: BlockIdx) -> crate:: Ok(()) } -/// flowgraph.c normalize_jumps -fn normalize_jumps(blocks: &mut Blocks) -> crate::InternalResult<()> { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - blocks[current.idx()].visited = false; - current = blocks[current.idx()].next; - } - - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - blocks[idx].visited = true; - normalize_jumps_in_block(blocks, current)?; - current = blocks[idx].next; - } - Ok(()) -} - /// flowgraph.c basicblock_inline_small_or_no_lineno_blocks fn basicblock_inline_small_or_no_lineno_blocks( blocks: &mut Blocks, @@ -5638,7 +5796,7 @@ fn basicblock_inline_small_or_no_lineno_blocks( let last = basicblock_last_instr_mut(&mut blocks[block_idx]) .expect("non-empty block has last instruction"); set_to_nop(last); - basicblock_append_block_instructions(blocks, block_idx, target)?; + blocks.basicblock_append_block_instructions(block_idx, target)?; if no_lineno_no_fallthrough { let last = basicblock_last_instr_mut(&mut blocks[block_idx]).unwrap(); if last.instr.is_unconditional_jump() @@ -5839,27 +5997,6 @@ fn remove_redundant_nops_and_jumps(blocks: &mut Blocks) -> crate::InternalResult Ok(()) } -/// flowgraph.c make_cfg_traversal_stack -fn make_cfg_traversal_stack(blocks: &mut Blocks) -> crate::InternalResult { - debug_assert!(!blocks.is_empty()); - let mut nblocks = 0; - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - blocks[current.idx()].visited = false; - nblocks += 1; - current = blocks[current.idx()].next; - } - debug_assert!(nblocks > 0); - let mut stack = Vec::new(); - stack - .try_reserve_exact(nblocks) - .map_err(|_| InternalError::MalformedControlFlowGraph)?; - stack.resize(nblocks, BlockIdx::NULL); - let stack = CfgTraversalStack { stack, sp: 0 }; - debug_assert_eq!(stack.capacity(), nblocks); - Ok(stack) -} - fn blocks_new_block(blocks: &mut Blocks) -> crate::InternalResult { blocks .try_reserve(1) @@ -6284,7 +6421,7 @@ fn add_checks_for_loads_of_uninitialized_variables( nlocals = LOCAL_UNSAFE_MASK_BITS; } - let mut worklist = make_cfg_traversal_stack(blocks)?; + let mut worklist = blocks.make_cfg_traversal_stack()?; let mut start_mask = 0u64; for i in nparams..nlocals { start_mask |= 1u64 << i; @@ -6405,14 +6542,6 @@ fn basicblock_has_no_lineno(block: &Block) -> bool { true } -/// flowgraph.c copy_basicblock -fn copy_basicblock(blocks: &mut Blocks, block_idx: BlockIdx) -> crate::InternalResult { - debug_assert!(bb_no_fallthrough(&blocks[block_idx.idx()])); - let result = blocks_new_block(blocks)?; - basicblock_append_block_instructions(blocks, result, block_idx)?; - Ok(result) -} - /// flowgraph.c get_max_label fn get_max_label(blocks: &Blocks) -> i32 { let mut lbl = -1; @@ -6425,110 +6554,6 @@ fn get_max_label(blocks: &Blocks) -> i32 { lbl } -fn duplicate_exits_without_lineno(blocks: &mut Blocks) -> crate::InternalResult<()> { - let mut next_lbl = get_max_label(blocks) + 1; - - let entryblock = BlockIdx(0); - let mut b = entryblock; - while b != BlockIdx::NULL { - let Some(last) = basicblock_last_instr(&blocks[b]).copied() else { - b = blocks[b].next; - continue; - }; - if is_jump(&last) { - debug_assert!(last.target != BlockIdx::NULL); - let target = next_nonempty_block(blocks, last.target); - debug_assert!(target != BlockIdx::NULL); - if is_exit_or_eval_check_without_lineno(&blocks[target]) - && blocks[target].predecessors > 1 - { - let new_target = copy_basicblock(blocks, target)?; - instr_set_location( - &mut blocks[new_target].instructions[0], - instr_location(&last), - ); - let last_mut = basicblock_last_instr_mut(&mut blocks[b]).unwrap(); - last_mut.target = new_target; - blocks[target].predecessors -= 1; - blocks[new_target].predecessors = 1; - blocks[new_target].next = blocks[target].next; - blocks[new_target].cpython_label = InstructionSequenceLabel(next_lbl); - next_lbl += 1; - blocks[target].next = new_target; - } - } - b = blocks[b].next; - } - - b = entryblock; - while b != BlockIdx::NULL { - let next = blocks[b].next; - if bb_has_fallthrough(&blocks[b]) - && next != BlockIdx::NULL - && blocks[b].instruction_used != 0 - && is_exit_or_eval_check_without_lineno(&blocks[next]) - { - let last = *basicblock_last_instr(&blocks[b]).expect("block has instructions"); - instr_set_location(&mut blocks[next].instructions[0], instr_location(&last)); - } - b = blocks[b].next; - } - Ok(()) -} - -fn propagate_line_numbers(blocks: &mut Blocks) { - let mut current = BlockIdx(0); - while current != BlockIdx::NULL { - let idx = current.idx(); - let Some(last) = basicblock_last_instr(&blocks[idx]).copied() else { - current = blocks[idx].next; - continue; - }; - - let mut prev_location = no_instruction_location(); - for i in 0..blocks[idx].instruction_used { - if instruction_is_no_location(&blocks[idx].instructions[i]) { - instr_set_location(&mut blocks[idx].instructions[i], prev_location); - } else { - prev_location = instr_location(&blocks[idx].instructions[i]); - } - } - - let next = blocks[idx].next; - if bb_has_fallthrough(&blocks[idx]) { - debug_assert!(next != BlockIdx::NULL); - if next != BlockIdx::NULL - && blocks[next].predecessors == 1 - && blocks[next].instruction_used != 0 - && instruction_is_no_location(&blocks[next].instructions[0]) - { - instr_set_location(&mut blocks[next].instructions[0], prev_location); - } - } - - if is_jump(&last) { - let target = last.target; - debug_assert!(target != BlockIdx::NULL); - if blocks[target].predecessors == 1 { - let instr = basicblock_raw_first_instr_mut(&mut blocks[target]); - if instruction_is_no_location(instr) { - instr_set_location(instr, prev_location); - } - } - } - current = blocks[current].next; - } -} - -fn resolve_line_numbers( - blocks: &mut Blocks, - _firstlineno: OneIndexed, -) -> crate::InternalResult<()> { - duplicate_exits_without_lineno(blocks)?; - propagate_line_numbers(blocks); - Ok(()) -} - /// flowgraph.c make_except_stack #[allow(clippy::unnecessary_wraps)] fn make_except_stack() -> crate::InternalResult { @@ -6592,7 +6617,7 @@ fn pop_except_block(stack: &mut CfgExceptStack, blocks: &Blocks) -> Option crate::InternalResult<()> { - let mut todo = make_cfg_traversal_stack(blocks)?; + let mut todo = blocks.make_cfg_traversal_stack()?; todo.push(BlockIdx(0)); blocks[0].visited = true; @@ -7014,7 +7039,7 @@ mod tests { blocks[0].visited = true; blocks[1].visited = true; - let mut stack = make_cfg_traversal_stack(&mut blocks).unwrap(); + let mut stack = blocks.make_cfg_traversal_stack().unwrap(); assert!(!blocks[0].visited); assert!(!blocks[1].visited); assert!(stack.capacity() >= 2); @@ -7230,7 +7255,8 @@ mod tests { basicblock_clear(&mut blocks[0]); test_block_push(&mut blocks[1], test_instr(Instruction::PopTop, 42)); - basicblock_append_block_instructions(&mut blocks, BlockIdx::new(0), BlockIdx::new(1)) + blocks + .basicblock_append_block_instructions(BlockIdx::new(0), BlockIdx::new(1)) .expect("basicblock_append_block_instructions succeeds"); // CPython `basicblock_append_instructions()` obtains a slot with @@ -7252,7 +7278,8 @@ mod tests { blocks[0].next = BlockIdx::new(1); let mut instr_sequence = instruction_sequence_new(); - cfg_to_instruction_sequence(&mut blocks, &mut instr_sequence) + blocks + .cfg_to_instruction_sequence(&mut instr_sequence) .expect("non-target NOP should ignore stale CPython i_target"); } @@ -7265,7 +7292,7 @@ mod tests { let mut blocks = Blocks::from([block]); let mut instr_sequence = instruction_sequence_new(); - let _ = cfg_to_instruction_sequence(&mut blocks, &mut instr_sequence); + let _ = blocks.cfg_to_instruction_sequence(&mut instr_sequence); } #[test] @@ -7381,7 +7408,9 @@ mod tests { test_block_push(&mut block, test_instr(Instruction::PopTop, 10)); let mut code = test_code_info(block); - optimize_load_fast(&mut code.blocks).expect("optimize_load_fast succeeds"); + code.blocks + .optimize_load_fast() + .expect("optimize_load_fast succeeds"); // CPython `optimize_load_fast()` shadows the outer instruction index in // the produced-value loop for GET_LEN, so the produced ref is recorded @@ -7471,8 +7500,12 @@ mod tests { test_block_push(&mut blocks[2], test_instr(Instruction::ReturnValue, 30)); blocks[2].instructions[0].lineno_override = Some(NO_LOCATION_OVERRIDE); - remove_unreachable(&mut blocks).expect("remove_unreachable succeeds"); - resolve_line_numbers(&mut blocks, OneIndexed::MIN).expect("resolve_line_numbers succeeds"); + blocks + .remove_unreachable() + .expect("remove_unreachable succeeds"); + blocks + .resolve_line_numbers(OneIndexed::MIN) + .expect("resolve_line_numbers succeeds"); // CPython `duplicate_exits_without_lineno()` copies a shared exit block // reached by jumps so each copy can inherit its sole predecessor's line. @@ -7495,10 +7528,12 @@ mod tests { block.instructions[1].lineno_override = Some(NEXT_LOCATION_OVERRIDE); test_block_push(&mut block, test_instr(Instruction::ReturnValue, 30)); block.instructions[2].lineno_override = Some(NO_LOCATION_OVERRIDE); - let mut blocks = [block].into(); + let mut blocks = Blocks::from([block]); - remove_unreachable(&mut blocks).expect("remove_unreachable succeeds"); - propagate_line_numbers(&mut blocks); + blocks + .remove_unreachable() + .expect("remove_unreachable succeeds"); + blocks.propagate_line_numbers(); // CPython `propagate_line_numbers()` only copies over NO_LOCATION // (`lineno == NO_LOCATION`). `NEXT_LOCATION` (`lineno == -2`) becomes the @@ -7524,8 +7559,10 @@ mod tests { basicblock_clear(&mut blocks[1]); test_block_push(&mut blocks[2], test_instr(Instruction::ReturnValue, 30)); - remove_unreachable(&mut blocks).expect("remove_unreachable succeeds"); - propagate_line_numbers(&mut blocks); + blocks + .remove_unreachable() + .expect("remove_unreachable succeeds"); + blocks.propagate_line_numbers(); // CPython `propagate_line_numbers()` directly reads `target->b_instr[0]` // for jump targets without checking `b_iused`. If @@ -7568,7 +7605,8 @@ mod tests { test_block_push(&mut blocks[3], test_instr(Instruction::ReturnValue, 40)); let mut metadata = test_code_info(Block::default()).metadata; - optimize_basic_block(&mut blocks, &mut metadata, BlockIdx::new(0)) + blocks + .optimize_basic_block(&mut metadata, BlockIdx::new(0)) .expect("valid jump chain"); // CPython `optimize_basic_block()` continues after `jump_thread()`, so