diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 4e92610b15b..87d002c98b2 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -9,7 +9,7 @@ use num_traits::{ToPrimitive, Zero}; use rustpython_compiler_core::{ OneIndexed, SourceLocation, bytecode::{ - AnyInstruction, AnyOpcode, Arg, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, + AnyInstruction, AnyOpcode, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, ExceptionTableEntry, InstrDisplayContext, Instruction, InstructionMetadata, IntrinsicFunction1, Label, OpArg, Opcode, PseudoInstruction, PseudoOpcode, PyCodeLocationInfoKind, encode_exception_table, @@ -163,14 +163,14 @@ fn is_named_except_cleanup_normal_exit_block(block: &Block) -> bool { } let tail = &block.instructions[len - 5..]; matches!(tail[0].instr.real(), Some(Instruction::PopExcept)) - && matches!(tail[1].instr.real(), Some(Instruction::LoadConst { .. })) + && matches!(tail[1].instr.real_opcode(), Some(Opcode::LoadConst)) && matches!( - tail[2].instr.real(), - Some(Instruction::StoreName { .. } | Instruction::StoreFast { .. }) + tail[2].instr.real_opcode(), + Some(Opcode::StoreName | Opcode::StoreFast) ) && matches!( - tail[3].instr.real(), - Some(Instruction::DeleteName { .. } | Instruction::DeleteFast { .. }) + tail[3].instr.real_opcode(), + Some(Opcode::DeleteName | Opcode::DeleteFast) ) && tail[4].instr.is_unconditional_jump() } @@ -762,8 +762,8 @@ impl CodeInfo { .iter() .take_while(|info| { matches!( - info.instr.real(), - Some(Instruction::MakeCell { .. } | Instruction::CopyFreeVars { .. }) + info.instr.real_opcode(), + Some(Opcode::MakeCell | Opcode::CopyFreeVars) ) }) .count(); @@ -778,7 +778,7 @@ impl CodeInfo { if nfrees > 0 { entry.instructions.push(InstructionInfo { - instr: Instruction::CopyFreeVars { n: Arg::marker() }.into(), + instr: Opcode::CopyFreeVars.into(), arg: OpArg::new(nfrees as u32), ..anchor }); @@ -795,7 +795,7 @@ impl CodeInfo { } for oldindex in sorted.into_iter().flatten() { entry.instructions.push(InstructionInfo { - instr: Instruction::MakeCell { i: Arg::marker() }.into(), + instr: Opcode::MakeCell.into(), arg: OpArg::new(oldindex as u32), ..anchor }); @@ -857,47 +857,47 @@ impl CodeInfo { op: Instruction, intrinsic: Option, ) -> Option { - match (operand, op, intrinsic) { - (ConstantData::Integer { value }, Instruction::UnaryNegative, None) => { + match (operand, op.into(), intrinsic) { + (ConstantData::Integer { value }, Opcode::UnaryNegative, None) => { Some(ConstantData::Integer { value: -value }) } - (ConstantData::Float { value }, Instruction::UnaryNegative, None) => { + (ConstantData::Float { value }, Opcode::UnaryNegative, None) => { Some(ConstantData::Float { value: -value }) } - (ConstantData::Complex { value }, Instruction::UnaryNegative, None) => { + (ConstantData::Complex { value }, Opcode::UnaryNegative, None) => { Some(ConstantData::Complex { value: -value }) } - (ConstantData::Boolean { value }, Instruction::UnaryNegative, None) => { + (ConstantData::Boolean { value }, Opcode::UnaryNegative, None) => { Some(ConstantData::Integer { value: BigInt::from(-i32::from(*value)), }) } - (ConstantData::Integer { value }, Instruction::UnaryInvert, None) => { + (ConstantData::Integer { value }, Opcode::UnaryInvert, None) => { Some(ConstantData::Integer { value: !value }) } - (ConstantData::Boolean { .. }, Instruction::UnaryInvert, None) => None, + (ConstantData::Boolean { .. }, Opcode::UnaryInvert, None) => None, ( ConstantData::Integer { value }, - Instruction::CallIntrinsic1 { .. }, + Opcode::CallIntrinsic1, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Integer { value: value.clone(), }), ( ConstantData::Float { value }, - Instruction::CallIntrinsic1 { .. }, + Opcode::CallIntrinsic1, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Float { value: *value }), ( ConstantData::Boolean { value }, - Instruction::CallIntrinsic1 { .. }, + Opcode::CallIntrinsic1, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Integer { value: BigInt::from(i32::from(*value)), }), ( ConstantData::Complex { value }, - Instruction::CallIntrinsic1 { .. }, + Opcode::CallIntrinsic1, Some(oparg::IntrinsicFunction1::UnaryPositive), ) => Some(ConstantData::Complex { value: *value }), _ => None, @@ -919,12 +919,7 @@ impl CodeInfo { oparg::IntrinsicFunction1::UnaryPositive ) => { - ( - Instruction::CallIntrinsic1 { - func: Arg::marker(), - }, - Some(func.get(instr.arg)), - ) + (Opcode::CallIntrinsic1.into(), Some(func.get(instr.arg))) } _ => { i += 1; @@ -955,10 +950,7 @@ impl CodeInfo { block.instructions[idx].end_location = block.instructions[i].end_location; prev = idx; } - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); block.instructions[i].folded_from_nonliteral_expr = false; i = i.saturating_sub(1); @@ -1038,7 +1030,7 @@ impl CodeInfo { for block in &mut self.blocks { let mut i = 0; while i < block.instructions.len() { - let Some(Instruction::BinaryOp { .. }) = block.instructions[i].instr.real() else { + let Some(Opcode::BinaryOp) = block.instructions[i].instr.real_opcode() else { i += 1; continue; }; @@ -1086,10 +1078,7 @@ impl CodeInfo { for &idx in &operand_indices { nop_out_no_location(&mut block.instructions[idx]); } - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); block.instructions[i].folded_from_nonliteral_expr = folded_from_nonliteral_expr; i = i.saturating_sub(1); // re-check with previous instruction @@ -1101,8 +1090,8 @@ impl CodeInfo { } fn get_const_value_from_dummy(info: &InstructionInfo) -> Option<()> { - match info.instr.real() { - Some(Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }) => Some(()), + match info.instr.real_opcode() { + Some(Opcode::LoadConst | Opcode::LoadSmallInt) => Some(()), _ => None, } } @@ -1111,12 +1100,12 @@ impl CodeInfo { metadata: &CodeUnitMetadata, info: &InstructionInfo, ) -> Option { - match info.instr.real() { - Some(Instruction::LoadConst { .. }) => { + match info.instr.real_opcode() { + Some(Opcode::LoadConst) => { let idx = u32::from(info.arg) as usize; metadata.consts.get_index(idx).cloned() } - Some(Instruction::LoadSmallInt { .. }) => { + Some(Opcode::LoadSmallInt) => { let v = u32::from(info.arg) as i32; Some(ConstantData::Integer { value: BigInt::from(v), @@ -1453,7 +1442,7 @@ impl CodeInfo { while i < block.instructions.len() { let instr = &block.instructions[i]; // Look for BUILD_TUPLE - let Some(Instruction::BuildTuple { .. }) = instr.instr.real() else { + let Some(Opcode::BuildTuple) = instr.instr.real_opcode() else { i += 1; continue; }; @@ -1462,14 +1451,13 @@ impl CodeInfo { if block .instructions .get(i + 1) - .and_then(|next| next.instr.real()) + .and_then(|next| next.instr.real_opcode()) .is_some_and(|next| { matches!( next, - Instruction::UnpackSequence { .. } - if usize::try_from(u32::from(block.instructions[i + 1].arg)) - .ok() - == Some(tuple_size) + Opcode::UnpackSequence if + usize::try_from(u32::from(block.instructions[i + 1].arg)) + .ok() == Some(tuple_size) ) }) { @@ -1502,8 +1490,9 @@ impl CodeInfo { all_const = false; break; } - match load_instr.instr.real() { - Some(Instruction::LoadConst { .. }) => { + + match load_instr.instr.real_opcode() { + Some(Opcode::LoadConst) => { let const_idx = u32::from(load_instr.arg) as usize; if let Some(constant) = self.metadata.consts.get_index(const_idx).cloned() @@ -1514,7 +1503,7 @@ impl CodeInfo { break; } } - Some(Instruction::LoadSmallInt { .. }) => { + Some(Opcode::LoadSmallInt) => { // arg is the i32 value stored as u32 (two's complement) let value = u32::from(load_instr.arg) as i32; elements.push(ConstantData::Integer { @@ -1565,7 +1554,7 @@ impl CodeInfo { let mut i = 0; while i < block.instructions.len() { let instr = &block.instructions[i]; - let Some(Instruction::BuildList { .. }) = instr.instr.real() else { + let Some(Opcode::BuildList) = instr.instr.real_opcode() else { i += 1; continue; }; @@ -1582,6 +1571,7 @@ impl CodeInfo { i += 1; continue; }; + if list_size < MIN_CONST_SEQUENCE_SIZE { i += 1; continue; @@ -1597,19 +1587,13 @@ impl CodeInfo { let build_idx = operand_indices[0]; let const_idx_slot = operand_indices[1]; - block.instructions[build_idx].instr = Instruction::BuildList { - count: Arg::marker(), - } - .into(); + block.instructions[build_idx].instr = Opcode::BuildList.into(); block.instructions[build_idx].arg = OpArg::new(0); block.instructions[build_idx].location = folded_loc; block.instructions[build_idx].end_location = end_loc; block.instructions[build_idx].except_handler = eh; - block.instructions[const_idx_slot].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[const_idx_slot].instr = Opcode::LoadConst.into(); block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); block.instructions[const_idx_slot].location = folded_loc; block.instructions[const_idx_slot].end_location = end_loc; @@ -1660,21 +1644,21 @@ impl CodeInfo { if let Some(non_nop4) = Self::get_non_nop_instr_indices(block, i, 4) { let is_build_list = non_nop4[0] == i && matches!( - block.instructions[non_nop4[0]].instr.real(), - Some(Instruction::BuildList { .. }) + block.instructions[non_nop4[0]].instr.real_opcode(), + Some(Opcode::BuildList) ) && u32::from(block.instructions[non_nop4[0]].arg) == 0; let is_const = matches!( - block.instructions[non_nop4[1]].instr.real(), - Some(Instruction::LoadConst { .. }) + block.instructions[non_nop4[1]].instr.real_opcode(), + Some(Opcode::LoadConst) ); let is_list_extend = matches!( - block.instructions[non_nop4[2]].instr.real(), - Some(Instruction::ListExtend { .. }) + block.instructions[non_nop4[2]].instr.real_opcode(), + Some(Opcode::ListExtend) ) && u32::from(block.instructions[non_nop4[2]].arg) == 1; let uses_iter_or_contains = matches!( - block.instructions[non_nop4[3]].instr.real(), - Some(Instruction::GetIter | Instruction::ContainsOp { .. }) + block.instructions[non_nop4[3]].instr.real_opcode(), + Some(Opcode::GetIter | Opcode::ContainsOp) ); if is_build_list && is_const && is_list_extend && uses_iter_or_contains { @@ -1689,13 +1673,13 @@ impl CodeInfo { let is_build_set = non_nop4[0] == i && matches!( - block.instructions[non_nop4[0]].instr.real(), - Some(Instruction::BuildSet { .. }) + block.instructions[non_nop4[0]].instr.real_opcode(), + Some(Opcode::BuildSet) ) && u32::from(block.instructions[non_nop4[0]].arg) == 0; let is_set_update = matches!( - block.instructions[non_nop4[2]].instr.real(), - Some(Instruction::SetUpdate { .. }) + block.instructions[non_nop4[2]].instr.real_opcode(), + Some(Opcode::SetUpdate) ) && u32::from(block.instructions[non_nop4[2]].arg) == 1; if is_build_set && is_const && is_set_update && uses_iter_or_contains { @@ -1715,17 +1699,18 @@ impl CodeInfo { }; let uses_iter_or_contains = non_nop2[0] == i && matches!( - block.instructions[non_nop2[1]].instr.real(), - Some(Instruction::GetIter | Instruction::ContainsOp { .. }) + block.instructions[non_nop2[1]].instr.real_opcode(), + Some(Opcode::GetIter | Opcode::ContainsOp) ); + if !uses_iter_or_contains { i += 1; continue; } if matches!( - block.instructions[i].instr.real(), - Some(Instruction::BuildList { .. }) + block.instructions[i].instr.real_opcode(), + Some(Opcode::BuildList) ) { let seq_size = u32::from(block.instructions[i].arg) as usize; if seq_size > STACK_USE_GUIDELINE { @@ -1759,8 +1744,8 @@ impl CodeInfo { block.instructions[i].instr = Opcode::BuildTuple.into(); i += 2; } else if matches!( - block.instructions[i].instr.real(), - Some(Instruction::BuildSet { .. }) + block.instructions[i].instr.real_opcode(), + Some(Opcode::BuildSet) ) { let seq_size = u32::from(block.instructions[i].arg) as usize; if seq_size > STACK_USE_GUIDELINE { @@ -1805,7 +1790,7 @@ impl CodeInfo { let mut i = 0; while i < block.instructions.len() { let instr = &block.instructions[i]; - let Some(Instruction::BuildSet { .. }) = instr.instr.real() else { + let Some(Opcode::BuildSet) = instr.instr.real_opcode() else { i += 1; continue; }; @@ -1832,19 +1817,13 @@ impl CodeInfo { let build_idx = operand_indices[0]; let const_idx_slot = operand_indices[1]; - block.instructions[build_idx].instr = Instruction::BuildSet { - count: Arg::marker(), - } - .into(); + block.instructions[build_idx].instr = Opcode::BuildSet.into(); block.instructions[build_idx].arg = OpArg::new(0); block.instructions[build_idx].location = folded_loc; block.instructions[build_idx].end_location = end_loc; block.instructions[build_idx].except_handler = eh; - block.instructions[const_idx_slot].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[const_idx_slot].instr = Opcode::LoadConst.into(); block.instructions[const_idx_slot].arg = OpArg::new(const_idx as u32); block.instructions[const_idx_slot].location = folded_loc; block.instructions[const_idx_slot].end_location = end_loc; @@ -1873,12 +1852,11 @@ impl CodeInfo { let instructions = &mut block.instructions; let len = instructions.len(); for i in 0..len.saturating_sub(1) { - let Some(Instruction::BuildTuple { .. }) = instructions[i].instr.real() else { + let Some(Opcode::BuildTuple) = instructions[i].instr.real_opcode() else { continue; }; let n = u32::from(instructions[i].arg); - let Some(Instruction::UnpackSequence { .. }) = instructions[i + 1].instr.real() - else { + let Some(Opcode::UnpackSequence) = instructions[i + 1].instr.real_opcode() else { continue; }; if u32::from(instructions[i + 1].arg) != n { @@ -1917,17 +1895,17 @@ impl CodeInfo { const VISITED: i32 = -1; /// Instruction classes that are safe to reorder around SWAP. - fn is_swappable(instr: &AnyInstruction) -> bool { + const fn is_swappable(instr: AnyInstruction) -> bool { matches!( - (*instr).into(), - AnyOpcode::Real(Opcode::StoreFast | Opcode::PopTop) + instr.real_opcode(), + Some(Opcode::StoreFast | Opcode::PopTop) ) } /// Variable index that a STORE_FAST writes to, or None. - fn stores_to(info: &InstructionInfo) -> Option { - match info.instr.into() { - AnyOpcode::Real(Opcode::StoreFast) => Some(u32::from(info.arg)), + fn stores_to(info: InstructionInfo) -> Option { + match info.instr.real_opcode() { + Some(Opcode::StoreFast) => Some(u32::from(info.arg)), _ => None, } } @@ -1942,20 +1920,26 @@ impl CodeInfo { ) -> Option { loop { i += 1; + if i >= instructions.len() { return None; } + let info = &instructions[i]; let info_lineno = info.location.line.get() as i32; + if lineno >= 0 && info_lineno > 0 && info_lineno != lineno { return None; } - if matches!(info.instr, AnyInstruction::Real(Instruction::Nop)) { + + if matches!(info.instr.real_opcode(), Some(Opcode::Nop)) { continue; } - if is_swappable(&info.instr) { + + if is_swappable(info.instr) { return Some(i); } + return None; } } @@ -1963,7 +1947,7 @@ impl CodeInfo { fn optimize_swap_block(instructions: &mut [InstructionInfo]) { let mut i = 0usize; while i < instructions.len() { - let AnyInstruction::Real(Instruction::Swap { .. }) = instructions[i].instr else { + let Some(Opcode::Swap) = instructions[i].instr.real_opcode() else { i += 1; continue; }; @@ -1973,14 +1957,14 @@ impl CodeInfo { let mut more = false; while i + len < instructions.len() { let info = &instructions[i + len]; - match info.instr.real() { - Some(Instruction::Swap { .. }) => { + match info.instr.real_opcode() { + Some(Opcode::Swap) => { let oparg = u32::from(info.arg) as usize; depth = depth.max(oparg); more |= len > 0; len += 1; } - Some(Instruction::Nop) => { + Some(Opcode::Nop) => { len += 1; } _ => break, @@ -1994,7 +1978,7 @@ impl CodeInfo { let mut stack: Vec = (0..depth as i32).collect(); for info in &instructions[i..i + len] { - if matches!(info.instr.real(), Some(Instruction::Swap { .. })) { + if matches!(info.instr.real_opcode(), Some(Opcode::Swap)) { let oparg = u32::from(info.arg) as usize; stack.swap(0, oparg - 1); } @@ -2034,11 +2018,9 @@ impl CodeInfo { fn apply_from(instructions: &mut [InstructionInfo], mut i: isize) { while i >= 0 { let idx = i as usize; - let swap_arg = match instructions[idx].instr.real() { - Some(Instruction::Swap { .. }) => u32::from(instructions[idx].arg), - Some( - Instruction::Nop | Instruction::PopTop | Instruction::StoreFast { .. }, - ) => { + let swap_arg = match instructions[idx].instr.real_opcode() { + Some(Opcode::Swap) => u32::from(instructions[idx].arg), + Some(Opcode::Nop | Opcode::PopTop | Opcode::StoreFast) => { i -= 1; continue; } @@ -2061,19 +2043,20 @@ impl CodeInfo { k = next; } - let store_j = stores_to(&instructions[j]); - let store_k = stores_to(&instructions[k]); + let store_j = stores_to(instructions[j]); + let store_k = stores_to(instructions[k]); if store_j.is_some() || store_k.is_some() { if store_j == store_k { return; } - let conflict = instructions[(j + 1)..k].iter().any(|info| { + let conflict = instructions[(j + 1)..k].iter().any(|&info| { if let Some(store_idx) = stores_to(info) { Some(store_idx) == store_j || Some(store_idx) == store_k } else { false } }); + if conflict { return; } @@ -2091,8 +2074,8 @@ impl CodeInfo { let len = block.instructions.len(); for i in 0..len { if matches!( - block.instructions[i].instr.real(), - Some(Instruction::Swap { .. }) + block.instructions[i].instr.real_opcode(), + Some(Opcode::Swap) ) { apply_from(&mut block.instructions, i as isize); } @@ -2117,8 +2100,8 @@ impl CodeInfo { while i < len { // Look for UNPACK_SEQUENCE or UNPACK_EX let is_unpack = matches!( - instructions[i].instr.into(), - AnyOpcode::Real(Opcode::UnpackSequence | Opcode::UnpackEx) + instructions[i].instr.real_opcode(), + Some(Opcode::UnpackSequence | Opcode::UnpackEx) ); if !is_unpack { i += 1; @@ -2129,8 +2112,8 @@ impl CodeInfo { let mut run_end = run_start; while run_end < len && matches!( - instructions[run_end].instr.into(), - AnyOpcode::Real(Opcode::StoreFast) + instructions[run_end].instr.real_opcode(), + Some(Opcode::StoreFast) ) { run_end += 1; @@ -2159,13 +2142,10 @@ impl CodeInfo { for i in 0..instructions.len().saturating_sub(1) { let lhs = &instructions[i]; let rhs = &instructions[i + 1]; - let preceded_by_swap = i > 0 - && matches!( - instructions[i - 1].instr.real(), - Some(Instruction::Swap { .. }) - ); - if !matches!(lhs.instr.real(), Some(Instruction::StoreFast { .. })) - || !matches!(rhs.instr.real(), Some(Instruction::StoreFast { .. })) + let preceded_by_swap = + i > 0 && matches!(instructions[i - 1].instr.real_opcode(), Some(Opcode::Swap)); + if !matches!(lhs.instr.real_opcode(), Some(Opcode::StoreFast)) + || !matches!(rhs.instr.real_opcode(), Some(Opcode::StoreFast)) || u32::from(lhs.arg) != u32::from(rhs.arg) || instruction_lineno(lhs) != instruction_lineno(rhs) || preceded_by_swap @@ -2219,9 +2199,9 @@ impl CodeInfo { }; if let Some(is_true) = const_truthiness(curr_instr, curr.arg, &self.metadata) { - let jump_if_true = match next_instr { - Instruction::PopJumpIfTrue { .. } => Some(true), - Instruction::PopJumpIfFalse { .. } => Some(false), + let jump_if_true = match next_instr.into() { + Opcode::PopJumpIfTrue => Some(true), + Opcode::PopJumpIfFalse => Some(false), _ => None, }; if let Some(jump_if_true) = jump_if_true { @@ -2232,10 +2212,7 @@ impl CodeInfo { }; set_to_nop(&mut block.instructions[i]); if is_true == jump_if_true { - block.instructions[i + 1].instr = PseudoInstruction::Jump { - delta: Arg::marker(), - } - .into(); + block.instructions[i + 1].instr = PseudoOpcode::Jump.into(); block.instructions[i + 1].arg = OpArg::new(u32::from(target)); } else { set_to_nop(&mut block.instructions[i + 1]); @@ -2257,8 +2234,8 @@ impl CodeInfo { } if matches!( - block.instructions[jump_idx].instr.real(), - Some(Instruction::ToBool) + block.instructions[jump_idx].instr.real_opcode(), + Some(Opcode::ToBool) ) { set_to_nop(&mut block.instructions[jump_idx]); jump_idx += 1; @@ -2294,13 +2271,9 @@ impl CodeInfo { set_to_nop(&mut block.instructions[i]); set_to_nop(&mut block.instructions[i + 1]); block.instructions[jump_idx].instr = if invert { - Instruction::PopJumpIfNotNone { - delta: Arg::marker(), - } + Opcode::PopJumpIfNotNone } else { - Instruction::PopJumpIfNone { - delta: Arg::marker(), - } + Opcode::PopJumpIfNone } .into(); block.instructions[jump_idx].arg = OpArg::new(u32::from(delta)); @@ -2309,10 +2282,8 @@ impl CodeInfo { } } - if matches!( - curr_instr, - Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. } - ) && matches!(next_instr, Instruction::PopTop) + if matches!(curr_instr.into(), Opcode::LoadConst | Opcode::LoadSmallInt) + && matches!(next_instr, Instruction::PopTop) { set_to_nop(&mut block.instructions[i]); set_to_nop(&mut block.instructions[i + 1]); @@ -2344,12 +2315,7 @@ impl CodeInfo { .metadata .consts .insert_full(ConstantData::Boolean { value }); - Some(( - Instruction::LoadConst { - consti: Arg::marker(), - }, - OpArg::new(const_idx as u32), - )) + Some((Opcode::LoadConst.into(), OpArg::new(const_idx as u32))) } else { None } @@ -2401,8 +2367,8 @@ impl CodeInfo { let curr = &block.instructions[i]; let next = &block.instructions[i + 1]; - let (Some(Instruction::LoadGlobal { .. }), Some(Instruction::PushNull)) = - (curr.instr.real(), next.instr.real()) + let (Some(Opcode::LoadGlobal), Some(Opcode::PushNull)) = + (curr.instr.real_opcode(), next.instr.real_opcode()) else { i += 1; continue; @@ -2436,11 +2402,8 @@ impl CodeInfo { }; let redundant = matches!( - (curr_instr, next_instr), - ( - Instruction::LoadConst { .. } | Instruction::LoadSmallInt { .. }, - Instruction::PopTop - ) + (curr_instr.into(), next_instr.into()), + (Opcode::LoadConst | Opcode::LoadSmallInt, Opcode::PopTop) ) || matches!(curr_instr, Instruction::Copy { i } if i.get(curr.arg) == 1) && matches!(next_instr, Instruction::PopTop); @@ -2461,7 +2424,7 @@ impl CodeInfo { for block in &mut self.blocks { for instr in &mut block.instructions { // Check if it's a LOAD_CONST instruction - let Some(Instruction::LoadConst { .. }) = instr.instr.real() else { + let Some(Opcode::LoadConst) = instr.instr.real_opcode() else { continue; }; @@ -2502,7 +2465,7 @@ impl CodeInfo { for block in &self.blocks { for instr in &block.instructions { - if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + if let Some(Opcode::LoadConst) = instr.instr.real_opcode() { let idx = u32::from(instr.arg) as usize; if idx < nconsts { used[idx] = true; @@ -2539,7 +2502,7 @@ impl CodeInfo { // Update LOAD_CONST instruction arguments for block in &mut self.blocks { for instr in &mut block.instructions { - if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + if let Some(Opcode::LoadConst) = instr.instr.real_opcode() { let old_idx = u32::from(instr.arg) as usize; if old_idx < nconsts { instr.arg = OpArg::new(old_to_new[old_idx] as u32); @@ -2557,10 +2520,10 @@ impl CodeInfo { .instructions .iter() .rev() - .filter_map(|info| info.instr.real()); + .filter_map(|info| info.instr.real_opcode()); matches!( (reals.next(), reals.next()), - (Some(Instruction::PopIter), Some(Instruction::EndFor)) + (Some(Opcode::PopIter), Some(Opcode::EndFor)) ) } @@ -2619,8 +2582,8 @@ impl CodeInfo { continue; } - match (curr.instr.real(), next.instr.real()) { - (Some(Instruction::LoadFast { .. }), Some(Instruction::LoadFast { .. })) => { + match (curr.instr.real_opcode(), next.instr.real_opcode()) { + (Some(Opcode::LoadFast), Some(Opcode::LoadFast)) => { let idx1 = u32::from(curr.arg); let idx2 = u32::from(next.arg); if idx1 >= 16 || idx2 >= 16 { @@ -2628,14 +2591,11 @@ impl CodeInfo { continue; } let packed = (idx1 << 4) | idx2; - block.instructions[i].instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadFastLoadFast.into(); block.instructions[i].arg = OpArg::new(packed); block.instructions.remove(i + 1); } - (Some(Instruction::StoreFast { .. }), Some(Instruction::LoadFast { .. })) => { + (Some(Opcode::StoreFast), Some(Opcode::LoadFast)) => { let store_idx = u32::from(curr.arg); let load_idx = u32::from(next.arg); if store_idx >= 16 || load_idx >= 16 { @@ -2643,14 +2603,11 @@ impl CodeInfo { continue; } let packed = (store_idx << 4) | load_idx; - block.instructions[i].instr = Instruction::StoreFastLoadFast { - var_nums: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::StoreFastLoadFast.into(); block.instructions[i].arg = OpArg::new(packed); block.instructions.remove(i + 1); } - (Some(Instruction::StoreFast { .. }), Some(Instruction::StoreFast { .. })) => { + (Some(Opcode::StoreFast), Some(Opcode::StoreFast)) => { let idx1 = u32::from(curr.arg); let idx2 = u32::from(next.arg); if idx1 >= 16 || idx2 >= 16 { @@ -2658,10 +2615,7 @@ impl CodeInfo { continue; } let packed = (idx1 << 4) | idx2; - block.instructions[i].instr = Instruction::StoreFastStoreFast { - var_nums: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::StoreFastStoreFast.into(); block.instructions[i].arg = OpArg::new(packed); block.instructions.remove(i + 1); } @@ -3007,21 +2961,17 @@ impl CodeInfo { if instr_flags[i] != 0 { continue; } - match info.instr.real() { - Some(Instruction::LoadFast { .. }) => { - info.instr = Instruction::LoadFastBorrow { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastLoadFast { .. }) => { - info.instr = Instruction::LoadFastBorrowLoadFastBorrow { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + info.instr = match opcode { + Opcode::LoadFast => Opcode::LoadFastBorrow, + Opcode::LoadFastLoadFast => Opcode::LoadFastBorrowLoadFastBorrow, + _ => continue, } + .into(); } } } @@ -3083,27 +3033,28 @@ impl CodeInfo { for block in &mut self.blocks { let len = block.instructions.len(); for i in 0..len { - let Some(Instruction::LoadFastBorrow { .. }) = block.instructions[i].instr.real() - else { + let Some(Opcode::LoadFastBorrow) = block.instructions[i].instr.real_opcode() else { continue; }; + let tail = &block.instructions[i + 1..]; if tail.len() < 3 { continue; } + if !matches!(tail[0].instr.real(), Some(Instruction::Swap { .. })) { continue; } + if !matches!(tail[1].instr.real(), Some(Instruction::PopExcept)) { continue; } + if !matches!(tail[2].instr.real(), Some(Instruction::ReturnValue)) { continue; } - block.instructions[i].instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); + + block.instructions[i].instr = Opcode::LoadFast.into(); } } } @@ -3124,21 +3075,16 @@ impl CodeInfo { fn deoptimize_block_borrows(block: &mut Block) { for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + info.instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast, + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, + _ => continue, } + .into(); } } @@ -3153,28 +3099,29 @@ impl CodeInfo { return false; } let starts_with_load_fast = matches!( - infos[0].instr.real(), - Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) + infos[0].instr.real_opcode(), + Some(Opcode::LoadFast | Opcode::LoadFastBorrow) ); + if !starts_with_load_fast { return false; } matches!( - infos.get(1).and_then(|info| info.instr.real()), + infos.get(1).and_then(|info| info.instr.real_opcode()), Some( - Instruction::PopJumpIfFalse { .. } - | Instruction::PopJumpIfTrue { .. } - | Instruction::PopJumpIfNone { .. } - | Instruction::PopJumpIfNotNone { .. } + Opcode::PopJumpIfFalse + | Opcode::PopJumpIfTrue + | Opcode::PopJumpIfNone + | Opcode::PopJumpIfNotNone ) ) || (matches!(infos[1].instr.real(), Some(Instruction::ToBool)) && matches!( - infos.get(2).and_then(|info| info.instr.real()), + infos.get(2).and_then(|info| info.instr.real_opcode()), Some( - Instruction::PopJumpIfFalse { .. } - | Instruction::PopJumpIfTrue { .. } - | Instruction::PopJumpIfNone { .. } - | Instruction::PopJumpIfNotNone { .. } + Opcode::PopJumpIfFalse + | Opcode::PopJumpIfTrue + | Opcode::PopJumpIfNone + | Opcode::PopJumpIfNotNone ) )) } @@ -3294,12 +3241,12 @@ impl CodeInfo { return None; } if !matches!( - infos[2].instr.real(), + infos[2].instr.real_opcode(), Some( - Instruction::PopJumpIfFalse { .. } - | Instruction::PopJumpIfTrue { .. } - | Instruction::PopJumpIfNone { .. } - | Instruction::PopJumpIfNotNone { .. } + Opcode::PopJumpIfFalse + | Opcode::PopJumpIfTrue + | Opcode::PopJumpIfNone + | Opcode::PopJumpIfNotNone ) ) { return None; @@ -3309,21 +3256,16 @@ impl CodeInfo { fn deoptimize_block_borrows(block: &mut Block) { for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + info.instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast, + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, + _ => continue, } + .into(); } } @@ -3412,7 +3354,7 @@ impl CodeInfo { if block .instructions .iter() - .any(|info| matches!(info.instr.real(), Some(Instruction::RaiseVarargs { .. }))) + .any(|info| matches!(info.instr.real_opcode(), Some(Opcode::RaiseVarargs))) { return true; } @@ -3576,21 +3518,16 @@ impl CodeInfo { fn deoptimize_block_borrows(block: &mut Block) { for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + info.instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast, + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, + _ => continue, } + .into(); } } @@ -3614,26 +3551,27 @@ impl CodeInfo { } } - let seeds: Vec<_> = - self.blocks - .iter() - .enumerate() - .filter_map(|(idx, block)| { - let prev_protected = predecessors[idx].iter().any(|pred| { - self.blocks[pred.idx()] - .instructions - .iter() - .any(|info| info.except_handler.is_some()) - }); - (!block_is_exceptional(block) - && trailing_conditional_jump_index(block).is_some() - && prev_protected - && block.instructions.iter().any(|info| { - matches!(info.instr.real(), Some(Instruction::Call { .. })) - })) - .then_some(BlockIdx::new(idx as u32)) - }) - .collect(); + let seeds: Vec<_> = self + .blocks + .iter() + .enumerate() + .filter_map(|(idx, block)| { + let prev_protected = predecessors[idx].iter().any(|pred| { + self.blocks[pred.idx()] + .instructions + .iter() + .any(|info| info.except_handler.is_some()) + }); + (!block_is_exceptional(block) + && trailing_conditional_jump_index(block).is_some() + && prev_protected + && block + .instructions + .iter() + .any(|info| matches!(info.instr.real_opcode(), Some(Opcode::Call)))) + .then_some(BlockIdx::new(idx as u32)) + }) + .collect(); let mut visited = vec![false; self.blocks.len()]; for seed in seeds { @@ -3735,21 +3673,17 @@ impl CodeInfo { if !info.folded_from_nonliteral_expr { continue; } - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + info.instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast, + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, + _ => continue, } + .into(); } } } @@ -3758,26 +3692,22 @@ impl CodeInfo { for block in &mut self.blocks { let mut in_exception_state = false; for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::PushExcInfo) => { + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + match opcode { + Opcode::PushExcInfo => { in_exception_state = true; } - Some(Instruction::PopExcept | Instruction::Reraise { .. }) => { + Opcode::PopExcept | Opcode::Reraise => { in_exception_state = false; } - Some(Instruction::LoadFastBorrow { .. }) if in_exception_state => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); + Opcode::LoadFastBorrow if in_exception_state => { + info.instr = Opcode::LoadFast.into(); } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) - if in_exception_state => - { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); + Opcode::LoadFastBorrowLoadFastBorrow if in_exception_state => { + info.instr = Opcode::LoadFastLoadFast.into(); } _ => {} } @@ -3789,28 +3719,25 @@ impl CodeInfo { fn deoptimize_block_borrows(block: &mut Block, after_import_only: bool) { let mut after_import = !after_import_only; for info in &mut block.instructions { - if matches!(info.instr.real(), Some(Instruction::ImportName { .. })) { + if matches!(info.instr.real_opcode(), Some(Opcode::ImportName)) { after_import = true; continue; } + if !after_import { continue; } - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + info.instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast, + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, + _ => continue, } + .into(); } } @@ -3822,11 +3749,11 @@ impl CodeInfo { let jumps_to_target = block.instructions.iter().any(|info| { info.target == target && matches!( - info.instr.real(), + info.instr.real_opcode(), Some( - Instruction::JumpForward { .. } - | Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } + Opcode::JumpForward + | Opcode::JumpBackward + | Opcode::JumpBackwardNoInterrupt ) ) }); @@ -3908,21 +3835,16 @@ impl CodeInfo { fn deoptimize_borrow_after_protected_store_tail(&mut self) { fn deoptimize_block_borrows_from(block: &mut Block, start: usize) { for info in block.instructions.iter_mut().skip(start) { - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + info.instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast, + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, + _ => continue, } + .into(); } } @@ -3934,11 +3856,11 @@ impl CodeInfo { let jumps_to_target = block.instructions.iter().any(|info| { info.target == target && matches!( - info.instr.real(), + info.instr.real_opcode(), Some( - Instruction::JumpForward { .. } - | Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } + Opcode::JumpForward + | Opcode::JumpBackward + | Opcode::JumpBackwardNoInterrupt ) ) }); @@ -3971,11 +3893,11 @@ impl CodeInfo { && info.target != BlockIdx::NULL && in_segment[info.target.idx()] && matches!( - info.instr.real(), + info.instr.real_opcode(), Some( - Instruction::JumpForward { .. } - | Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } + Opcode::JumpForward + | Opcode::JumpBackward + | Opcode::JumpBackwardNoInterrupt ) ) { @@ -3991,12 +3913,8 @@ impl CodeInfo { fn block_has_tail_deopt_trigger_from(block: &Block, start: usize) -> bool { block.instructions.iter().skip(start).any(|info| { matches!( - info.instr.real(), - Some( - Instruction::Call { .. } - | Instruction::CallKw { .. } - | Instruction::StoreAttr { .. } - ) + info.instr.real_opcode(), + Some(Opcode::Call | Opcode::CallKw | Opcode::StoreAttr) ) }) } @@ -4080,8 +3998,8 @@ impl CodeInfo { .any(|info| info.except_handler.is_some()) || !block.instructions.iter().any(|info| { matches!( - info.instr.real(), - Some(Instruction::Call { .. } | Instruction::CallKw { .. }) + info.instr.real_opcode(), + Some(Opcode::Call | Opcode::CallKw) ) }) || !block_has_exception_match_handler(&self.blocks, block) @@ -4252,10 +4170,10 @@ impl CodeInfo { let block = &self.blocks[block_idx]; let len = block.instructions.len(); for i in 0..len { - let Some(Instruction::LoadFastBorrow { .. }) = block.instructions[i].instr.real() - else { + let Some(Opcode::LoadFastBorrow) = block.instructions[i].instr.real_opcode() else { continue; }; + let Some(Instruction::LoadAttr { namei }) = block .instructions .get(i + 1) @@ -4275,17 +4193,17 @@ impl CodeInfo { loop { let scan_block = &self.blocks[scan_block_idx]; for info in scan_block.instructions.iter().skip(scan_start) { - match info.instr.real() { + match info.instr.real_opcode() { Some( - Instruction::LoadConst { .. } - | Instruction::LoadSmallInt { .. } - | Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadAttr { .. } - | Instruction::Nop, + Opcode::LoadConst + | Opcode::LoadSmallInt + | Opcode::LoadFast + | Opcode::LoadFastBorrow + | Opcode::LoadAttr + | Opcode::Nop, ) => {} - Some(Instruction::BuildTuple { .. }) => saw_build_tuple = true, - Some(Instruction::MatchKeys) => { + Some(Opcode::BuildTuple) => saw_build_tuple = true, + Some(Opcode::MatchKeys) => { saw_match_keys = true; break; } @@ -4319,10 +4237,7 @@ impl CodeInfo { } for (block_idx, instr_idx) in to_deopt { - self.blocks[block_idx].instructions[instr_idx].instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); + self.blocks[block_idx].instructions[instr_idx].instr = Opcode::LoadFast.into(); } } @@ -4338,28 +4253,19 @@ impl CodeInfo { } fn deoptimize_borrow(info: &mut InstructionInfo) { - match info.instr.real() { - Some(Instruction::LoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + let Some(opcode) = info.instr.real_opcode() else { + return; + }; + + info.instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast.into(), + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast.into(), + _ => info.instr, } } - fn is_attr_load(instr: Instruction) -> bool { - matches!( - instr, - Instruction::LoadAttr { .. } | Instruction::LoadSuperAttr { .. } - ) + const fn is_attr_load(instr: Instruction) -> bool { + matches!(instr.opcode(), Opcode::LoadAttr | Opcode::LoadSuperAttr) } fn attr_load_is_method(info: InstructionInfo) -> bool { @@ -4370,15 +4276,15 @@ impl CodeInfo { } } - fn is_subscript_index_setup(instr: Instruction) -> bool { + const fn is_subscript_index_setup(instr: Instruction) -> bool { matches!( - instr, - Instruction::LoadConst { .. } - | Instruction::LoadSmallInt { .. } - | Instruction::LoadFast { .. } - | Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastCheck { .. } - | Instruction::Nop + instr.opcode(), + Opcode::LoadConst + | Opcode::LoadSmallInt + | Opcode::LoadFast + | Opcode::LoadFastBorrow + | Opcode::LoadFastCheck + | Opcode::Nop ) } @@ -4409,11 +4315,11 @@ impl CodeInfo { let (_, next_info) = real_instrs.get(cursor)?; - match next_info.instr.real() { - Some(Instruction::GetIter) => Some(DeoptKind::ReturnIter { + match next_info.instr.real_opcode() { + Some(Opcode::GetIter) => Some(DeoptKind::ReturnIter { tail_start_idx: cursor + 1, }), - Some(Instruction::Call { .. } | Instruction::CallKw { .. }) => real_instrs + Some(Opcode::Call | Opcode::CallKw) => real_instrs .get(cursor + 1) .and_then(|(_, info)| info.instr.real()) .and_then(|instr| { @@ -4458,18 +4364,21 @@ impl CodeInfo { } let block = &blocks[block_idx.idx()]; for info in block.instructions.iter().skip(current_start) { - match info.instr.real() { - Some(Instruction::ReturnValue) => return true, - Some( - Instruction::StoreFast { .. } - | Instruction::StoreFastLoadFast { .. } - | Instruction::StoreFastStoreFast { .. } - | Instruction::DeleteFast { .. } - | Instruction::LoadFastAndClear { .. }, - ) => return false, + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + match opcode { + Opcode::ReturnValue => return true, + Opcode::StoreFast + | Opcode::StoreFastLoadFast + | Opcode::StoreFastStoreFast + | Opcode::DeleteFast + | Opcode::LoadFastAndClear => return false, _ => {} } } + block_idx = block.next; current_start = 0; } @@ -4561,8 +4470,8 @@ impl CodeInfo { let mut to_deopt = Vec::new(); for (real_idx, (instr_idx, info)) in real_instrs.iter().enumerate() { let is_attr_chain_root = matches!( - info.instr.real(), - Some(Instruction::LoadFast { .. } | Instruction::LoadFastBorrow { .. }) + info.instr.real_opcode(), + Some(Opcode::LoadFast | Opcode::LoadFastBorrow) ); if info.except_handler.is_some() || !is_attr_chain_root { continue; @@ -4590,9 +4499,11 @@ impl CodeInfo { { continue; } - if matches!(info.instr.real(), Some(Instruction::LoadFastBorrow { .. })) { + + if matches!(info.instr.real_opcode(), Some(Opcode::LoadFastBorrow)) { to_deopt.push(*instr_idx); } + if let DeoptKind::Subscript { binary_op_idx } = deopt_kind { for (extra_instr_idx, extra_info) in real_instrs .iter() @@ -4600,28 +4511,22 @@ impl CodeInfo { .take(binary_op_idx.saturating_sub(real_idx + 1)) .map(|(idx, info)| (*idx, *info)) { - if matches!( - extra_info.instr.real(), - Some(Instruction::LoadFastBorrow { .. }) - ) { + if matches!(extra_info.instr.real_opcode(), Some(Opcode::LoadFastBorrow)) { to_deopt.push(extra_instr_idx); } } if matches!( real_instrs .get(binary_op_idx + 1) - .and_then(|(_, info)| info.instr.real()), - Some(Instruction::StoreFast { .. }) + .and_then(|(_, info)| info.instr.real_opcode()), + Some(Opcode::StoreFast) ) { for (extra_instr_idx, extra_info) in real_instrs.iter().skip(binary_op_idx + 2) { if matches!( - extra_info.instr.real(), - Some( - Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } - ) + extra_info.instr.real_opcode(), + Some(Opcode::LoadFastBorrow | Opcode::LoadFastBorrowLoadFastBorrow) ) { to_deopt.push(*extra_instr_idx); } @@ -4648,10 +4553,10 @@ impl CodeInfo { .enumerate() { if matches!( - tail_info.instr.real(), + tail_info.instr.real_opcode(), Some( - Instruction::LoadFastBorrow { .. } - | Instruction::LoadFastBorrowLoadFastBorrow { .. } + Opcode::LoadFastBorrow + | Opcode::LoadFastBorrowLoadFastBorrow ) ) { cross_block_deopts.push((tail_block_idx, tail_instr_idx)); @@ -4667,26 +4572,19 @@ impl CodeInfo { } } for (block_idx, instr_idx) in cross_block_deopts { - match self.blocks[block_idx.idx()].instructions[instr_idx] + let Some(opcode) = self.blocks[block_idx.idx()].instructions[instr_idx] .instr - .real() - { - Some(Instruction::LoadFastBorrow { .. }) => { - self.blocks[block_idx.idx()].instructions[instr_idx].instr = - Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); - } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) => { - self.blocks[block_idx.idx()].instructions[instr_idx].instr = - Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); - } - _ => {} + .real_opcode() + else { + continue; + }; + + self.blocks[block_idx.idx()].instructions[instr_idx].instr = match opcode { + Opcode::LoadFastBorrow => Opcode::LoadFast, + Opcode::LoadFastBorrowLoadFastBorrow => Opcode::LoadFastLoadFast, + _ => continue, } + .into(); } } @@ -4702,10 +4600,10 @@ impl CodeInfo { fn is_cleanup_restore_prefix(instructions: &[InstructionInfo]) -> bool { let mut saw_pop_iter = false; for info in instructions { - match info.instr.real() { - Some(Instruction::EndFor) if !saw_pop_iter => {} - Some(Instruction::PopIter) if !saw_pop_iter => saw_pop_iter = true, - Some(Instruction::Swap { .. } | Instruction::PopTop) if saw_pop_iter => {} + match info.instr.real_opcode() { + Some(Opcode::EndFor) if !saw_pop_iter => {} + Some(Opcode::PopIter) if !saw_pop_iter => saw_pop_iter = true, + Some(Opcode::Swap | Opcode::PopTop) if saw_pop_iter => {} _ => return false, } } @@ -4743,27 +4641,23 @@ impl CodeInfo { for (i, info) in block.instructions.iter().copied().enumerate() { if !in_restore_prefix && matches!( - info.instr.real(), - Some( - Instruction::StoreFast { .. } | Instruction::StoreFastStoreFast { .. } - ) + info.instr.real_opcode(), + Some(Opcode::StoreFast | Opcode::StoreFastStoreFast) ) && !new_instructions.is_empty() && (new_instructions.iter().all(|prev: &InstructionInfo| { matches!( - prev.instr.real(), - Some(Instruction::Swap { .. } | Instruction::PopTop) + prev.instr.real_opcode(), + Some(Opcode::Swap | Opcode::PopTop) ) }) || is_cleanup_restore_prefix(&new_instructions)) { in_restore_prefix = true; } - let expand = matches!( - info.instr.real(), - Some(Instruction::StoreFastStoreFast { .. }) - ) && (is_cleanup_restore_prefix(&new_instructions) - || (i == 0 && starts_after_cleanup[block_idx]) - || in_restore_prefix); + let expand = matches!(info.instr.real_opcode(), Some(Opcode::StoreFastStoreFast)) + && (is_cleanup_restore_prefix(&new_instructions) + || (i == 0 && starts_after_cleanup[block_idx]) + || in_restore_prefix); if expand { let Some(Instruction::StoreFastStoreFast { var_nums }) = info.instr.real() @@ -4774,25 +4668,18 @@ impl CodeInfo { let (idx1, idx2) = packed.indexes(); let mut first = info; - first.instr = Instruction::StoreFast { - var_num: Arg::marker(), - } - .into(); + first.instr = Opcode::StoreFast.into(); first.arg = OpArg::new(u32::from(idx1)); new_instructions.push(first); let mut second = info; - second.instr = Instruction::StoreFast { - var_num: Arg::marker(), - } - .into(); + second.instr = Opcode::StoreFast.into(); second.arg = OpArg::new(u32::from(idx2)); new_instructions.push(second); continue; } - in_restore_prefix &= - matches!(info.instr.real(), Some(Instruction::StoreFast { .. })); + in_restore_prefix &= matches!(info.instr.real_opcode(), Some(Opcode::StoreFast)); new_instructions.push(info); } block.instructions = new_instructions; @@ -4849,7 +4736,7 @@ impl CodeInfo { worklist.push(target); } } - if matches!(info.instr.real(), Some(Instruction::ForIter { .. })) + if matches!(info.instr.real_opcode(), Some(Opcode::ForIter)) && info.target != BlockIdx::NULL && merge_unsafe_mask(&mut in_masks[info.target.idx()], &unsafe_mask) { @@ -4950,10 +4837,11 @@ impl CodeInfo { let mut second = info; second.instr = if needs_check_2 { - Opcode::LoadFastCheck.into() + Opcode::LoadFastCheck } else { - Opcode::LoadFast.into() - }; + Opcode::LoadFast + } + .into(); second.arg = OpArg::new(idx2 as u32); new_instructions.push(first); @@ -6249,20 +6137,21 @@ fn normalize_jumps(blocks: &mut Vec) { info.instr = match info.instr.into() { AnyOpcode::Pseudo(PseudoOpcode::Jump) => { if target_pos > source_pos { - Opcode::JumpForward.into() + Opcode::JumpForward } else { - Opcode::JumpBackward.into() + Opcode::JumpBackward } } AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { if target_pos > source_pos { - Opcode::JumpForward.into() + Opcode::JumpForward } else { - Opcode::JumpBackwardNoInterrupt.into() + Opcode::JumpBackwardNoInterrupt } } - _ => info.instr, - }; + _ => continue, + } + .into(); } } } @@ -6810,24 +6699,22 @@ fn deoptimize_borrow_after_push_exc_info_in_blocks(blocks: &mut [Block]) { while current != BlockIdx::NULL { let block = &mut blocks[current.idx()]; for info in &mut block.instructions { - match info.instr.real() { - Some(Instruction::PushExcInfo) => { + let Some(opcode) = info.instr.real_opcode() else { + continue; + }; + + match opcode { + Opcode::PushExcInfo => { in_exception_state = true; } - Some(Instruction::PopExcept | Instruction::Reraise { .. }) => { + Opcode::PopExcept | Opcode::Reraise => { in_exception_state = false; } - Some(Instruction::LoadFastBorrow { .. }) if in_exception_state => { - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); + Opcode::LoadFastBorrow if in_exception_state => { + info.instr = Opcode::LoadFast.into(); } - Some(Instruction::LoadFastBorrowLoadFastBorrow { .. }) if in_exception_state => { - info.instr = Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - } - .into(); + Opcode::LoadFastBorrowLoadFastBorrow if in_exception_state => { + info.instr = Opcode::LoadFastLoadFast.into(); } _ => {} } @@ -6848,7 +6735,7 @@ fn next_nonempty_block(blocks: &[Block], mut idx: BlockIdx) -> BlockIdx { } fn is_load_const_none(instr: &InstructionInfo, metadata: &CodeUnitMetadata) -> bool { - matches!(instr.instr.real(), Some(Instruction::LoadConst { .. })) + matches!(instr.instr.real_opcode(), Some(Opcode::LoadConst)) && matches!( metadata.consts.get_index(u32::from(instr.arg) as usize), Some(ConstantData::None) @@ -6997,7 +6884,7 @@ fn is_exception_cleanup_block(block: &Block) -> bool { && block .instructions .last() - .is_some_and(|instr| matches!(instr.instr.real(), Some(Instruction::Reraise { .. }))) + .is_some_and(|instr| matches!(instr.instr.real_opcode(), Some(Opcode::Reraise))) } fn is_with_suppress_exit_block(block: &Block) -> bool { @@ -7030,14 +6917,11 @@ fn block_contains_suspension_point(block: &Block) -> bool { block .instructions .iter() - .filter_map(|info| info.instr.real()) + .filter_map(|info| info.instr.real_opcode()) .any(|instr| { matches!( instr, - Instruction::YieldValue { .. } - | Instruction::GetAwaitable { .. } - | Instruction::GetANext - | Instruction::EndAsyncFor + Opcode::YieldValue | Opcode::GetAwaitable | Opcode::GetANext | Opcode::EndAsyncFor ) }) } @@ -7202,8 +7086,8 @@ fn reorder_conditional_exit_and_jump_blocks(blocks: &mut [Block]) { continue; } if !matches!( - blocks[jump_block.idx()].instructions[0].instr.real(), - Some(Instruction::JumpForward { .. }) + blocks[jump_block.idx()].instructions[0].instr.real_opcode(), + Some(Opcode::JumpForward) ) { current = next; continue; @@ -7270,10 +7154,7 @@ fn reorder_conditional_jump_and_exit_blocks(blocks: &mut [Block]) { continue; } let jump_instr = blocks[jump_block.idx()].instructions[0]; - if !matches!( - jump_instr.instr.real(), - Some(Instruction::JumpForward { .. }) - ) { + if !matches!(jump_instr.instr.real_opcode(), Some(Opcode::JumpForward)) { current = next; continue; } @@ -7386,7 +7267,7 @@ fn reorder_conditional_chain_and_jump_back_blocks(blocks: &mut Vec) { blocks, ); let allow_true_path_jump_back_reorder = - matches!(last.instr.real(), Some(Instruction::PopJumpIfTrue { .. })) + matches!(last.instr.real_opcode(), Some(Opcode::PopJumpIfTrue)) && (chain_has_suspension_point || chain_starts_with_false_path_jump || chain_is_single_exit_block @@ -7522,7 +7403,7 @@ fn reorder_jump_over_exception_cleanup_blocks(blocks: &mut [Block]) { current = next; continue; }; - if !matches!(last.instr.real(), Some(Instruction::JumpForward { .. })) + if !matches!(last.instr.real_opcode(), Some(Opcode::JumpForward)) || last.target == BlockIdx::NULL { current = next; @@ -8032,15 +7913,9 @@ fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { let last_insts = &blocks[last_block.idx()].instructions; // Only apply when the last block is EXACTLY a return-None epilogue. let is_return_block = last_insts.len() == 2 - && matches!( - last_insts[0].instr, - AnyInstruction::Real(Instruction::LoadConst { .. }) - ) + && matches!(last_insts[0].instr.real_opcode(), Some(Opcode::LoadConst)) && is_load_const_none(&last_insts[0], metadata) - && matches!( - last_insts[1].instr, - AnyInstruction::Real(Instruction::ReturnValue) - ); + && matches!(last_insts[1].instr.real(), Some(Instruction::ReturnValue)); if !is_return_block { return; } @@ -8067,11 +7942,11 @@ fn duplicate_end_returns(blocks: &mut Vec, metadata: &CodeUnitMetadata) { let already_has_return = block.instructions.len() >= 2 && { let n = block.instructions.len(); matches!( - block.instructions[n - 2].instr, - AnyInstruction::Real(Instruction::LoadConst { .. }) + block.instructions[n - 2].instr.real_opcode(), + Some(Opcode::LoadConst) ) && matches!( - block.instructions[n - 1].instr, - AnyInstruction::Real(Instruction::ReturnValue) + block.instructions[n - 1].instr.real(), + Some(Instruction::ReturnValue) ) }; if !block.except_handler @@ -8204,12 +8079,12 @@ fn is_named_except_cleanup_return_block(block: &Block, metadata: &CodeUnitMetada if matches!(pop_except.instr.real(), Some(Instruction::PopExcept)) && is_load_const_none(load_none1, metadata) && matches!( - store.instr.real(), - Some(Instruction::StoreFast { .. } | Instruction::StoreName { .. }) + store.instr.real_opcode(), + Some(Opcode::StoreFast | Opcode::StoreName ) ) && matches!( - delete.instr.real(), - Some(Instruction::DeleteFast { .. } | Instruction::DeleteName { .. }) + delete.instr.real_opcode(), + Some(Opcode::DeleteFast | Opcode::DeleteName ) ) && is_load_const_none(load_none2, metadata) && matches!(ret.instr.real(), Some(Instruction::ReturnValue)) @@ -8283,8 +8158,8 @@ fn duplicate_named_except_cleanup_returns(blocks: &mut Vec, metadata: &Co fn is_const_return_block(block: &Block) -> bool { block.instructions.len() == 2 && matches!( - block.instructions[0].instr.real(), - Some(Instruction::LoadConst { .. }) + block.instructions[0].instr.real_opcode(), + Some(Opcode::LoadConst) ) && matches!( block.instructions[1].instr.real(), @@ -8367,11 +8242,8 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { if is_push { // Determine preserve_lasti from instruction type (push_except_block) let preserve_lasti = matches!( - blocks[bi].instructions[i].instr.pseudo(), - Some( - PseudoInstruction::SetupWith { .. } - | PseudoInstruction::SetupCleanup { .. } - ) + blocks[bi].instructions[i].instr.pseudo_opcode(), + Some(PseudoOpcode::SetupWith | PseudoOpcode::SetupCleanup) ); // Set preserve_lasti on handler block @@ -8421,9 +8293,7 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) { // - plain yield outside try: depth=1 → DEPTH1 set // - yield inside try: depth=2+ → no DEPTH1 // - yield-from/await: has internal SETUP_FINALLY → depth=2+ → no DEPTH1 - if let Some(Instruction::YieldValue { .. }) = - blocks[bi].instructions[i].instr.real() - { + if let Some(Opcode::YieldValue) = blocks[bi].instructions[i].instr.real_opcode() { last_yield_except_depth = stack.len() as i32; } diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 5aec1cced96..3a0e9a2132b 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -1344,12 +1344,14 @@ define_opcodes!( ); impl PseudoInstruction { - /// Returns true if this is a block push pseudo instruction - /// (SETUP_FINALLY, SETUP_CLEANUP, or SETUP_WITH). - pub fn is_block_push(&self) -> bool { + /// Returns true if self is one of: + /// - [`PseudoInstruction::SetupCleanup`] + /// - [`PseudoInstruction::SetupFinally`] + /// - [`PseudoInstruction::SetupWith`] + pub const fn is_block_push(&self) -> bool { matches!( - self, - Self::SetupCleanup { .. } | Self::SetupFinally { .. } | Self::SetupWith { .. } + self.opcode(), + PseudoOpcode::SetupCleanup | PseudoOpcode::SetupFinally | PseudoOpcode::SetupWith ) } } @@ -1527,7 +1529,7 @@ impl InstructionMetadata for AnyInstruction { } impl AnyInstruction { - /// Gets the inner value of [`Self::Real`]. + /// Inner value of [`Self::Real`]. pub const fn real(self) -> Option { match self { Self::Real(ins) => Some(ins), @@ -1535,7 +1537,7 @@ impl AnyInstruction { } } - /// Gets the inner value of [`Self::Pseudo`]. + /// Inner value of [`Self::Pseudo`]. pub const fn pseudo(self) -> Option { match self { Self::Pseudo(ins) => Some(ins), @@ -1543,6 +1545,22 @@ impl AnyInstruction { } } + /// Get [`Self::Real`] as [`Opcode`]. + pub const fn real_opcode(self) -> Option { + match self.real() { + Some(ins) => Some(ins.opcode()), + _ => None, + } + } + + /// Get [`Self::Pseudo`] as [`PseudoOpcode`]. + pub const fn pseudo_opcode(self) -> Option { + match self.pseudo() { + Some(ins) => Some(ins.opcode()), + _ => None, + } + } + /// Same as [`Self::real`] but panics if wasn't called on [`Self::Real`]. /// /// # Panics @@ -1563,15 +1581,14 @@ impl AnyInstruction { .expect("Expected AnyInstruction::Pseudo, found AnyInstruction::Real") } - /// Returns true if this is a block push pseudo instruction - /// (SETUP_FINALLY, SETUP_CLEANUP, or SETUP_WITH). - pub fn is_block_push(&self) -> bool { - matches!(self, Self::Pseudo(p) if p.is_block_push()) + /// Returns true if this is a [`PseudoInstruction::PopBlock`]. + pub const fn is_pop_block(self) -> bool { + matches!(self, Self::Pseudo(PseudoInstruction::PopBlock)) } - /// Returns true if this is a POP_BLOCK pseudo instruction. - pub fn is_pop_block(&self) -> bool { - matches!(self, Self::Pseudo(PseudoInstruction::PopBlock)) + /// See [`PseudoInstruction::is_block_push`]. + pub const fn is_block_push(self) -> bool { + matches!(self, Self::Pseudo(p) if p.is_block_push()) } }